shinalok
86370f6c1e
- FastEmbedSparse(Qdrant/bm25) 기반 sparse 임베딩 추가 (fastembed 패키지) - IngestionService: HYBRID_SEARCH_ENABLED 시 dense + sparse 동시 저장 (RetrievalMode.HYBRID) - _ensure_collection_schema(): sparse vector 미설정 컬렉션 자동 삭제·재생성 - RetrieverService: hybrid 스토어 + dense 폴백 구조, Qdrant 내장 RRF로 결과 통합 - container.py: sparse_embeddings Singleton 프로바이더, ingestion/retriever 양쪽 주입 - .env.example: HYBRID_SEARCH_ENABLED, SPARSE_MODEL_ID 항목 추가 활성화: .env에 HYBRID_SEARCH_ENABLED=true 설정 후 기존 문서 재수집 필요 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
from langchain_core.documents import Document
|
|
from langchain_qdrant import QdrantVectorStore, RetrievalMode
|
|
from qdrant_client import QdrantClient
|
|
from qdrant_client.models import Filter, FieldCondition, MatchValue, FilterSelector
|
|
|
|
|
|
class RetrieverService:
|
|
"""Qdrant 벡터 검색 서비스. LangGraph Tool 및 직접 검색 모두 지원."""
|
|
|
|
def __init__(
|
|
self,
|
|
embeddings,
|
|
qdrant_url: str,
|
|
collection_name: str,
|
|
top_k: int,
|
|
reranker=None,
|
|
rerank_fetch_k: int = 10,
|
|
sparse_embeddings=None,
|
|
):
|
|
self._client = QdrantClient(url=qdrant_url)
|
|
self._collection_name = collection_name
|
|
self._top_k = top_k
|
|
self._reranker = reranker
|
|
self._rerank_fetch_k = rerank_fetch_k
|
|
self._sparse_embeddings = sparse_embeddings
|
|
|
|
# Dense-only store — hybrid 실패 시 폴백으로도 사용
|
|
self._dense_store = QdrantVectorStore(
|
|
client=self._client,
|
|
collection_name=collection_name,
|
|
embedding=embeddings,
|
|
)
|
|
|
|
if sparse_embeddings:
|
|
self._store = QdrantVectorStore(
|
|
client=self._client,
|
|
collection_name=collection_name,
|
|
embedding=embeddings,
|
|
sparse_embedding=sparse_embeddings,
|
|
retrieval_mode=RetrievalMode.HYBRID,
|
|
)
|
|
else:
|
|
self._store = self._dense_store
|
|
|
|
def as_retriever(self):
|
|
return self._store.as_retriever(search_kwargs={"k": self._top_k})
|
|
|
|
def search(self, query: str) -> list[Document]:
|
|
fetch_k = self._rerank_fetch_k if self._reranker else self._top_k
|
|
try:
|
|
docs = self._store.similarity_search(query, k=fetch_k)
|
|
except Exception as e:
|
|
if self._sparse_embeddings:
|
|
# 컬렉션에 sparse vector 없음 → dense 폴백 (재수집 필요)
|
|
print(f"[Hybrid] 검색 실패, dense 폴백 (문서 재수집 필요): {e}")
|
|
docs = self._dense_store.similarity_search(query, k=fetch_k)
|
|
else:
|
|
raise
|
|
if self._reranker:
|
|
docs = self._reranker.rerank(query, docs, top_k=self._top_k)
|
|
return docs
|
|
|
|
def list_documents(self) -> list[str]:
|
|
"""Qdrant에 저장된 고유 파일 경로 목록을 반환한다."""
|
|
sources: set[str] = set()
|
|
offset = None
|
|
while True:
|
|
results, next_offset = self._client.scroll(
|
|
collection_name=self._collection_name,
|
|
with_payload=True,
|
|
limit=200,
|
|
offset=offset,
|
|
)
|
|
for point in results:
|
|
src = (point.payload or {}).get("metadata", {}).get("source", "")
|
|
if src:
|
|
sources.add(src)
|
|
if next_offset is None:
|
|
break
|
|
offset = next_offset
|
|
return sorted(sources)
|
|
|
|
def delete_document(self, source: str) -> None:
|
|
"""파일 경로로 저장된 모든 청크를 Qdrant에서 삭제한다."""
|
|
try:
|
|
self._client.delete(
|
|
collection_name=self._collection_name,
|
|
points_selector=FilterSelector(
|
|
filter=Filter(
|
|
must=[FieldCondition(
|
|
key="metadata.source",
|
|
match=MatchValue(value=source),
|
|
)]
|
|
)
|
|
),
|
|
)
|
|
except Exception:
|
|
pass
|