Implement Phase 18: Hybrid Search (BM25 + Vector)
- FastEmbedSparse(Qdrant/bm25) 기반 sparse 임베딩 추가 (fastembed 패키지) - IngestionService: HYBRID_SEARCH_ENABLED 시 dense + sparse 동시 저장 (RetrievalMode.HYBRID) - _ensure_collection_schema(): sparse vector 미설정 컬렉션 자동 삭제·재생성 - RetrieverService: hybrid 스토어 + dense 폴백 구조, Qdrant 내장 RRF로 결과 통합 - container.py: sparse_embeddings Singleton 프로바이더, ingestion/retriever 양쪽 주입 - .env.example: HYBRID_SEARCH_ENABLED, SPARSE_MODEL_ID 항목 추가 활성화: .env에 HYBRID_SEARCH_ENABLED=true 설정 후 기존 문서 재수집 필요 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
from langchain_core.documents import Document
|
||||
from langchain_qdrant import QdrantVectorStore
|
||||
from langchain_qdrant import QdrantVectorStore, RetrievalMode
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import Filter, FieldCondition, MatchValue, FilterSelector
|
||||
|
||||
@@ -15,24 +15,47 @@ class RetrieverService:
|
||||
top_k: int,
|
||||
reranker=None,
|
||||
rerank_fetch_k: int = 10,
|
||||
sparse_embeddings=None,
|
||||
):
|
||||
self._client = QdrantClient(url=qdrant_url)
|
||||
self._collection_name = collection_name
|
||||
self._store = QdrantVectorStore(
|
||||
self._top_k = top_k
|
||||
self._reranker = reranker
|
||||
self._rerank_fetch_k = rerank_fetch_k
|
||||
self._sparse_embeddings = sparse_embeddings
|
||||
|
||||
# Dense-only store — hybrid 실패 시 폴백으로도 사용
|
||||
self._dense_store = QdrantVectorStore(
|
||||
client=self._client,
|
||||
collection_name=collection_name,
|
||||
embedding=embeddings,
|
||||
)
|
||||
self._top_k = top_k
|
||||
self._reranker = reranker
|
||||
self._rerank_fetch_k = rerank_fetch_k
|
||||
|
||||
if sparse_embeddings:
|
||||
self._store = QdrantVectorStore(
|
||||
client=self._client,
|
||||
collection_name=collection_name,
|
||||
embedding=embeddings,
|
||||
sparse_embedding=sparse_embeddings,
|
||||
retrieval_mode=RetrievalMode.HYBRID,
|
||||
)
|
||||
else:
|
||||
self._store = self._dense_store
|
||||
|
||||
def as_retriever(self):
|
||||
return self._store.as_retriever(search_kwargs={"k": self._top_k})
|
||||
|
||||
def search(self, query: str) -> list[Document]:
|
||||
fetch_k = self._rerank_fetch_k if self._reranker else self._top_k
|
||||
docs = self._store.similarity_search(query, k=fetch_k)
|
||||
try:
|
||||
docs = self._store.similarity_search(query, k=fetch_k)
|
||||
except Exception as e:
|
||||
if self._sparse_embeddings:
|
||||
# 컬렉션에 sparse vector 없음 → dense 폴백 (재수집 필요)
|
||||
print(f"[Hybrid] 검색 실패, dense 폴백 (문서 재수집 필요): {e}")
|
||||
docs = self._dense_store.similarity_search(query, k=fetch_k)
|
||||
else:
|
||||
raise
|
||||
if self._reranker:
|
||||
docs = self._reranker.rerank(query, docs, top_k=self._top_k)
|
||||
return docs
|
||||
|
||||
Reference in New Issue
Block a user