Implement Phase 18: Hybrid Search (BM25 + Vector)

- FastEmbedSparse(Qdrant/bm25) 기반 sparse 임베딩 추가 (fastembed 패키지)
- IngestionService: HYBRID_SEARCH_ENABLED 시 dense + sparse 동시 저장 (RetrievalMode.HYBRID)
  - _ensure_collection_schema(): sparse vector 미설정 컬렉션 자동 삭제·재생성
- RetrieverService: hybrid 스토어 + dense 폴백 구조, Qdrant 내장 RRF로 결과 통합
- container.py: sparse_embeddings Singleton 프로바이더, ingestion/retriever 양쪽 주입
- .env.example: HYBRID_SEARCH_ENABLED, SPARSE_MODEL_ID 항목 추가

활성화: .env에 HYBRID_SEARCH_ENABLED=true 설정 후 기존 문서 재수집 필요

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

services/rag/ingestion_service.py

@@ -1,6 +1,6 @@
 from langchain_community.document_loaders import PDFPlumberLoader, TextLoader
 from langchain_experimental.text_splitter import SemanticChunker
-from langchain_qdrant import QdrantVectorStore
+from langchain_qdrant import QdrantVectorStore, RetrievalMode
 from qdrant_client import QdrantClient
 from qdrant_client.models import Filter, FieldCondition, MatchValue, FilterSelector
 
@@ -15,10 +15,12 @@ class IngestionService:
         collection_name: str,
         breakpoint_threshold_type: str = "percentile",
         buffer_size: int = 1,
+        sparse_embeddings=None,
     ):
         self._embeddings = embeddings
         self._qdrant_url = qdrant_url
         self._collection_name = collection_name
+        self._sparse_embeddings = sparse_embeddings
         self._splitter = SemanticChunker(
             embeddings=embeddings,
             breakpoint_threshold_type=breakpoint_threshold_type,
@@ -26,6 +28,18 @@ class IngestionService:
         )
         self._client = QdrantClient(url=qdrant_url)
 
+    def _ensure_collection_schema(self) -> None:
+        """Hybrid 모드 전환 시 컬렉션에 sparse vector 설정이 없으면 삭제해 재생성을 유도한다."""
+        if not self._sparse_embeddings:
+            return
+        try:
+            info = self._client.get_collection(self._collection_name)
+            if not info.config.params.sparse_vectors:
+                print(f"[Hybrid] '{self._collection_name}' 컬렉션에 sparse vector 설정이 없어 재생성합니다.")
+                self._client.delete_collection(self._collection_name)
+        except Exception:
+            pass  # 컬렉션 미존재 시 무시
+
     def _delete_by_source(self, source_path: str) -> None:
         """같은 파일 경로로 저장된 기존 청크를 모두 삭제한다."""
         try:
@@ -46,6 +60,7 @@ class IngestionService:
             pass  # 컬렉션이 없을 때(최초 수집) 무시
 
     def ingest(self, file_paths: list[str]) -> int:
+        self._ensure_collection_schema()
         docs = []
         for path in file_paths:
             self._delete_by_source(path)
@@ -53,10 +68,14 @@ class IngestionService:
             docs.extend(loader.load())
 
         chunks = self._splitter.split_documents(docs)
-        QdrantVectorStore.from_documents(
+        kwargs = dict(
             documents=chunks,
             embedding=self._embeddings,
             url=self._qdrant_url,
             collection_name=self._collection_name,
         )
+        if self._sparse_embeddings:
+            kwargs["sparse_embedding"] = self._sparse_embeddings
+            kwargs["retrieval_mode"] = RetrievalMode.HYBRID
+        QdrantVectorStore.from_documents(**kwargs)
         return len(chunks)

services/rag/retriever_service.py

@@ -1,5 +1,5 @@
 from langchain_core.documents import Document
-from langchain_qdrant import QdrantVectorStore
+from langchain_qdrant import QdrantVectorStore, RetrievalMode
 from qdrant_client import QdrantClient
 from qdrant_client.models import Filter, FieldCondition, MatchValue, FilterSelector
 
@@ -15,24 +15,47 @@ class RetrieverService:
         top_k: int,
         reranker=None,
         rerank_fetch_k: int = 10,
+        sparse_embeddings=None,
     ):
         self._client = QdrantClient(url=qdrant_url)
         self._collection_name = collection_name
-        self._store = QdrantVectorStore(
+        self._top_k = top_k
+        self._reranker = reranker
+        self._rerank_fetch_k = rerank_fetch_k
+        self._sparse_embeddings = sparse_embeddings
+
+        # Dense-only store — hybrid 실패 시 폴백으로도 사용
+        self._dense_store = QdrantVectorStore(
             client=self._client,
             collection_name=collection_name,
             embedding=embeddings,
         )
-        self._top_k = top_k
-        self._reranker = reranker
-        self._rerank_fetch_k = rerank_fetch_k
+
+        if sparse_embeddings:
+            self._store = QdrantVectorStore(
+                client=self._client,
+                collection_name=collection_name,
+                embedding=embeddings,
+                sparse_embedding=sparse_embeddings,
+                retrieval_mode=RetrievalMode.HYBRID,
+            )
+        else:
+            self._store = self._dense_store
 
     def as_retriever(self):
         return self._store.as_retriever(search_kwargs={"k": self._top_k})
 
     def search(self, query: str) -> list[Document]:
         fetch_k = self._rerank_fetch_k if self._reranker else self._top_k
-        docs = self._store.similarity_search(query, k=fetch_k)
+        try:
+            docs = self._store.similarity_search(query, k=fetch_k)
+        except Exception as e:
+            if self._sparse_embeddings:
+                # 컬렉션에 sparse vector 없음 → dense 폴백 (재수집 필요)
+                print(f"[Hybrid] 검색 실패, dense 폴백 (문서 재수집 필요): {e}")
+                docs = self._dense_store.similarity_search(query, k=fetch_k)
+            else:
+                raise
         if self._reranker:
             docs = self._reranker.rerank(query, docs, top_k=self._top_k)
         return docs