biblicus 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

biblicus/backends/__init__.py

@@ -7,8 +7,10 @@ from __future__ import annotations
 from typing import Dict, Type
 
 from .base import RetrievalBackend
+from .hybrid import HybridBackend
 from .scan import ScanBackend
 from .sqlite_full_text_search import SqliteFullTextSearchBackend
+from .vector import VectorBackend
 
 
 def available_backends() -> Dict[str, Type[RetrievalBackend]]:
@@ -19,8 +21,10 @@ def available_backends() -> Dict[str, Type[RetrievalBackend]]:
     :rtype: dict[str, Type[RetrievalBackend]]
     """
     return {
+        HybridBackend.backend_id: HybridBackend,
         ScanBackend.backend_id: ScanBackend,
         SqliteFullTextSearchBackend.backend_id: SqliteFullTextSearchBackend,
+        VectorBackend.backend_id: VectorBackend,
     }
 
 

biblicus/backends/hybrid.py

@@ -0,0 +1,284 @@
+"""
+Hybrid retrieval backend combining lexical and vector results.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest
+from ..time import utc_now_iso
+
+
+class HybridRecipeConfig(BaseModel):
+    """
+    Configuration for hybrid retrieval fusion.
+
+    :ivar lexical_backend: Backend identifier for lexical retrieval.
+    :vartype lexical_backend: str
+    :ivar embedding_backend: Backend identifier for embedding retrieval.
+    :vartype embedding_backend: str
+    :ivar lexical_weight: Weight for lexical scores.
+    :vartype lexical_weight: float
+    :ivar embedding_weight: Weight for embedding scores.
+    :vartype embedding_weight: float
+    :ivar lexical_config: Optional lexical backend configuration.
+    :vartype lexical_config: dict[str, object]
+    :ivar embedding_config: Optional embedding backend configuration.
+    :vartype embedding_config: dict[str, object]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    lexical_backend: str = Field(default="sqlite-full-text-search", min_length=1)
+    embedding_backend: str = Field(default="vector", min_length=1)
+    lexical_weight: float = Field(default=0.5, ge=0, le=1)
+    embedding_weight: float = Field(default=0.5, ge=0, le=1)
+    lexical_config: Dict[str, object] = Field(default_factory=dict)
+    embedding_config: Dict[str, object] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate_weights(self) -> "HybridRecipeConfig":
+        if abs((self.lexical_weight + self.embedding_weight) - 1.0) > 1e-6:
+            raise ValueError("weights must sum to 1")
+        return self
+
+
+class HybridBackend:
+    """
+    Hybrid backend that fuses lexical and embedding retrieval.
+
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    """
+
+    backend_id = "hybrid"
+
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
+        """
+        Build or register a hybrid retrieval run.
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+        recipe_config = HybridRecipeConfig.model_validate(config)
+        _ensure_backend_supported(recipe_config)
+        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
+        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
+        lexical_run = lexical_backend.build_run(
+            corpus, recipe_name=f"{recipe_name}-lexical", config=recipe_config.lexical_config
+        )
+        embedding_run = embedding_backend.build_run(
+            corpus, recipe_name=f"{recipe_name}-embedding", config=recipe_config.embedding_config
+        )
+        recipe = create_recipe_manifest(
+            backend_id=self.backend_id,
+            name=recipe_name,
+            config=recipe_config.model_dump(),
+        )
+        stats = {
+            "lexical_run_id": lexical_run.run_id,
+            "embedding_run_id": embedding_run.run_id,
+        }
+        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
+        corpus.write_run(run)
+        return run
+
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Query using both lexical and embedding backends and fuse scores.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+        recipe_config = HybridRecipeConfig.model_validate(run.recipe.config)
+        _ensure_backend_supported(recipe_config)
+        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
+        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
+        lexical_run_id = run.stats.get("lexical_run_id")
+        embedding_run_id = run.stats.get("embedding_run_id")
+        if not lexical_run_id or not embedding_run_id:
+            raise ValueError("Hybrid run missing lexical or embedding run identifiers")
+        lexical_run = corpus.load_run(str(lexical_run_id))
+        embedding_run = corpus.load_run(str(embedding_run_id))
+        component_budget = _expand_component_budget(budget)
+        lexical_result = lexical_backend.query(
+            corpus, run=lexical_run, query_text=query_text, budget=component_budget
+        )
+        embedding_result = embedding_backend.query(
+            corpus, run=embedding_run, query_text=query_text, budget=component_budget
+        )
+        candidates = _fuse_evidence(
+            lexical_result.evidence,
+            embedding_result.evidence,
+            lexical_weight=recipe_config.lexical_weight,
+            embedding_weight=recipe_config.embedding_weight,
+        )
+        sorted_candidates = sorted(
+            candidates,
+            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        )
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(sorted_candidates, start=1)
+        ]
+        evidence = apply_budget(ranked, budget)
+        stats = {
+            "candidates": len(sorted_candidates),
+            "returned": len(evidence),
+            "fusion_weights": {
+                "lexical": recipe_config.lexical_weight,
+                "embedding": recipe_config.embedding_weight,
+            },
+        }
+        return RetrievalResult(
+            query_text=query_text,
+            budget=budget,
+            run_id=run.run_id,
+            recipe_id=run.recipe.recipe_id,
+            backend_id=self.backend_id,
+            generated_at=utc_now_iso(),
+            evidence=evidence,
+            stats=stats,
+        )
+
+
+def _ensure_backend_supported(recipe_config: HybridRecipeConfig) -> None:
+    """
+    Validate that hybrid backends do not reference the hybrid backend itself.
+
+    :param recipe_config: Parsed hybrid recipe configuration.
+    :type recipe_config: HybridRecipeConfig
+    :return: None.
+    :rtype: None
+    :raises ValueError: If hybrid is used as a component backend.
+    """
+    if recipe_config.lexical_backend == HybridBackend.backend_id:
+        raise ValueError("Hybrid backend cannot use itself as the lexical backend")
+    if recipe_config.embedding_backend == HybridBackend.backend_id:
+        raise ValueError("Hybrid backend cannot use itself as the embedding backend")
+
+
+def _resolve_backend(backend_id: str):
+    """
+    Resolve a backend by identifier.
+
+    :param backend_id: Backend identifier.
+    :type backend_id: str
+    :return: Backend instance.
+    :rtype: object
+    """
+    from . import get_backend
+
+    return get_backend(backend_id)
+
+
+def _expand_component_budget(budget: QueryBudget, *, multiplier: int = 5) -> QueryBudget:
+    """
+    Expand a final budget to collect more candidates for fusion.
+
+    :param budget: Final evidence budget.
+    :type budget: QueryBudget
+    :param multiplier: Candidate expansion multiplier.
+    :type multiplier: int
+    :return: Expanded budget for component backends.
+    :rtype: QueryBudget
+    """
+    max_total_characters = budget.max_total_characters
+    expanded_characters = (
+        max_total_characters * multiplier if max_total_characters is not None else None
+    )
+    return QueryBudget(
+        max_total_items=budget.max_total_items * multiplier,
+        max_total_characters=expanded_characters,
+        max_items_per_source=budget.max_items_per_source,
+    )
+
+
+def _fuse_evidence(
+    lexical: List[Evidence],
+    embedding: List[Evidence],
+    *,
+    lexical_weight: float,
+    embedding_weight: float,
+) -> List[Evidence]:
+    """
+    Fuse lexical and embedding evidence lists into hybrid candidates.
+
+    :param lexical: Lexical evidence list.
+    :type lexical: list[Evidence]
+    :param embedding: Embedding evidence list.
+    :type embedding: list[Evidence]
+    :param lexical_weight: Lexical score weight.
+    :type lexical_weight: float
+    :param embedding_weight: Embedding score weight.
+    :type embedding_weight: float
+    :return: Hybrid evidence list.
+    :rtype: list[Evidence]
+    """
+    merged: Dict[str, Dict[str, Optional[Evidence]]] = {}
+    for evidence_item in lexical:
+        merged.setdefault(evidence_item.item_id, {})["lexical"] = evidence_item
+    for evidence_item in embedding:
+        merged.setdefault(evidence_item.item_id, {})["embedding"] = evidence_item
+
+    candidates: List[Evidence] = []
+    for item_id, sources in merged.items():
+        lexical_evidence = sources.get("lexical")
+        embedding_evidence = sources.get("embedding")
+        lexical_score = lexical_evidence.score if lexical_evidence else 0.0
+        embedding_score = embedding_evidence.score if embedding_evidence else 0.0
+        combined_score = (lexical_score * lexical_weight) + (embedding_score * embedding_weight)
+        base_evidence = lexical_evidence or embedding_evidence
+        candidates.append(
+            Evidence(
+                item_id=item_id,
+                source_uri=base_evidence.source_uri,
+                media_type=base_evidence.media_type,
+                score=combined_score,
+                rank=1,
+                text=base_evidence.text,
+                content_ref=base_evidence.content_ref,
+                span_start=base_evidence.span_start,
+                span_end=base_evidence.span_end,
+                stage="hybrid",
+                stage_scores={"lexical": lexical_score, "embedding": embedding_score},
+                recipe_id="",
+                run_id="",
+                hash=base_evidence.hash,
+            )
+        )
+    return candidates

biblicus/backends/sqlite_full_text_search.py

@@ -6,9 +6,9 @@ from __future__ import annotations
 
 import sqlite3
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from ..constants import CORPUS_DIR_NAME, RUNS_DIR_NAME
 from ..corpus import Corpus
@@ -35,6 +35,28 @@ class SqliteFullTextSearchRecipeConfig(BaseModel):
     :vartype chunk_overlap: int
     :ivar snippet_characters: Maximum characters to include in evidence snippets.
     :vartype snippet_characters: int
+    :ivar bm25_k1: BM25 k1 tuning parameter.
+    :vartype bm25_k1: float
+    :ivar bm25_b: BM25 b tuning parameter.
+    :vartype bm25_b: float
+    :ivar ngram_min: Minimum n-gram size for lexical tuning.
+    :vartype ngram_min: int
+    :ivar ngram_max: Maximum n-gram size for lexical tuning.
+    :vartype ngram_max: int
+    :ivar stop_words: Optional stop word policy or list.
+    :vartype stop_words: str or list[str] or None
+    :ivar field_weight_title: Relative weight for title field matches.
+    :vartype field_weight_title: float
+    :ivar field_weight_body: Relative weight for body field matches.
+    :vartype field_weight_body: float
+    :ivar field_weight_tags: Relative weight for tag field matches.
+    :vartype field_weight_tags: float
+    :ivar rerank_enabled: Whether to apply reranking to retrieved candidates.
+    :vartype rerank_enabled: bool
+    :ivar rerank_model: Reranker model identifier for metadata.
+    :vartype rerank_model: str or None
+    :ivar rerank_top_k: Number of candidates to rerank.
+    :vartype rerank_top_k: int
     :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
     :vartype extraction_run: str or None
     """
@@ -44,8 +66,81 @@ class SqliteFullTextSearchRecipeConfig(BaseModel):
     chunk_size: int = Field(default=800, ge=1)
     chunk_overlap: int = Field(default=200, ge=0)
     snippet_characters: int = Field(default=400, ge=1)
+    bm25_k1: float = Field(default=1.2, gt=0)
+    bm25_b: float = Field(default=0.75, ge=0, le=1)
+    ngram_min: int = Field(default=1, ge=1)
+    ngram_max: int = Field(default=1, ge=1)
+    stop_words: Optional[Union[str, List[str]]] = None
+    field_weight_title: float = Field(default=1.0, ge=0)
+    field_weight_body: float = Field(default=1.0, ge=0)
+    field_weight_tags: float = Field(default=1.0, ge=0)
+    rerank_enabled: bool = False
+    rerank_model: Optional[str] = None
+    rerank_top_k: int = Field(default=10, ge=1)
     extraction_run: Optional[str] = None
 
+    @field_validator("stop_words")
+    @classmethod
+    def _validate_stop_words(
+        cls, value: Optional[Union[str, List[str]]]
+    ) -> Optional[Union[str, List[str]]]:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            if value.lower() == "english":
+                return "english"
+            raise ValueError("stop_words must be 'english' or a list of strings")
+        if not value:
+            raise ValueError("stop_words list must not be empty")
+        if any(not isinstance(token, str) or not token.strip() for token in value):
+            raise ValueError("stop_words list must contain non-empty strings")
+        return value
+
+    @model_validator(mode="after")
+    def _validate_ngram_range(self) -> "SqliteFullTextSearchRecipeConfig":
+        if self.ngram_min > self.ngram_max:
+            raise ValueError("Invalid ngram range: ngram_min must be <= ngram_max")
+        if self.rerank_enabled and not self.rerank_model:
+            raise ValueError("Rerank enabled requires rerank_model")
+        return self
+
+
+_ENGLISH_STOP_WORDS: Set[str] = {
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "but",
+    "by",
+    "for",
+    "if",
+    "in",
+    "into",
+    "is",
+    "it",
+    "no",
+    "not",
+    "of",
+    "on",
+    "or",
+    "such",
+    "that",
+    "the",
+    "their",
+    "then",
+    "there",
+    "these",
+    "they",
+    "this",
+    "to",
+    "was",
+    "will",
+    "with",
+}
+
 
 class SqliteFullTextSearchBackend:
     """
@@ -118,29 +213,39 @@ class SqliteFullTextSearchBackend:
         :rtype: RetrievalResult
         """
         recipe_config = SqliteFullTextSearchRecipeConfig.model_validate(run.recipe.config)
+        query_tokens = _tokenize_query(query_text)
+        stop_words = _resolve_stop_words(recipe_config.stop_words)
+        filtered_tokens = _apply_stop_words(query_tokens, stop_words)
+        if not filtered_tokens:
+            return RetrievalResult(
+                query_text=query_text,
+                budget=budget,
+                run_id=run.run_id,
+                recipe_id=run.recipe.recipe_id,
+                backend_id=self.backend_id,
+                generated_at=utc_now_iso(),
+                evidence=[],
+                stats={"candidates": 0, "returned": 0},
+            )
         db_path = _resolve_run_db_path(corpus, run)
         candidates = _query_full_text_search_index(
             db_path=db_path,
-            query_text=query_text,
+            query_text=" ".join(filtered_tokens),
             limit=_candidate_limit(budget.max_total_items),
             snippet_characters=recipe_config.snippet_characters,
         )
-        sorted_candidates = sorted(
-            candidates,
-            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        sorted_candidates = _rank_candidates(candidates)
+        evidence = _apply_rerank_if_enabled(
+            sorted_candidates,
+            query_tokens=filtered_tokens,
+            run=run,
+            budget=budget,
+            rerank_enabled=recipe_config.rerank_enabled,
+            rerank_top_k=recipe_config.rerank_top_k,
         )
-        ranked = [
-            evidence_item.model_copy(
-                update={
-                    "rank": index,
-                    "recipe_id": run.recipe.recipe_id,
-                    "run_id": run.run_id,
-                }
-            )
-            for index, evidence_item in enumerate(sorted_candidates, start=1)
-        ]
-        evidence = apply_budget(ranked, budget)
-        stats = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        stats: Dict[str, object] = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        if recipe_config.rerank_enabled:
+            stats["reranked_candidates"] = min(len(sorted_candidates), recipe_config.rerank_top_k)
         return RetrievalResult(
             query_text=query_text,
             budget=budget,
@@ -165,6 +270,147 @@ def _candidate_limit(max_total_items: int) -> int:
     return max_total_items * 5
 
 
+def _tokenize_query(query_text: str) -> List[str]:
+    """
+    Tokenize a query string into lowercased terms.
+
+    :param query_text: Raw query text.
+    :type query_text: str
+    :return: Token list.
+    :rtype: list[str]
+    """
+    return [token for token in query_text.lower().split() if token]
+
+
+def _resolve_stop_words(value: Optional[Union[str, List[str]]]) -> Set[str]:
+    """
+    Resolve stop words based on a configuration value.
+
+    :param value: Stop word configuration.
+    :type value: str or list[str] or None
+    :return: Stop word set.
+    :rtype: set[str]
+    """
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        return set(_ENGLISH_STOP_WORDS)
+    return {token.strip().lower() for token in value if token.strip()}
+
+
+def _apply_stop_words(tokens: List[str], stop_words: Set[str]) -> List[str]:
+    """
+    Filter query tokens by a stop word list.
+
+    :param tokens: Token list.
+    :type tokens: list[str]
+    :param stop_words: Stop word set.
+    :type stop_words: set[str]
+    :return: Filtered token list.
+    :rtype: list[str]
+    """
+    if not stop_words:
+        return tokens
+    return [token for token in tokens if token not in stop_words]
+
+
+def _rank_candidates(candidates: List[Evidence]) -> List[Evidence]:
+    """
+    Sort evidence candidates by descending score.
+
+    :param candidates: Evidence list to sort.
+    :type candidates: list[Evidence]
+    :return: Sorted evidence list.
+    :rtype: list[Evidence]
+    """
+    return sorted(
+        candidates, key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id)
+    )
+
+
+def _rerank_score(text: str, query_tokens: List[str]) -> float:
+    """
+    Compute a simple rerank score using token overlap.
+
+    :param text: Candidate text.
+    :type text: str
+    :param query_tokens: Query tokens.
+    :type query_tokens: list[str]
+    :return: Rerank score.
+    :rtype: float
+    """
+    lower_text = text.lower() if text else ""
+    return float(sum(1 for token in query_tokens if token in lower_text))
+
+
+def _apply_rerank_if_enabled(
+    candidates: List[Evidence],
+    *,
+    query_tokens: List[str],
+    run: RetrievalRun,
+    budget: QueryBudget,
+    rerank_enabled: bool,
+    rerank_top_k: int,
+) -> List[Evidence]:
+    """
+    Rerank candidates when enabled, otherwise apply the budget to ranked results.
+
+    :param candidates: Ranked candidate evidence.
+    :type candidates: list[Evidence]
+    :param query_tokens: Query tokens used for reranking.
+    :type query_tokens: list[str]
+    :param run: Retrieval run to annotate evidence with.
+    :type run: RetrievalRun
+    :param budget: Evidence selection budget.
+    :type budget: QueryBudget
+    :param rerank_enabled: Whether reranking is enabled.
+    :type rerank_enabled: bool
+    :param rerank_top_k: Maximum candidates to rerank.
+    :type rerank_top_k: int
+    :return: Evidence list respecting budget.
+    :rtype: list[Evidence]
+    """
+    if not rerank_enabled:
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(candidates, start=1)
+        ]
+        return apply_budget(ranked, budget)
+
+    rerank_limit = min(len(candidates), rerank_top_k)
+    rerank_candidates = candidates[:rerank_limit]
+    reranked: List[Evidence] = []
+    for evidence_item in rerank_candidates:
+        rerank_score = _rerank_score(evidence_item.text or "", query_tokens)
+        reranked.append(
+            evidence_item.model_copy(
+                update={
+                    "score": rerank_score,
+                    "stage": "rerank",
+                    "stage_scores": {"retrieve": evidence_item.score, "rerank": rerank_score},
+                }
+            )
+        )
+    reranked_sorted = _rank_candidates(reranked)
+    ranked = [
+        evidence_item.model_copy(
+            update={
+                "rank": index,
+                "recipe_id": run.recipe.recipe_id,
+                "run_id": run.run_id,
+            }
+        )
+        for index, evidence_item in enumerate(reranked_sorted, start=1)
+    ]
+    return apply_budget(ranked, budget)
+
+
 def _resolve_run_db_path(corpus: Corpus, run: RetrievalRun) -> Path:
     """
     Resolve the SQLite index path for a retrieval run.
@@ -213,8 +459,7 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
     :return: None.
     :rtype: None
     """
-    conn.execute(
-        """
+    conn.execute("""
         CREATE VIRTUAL TABLE chunks_full_text_search USING fts5(
             content,
             item_id UNINDEXED,
@@ -225,8 +470,7 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
             start_offset UNINDEXED,
             end_offset UNINDEXED
         )
-        """
-    )
+        """)
 
 
 def _build_full_text_search_index(