biblicus 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

biblicus/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.10.0"
+__version__ = "0.11.0"

biblicus/analysis/profiling.py

@@ -266,7 +266,7 @@ def _build_extracted_text_report(
             empty_items += 1
             continue
         nonempty_items += 1
-        text_lengths.append(len(text_value))
+        text_lengths.append(len(stripped))
 
     sampled_lengths = _apply_sample(text_lengths, config.sample_size)
     characters_distribution = _build_distribution(sampled_lengths, config.percentiles)

biblicus/backends/__init__.py

@@ -7,8 +7,10 @@ from __future__ import annotations
 from typing import Dict, Type
 
 from .base import RetrievalBackend
+from .hybrid import HybridBackend
 from .scan import ScanBackend
 from .sqlite_full_text_search import SqliteFullTextSearchBackend
+from .vector import VectorBackend
 
 
 def available_backends() -> Dict[str, Type[RetrievalBackend]]:
@@ -19,8 +21,10 @@ def available_backends() -> Dict[str, Type[RetrievalBackend]]:
     :rtype: dict[str, Type[RetrievalBackend]]
     """
     return {
+        HybridBackend.backend_id: HybridBackend,
         ScanBackend.backend_id: ScanBackend,
         SqliteFullTextSearchBackend.backend_id: SqliteFullTextSearchBackend,
+        VectorBackend.backend_id: VectorBackend,
     }
 
 

biblicus/backends/hybrid.py

@@ -0,0 +1,284 @@
+"""
+Hybrid retrieval backend combining lexical and vector results.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest
+from ..time import utc_now_iso
+
+
+class HybridRecipeConfig(BaseModel):
+    """
+    Configuration for hybrid retrieval fusion.
+
+    :ivar lexical_backend: Backend identifier for lexical retrieval.
+    :vartype lexical_backend: str
+    :ivar embedding_backend: Backend identifier for embedding retrieval.
+    :vartype embedding_backend: str
+    :ivar lexical_weight: Weight for lexical scores.
+    :vartype lexical_weight: float
+    :ivar embedding_weight: Weight for embedding scores.
+    :vartype embedding_weight: float
+    :ivar lexical_config: Optional lexical backend configuration.
+    :vartype lexical_config: dict[str, object]
+    :ivar embedding_config: Optional embedding backend configuration.
+    :vartype embedding_config: dict[str, object]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    lexical_backend: str = Field(default="sqlite-full-text-search", min_length=1)
+    embedding_backend: str = Field(default="vector", min_length=1)
+    lexical_weight: float = Field(default=0.5, ge=0, le=1)
+    embedding_weight: float = Field(default=0.5, ge=0, le=1)
+    lexical_config: Dict[str, object] = Field(default_factory=dict)
+    embedding_config: Dict[str, object] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _validate_weights(self) -> "HybridRecipeConfig":
+        if abs((self.lexical_weight + self.embedding_weight) - 1.0) > 1e-6:
+            raise ValueError("weights must sum to 1")
+        return self
+
+
+class HybridBackend:
+    """
+    Hybrid backend that fuses lexical and embedding retrieval.
+
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    """
+
+    backend_id = "hybrid"
+
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
+        """
+        Build or register a hybrid retrieval run.
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+        recipe_config = HybridRecipeConfig.model_validate(config)
+        _ensure_backend_supported(recipe_config)
+        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
+        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
+        lexical_run = lexical_backend.build_run(
+            corpus, recipe_name=f"{recipe_name}-lexical", config=recipe_config.lexical_config
+        )
+        embedding_run = embedding_backend.build_run(
+            corpus, recipe_name=f"{recipe_name}-embedding", config=recipe_config.embedding_config
+        )
+        recipe = create_recipe_manifest(
+            backend_id=self.backend_id,
+            name=recipe_name,
+            config=recipe_config.model_dump(),
+        )
+        stats = {
+            "lexical_run_id": lexical_run.run_id,
+            "embedding_run_id": embedding_run.run_id,
+        }
+        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
+        corpus.write_run(run)
+        return run
+
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Query using both lexical and embedding backends and fuse scores.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+        recipe_config = HybridRecipeConfig.model_validate(run.recipe.config)
+        _ensure_backend_supported(recipe_config)
+        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
+        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
+        lexical_run_id = run.stats.get("lexical_run_id")
+        embedding_run_id = run.stats.get("embedding_run_id")
+        if not lexical_run_id or not embedding_run_id:
+            raise ValueError("Hybrid run missing lexical or embedding run identifiers")
+        lexical_run = corpus.load_run(str(lexical_run_id))
+        embedding_run = corpus.load_run(str(embedding_run_id))
+        component_budget = _expand_component_budget(budget)
+        lexical_result = lexical_backend.query(
+            corpus, run=lexical_run, query_text=query_text, budget=component_budget
+        )
+        embedding_result = embedding_backend.query(
+            corpus, run=embedding_run, query_text=query_text, budget=component_budget
+        )
+        candidates = _fuse_evidence(
+            lexical_result.evidence,
+            embedding_result.evidence,
+            lexical_weight=recipe_config.lexical_weight,
+            embedding_weight=recipe_config.embedding_weight,
+        )
+        sorted_candidates = sorted(
+            candidates,
+            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        )
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(sorted_candidates, start=1)
+        ]
+        evidence = apply_budget(ranked, budget)
+        stats = {
+            "candidates": len(sorted_candidates),
+            "returned": len(evidence),
+            "fusion_weights": {
+                "lexical": recipe_config.lexical_weight,
+                "embedding": recipe_config.embedding_weight,
+            },
+        }
+        return RetrievalResult(
+            query_text=query_text,
+            budget=budget,
+            run_id=run.run_id,
+            recipe_id=run.recipe.recipe_id,
+            backend_id=self.backend_id,
+            generated_at=utc_now_iso(),
+            evidence=evidence,
+            stats=stats,
+        )
+
+
+def _ensure_backend_supported(recipe_config: HybridRecipeConfig) -> None:
+    """
+    Validate that hybrid backends do not reference the hybrid backend itself.
+
+    :param recipe_config: Parsed hybrid recipe configuration.
+    :type recipe_config: HybridRecipeConfig
+    :return: None.
+    :rtype: None
+    :raises ValueError: If hybrid is used as a component backend.
+    """
+    if recipe_config.lexical_backend == HybridBackend.backend_id:
+        raise ValueError("Hybrid backend cannot use itself as the lexical backend")
+    if recipe_config.embedding_backend == HybridBackend.backend_id:
+        raise ValueError("Hybrid backend cannot use itself as the embedding backend")
+
+
+def _resolve_backend(backend_id: str):
+    """
+    Resolve a backend by identifier.
+
+    :param backend_id: Backend identifier.
+    :type backend_id: str
+    :return: Backend instance.
+    :rtype: object
+    """
+    from . import get_backend
+
+    return get_backend(backend_id)
+
+
+def _expand_component_budget(budget: QueryBudget, *, multiplier: int = 5) -> QueryBudget:
+    """
+    Expand a final budget to collect more candidates for fusion.
+
+    :param budget: Final evidence budget.
+    :type budget: QueryBudget
+    :param multiplier: Candidate expansion multiplier.
+    :type multiplier: int
+    :return: Expanded budget for component backends.
+    :rtype: QueryBudget
+    """
+    max_total_characters = budget.max_total_characters
+    expanded_characters = (
+        max_total_characters * multiplier if max_total_characters is not None else None
+    )
+    return QueryBudget(
+        max_total_items=budget.max_total_items * multiplier,
+        max_total_characters=expanded_characters,
+        max_items_per_source=budget.max_items_per_source,
+    )
+
+
+def _fuse_evidence(
+    lexical: List[Evidence],
+    embedding: List[Evidence],
+    *,
+    lexical_weight: float,
+    embedding_weight: float,
+) -> List[Evidence]:
+    """
+    Fuse lexical and embedding evidence lists into hybrid candidates.
+
+    :param lexical: Lexical evidence list.
+    :type lexical: list[Evidence]
+    :param embedding: Embedding evidence list.
+    :type embedding: list[Evidence]
+    :param lexical_weight: Lexical score weight.
+    :type lexical_weight: float
+    :param embedding_weight: Embedding score weight.
+    :type embedding_weight: float
+    :return: Hybrid evidence list.
+    :rtype: list[Evidence]
+    """
+    merged: Dict[str, Dict[str, Optional[Evidence]]] = {}
+    for evidence_item in lexical:
+        merged.setdefault(evidence_item.item_id, {})["lexical"] = evidence_item
+    for evidence_item in embedding:
+        merged.setdefault(evidence_item.item_id, {})["embedding"] = evidence_item
+
+    candidates: List[Evidence] = []
+    for item_id, sources in merged.items():
+        lexical_evidence = sources.get("lexical")
+        embedding_evidence = sources.get("embedding")
+        lexical_score = lexical_evidence.score if lexical_evidence else 0.0
+        embedding_score = embedding_evidence.score if embedding_evidence else 0.0
+        combined_score = (lexical_score * lexical_weight) + (embedding_score * embedding_weight)
+        base_evidence = lexical_evidence or embedding_evidence
+        candidates.append(
+            Evidence(
+                item_id=item_id,
+                source_uri=base_evidence.source_uri,
+                media_type=base_evidence.media_type,
+                score=combined_score,
+                rank=1,
+                text=base_evidence.text,
+                content_ref=base_evidence.content_ref,
+                span_start=base_evidence.span_start,
+                span_end=base_evidence.span_end,
+                stage="hybrid",
+                stage_scores={"lexical": lexical_score, "embedding": embedding_score},
+                recipe_id="",
+                run_id="",
+                hash=base_evidence.hash,
+            )
+        )
+    return candidates

biblicus/backends/sqlite_full_text_search.py

@@ -6,9 +6,9 @@ from __future__ import annotations
 
 import sqlite3
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from ..constants import CORPUS_DIR_NAME, RUNS_DIR_NAME
 from ..corpus import Corpus
@@ -35,6 +35,28 @@ class SqliteFullTextSearchRecipeConfig(BaseModel):
     :vartype chunk_overlap: int
     :ivar snippet_characters: Maximum characters to include in evidence snippets.
     :vartype snippet_characters: int
+    :ivar bm25_k1: BM25 k1 tuning parameter.
+    :vartype bm25_k1: float
+    :ivar bm25_b: BM25 b tuning parameter.
+    :vartype bm25_b: float
+    :ivar ngram_min: Minimum n-gram size for lexical tuning.
+    :vartype ngram_min: int
+    :ivar ngram_max: Maximum n-gram size for lexical tuning.
+    :vartype ngram_max: int
+    :ivar stop_words: Optional stop word policy or list.
+    :vartype stop_words: str or list[str] or None
+    :ivar field_weight_title: Relative weight for title field matches.
+    :vartype field_weight_title: float
+    :ivar field_weight_body: Relative weight for body field matches.
+    :vartype field_weight_body: float
+    :ivar field_weight_tags: Relative weight for tag field matches.
+    :vartype field_weight_tags: float
+    :ivar rerank_enabled: Whether to apply reranking to retrieved candidates.
+    :vartype rerank_enabled: bool
+    :ivar rerank_model: Reranker model identifier for metadata.
+    :vartype rerank_model: str or None
+    :ivar rerank_top_k: Number of candidates to rerank.
+    :vartype rerank_top_k: int
     :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
     :vartype extraction_run: str or None
     """
@@ -44,8 +66,81 @@ class SqliteFullTextSearchRecipeConfig(BaseModel):
     chunk_size: int = Field(default=800, ge=1)
     chunk_overlap: int = Field(default=200, ge=0)
     snippet_characters: int = Field(default=400, ge=1)
+    bm25_k1: float = Field(default=1.2, gt=0)
+    bm25_b: float = Field(default=0.75, ge=0, le=1)
+    ngram_min: int = Field(default=1, ge=1)
+    ngram_max: int = Field(default=1, ge=1)
+    stop_words: Optional[Union[str, List[str]]] = None
+    field_weight_title: float = Field(default=1.0, ge=0)
+    field_weight_body: float = Field(default=1.0, ge=0)
+    field_weight_tags: float = Field(default=1.0, ge=0)
+    rerank_enabled: bool = False
+    rerank_model: Optional[str] = None
+    rerank_top_k: int = Field(default=10, ge=1)
     extraction_run: Optional[str] = None
 
+    @field_validator("stop_words")
+    @classmethod
+    def _validate_stop_words(
+        cls, value: Optional[Union[str, List[str]]]
+    ) -> Optional[Union[str, List[str]]]:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            if value.lower() == "english":
+                return "english"
+            raise ValueError("stop_words must be 'english' or a list of strings")
+        if not value:
+            raise ValueError("stop_words list must not be empty")
+        if any(not isinstance(token, str) or not token.strip() for token in value):
+            raise ValueError("stop_words list must contain non-empty strings")
+        return value
+
+    @model_validator(mode="after")
+    def _validate_ngram_range(self) -> "SqliteFullTextSearchRecipeConfig":
+        if self.ngram_min > self.ngram_max:
+            raise ValueError("Invalid ngram range: ngram_min must be <= ngram_max")
+        if self.rerank_enabled and not self.rerank_model:
+            raise ValueError("Rerank enabled requires rerank_model")
+        return self
+
+
+_ENGLISH_STOP_WORDS: Set[str] = {
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "but",
+    "by",
+    "for",
+    "if",
+    "in",
+    "into",
+    "is",
+    "it",
+    "no",
+    "not",
+    "of",
+    "on",
+    "or",
+    "such",
+    "that",
+    "the",
+    "their",
+    "then",
+    "there",
+    "these",
+    "they",
+    "this",
+    "to",
+    "was",
+    "will",
+    "with",
+}
+
 
 class SqliteFullTextSearchBackend:
     """
@@ -118,29 +213,39 @@ class SqliteFullTextSearchBackend:
         :rtype: RetrievalResult
         """
         recipe_config = SqliteFullTextSearchRecipeConfig.model_validate(run.recipe.config)
+        query_tokens = _tokenize_query(query_text)
+        stop_words = _resolve_stop_words(recipe_config.stop_words)
+        filtered_tokens = _apply_stop_words(query_tokens, stop_words)
+        if not filtered_tokens:
+            return RetrievalResult(
+                query_text=query_text,
+                budget=budget,
+                run_id=run.run_id,
+                recipe_id=run.recipe.recipe_id,
+                backend_id=self.backend_id,
+                generated_at=utc_now_iso(),
+                evidence=[],
+                stats={"candidates": 0, "returned": 0},
+            )
         db_path = _resolve_run_db_path(corpus, run)
         candidates = _query_full_text_search_index(
             db_path=db_path,
-            query_text=query_text,
+            query_text=" ".join(filtered_tokens),
             limit=_candidate_limit(budget.max_total_items),
             snippet_characters=recipe_config.snippet_characters,
         )
-        sorted_candidates = sorted(
-            candidates,
-            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        sorted_candidates = _rank_candidates(candidates)
+        evidence = _apply_rerank_if_enabled(
+            sorted_candidates,
+            query_tokens=filtered_tokens,
+            run=run,
+            budget=budget,
+            rerank_enabled=recipe_config.rerank_enabled,
+            rerank_top_k=recipe_config.rerank_top_k,
         )
-        ranked = [
-            evidence_item.model_copy(
-                update={
-                    "rank": index,
-                    "recipe_id": run.recipe.recipe_id,
-                    "run_id": run.run_id,
-                }
-            )
-            for index, evidence_item in enumerate(sorted_candidates, start=1)
-        ]
-        evidence = apply_budget(ranked, budget)
-        stats = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        stats: Dict[str, object] = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        if recipe_config.rerank_enabled:
+            stats["reranked_candidates"] = min(len(sorted_candidates), recipe_config.rerank_top_k)
         return RetrievalResult(
             query_text=query_text,
             budget=budget,
@@ -165,6 +270,147 @@ def _candidate_limit(max_total_items: int) -> int:
     return max_total_items * 5
 
 
+def _tokenize_query(query_text: str) -> List[str]:
+    """
+    Tokenize a query string into lowercased terms.
+
+    :param query_text: Raw query text.
+    :type query_text: str
+    :return: Token list.
+    :rtype: list[str]
+    """
+    return [token for token in query_text.lower().split() if token]
+
+
+def _resolve_stop_words(value: Optional[Union[str, List[str]]]) -> Set[str]:
+    """
+    Resolve stop words based on a configuration value.
+
+    :param value: Stop word configuration.
+    :type value: str or list[str] or None
+    :return: Stop word set.
+    :rtype: set[str]
+    """
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        return set(_ENGLISH_STOP_WORDS)
+    return {token.strip().lower() for token in value if token.strip()}
+
+
+def _apply_stop_words(tokens: List[str], stop_words: Set[str]) -> List[str]:
+    """
+    Filter query tokens by a stop word list.
+
+    :param tokens: Token list.
+    :type tokens: list[str]
+    :param stop_words: Stop word set.
+    :type stop_words: set[str]
+    :return: Filtered token list.
+    :rtype: list[str]
+    """
+    if not stop_words:
+        return tokens
+    return [token for token in tokens if token not in stop_words]
+
+
+def _rank_candidates(candidates: List[Evidence]) -> List[Evidence]:
+    """
+    Sort evidence candidates by descending score.
+
+    :param candidates: Evidence list to sort.
+    :type candidates: list[Evidence]
+    :return: Sorted evidence list.
+    :rtype: list[Evidence]
+    """
+    return sorted(
+        candidates, key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id)
+    )
+
+
+def _rerank_score(text: str, query_tokens: List[str]) -> float:
+    """
+    Compute a simple rerank score using token overlap.
+
+    :param text: Candidate text.
+    :type text: str
+    :param query_tokens: Query tokens.
+    :type query_tokens: list[str]
+    :return: Rerank score.
+    :rtype: float
+    """
+    lower_text = text.lower() if text else ""
+    return float(sum(1 for token in query_tokens if token in lower_text))
+
+
+def _apply_rerank_if_enabled(
+    candidates: List[Evidence],
+    *,
+    query_tokens: List[str],
+    run: RetrievalRun,
+    budget: QueryBudget,
+    rerank_enabled: bool,
+    rerank_top_k: int,
+) -> List[Evidence]:
+    """
+    Rerank candidates when enabled, otherwise apply the budget to ranked results.
+
+    :param candidates: Ranked candidate evidence.
+    :type candidates: list[Evidence]
+    :param query_tokens: Query tokens used for reranking.
+    :type query_tokens: list[str]
+    :param run: Retrieval run to annotate evidence with.
+    :type run: RetrievalRun
+    :param budget: Evidence selection budget.
+    :type budget: QueryBudget
+    :param rerank_enabled: Whether reranking is enabled.
+    :type rerank_enabled: bool
+    :param rerank_top_k: Maximum candidates to rerank.
+    :type rerank_top_k: int
+    :return: Evidence list respecting budget.
+    :rtype: list[Evidence]
+    """
+    if not rerank_enabled:
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(candidates, start=1)
+        ]
+        return apply_budget(ranked, budget)
+
+    rerank_limit = min(len(candidates), rerank_top_k)
+    rerank_candidates = candidates[:rerank_limit]
+    reranked: List[Evidence] = []
+    for evidence_item in rerank_candidates:
+        rerank_score = _rerank_score(evidence_item.text or "", query_tokens)
+        reranked.append(
+            evidence_item.model_copy(
+                update={
+                    "score": rerank_score,
+                    "stage": "rerank",
+                    "stage_scores": {"retrieve": evidence_item.score, "rerank": rerank_score},
+                }
+            )
+        )
+    reranked_sorted = _rank_candidates(reranked)
+    ranked = [
+        evidence_item.model_copy(
+            update={
+                "rank": index,
+                "recipe_id": run.recipe.recipe_id,
+                "run_id": run.run_id,
+            }
+        )
+        for index, evidence_item in enumerate(reranked_sorted, start=1)
+    ]
+    return apply_budget(ranked, budget)
+
+
 def _resolve_run_db_path(corpus: Corpus, run: RetrievalRun) -> Path:
     """
     Resolve the SQLite index path for a retrieval run.

biblicus/backends/vector.py

@@ -0,0 +1,460 @@
+"""
+Deterministic term-frequency vector retrieval backend.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..corpus import Corpus
+from ..frontmatter import parse_front_matter
+from ..models import (
+    Evidence,
+    ExtractionRunReference,
+    QueryBudget,
+    RetrievalResult,
+    RetrievalRun,
+    parse_extraction_run_reference,
+)
+from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
+from ..time import utc_now_iso
+
+
+class VectorRecipeConfig(BaseModel):
+    """
+    Configuration for the vector retrieval backend.
+
+    :ivar snippet_characters: Maximum characters to include in evidence snippets.
+    :vartype snippet_characters: int
+    :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
+    :vartype extraction_run: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    snippet_characters: int = Field(default=400, ge=1)
+    extraction_run: Optional[str] = None
+
+
+class VectorBackend:
+    """
+    Deterministic vector backend using term-frequency cosine similarity.
+
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    """
+
+    backend_id = "vector"
+
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
+        """
+        Register a vector backend run (no materialization).
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+        recipe_config = VectorRecipeConfig.model_validate(config)
+        catalog = corpus.load_catalog()
+        recipe = create_recipe_manifest(
+            backend_id=self.backend_id,
+            name=recipe_name,
+            config=recipe_config.model_dump(),
+        )
+        stats = {
+            "items": len(catalog.items),
+            "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config),
+        }
+        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
+        corpus.write_run(run)
+        return run
+
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Query the corpus using term-frequency cosine similarity.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+        recipe_config = VectorRecipeConfig.model_validate(run.recipe.config)
+        query_tokens = _tokenize_text(query_text)
+        if not query_tokens:
+            return RetrievalResult(
+                query_text=query_text,
+                budget=budget,
+                run_id=run.run_id,
+                recipe_id=run.recipe.recipe_id,
+                backend_id=self.backend_id,
+                generated_at=utc_now_iso(),
+                evidence=[],
+                stats={"candidates": 0, "returned": 0},
+            )
+        query_vector = _term_frequencies(query_tokens)
+        query_norm = _vector_norm(query_vector)
+        catalog = corpus.load_catalog()
+        extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+        scored_candidates = _score_items(
+            corpus,
+            catalog.items.values(),
+            query_tokens=query_tokens,
+            query_vector=query_vector,
+            query_norm=query_norm,
+            snippet_characters=recipe_config.snippet_characters,
+            extraction_reference=extraction_reference,
+        )
+        sorted_candidates = sorted(
+            scored_candidates,
+            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        )
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(sorted_candidates, start=1)
+        ]
+        evidence = apply_budget(ranked, budget)
+        stats = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        return RetrievalResult(
+            query_text=query_text,
+            budget=budget,
+            run_id=run.run_id,
+            recipe_id=run.recipe.recipe_id,
+            backend_id=self.backend_id,
+            generated_at=utc_now_iso(),
+            evidence=evidence,
+            stats=stats,
+        )
+
+
+def _resolve_extraction_reference(
+    corpus: Corpus, recipe_config: VectorRecipeConfig
+) -> Optional[ExtractionRunReference]:
+    """
+    Resolve an extraction run reference from a recipe config.
+
+    :param corpus: Corpus associated with the recipe.
+    :type corpus: Corpus
+    :param recipe_config: Parsed vector recipe configuration.
+    :type recipe_config: VectorRecipeConfig
+    :return: Parsed extraction reference or None.
+    :rtype: ExtractionRunReference or None
+    :raises FileNotFoundError: If an extraction run is referenced but not present.
+    """
+    if not recipe_config.extraction_run:
+        return None
+    extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
+    run_dir = corpus.extraction_run_dir(
+        extractor_id=extraction_reference.extractor_id,
+        run_id=extraction_reference.run_id,
+    )
+    if not run_dir.is_dir():
+        raise FileNotFoundError(f"Missing extraction run: {extraction_reference.as_string()}")
+    return extraction_reference
+
+
+def _count_text_items(
+    corpus: Corpus, items: Iterable[object], recipe_config: VectorRecipeConfig
+) -> int:
+    """
+    Count catalog items that represent text content.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
+    :param items: Catalog items to inspect.
+    :type items: Iterable[object]
+    :param recipe_config: Parsed vector recipe configuration.
+    :type recipe_config: VectorRecipeConfig
+    :return: Number of text items.
+    :rtype: int
+    """
+    text_item_count = 0
+    extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+    for catalog_item in items:
+        item_id = str(getattr(catalog_item, "id", ""))
+        if extraction_reference and item_id:
+            extracted_text = corpus.read_extracted_text(
+                extractor_id=extraction_reference.extractor_id,
+                run_id=extraction_reference.run_id,
+                item_id=item_id,
+            )
+            if isinstance(extracted_text, str) and extracted_text.strip():
+                text_item_count += 1
+                continue
+        media_type = getattr(catalog_item, "media_type", "")
+        if media_type == "text/markdown" or str(media_type).startswith("text/"):
+            text_item_count += 1
+    return text_item_count
+
+
+def _tokenize_text(text: str) -> List[str]:
+    """
+    Tokenize text into lowercase word tokens.
+
+    :param text: Input text.
+    :type text: str
+    :return: Token list.
+    :rtype: list[str]
+    """
+    return re.findall(r"[a-z0-9]+", text.lower())
+
+
+def _term_frequencies(tokens: List[str]) -> Dict[str, float]:
+    """
+    Build term frequency weights from tokens.
+
+    :param tokens: Token list.
+    :type tokens: list[str]
+    :return: Term frequency mapping.
+    :rtype: dict[str, float]
+    """
+    frequencies: Dict[str, float] = {}
+    for token in tokens:
+        frequencies[token] = frequencies.get(token, 0.0) + 1.0
+    return frequencies
+
+
+def _vector_norm(vector: Dict[str, float]) -> float:
+    """
+    Compute the Euclidean norm of a term-frequency vector.
+
+    :param vector: Term frequency mapping.
+    :type vector: dict[str, float]
+    :return: Vector norm.
+    :rtype: float
+    """
+    return math.sqrt(sum(value * value for value in vector.values()))
+
+
+def _cosine_similarity(
+    left: Dict[str, float],
+    *,
+    left_norm: float,
+    right: Dict[str, float],
+    right_norm: float,
+) -> float:
+    """
+    Compute cosine similarity between two term-frequency vectors.
+
+    :param left: Left term-frequency vector.
+    :type left: dict[str, float]
+    :param left_norm: Precomputed left vector norm.
+    :type left_norm: float
+    :param right: Right term-frequency vector.
+    :type right: dict[str, float]
+    :param right_norm: Precomputed right vector norm.
+    :type right_norm: float
+    :return: Cosine similarity score.
+    :rtype: float
+    """
+    dot = 0.0
+    if len(left) < len(right):
+        for token, value in left.items():
+            dot += value * right.get(token, 0.0)
+    else:
+        for token, value in right.items():
+            dot += value * left.get(token, 0.0)
+    return dot / (left_norm * right_norm)
+
+
+def _load_text_from_item(
+    corpus: Corpus,
+    *,
+    item_id: str,
+    relpath: str,
+    media_type: str,
+    extraction_reference: Optional[ExtractionRunReference],
+) -> Optional[str]:
+    """
+    Load a text payload from a catalog item.
+
+    :param corpus: Corpus containing the item.
+    :type corpus: Corpus
+    :param item_id: Item identifier.
+    :type item_id: str
+    :param relpath: Relative path to the stored content.
+    :type relpath: str
+    :param media_type: Media type for the stored content.
+    :type media_type: str
+    :param extraction_reference: Optional extraction run reference.
+    :type extraction_reference: ExtractionRunReference or None
+    :return: Text payload or None if not decodable as text.
+    :rtype: str or None
+    """
+    if extraction_reference:
+        extracted_text = corpus.read_extracted_text(
+            extractor_id=extraction_reference.extractor_id,
+            run_id=extraction_reference.run_id,
+            item_id=item_id,
+        )
+        if isinstance(extracted_text, str) and extracted_text.strip():
+            return extracted_text
+
+    content_path = corpus.root / relpath
+    raw_bytes = content_path.read_bytes()
+    if media_type == "text/markdown":
+        markdown_text = raw_bytes.decode("utf-8")
+        parsed_document = parse_front_matter(markdown_text)
+        return parsed_document.body
+    if media_type.startswith("text/"):
+        return raw_bytes.decode("utf-8")
+    return None
+
+
+def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]:
+    """
+    Locate the earliest token match span in a text payload.
+
+    :param text: Text to scan.
+    :type text: str
+    :param tokens: Query tokens.
+    :type tokens: list[str]
+    :return: Start/end span for the earliest match, or None if no matches.
+    :rtype: tuple[int, int] or None
+    """
+    lower_text = text.lower()
+    best_start: Optional[int] = None
+    best_end: Optional[int] = None
+    for token in tokens:
+        if not token:
+            continue
+        token_start = lower_text.find(token)
+        if token_start == -1:
+            continue
+        token_end = token_start + len(token)
+        if best_start is None or token_start < best_start:
+            best_start = token_start
+            best_end = token_end
+    if best_start is None or best_end is None:
+        return None
+    return best_start, best_end
+
+
+def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int) -> str:
+    """
+    Build a snippet around a match span, constrained by a character budget.
+
+    :param text: Source text to slice.
+    :type text: str
+    :param span: Match span to center on.
+    :type span: tuple[int, int] or None
+    :param max_chars: Maximum snippet length.
+    :type max_chars: int
+    :return: Snippet text.
+    :rtype: str
+    """
+    if not text:
+        return ""
+    if span is None:
+        return text[:max_chars]
+    span_start, span_end = span
+    half_window = max_chars // 2
+    snippet_start = max(span_start - half_window, 0)
+    snippet_end = min(span_end + half_window, len(text))
+    return text[snippet_start:snippet_end]
+
+
+def _score_items(
+    corpus: Corpus,
+    items: Iterable[object],
+    *,
+    query_tokens: List[str],
+    query_vector: Dict[str, float],
+    query_norm: float,
+    snippet_characters: int,
+    extraction_reference: Optional[ExtractionRunReference],
+) -> List[Evidence]:
+    """
+    Score catalog items and return evidence candidates.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
+    :param items: Catalog items to score.
+    :type items: Iterable[object]
+    :param query_tokens: Tokenized query text.
+    :type query_tokens: list[str]
+    :param query_vector: Query term-frequency vector.
+    :type query_vector: dict[str, float]
+    :param query_norm: Query vector norm.
+    :type query_norm: float
+    :param snippet_characters: Snippet length budget.
+    :type snippet_characters: int
+    :param extraction_reference: Optional extraction run reference.
+    :type extraction_reference: ExtractionRunReference or None
+    :return: Evidence candidates with provisional ranks.
+    :rtype: list[Evidence]
+    """
+    evidence_items: List[Evidence] = []
+    for catalog_item in items:
+        media_type = getattr(catalog_item, "media_type", "")
+        relpath = getattr(catalog_item, "relpath", "")
+        item_id = str(getattr(catalog_item, "id", ""))
+        item_text = _load_text_from_item(
+            corpus,
+            item_id=item_id,
+            relpath=relpath,
+            media_type=str(media_type),
+            extraction_reference=extraction_reference,
+        )
+        if item_text is None:
+            continue
+        tokens = _tokenize_text(item_text)
+        if not tokens:
+            continue
+        vector = _term_frequencies(tokens)
+        similarity = _cosine_similarity(
+            query_vector, left_norm=query_norm, right=vector, right_norm=_vector_norm(vector)
+        )
+        if similarity <= 0:
+            continue
+        span = _find_first_match(item_text, query_tokens)
+        snippet = _build_snippet(item_text, span, max_chars=snippet_characters)
+        span_start = span[0] if span else None
+        span_end = span[1] if span else None
+        evidence_items.append(
+            Evidence(
+                item_id=str(getattr(catalog_item, "id")),
+                source_uri=getattr(catalog_item, "source_uri", None),
+                media_type=str(media_type),
+                score=float(similarity),
+                rank=1,
+                text=snippet,
+                content_ref=None,
+                span_start=span_start,
+                span_end=span_end,
+                stage="vector",
+                recipe_id="",
+                run_id="",
+                hash=hash_text(snippet),
+            )
+        )
+    return evidence_items

biblicus/models.py

@@ -263,6 +263,8 @@ class Evidence(BaseModel):
     :vartype span_end: int or None
     :ivar stage: Retrieval stage label (for example, scan, full-text search, rerank).
     :vartype stage: str
+    :ivar stage_scores: Optional per-stage scores for multi-stage retrieval.
+    :vartype stage_scores: dict[str, float] or None
     :ivar recipe_id: Recipe identifier used to create the run.
     :vartype recipe_id: str
     :ivar run_id: Retrieval run identifier.
@@ -283,6 +285,7 @@ class Evidence(BaseModel):
     span_start: Optional[int] = None
     span_end: Optional[int] = None
     stage: str
+    stage_scores: Optional[Dict[str, float]] = None
     recipe_id: str
     run_id: str
     hash: Optional[str] = None

{biblicus-0.10.0.dist-info → biblicus-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.10.0
+Version: 0.11.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -493,6 +493,12 @@ Two backends are included.
 
 For detailed documentation including configuration options, performance characteristics, and usage examples, see the [Backend Reference][backend-reference].
 
+## Retrieval documentation
+
+For the retrieval pipeline overview and run artifacts, see `docs/RETRIEVAL.md`. For retrieval quality upgrades
+(tuned lexical baseline, reranking, hybrid retrieval), see `docs/RETRIEVAL_QUALITY.md`. For evaluation workflows
+and dataset formats, see `docs/RETRIEVAL_EVALUATION.md`.
+
 ## Extraction backends
 
 These extractors are built in. Optional ones require extra dependencies. See [text extraction documentation][text-extraction] for details.

{biblicus-0.10.0.dist-info → biblicus-0.11.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-biblicus/__init__.py,sha256=BejOPHIlCnT74pu9fNuLm14HsmWjGqCIwpfD9hDOqSo,496
+biblicus/__init__.py,sha256=sT0PFc3DRGFRcN7Zx4Yooc8OzmLvaj1-ZjbvFHce8lU,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
 biblicus/cli.py,sha256=aH3plnednnYgcPnSoYQf200nboKc6N-tuc3FuLPQEcU,35132
 biblicus/constants.py,sha256=-JaHI3Dngte2drawx93cGWxFVobbgIuaVhmjUJpf4GI,333
@@ -16,7 +16,7 @@ biblicus/hooks.py,sha256=OHQOmOi7rUcQqYWVeod4oPe8nVLepD7F_SlN7O_-BsE,7863
 biblicus/ignore.py,sha256=fyjt34E6tWNNrm1FseOhgH2MgryyVBQVzxhKL5s4aio,1800
 biblicus/inference.py,sha256=_k00AIPoXD2lruiTB-JUagtY4f_WKcdzA3axwiq1tck,3512
 biblicus/knowledge_base.py,sha256=JmlJw8WD_fgstuq1PyWVzU9kzvVzyv7_xOvhS70xwUw,6654
-biblicus/models.py,sha256=vlvPP7AOZGtnHSq47-s9YW-fqLwjgYR6NBcSfeC8YKk,15665
+biblicus/models.py,sha256=r28O6cg3d1bjJnKqpLieVLTgtXTfzb_60wMORvVuDN0,15846
 biblicus/retrieval.py,sha256=A1SI4WK5cX-WbtN6FJ0QQxqlEOtQhddLrL0LZIuoTC4,4180
 biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
 biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
@@ -30,13 +30,15 @@ biblicus/analysis/__init__.py,sha256=Z4Wb4d-EoUuGHkcfRm9ILuZ8vr9FBqRxC0u1i6Fp_0w
 biblicus/analysis/base.py,sha256=gB4ilvyMpiWU1m_ydy2dIHGP96ZFIFvVUL9iVDZKPJM,1265
 biblicus/analysis/llm.py,sha256=VjkZDKauHCDfj-TP-bTbI6a9WAXEIDe8bEiwErPx9xc,3309
 biblicus/analysis/models.py,sha256=LuR52w27JRzV-Mr-WAOduZrBOCTrp5uYkMc46QHTRrI,27300
-biblicus/analysis/profiling.py,sha256=z4w14LVJrTEXcQ3PBNwwb_61KuuwQgXw4-EiAaxOQ4Y,10672
+biblicus/analysis/profiling.py,sha256=v2B4Tn9WiXRRP_wIADBPRQVKkMc92KXCas7OBa7n0LU,10670
 biblicus/analysis/schema.py,sha256=MCiAQJmijVk8iM8rOUYbzyaDwsMR-Oo86iZU5NCbDMM,435
 biblicus/analysis/topic_modeling.py,sha256=ZGXvm2MyU6plxz2FE1RQU-3bra6QZ-t8EJj8kG1TW0M,19438
-biblicus/backends/__init__.py,sha256=wLXIumV51l6ZIKzjoKKeU7AgIxGOryG7T7ls3a_Fv98,1212
+biblicus/backends/__init__.py,sha256=3HJY0oMm8pFFVGC4Z-dlPRHhIPVDdUzsa4IMjKP_9dI,1378
 biblicus/backends/base.py,sha256=Erfj9dXg0nkRKnEcNjHR9_0Ddb2B1NvbmRksVm_g1dU,1776
+biblicus/backends/hybrid.py,sha256=CXh6QrlE0RsTJjSlZRdtomLlILfkglBDQG3YVa8RpFU,10589
 biblicus/backends/scan.py,sha256=hdNnQWqi5IH6j95w30BZHxLJ0W9PTaOkqfWJuxCCEMI,12478
-biblicus/backends/sqlite_full_text_search.py,sha256=XFuIbEHYWMD9JkjgRZcgYH3kP3b4hRnJ3PwP8rSFjUU,16502
+biblicus/backends/sqlite_full_text_search.py,sha256=VAn4fDdfiaS1Rn6zHlYz3E10_3vMU9P94QU8cL0l8Mk,24466
+biblicus/backends/vector.py,sha256=3RdxSBPb1kOX4Sfd4d1qXFW9ecuiRvGpOHadLCbeh1g,15183
 biblicus/extractors/__init__.py,sha256=ci3oldbdQZ8meAfHccM48CqQtZsPSRg3HkPrBSZF15M,2673
 biblicus/extractors/base.py,sha256=ka-nz_1zHPr4TS9sU4JfOoY-PJh7lbHPBOEBrbQFGSc,2171
 biblicus/extractors/deepgram_stt.py,sha256=VI71i4lbE-EFHcvpNcCPRpT8z7A5IuaSrT1UaPyZ8UY,6323
@@ -55,9 +57,9 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.10.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.10.0.dist-info/METADATA,sha256=xZ7scJLdlKHRtm0EU5Ravq5ih2mS2KNfMbbLXNqZ8Ek,27455
-biblicus-0.10.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.10.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.10.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.10.0.dist-info/RECORD,,
+biblicus-0.11.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.11.0.dist-info/METADATA,sha256=zrJESYGfGLu7Iq1I--GPIkEY9gXDb9szBIuenlWor7I,27765
+biblicus-0.11.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.11.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.11.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.11.0.dist-info/RECORD,,