biblicus 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

biblicus/backends/vector.py

@@ -0,0 +1,460 @@
+"""
+Deterministic term-frequency vector retrieval backend.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..corpus import Corpus
+from ..frontmatter import parse_front_matter
+from ..models import (
+    Evidence,
+    ExtractionRunReference,
+    QueryBudget,
+    RetrievalResult,
+    RetrievalRun,
+    parse_extraction_run_reference,
+)
+from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
+from ..time import utc_now_iso
+
+
+class VectorRecipeConfig(BaseModel):
+    """
+    Configuration for the vector retrieval backend.
+
+    :ivar snippet_characters: Maximum characters to include in evidence snippets.
+    :vartype snippet_characters: int
+    :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
+    :vartype extraction_run: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    snippet_characters: int = Field(default=400, ge=1)
+    extraction_run: Optional[str] = None
+
+
+class VectorBackend:
+    """
+    Deterministic vector backend using term-frequency cosine similarity.
+
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    """
+
+    backend_id = "vector"
+
+    def build_run(
+        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
+    ) -> RetrievalRun:
+        """
+        Register a vector backend run (no materialization).
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+        recipe_config = VectorRecipeConfig.model_validate(config)
+        catalog = corpus.load_catalog()
+        recipe = create_recipe_manifest(
+            backend_id=self.backend_id,
+            name=recipe_name,
+            config=recipe_config.model_dump(),
+        )
+        stats = {
+            "items": len(catalog.items),
+            "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config),
+        }
+        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
+        corpus.write_run(run)
+        return run
+
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Query the corpus using term-frequency cosine similarity.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+        recipe_config = VectorRecipeConfig.model_validate(run.recipe.config)
+        query_tokens = _tokenize_text(query_text)
+        if not query_tokens:
+            return RetrievalResult(
+                query_text=query_text,
+                budget=budget,
+                run_id=run.run_id,
+                recipe_id=run.recipe.recipe_id,
+                backend_id=self.backend_id,
+                generated_at=utc_now_iso(),
+                evidence=[],
+                stats={"candidates": 0, "returned": 0},
+            )
+        query_vector = _term_frequencies(query_tokens)
+        query_norm = _vector_norm(query_vector)
+        catalog = corpus.load_catalog()
+        extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+        scored_candidates = _score_items(
+            corpus,
+            catalog.items.values(),
+            query_tokens=query_tokens,
+            query_vector=query_vector,
+            query_norm=query_norm,
+            snippet_characters=recipe_config.snippet_characters,
+            extraction_reference=extraction_reference,
+        )
+        sorted_candidates = sorted(
+            scored_candidates,
+            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        )
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(sorted_candidates, start=1)
+        ]
+        evidence = apply_budget(ranked, budget)
+        stats = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        return RetrievalResult(
+            query_text=query_text,
+            budget=budget,
+            run_id=run.run_id,
+            recipe_id=run.recipe.recipe_id,
+            backend_id=self.backend_id,
+            generated_at=utc_now_iso(),
+            evidence=evidence,
+            stats=stats,
+        )
+
+
+def _resolve_extraction_reference(
+    corpus: Corpus, recipe_config: VectorRecipeConfig
+) -> Optional[ExtractionRunReference]:
+    """
+    Resolve an extraction run reference from a recipe config.
+
+    :param corpus: Corpus associated with the recipe.
+    :type corpus: Corpus
+    :param recipe_config: Parsed vector recipe configuration.
+    :type recipe_config: VectorRecipeConfig
+    :return: Parsed extraction reference or None.
+    :rtype: ExtractionRunReference or None
+    :raises FileNotFoundError: If an extraction run is referenced but not present.
+    """
+    if not recipe_config.extraction_run:
+        return None
+    extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
+    run_dir = corpus.extraction_run_dir(
+        extractor_id=extraction_reference.extractor_id,
+        run_id=extraction_reference.run_id,
+    )
+    if not run_dir.is_dir():
+        raise FileNotFoundError(f"Missing extraction run: {extraction_reference.as_string()}")
+    return extraction_reference
+
+
+def _count_text_items(
+    corpus: Corpus, items: Iterable[object], recipe_config: VectorRecipeConfig
+) -> int:
+    """
+    Count catalog items that represent text content.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
+    :param items: Catalog items to inspect.
+    :type items: Iterable[object]
+    :param recipe_config: Parsed vector recipe configuration.
+    :type recipe_config: VectorRecipeConfig
+    :return: Number of text items.
+    :rtype: int
+    """
+    text_item_count = 0
+    extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+    for catalog_item in items:
+        item_id = str(getattr(catalog_item, "id", ""))
+        if extraction_reference and item_id:
+            extracted_text = corpus.read_extracted_text(
+                extractor_id=extraction_reference.extractor_id,
+                run_id=extraction_reference.run_id,
+                item_id=item_id,
+            )
+            if isinstance(extracted_text, str) and extracted_text.strip():
+                text_item_count += 1
+                continue
+        media_type = getattr(catalog_item, "media_type", "")
+        if media_type == "text/markdown" or str(media_type).startswith("text/"):
+            text_item_count += 1
+    return text_item_count
+
+
+def _tokenize_text(text: str) -> List[str]:
+    """
+    Tokenize text into lowercase word tokens.
+
+    :param text: Input text.
+    :type text: str
+    :return: Token list.
+    :rtype: list[str]
+    """
+    return re.findall(r"[a-z0-9]+", text.lower())
+
+
+def _term_frequencies(tokens: List[str]) -> Dict[str, float]:
+    """
+    Build term frequency weights from tokens.
+
+    :param tokens: Token list.
+    :type tokens: list[str]
+    :return: Term frequency mapping.
+    :rtype: dict[str, float]
+    """
+    frequencies: Dict[str, float] = {}
+    for token in tokens:
+        frequencies[token] = frequencies.get(token, 0.0) + 1.0
+    return frequencies
+
+
+def _vector_norm(vector: Dict[str, float]) -> float:
+    """
+    Compute the Euclidean norm of a term-frequency vector.
+
+    :param vector: Term frequency mapping.
+    :type vector: dict[str, float]
+    :return: Vector norm.
+    :rtype: float
+    """
+    return math.sqrt(sum(value * value for value in vector.values()))
+
+
+def _cosine_similarity(
+    left: Dict[str, float],
+    *,
+    left_norm: float,
+    right: Dict[str, float],
+    right_norm: float,
+) -> float:
+    """
+    Compute cosine similarity between two term-frequency vectors.
+
+    :param left: Left term-frequency vector.
+    :type left: dict[str, float]
+    :param left_norm: Precomputed left vector norm.
+    :type left_norm: float
+    :param right: Right term-frequency vector.
+    :type right: dict[str, float]
+    :param right_norm: Precomputed right vector norm.
+    :type right_norm: float
+    :return: Cosine similarity score.
+    :rtype: float
+    """
+    dot = 0.0
+    if len(left) < len(right):
+        for token, value in left.items():
+            dot += value * right.get(token, 0.0)
+    else:
+        for token, value in right.items():
+            dot += value * left.get(token, 0.0)
+    return dot / (left_norm * right_norm)
+
+
+def _load_text_from_item(
+    corpus: Corpus,
+    *,
+    item_id: str,
+    relpath: str,
+    media_type: str,
+    extraction_reference: Optional[ExtractionRunReference],
+) -> Optional[str]:
+    """
+    Load a text payload from a catalog item.
+
+    :param corpus: Corpus containing the item.
+    :type corpus: Corpus
+    :param item_id: Item identifier.
+    :type item_id: str
+    :param relpath: Relative path to the stored content.
+    :type relpath: str
+    :param media_type: Media type for the stored content.
+    :type media_type: str
+    :param extraction_reference: Optional extraction run reference.
+    :type extraction_reference: ExtractionRunReference or None
+    :return: Text payload or None if not decodable as text.
+    :rtype: str or None
+    """
+    if extraction_reference:
+        extracted_text = corpus.read_extracted_text(
+            extractor_id=extraction_reference.extractor_id,
+            run_id=extraction_reference.run_id,
+            item_id=item_id,
+        )
+        if isinstance(extracted_text, str) and extracted_text.strip():
+            return extracted_text
+
+    content_path = corpus.root / relpath
+    raw_bytes = content_path.read_bytes()
+    if media_type == "text/markdown":
+        markdown_text = raw_bytes.decode("utf-8")
+        parsed_document = parse_front_matter(markdown_text)
+        return parsed_document.body
+    if media_type.startswith("text/"):
+        return raw_bytes.decode("utf-8")
+    return None
+
+
+def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]:
+    """
+    Locate the earliest token match span in a text payload.
+
+    :param text: Text to scan.
+    :type text: str
+    :param tokens: Query tokens.
+    :type tokens: list[str]
+    :return: Start/end span for the earliest match, or None if no matches.
+    :rtype: tuple[int, int] or None
+    """
+    lower_text = text.lower()
+    best_start: Optional[int] = None
+    best_end: Optional[int] = None
+    for token in tokens:
+        if not token:
+            continue
+        token_start = lower_text.find(token)
+        if token_start == -1:
+            continue
+        token_end = token_start + len(token)
+        if best_start is None or token_start < best_start:
+            best_start = token_start
+            best_end = token_end
+    if best_start is None or best_end is None:
+        return None
+    return best_start, best_end
+
+
+def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int) -> str:
+    """
+    Build a snippet around a match span, constrained by a character budget.
+
+    :param text: Source text to slice.
+    :type text: str
+    :param span: Match span to center on.
+    :type span: tuple[int, int] or None
+    :param max_chars: Maximum snippet length.
+    :type max_chars: int
+    :return: Snippet text.
+    :rtype: str
+    """
+    if not text:
+        return ""
+    if span is None:
+        return text[:max_chars]
+    span_start, span_end = span
+    half_window = max_chars // 2
+    snippet_start = max(span_start - half_window, 0)
+    snippet_end = min(span_end + half_window, len(text))
+    return text[snippet_start:snippet_end]
+
+
+def _score_items(
+    corpus: Corpus,
+    items: Iterable[object],
+    *,
+    query_tokens: List[str],
+    query_vector: Dict[str, float],
+    query_norm: float,
+    snippet_characters: int,
+    extraction_reference: Optional[ExtractionRunReference],
+) -> List[Evidence]:
+    """
+    Score catalog items and return evidence candidates.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
+    :param items: Catalog items to score.
+    :type items: Iterable[object]
+    :param query_tokens: Tokenized query text.
+    :type query_tokens: list[str]
+    :param query_vector: Query term-frequency vector.
+    :type query_vector: dict[str, float]
+    :param query_norm: Query vector norm.
+    :type query_norm: float
+    :param snippet_characters: Snippet length budget.
+    :type snippet_characters: int
+    :param extraction_reference: Optional extraction run reference.
+    :type extraction_reference: ExtractionRunReference or None
+    :return: Evidence candidates with provisional ranks.
+    :rtype: list[Evidence]
+    """
+    evidence_items: List[Evidence] = []
+    for catalog_item in items:
+        media_type = getattr(catalog_item, "media_type", "")
+        relpath = getattr(catalog_item, "relpath", "")
+        item_id = str(getattr(catalog_item, "id", ""))
+        item_text = _load_text_from_item(
+            corpus,
+            item_id=item_id,
+            relpath=relpath,
+            media_type=str(media_type),
+            extraction_reference=extraction_reference,
+        )
+        if item_text is None:
+            continue
+        tokens = _tokenize_text(item_text)
+        if not tokens:
+            continue
+        vector = _term_frequencies(tokens)
+        similarity = _cosine_similarity(
+            query_vector, left_norm=query_norm, right=vector, right_norm=_vector_norm(vector)
+        )
+        if similarity <= 0:
+            continue
+        span = _find_first_match(item_text, query_tokens)
+        snippet = _build_snippet(item_text, span, max_chars=snippet_characters)
+        span_start = span[0] if span else None
+        span_end = span[1] if span else None
+        evidence_items.append(
+            Evidence(
+                item_id=str(getattr(catalog_item, "id")),
+                source_uri=getattr(catalog_item, "source_uri", None),
+                media_type=str(media_type),
+                score=float(similarity),
+                rank=1,
+                text=snippet,
+                content_ref=None,
+                span_start=span_start,
+                span_end=span_end,
+                stage="vector",
+                recipe_id="",
+                run_id="",
+                hash=hash_text(snippet),
+            )
+        )
+    return evidence_items

biblicus/cli.py

@@ -563,7 +563,9 @@ def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
     """
     input_text = sys.stdin.read()
     if not input_text.strip():
-        raise ValueError("Context pack build requires a retrieval result JavaScript Object Notation on standard input")
+        raise ValueError(
+            "Context pack build requires a retrieval result JavaScript Object Notation on standard input"
+        )
     retrieval_result = RetrievalResult.model_validate_json(input_text)
     join_with = bytes(arguments.join_with, "utf-8").decode("unicode_escape")
     policy = ContextPackPolicy(join_with=join_with)
@@ -685,6 +687,58 @@ def cmd_analyze_topics(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_analyze_profile(arguments: argparse.Namespace) -> int:
+    """
+    Run profiling analysis for a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    import yaml
+
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+
+    recipe_data: dict[str, object] = {}
+    if arguments.recipe is not None:
+        recipe_path = Path(arguments.recipe)
+        if not recipe_path.is_file():
+            raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
+        recipe_raw = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) or {}
+        if not isinstance(recipe_raw, dict):
+            raise ValueError("Profiling recipe must be a mapping/object")
+        recipe_data = recipe_raw
+
+    if arguments.extraction_run:
+        extraction_run = parse_extraction_run_reference(arguments.extraction_run)
+    else:
+        extraction_run = corpus.latest_extraction_run_reference()
+        if extraction_run is None:
+            raise ValueError("Profiling analysis requires an extraction run to supply text inputs")
+        print(
+            "Warning: using latest extraction run; pass --extraction-run for reproducibility.",
+            file=sys.stderr,
+        )
+
+    backend = get_analysis_backend("profiling")
+    try:
+        output = backend.run_analysis(
+            corpus,
+            recipe_name=arguments.recipe_name,
+            config=recipe_data,
+            extraction_run=extraction_run,
+        )
+    except ValidationError as exc:
+        raise ValueError(f"Invalid profiling recipe: {exc}") from exc
+    print(output.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -890,14 +944,20 @@ def build_parser() -> argparse.ArgumentParser:
 
     p_crawl = sub.add_parser("crawl", help="Crawl a website prefix into the corpus.")
     _add_common_corpus_arg(p_crawl)
-    p_crawl.add_argument("--root-url", required=True, help="Root uniform resource locator to fetch.")
+    p_crawl.add_argument(
+        "--root-url", required=True, help="Root uniform resource locator to fetch."
+    )
     p_crawl.add_argument(
         "--allowed-prefix",
         required=True,
         help="Uniform resource locator prefix that limits which links are eligible for crawl.",
     )
-    p_crawl.add_argument("--max-items", type=int, default=50, help="Maximum number of items to store.")
-    p_crawl.add_argument("--tags", default=None, help="Comma-separated tags to apply to stored items.")
+    p_crawl.add_argument(
+        "--max-items", type=int, default=50, help="Maximum number of items to store."
+    )
+    p_crawl.add_argument(
+        "--tags", default=None, help="Comma-separated tags to apply to stored items."
+    )
     p_crawl.add_argument("--tag", action="append", help="Repeatable tag to apply to stored items.")
     p_crawl.set_defaults(func=cmd_crawl)
 
@@ -923,6 +983,25 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_analyze_topics.set_defaults(func=cmd_analyze_topics)
 
+    p_analyze_profile = analyze_sub.add_parser("profile", help="Run profiling analysis.")
+    _add_common_corpus_arg(p_analyze_profile)
+    p_analyze_profile.add_argument(
+        "--recipe",
+        default=None,
+        help="Optional profiling recipe YAML file.",
+    )
+    p_analyze_profile.add_argument(
+        "--recipe-name",
+        default="default",
+        help="Human-readable recipe name.",
+    )
+    p_analyze_profile.add_argument(
+        "--extraction-run",
+        default=None,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_analyze_profile.set_defaults(func=cmd_analyze_profile)
+
     return parser
 
 

biblicus/corpus.py

@@ -622,7 +622,9 @@ class Corpus:
         data = json.loads(manifest_path.read_text(encoding="utf-8"))
         return ExtractionRunManifest.model_validate(data)
 
-    def list_extraction_runs(self, *, extractor_id: Optional[str] = None) -> List[ExtractionRunListEntry]:
+    def list_extraction_runs(
+        self, *, extractor_id: Optional[str] = None
+    ) -> List[ExtractionRunListEntry]:
         """
         List extraction runs stored under the corpus.
 
@@ -669,7 +671,9 @@ class Corpus:
                     )
                 )
 
-        entries.sort(key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True)
+        entries.sort(
+            key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True
+        )
         return entries
 
     def latest_extraction_run_reference(
@@ -1366,7 +1370,9 @@ class Corpus:
         """
         _ = filename
         item_id = str(uuid.uuid4())
-        destination_relpath = str(Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path)
+        destination_relpath = str(
+            Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path
+        )
         destination_path = (self.root / destination_relpath).resolve()
         destination_path.parent.mkdir(parents=True, exist_ok=True)
         destination_path.write_bytes(data)

biblicus/evidence_processing.py

@@ -99,7 +99,10 @@ class EvidenceRerankLongestText(EvidenceReranker):
         """
         return sorted(
             evidence,
-            key=lambda evidence_item: (-len((evidence_item.text or "").strip()), evidence_item.item_id),
+            key=lambda evidence_item: (
+                -len((evidence_item.text or "").strip()),
+                evidence_item.item_id,
+            ),
         )
 
 
@@ -198,4 +201,3 @@ def apply_evidence_filter(
     """
     evidence_filter = _EVIDENCE_FILTERS[filter_id]
     return evidence_filter.filter(query_text=query_text, evidence=evidence, config=config)
-

biblicus/extraction.py

@@ -345,7 +345,9 @@ def build_extraction_run(
     manifest = create_extraction_run_manifest(corpus, recipe=recipe)
     run_dir = corpus.extraction_run_dir(extractor_id=extractor_id, run_id=manifest.run_id)
     if run_dir.exists():
-        return corpus.load_extraction_run_manifest(extractor_id=extractor_id, run_id=manifest.run_id)
+        return corpus.load_extraction_run_manifest(
+            extractor_id=extractor_id, run_id=manifest.run_id
+        )
     run_dir.mkdir(parents=True, exist_ok=False)
 
     catalog = corpus.load_catalog()

biblicus/extractors/markitdown_text.py

@@ -29,6 +29,7 @@ class MarkItDownExtractorConfig(BaseModel):
 
     enable_plugins: bool = Field(default=False)
 
+
 class MarkItDownExtractor(TextExtractor):
     """
     Extractor plugin backed by the `markitdown` library.

biblicus/extractors/paddleocr_vl_text.py

@@ -152,9 +152,7 @@ class PaddleOcrVlExtractor(TextExtractor):
                 parsed_config.backend.api_provider,
                 config_override=parsed_config.backend.api_key,
             )
-            text, confidence = self._extract_via_api(
-                source_path, parsed_config, api_key
-            )
+            text, confidence = self._extract_via_api(source_path, parsed_config, api_key)
 
         return ExtractedText(
             text=text,

biblicus/models.py

@@ -263,6 +263,8 @@ class Evidence(BaseModel):
     :vartype span_end: int or None
     :ivar stage: Retrieval stage label (for example, scan, full-text search, rerank).
     :vartype stage: str
+    :ivar stage_scores: Optional per-stage scores for multi-stage retrieval.
+    :vartype stage_scores: dict[str, float] or None
     :ivar recipe_id: Recipe identifier used to create the run.
     :vartype recipe_id: str
     :ivar run_id: Retrieval run identifier.
@@ -283,6 +285,7 @@ class Evidence(BaseModel):
     span_start: Optional[int] = None
     span_end: Optional[int] = None
     stage: str
+    stage_scores: Optional[Dict[str, float]] = None
     recipe_id: str
     run_id: str
     hash: Optional[str] = None