biblicus 0.1.1__py3-none-any.whl

biblicus/__init__.py

@@ -0,0 +1,28 @@
+"""
+Biblicus public package interface.
+"""
+
+from .models import (
+    CorpusConfig,
+    Evidence,
+    IngestResult,
+    QueryBudget,
+    RecipeManifest,
+    RetrievalResult,
+    RetrievalRun,
+)
+from .corpus import Corpus
+
+__all__ = [
+    "__version__",
+    "Corpus",
+    "CorpusConfig",
+    "Evidence",
+    "IngestResult",
+    "QueryBudget",
+    "RecipeManifest",
+    "RetrievalResult",
+    "RetrievalRun",
+]
+
+__version__ = "0.1.1"

biblicus/__main__.py

@@ -0,0 +1,8 @@
+"""
+Biblicus module entry point.
+"""
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

biblicus/backends/__init__.py

@@ -0,0 +1,44 @@
+"""
+Backend registry for Biblicus retrieval engines.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Type
+
+from .base import RetrievalBackend
+from .scan import ScanBackend
+from .sqlite_full_text_search import SqliteFullTextSearchBackend
+
+
+def available_backends() -> Dict[str, Type[RetrievalBackend]]:
+    """
+    Return the registered retrieval backends.
+
+    :return: Mapping of backend identifiers to backend classes.
+    :rtype: dict[str, Type[RetrievalBackend]]
+    """
+
+    return {
+        ScanBackend.backend_id: ScanBackend,
+        SqliteFullTextSearchBackend.backend_id: SqliteFullTextSearchBackend,
+    }
+
+
+def get_backend(backend_id: str) -> RetrievalBackend:
+    """
+    Instantiate a retrieval backend by identifier.
+
+    :param backend_id: Backend identifier.
+    :type backend_id: str
+    :return: Backend instance.
+    :rtype: RetrievalBackend
+    :raises KeyError: If the backend identifier is unknown.
+    """
+
+    registry = available_backends()
+    backend_class = registry.get(backend_id)
+    if backend_class is None:
+        known = ", ".join(sorted(registry))
+        raise KeyError(f"Unknown backend '{backend_id}'. Known backends: {known}")
+    return backend_class()

biblicus/backends/base.py

@@ -0,0 +1,65 @@
+"""
+Backend interface for Biblicus retrieval engines.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from ..corpus import Corpus
+from ..models import QueryBudget, RetrievalResult, RetrievalRun
+
+
+class RetrievalBackend(ABC):
+    """
+    Abstract interface for retrieval backends.
+
+    :ivar backend_id: Identifier string for the backend.
+    :vartype backend_id: str
+    """
+
+    backend_id: str
+
+    @abstractmethod
+    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+        """
+        Build or register a retrieval run for the backend.
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human name for the recipe.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Run a retrieval query against a backend.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+
+        raise NotImplementedError

biblicus/backends/scan.py

@@ -0,0 +1,292 @@
+"""
+Naive full-scan retrieval backend.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..corpus import Corpus
+from ..frontmatter import parse_front_matter
+from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
+from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
+from ..time import utc_now_iso
+
+
+class ScanRecipeConfig(BaseModel):
+    """
+    Configuration for the naive scan backend.
+
+    :ivar snippet_characters: Maximum characters to include in evidence snippets.
+    :vartype snippet_characters: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    snippet_characters: int = Field(default=400, ge=1)
+
+
+class ScanBackend:
+    """
+    Naive backend that scans all text items at query time.
+
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    """
+
+    backend_id = "scan"
+
+    def build_run(self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]) -> RetrievalRun:
+        """
+        Register a scan backend run (no materialization).
+
+        :param corpus: Corpus to build against.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Backend-specific configuration values.
+        :type config: dict[str, object]
+        :return: Run manifest describing the build.
+        :rtype: RetrievalRun
+        """
+
+        recipe_config = ScanRecipeConfig.model_validate(config)
+        catalog = corpus.load_catalog()
+        recipe = create_recipe_manifest(
+            backend_id=self.backend_id,
+            name=recipe_name,
+            config=recipe_config.model_dump(),
+        )
+        stats = {"items": len(catalog.items), "text_items": _count_text_items(catalog.items.values())}
+        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
+        corpus.write_run(run)
+        return run
+
+    def query(
+        self,
+        corpus: Corpus,
+        *,
+        run: RetrievalRun,
+        query_text: str,
+        budget: QueryBudget,
+    ) -> RetrievalResult:
+        """
+        Query the corpus with a full scan.
+
+        :param corpus: Corpus associated with the run.
+        :type corpus: Corpus
+        :param run: Run manifest to use for querying.
+        :type run: RetrievalRun
+        :param query_text: Query text to execute.
+        :type query_text: str
+        :param budget: Evidence selection budget.
+        :type budget: QueryBudget
+        :return: Retrieval results containing evidence.
+        :rtype: RetrievalResult
+        """
+
+        recipe_config = ScanRecipeConfig.model_validate(run.recipe.config)
+        catalog = corpus.load_catalog()
+        query_tokens = _tokenize_query(query_text)
+        scored_candidates = _score_items(
+            corpus,
+            catalog.items.values(),
+            query_tokens,
+            recipe_config.snippet_characters,
+        )
+        sorted_candidates = sorted(
+            scored_candidates,
+            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
+        )
+        ranked = [
+            evidence_item.model_copy(
+                update={
+                    "rank": index,
+                    "recipe_id": run.recipe.recipe_id,
+                    "run_id": run.run_id,
+                }
+            )
+            for index, evidence_item in enumerate(sorted_candidates, start=1)
+        ]
+        evidence = apply_budget(ranked, budget)
+        stats = {"candidates": len(sorted_candidates), "returned": len(evidence)}
+        return RetrievalResult(
+            query_text=query_text,
+            budget=budget,
+            run_id=run.run_id,
+            recipe_id=run.recipe.recipe_id,
+            backend_id=self.backend_id,
+            generated_at=utc_now_iso(),
+            evidence=evidence,
+            stats=stats,
+        )
+
+
+def _count_text_items(items: Iterable[object]) -> int:
+    """
+    Count catalog items that represent text content.
+
+    :param items: Catalog items to inspect.
+    :type items: Iterable[object]
+    :return: Number of text items.
+    :rtype: int
+    """
+
+    text_item_count = 0
+    for catalog_item in items:
+        media_type = getattr(catalog_item, "media_type", "")
+        if media_type == "text/markdown" or str(media_type).startswith("text/"):
+            text_item_count += 1
+    return text_item_count
+
+
+def _tokenize_query(query_text: str) -> List[str]:
+    """
+    Tokenize a query string for naive text matching.
+
+    :param query_text: Raw query text.
+    :type query_text: str
+    :return: Lowercased non-empty tokens.
+    :rtype: list[str]
+    """
+
+    return [token for token in query_text.lower().split() if token]
+
+
+def _load_text_from_item(corpus: Corpus, relpath: str, media_type: str) -> Optional[str]:
+    """
+    Load a text payload from a catalog item.
+
+    :param corpus: Corpus containing the item.
+    :type corpus: Corpus
+    :param relpath: Relative path to the stored content.
+    :type relpath: str
+    :param media_type: Media type for the stored content.
+    :type media_type: str
+    :return: Text payload or None if not decodable as text.
+    :rtype: str or None
+    """
+
+    content_path = corpus.root / relpath
+    raw_bytes = content_path.read_bytes()
+    if media_type == "text/markdown":
+        markdown_text = raw_bytes.decode("utf-8")
+        parsed_document = parse_front_matter(markdown_text)
+        return parsed_document.body
+    if media_type.startswith("text/"):
+        return raw_bytes.decode("utf-8")
+    return None
+
+
+def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]:
+    """
+    Locate the earliest token match span in a text payload.
+
+    :param text: Text to scan.
+    :type text: str
+    :param tokens: Query tokens.
+    :type tokens: list[str]
+    :return: Start/end span for the earliest match, or None if no matches.
+    :rtype: tuple[int, int] or None
+    """
+
+    lower_text = text.lower()
+    best_start: Optional[int] = None
+    best_end: Optional[int] = None
+    for token in tokens:
+        if not token:
+            continue
+        token_start = lower_text.find(token)
+        if token_start == -1:
+            continue
+        token_end = token_start + len(token)
+        if best_start is None or token_start < best_start:
+            best_start = token_start
+            best_end = token_end
+    if best_start is None or best_end is None:
+        return None
+    return best_start, best_end
+
+
+def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int) -> str:
+    """
+    Build a snippet around a match span, constrained by a character budget.
+
+    :param text: Source text to slice.
+    :type text: str
+    :param span: Match span to center on.
+    :type span: tuple[int, int] or None
+    :param max_chars: Maximum snippet length.
+    :type max_chars: int
+    :return: Snippet text.
+    :rtype: str
+    """
+
+    if not text:
+        return ""
+    if span is None:
+        return text[:max_chars]
+    span_start, span_end = span
+    half_window = max_chars // 2
+    snippet_start = max(span_start - half_window, 0)
+    snippet_end = min(span_end + half_window, len(text))
+    return text[snippet_start:snippet_end]
+
+
+def _score_items(
+    corpus: Corpus,
+    items: Iterable[object],
+    tokens: List[str],
+    snippet_characters: int,
+) -> List[Evidence]:
+    """
+    Score catalog items by token frequency and return evidence candidates.
+
+    :param corpus: Corpus containing the items.
+    :type corpus: Corpus
+    :param items: Catalog items to score.
+    :type items: Iterable[object]
+    :param tokens: Query tokens to count.
+    :type tokens: list[str]
+    :param snippet_characters: Snippet length budget.
+    :type snippet_characters: int
+    :return: Evidence candidates with provisional ranks.
+    :rtype: list[Evidence]
+    """
+
+    evidence_items: List[Evidence] = []
+    for catalog_item in items:
+        media_type = getattr(catalog_item, "media_type", "")
+        relpath = getattr(catalog_item, "relpath", "")
+        item_text = _load_text_from_item(corpus, relpath, media_type)
+        if item_text is None:
+            continue
+        lower_text = item_text.lower()
+        match_score = sum(lower_text.count(token) for token in tokens)
+        if match_score <= 0:
+            continue
+        span = _find_first_match(item_text, tokens)
+        snippet = _build_snippet(item_text, span, max_chars=snippet_characters)
+        span_start = span[0] if span else None
+        span_end = span[1] if span else None
+        evidence_items.append(
+            Evidence(
+                item_id=str(getattr(catalog_item, "id")),
+                source_uri=getattr(catalog_item, "source_uri", None),
+                media_type=str(media_type),
+                score=float(match_score),
+                rank=1,
+                text=snippet,
+                content_ref=None,
+                span_start=span_start,
+                span_end=span_end,
+                stage="scan",
+                recipe_id="",
+                run_id="",
+                hash=hash_text(snippet),
+            )
+        )
+
+    return evidence_items