biblicus 0.15.1__py3-none-any.whl → 0.16.0__py3-none-any.whl

biblicus/chunking.py

@@ -0,0 +1,396 @@
+"""
+Chunking primitives for text retrieval backends.
+
+Chunking converts a document-sized text string into smaller spans that can be embedded or indexed.
+The chunk span offsets are expressed as character positions into the original text string.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class TokenSpan(BaseModel):
+    """
+    A token with its character span in a source string.
+
+    :ivar token: Token text.
+    :vartype token: str
+    :ivar span_start: Inclusive start character offset.
+    :vartype span_start: int
+    :ivar span_end: Exclusive end character offset.
+    :vartype span_end: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    token: str = Field(min_length=1)
+    span_start: int = Field(ge=0)
+    span_end: int = Field(ge=0)
+
+    @model_validator(mode="after")
+    def _validate_span(self) -> "TokenSpan":
+        if self.span_end <= self.span_start:
+            raise ValueError("token span_end must be greater than span_start")
+        return self
+
+
+class Tokenizer(ABC):
+    """
+    Interface for producing token spans from text.
+
+    Tokenizers are used by token-window chunking strategies to convert token indices into
+    stable character spans.
+
+    :ivar tokenizer_id: Tokenizer identifier.
+    :vartype tokenizer_id: str
+    """
+
+    tokenizer_id: str
+
+    @abstractmethod
+    def tokenize(self, text: str) -> List[TokenSpan]:
+        """
+        Tokenize a string and return spans for each token.
+
+        :param text: Input text.
+        :type text: str
+        :return: Token spans.
+        :rtype: list[TokenSpan]
+        """
+        raise NotImplementedError
+
+
+class WhitespaceTokenizer(Tokenizer):
+    """
+    Tokenizer that treats runs of non-whitespace characters as tokens.
+    """
+
+    tokenizer_id = "whitespace"
+
+    def tokenize(self, text: str) -> List[TokenSpan]:
+        """
+        Tokenize a string by whitespace boundaries.
+
+        :param text: Input text.
+        :type text: str
+        :return: Token spans for each non-whitespace token.
+        :rtype: list[TokenSpan]
+        """
+        import re
+
+        spans: List[TokenSpan] = []
+        for match in re.finditer(r"\S+", text):
+            spans.append(
+                TokenSpan(
+                    token=match.group(0),
+                    span_start=int(match.start()),
+                    span_end=int(match.end()),
+                )
+            )
+        return spans
+
+
+class TextChunk(BaseModel):
+    """
+    A chunk extracted from a larger text string.
+
+    :ivar chunk_id: Stable chunk identifier within a build.
+    :vartype chunk_id: int
+    :ivar item_id: Source item identifier.
+    :vartype item_id: str
+    :ivar span_start: Inclusive start character offset.
+    :vartype span_start: int
+    :ivar span_end: Exclusive end character offset.
+    :vartype span_end: int
+    :ivar text: Chunk text.
+    :vartype text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    chunk_id: int = Field(ge=0)
+    item_id: str = Field(min_length=1)
+    span_start: int = Field(ge=0)
+    span_end: int = Field(ge=0)
+    text: str
+
+    @model_validator(mode="after")
+    def _validate_span(self) -> "TextChunk":
+        if self.span_end <= self.span_start:
+            raise ValueError("chunk span_end must be greater than span_start")
+        if not isinstance(self.text, str) or not self.text:
+            raise ValueError("chunk text must be non-empty")
+        return self
+
+
+class Chunker(ABC):
+    """
+    Interface for converting text into chunks.
+
+    :ivar chunker_id: Chunker identifier.
+    :vartype chunker_id: str
+    """
+
+    chunker_id: str
+
+    @abstractmethod
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Split a text string into chunks.
+
+        :param item_id: Item identifier that produced the text.
+        :type item_id: str
+        :param text: Full text to chunk.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        raise NotImplementedError
+
+
+class FixedCharWindowChunker(Chunker):
+    """
+    Chunker that produces overlapping fixed-size character windows.
+    """
+
+    chunker_id = "fixed-char-window"
+
+    def __init__(self, *, window_characters: int, overlap_characters: int) -> None:
+        self._window_characters = int(window_characters)
+        self._overlap_characters = int(overlap_characters)
+        if self._window_characters <= 0:
+            raise ValueError("window_characters must be greater than 0")
+        if self._overlap_characters < 0:
+            raise ValueError("overlap_characters must be greater than or equal to 0")
+        if self._overlap_characters >= self._window_characters:
+            raise ValueError("overlap_characters must be less than window_characters")
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string into fixed-size character windows.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        position = 0
+        stride = self._window_characters - self._overlap_characters
+        while position < len(text):
+            span_start = position
+            span_end = min(position + self._window_characters, len(text))
+            chunk_text = text[span_start:span_end]
+            if chunk_text.strip():
+                chunks.append(
+                    TextChunk(
+                        chunk_id=chunk_id,
+                        item_id=item_id,
+                        span_start=span_start,
+                        span_end=span_end,
+                        text=chunk_text,
+                    )
+                )
+                chunk_id += 1
+            if span_end >= len(text):
+                position = len(text)
+            else:
+                position += stride
+        return chunks
+
+
+class ParagraphChunker(Chunker):
+    """
+    Chunker that produces paragraph spans separated by blank lines.
+    """
+
+    chunker_id = "paragraph"
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string by paragraph boundaries.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        import re
+
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        for match in re.finditer(r"(?:[^\n]|\n(?!\n))+", text):
+            span_start = int(match.start())
+            span_end = int(match.end())
+            chunk_text = text[span_start:span_end]
+            if not chunk_text.strip():
+                continue
+            chunks.append(
+                TextChunk(
+                    chunk_id=chunk_id,
+                    item_id=item_id,
+                    span_start=span_start,
+                    span_end=span_end,
+                    text=chunk_text,
+                )
+            )
+            chunk_id += 1
+        return chunks
+
+
+class FixedTokenWindowChunker(Chunker):
+    """
+    Chunker that produces overlapping fixed-size token windows.
+    """
+
+    chunker_id = "fixed-token-window"
+
+    def __init__(self, *, window_tokens: int, overlap_tokens: int, tokenizer: Tokenizer) -> None:
+        self._window_tokens = int(window_tokens)
+        self._overlap_tokens = int(overlap_tokens)
+        self._tokenizer = tokenizer
+        if self._window_tokens <= 0:
+            raise ValueError("window_tokens must be greater than 0")
+        if self._overlap_tokens < 0:
+            raise ValueError("overlap_tokens must be greater than or equal to 0")
+        if self._overlap_tokens >= self._window_tokens:
+            raise ValueError("overlap_tokens must be less than window_tokens")
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string into fixed-size token windows.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        token_spans = self._tokenizer.tokenize(text)
+        if not token_spans:
+            return []
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        stride = self._window_tokens - self._overlap_tokens
+        token_index = 0
+        while token_index < len(token_spans):
+            window_end = min(token_index + self._window_tokens, len(token_spans))
+            span_start = token_spans[token_index].span_start
+            span_end = token_spans[window_end - 1].span_end
+            chunk_text = text[span_start:span_end]
+            chunks.append(
+                TextChunk(
+                    chunk_id=chunk_id,
+                    item_id=item_id,
+                    span_start=span_start,
+                    span_end=span_end,
+                    text=chunk_text,
+                )
+            )
+            chunk_id += 1
+            if window_end >= len(token_spans):
+                token_index = len(token_spans)
+            else:
+                token_index += stride
+        return chunks
+
+
+class TokenizerConfig(BaseModel):
+    """
+    Configuration for tokenizer selection.
+
+    :ivar tokenizer_id: Tokenizer identifier.
+    :vartype tokenizer_id: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    tokenizer_id: str = Field(min_length=1)
+
+    def build_tokenizer(self) -> Tokenizer:
+        """
+        Build a tokenizer instance from this configuration.
+
+        :return: Tokenizer instance.
+        :rtype: Tokenizer
+        :raises ValueError: If the tokenizer identifier is unknown.
+        """
+        if self.tokenizer_id == WhitespaceTokenizer.tokenizer_id:
+            return WhitespaceTokenizer()
+        raise ValueError(f"Unknown tokenizer_id: {self.tokenizer_id!r}")
+
+
+class ChunkerConfig(BaseModel):
+    """
+    Configuration for chunker selection.
+
+    :ivar chunker_id: Chunker identifier.
+    :vartype chunker_id: str
+    :ivar window_characters: Window size for fixed character chunking.
+    :vartype window_characters: int or None
+    :ivar overlap_characters: Overlap size for fixed character chunking.
+    :vartype overlap_characters: int or None
+    :ivar window_tokens: Window size for fixed token chunking.
+    :vartype window_tokens: int or None
+    :ivar overlap_tokens: Overlap size for fixed token chunking.
+    :vartype overlap_tokens: int or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    chunker_id: str = Field(min_length=1)
+    window_characters: Optional[int] = Field(default=None, ge=1)
+    overlap_characters: Optional[int] = Field(default=None, ge=0)
+    window_tokens: Optional[int] = Field(default=None, ge=1)
+    overlap_tokens: Optional[int] = Field(default=None, ge=0)
+
+    def build_chunker(self, *, tokenizer: Optional[Tokenizer]) -> Chunker:
+        """
+        Build a chunker instance from this configuration.
+
+        :param tokenizer: Tokenizer used by token-window chunking strategies.
+        :type tokenizer: Tokenizer or None
+        :return: Chunker instance.
+        :rtype: Chunker
+        :raises ValueError: If required configuration is missing or unknown.
+        """
+        if self.chunker_id == FixedCharWindowChunker.chunker_id:
+            if self.window_characters is None or self.overlap_characters is None:
+                raise ValueError(
+                    "fixed-char-window requires window_characters and overlap_characters"
+                )
+            return FixedCharWindowChunker(
+                window_characters=self.window_characters,
+                overlap_characters=self.overlap_characters,
+            )
+        if self.chunker_id == ParagraphChunker.chunker_id:
+            return ParagraphChunker()
+        if self.chunker_id == FixedTokenWindowChunker.chunker_id:
+            if self.window_tokens is None or self.overlap_tokens is None:
+                raise ValueError("fixed-token-window requires window_tokens and overlap_tokens")
+            if tokenizer is None:
+                raise ValueError("tokenizer configuration is required for fixed-token-window")
+            return FixedTokenWindowChunker(
+                window_tokens=self.window_tokens,
+                overlap_tokens=self.overlap_tokens,
+                tokenizer=tokenizer,
+            )
+        raise ValueError(f"Unknown chunker_id: {self.chunker_id!r}")

biblicus/cli.py

@@ -8,7 +8,7 @@ import argparse
 import json
 import sys
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional
 
 from pydantic import ValidationError
 
@@ -239,15 +239,23 @@ def cmd_purge(arguments: argparse.Namespace) -> int:
     return 0
 
 
-def _parse_config_pairs(pairs: Optional[List[str]]) -> Dict[str, object]:
+def _parse_config_pairs(pairs: Optional[Iterable[str]]) -> Dict[str, object]:
     """
-    Parse repeated key=value config pairs.
+    Parse key=value pairs into a configuration mapping.
 
-    :param pairs: Config pairs supplied via the command-line interface.
-    :type pairs: list[str] or None
-    :return: Parsed config mapping.
+    This is used by a few command-line options that accept repeated key=value items.
+    Values are coerced to useful types in a predictable way:
+
+    - JSON objects/arrays (leading ``{`` or ``[``) are parsed as JSON.
+    - Whole numbers are parsed as integers.
+    - Other numeric forms are parsed as floats.
+    - Everything else remains a string.
+
+    :param pairs: Iterable of key=value strings.
+    :type pairs: Iterable[str] or None
+    :return: Parsed configuration mapping.
     :rtype: dict[str, object]
-    :raises ValueError: If any entry is not key=value.
+    :raises ValueError: If any entry is not a key=value pair or values are invalid.
     """
     config: Dict[str, object] = {}
     for item in pairs or []:
@@ -257,8 +265,14 @@ def _parse_config_pairs(pairs: Optional[List[str]]) -> Dict[str, object]:
         key = key.strip()
         if not key:
             raise ValueError("Config keys must be non-empty")
+        raw = raw.strip()
         value: object = raw
-        if raw.isdigit():
+        if raw.startswith("{") or raw.startswith("["):
+            try:
+                value = json.loads(raw)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"Config value must be valid JSON for key {key!r}") from exc
+        elif raw.isdigit():
             value = int(raw)
         else:
             try:
@@ -359,6 +373,7 @@ def _budget_from_args(arguments: argparse.Namespace) -> QueryBudget:
     """
     return QueryBudget(
         max_total_items=arguments.max_total_items,
+        offset=getattr(arguments, "offset", 0),
         max_total_characters=arguments.max_total_characters,
         max_items_per_source=arguments.max_items_per_source,
     )
@@ -373,13 +388,26 @@ def cmd_build(arguments: argparse.Namespace) -> int:
     :return: Exit code.
     :rtype: int
     """
+    from .recipes import apply_dotted_overrides, load_recipe_view, parse_dotted_overrides
+
     corpus = (
         Corpus.open(arguments.corpus)
         if getattr(arguments, "corpus", None)
         else Corpus.find(Path.cwd())
     )
     backend = get_backend(arguments.backend)
-    config = _parse_config_pairs(arguments.config)
+
+    base_config: Dict[str, object] = {}
+    if getattr(arguments, "recipe", None):
+        base_config = load_recipe_view(
+            arguments.recipe,
+            recipe_label="Recipe file",
+            mapping_error_message="Retrieval build recipe must be a mapping/object",
+        )
+
+    overrides = parse_dotted_overrides(arguments.config)
+    config = apply_dotted_overrides(base_config, overrides)
+
     run = backend.build_run(corpus, recipe_name=arguments.recipe_name, config=config)
     print(run.model_dump_json(indent=2))
     return 0
@@ -947,11 +975,17 @@ def build_parser() -> argparse.ArgumentParser:
         help="Backend identifier (for example, scan, sqlite-full-text-search).",
     )
     p_build.add_argument("--recipe-name", default="default", help="Human-readable recipe name.")
+    p_build.add_argument(
+        "--recipe",
+        default=None,
+        action="append",
+        help="Path to YAML recipe file (repeatable). If provided, recipes are composed in precedence order.",
+    )
     p_build.add_argument(
         "--config",
         action="append",
         default=None,
-        help="Backend config as key=value (repeatable).",
+        help="Backend config override as key=value (repeatable). Dotted keys create nested config mappings.",
     )
     p_build.set_defaults(func=cmd_build)
 
@@ -1030,6 +1064,12 @@ def build_parser() -> argparse.ArgumentParser:
     p_query.add_argument("--run", default=None, help="Run identifier (defaults to latest run).")
     p_query.add_argument("--backend", default=None, help="Validate backend identifier.")
     p_query.add_argument("--query", default=None, help="Query text (defaults to standard input).")
+    p_query.add_argument(
+        "--offset",
+        type=int,
+        default=0,
+        help="Skip this many ranked candidates before selecting evidence (pagination).",
+    )
     p_query.add_argument("--max-total-items", type=int, default=5)
     p_query.add_argument("--max-total-characters", type=int, default=2000)
     p_query.add_argument("--max-items-per-source", type=int, default=5)

biblicus/embedding_providers.py

@@ -0,0 +1,122 @@
+"""
+Embedding provider interfaces for retrieval backends.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from abc import ABC, abstractmethod
+from typing import Optional, Sequence
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EmbeddingProvider(ABC):
+    """
+    Interface for producing dense embedding vectors from text.
+
+    :ivar provider_id: Provider identifier.
+    :vartype provider_id: str
+    """
+
+    provider_id: str
+
+    @abstractmethod
+    def embed_texts(self, texts: Sequence[str]) -> np.ndarray:
+        """
+        Embed a batch of texts.
+
+        :param texts: Text inputs.
+        :type texts: Sequence[str]
+        :return: 2D float array with shape (len(texts), dimensions).
+        :rtype: numpy.ndarray
+        """
+        raise NotImplementedError
+
+
+def _l2_normalize_rows(matrix: np.ndarray) -> np.ndarray:
+    norms = np.linalg.norm(matrix, axis=1, keepdims=True)
+    norms = np.where(norms == 0, 1.0, norms)
+    return matrix / norms
+
+
+class HashEmbeddingProvider(EmbeddingProvider):
+    """
+    Deterministic embedding provider for tests and demos.
+
+    The output vectors are stable across runs and require no external services.
+    """
+
+    provider_id = "hash-embedding"
+
+    def __init__(self, *, dimensions: int, seed: str = "biblicus") -> None:
+        self._dimensions = int(dimensions)
+        self._seed = str(seed)
+        if self._dimensions <= 0:
+            raise ValueError("dimensions must be greater than 0")
+
+    def embed_texts(self, texts: Sequence[str]) -> np.ndarray:
+        """
+        Embed a batch of texts deterministically.
+
+        :param texts: Text inputs.
+        :type texts: Sequence[str]
+        :return: Normalized embedding matrix.
+        :rtype: numpy.ndarray
+        """
+        items = list(texts)
+        if not items:
+            return np.zeros((0, self._dimensions), dtype=np.float32)
+
+        vectors = np.zeros((len(items), self._dimensions), dtype=np.float32)
+        for row_index, text in enumerate(items):
+            vectors[row_index] = self._hash_to_vector(text)
+        return _l2_normalize_rows(vectors)
+
+    def _hash_to_vector(self, text: str) -> np.ndarray:
+        output = np.empty((self._dimensions,), dtype=np.float32)
+        remaining = self._dimensions
+        offset = 0
+        counter = 0
+        while remaining > 0:
+            digest = hashlib.sha256(f"{self._seed}:{counter}:{text}".encode("utf-8")).digest()
+            raw = np.frombuffer(digest, dtype=np.uint8).astype(np.float32)
+            raw = (raw / 255.0) * 2.0 - 1.0
+            take = min(remaining, raw.shape[0])
+            output[offset : offset + take] = raw[:take]
+            remaining -= take
+            offset += take
+            counter += 1
+        return output
+
+
+class EmbeddingProviderConfig(BaseModel):
+    """
+    Configuration for embedding provider selection.
+
+    :ivar provider_id: Provider identifier.
+    :vartype provider_id: str
+    :ivar dimensions: Dimensionality of produced vectors.
+    :vartype dimensions: int
+    :ivar seed: Optional deterministic seed for test providers.
+    :vartype seed: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    provider_id: str = Field(min_length=1)
+    dimensions: int = Field(ge=1)
+    seed: Optional[str] = None
+
+    def build_provider(self) -> EmbeddingProvider:
+        """
+        Build an embedding provider instance from this configuration.
+
+        :return: Embedding provider instance.
+        :rtype: EmbeddingProvider
+        :raises ValueError: If the provider identifier is unknown.
+        """
+        if self.provider_id == HashEmbeddingProvider.provider_id:
+            return HashEmbeddingProvider(dimensions=self.dimensions, seed=self.seed or "biblicus")
+        raise ValueError(f"Unknown embedding provider_id: {self.provider_id!r}")

biblicus/frontmatter.py

@@ -44,6 +44,8 @@ def parse_front_matter(text: str) -> FrontMatterDocument:
 
     raw_yaml = text[4:front_matter_end]
     body = text[front_matter_end + len("\n---\n") :]
+    if body.startswith("\n"):
+        body = body[1:]
 
     metadata = yaml.safe_load(raw_yaml) or {}
     if not isinstance(metadata, dict):

biblicus/models.py

@@ -224,8 +224,16 @@ class QueryBudget(BaseModel):
     """
     Evidence selection budget for retrieval.
 
+    The budget constrains the *returned* evidence. It intentionally does not
+    change how a backend scores candidates, only how many evidence items are
+    selected and how much text is allowed through.
+
     :ivar max_total_items: Maximum number of evidence items to return.
     :vartype max_total_items: int
+    :ivar offset: Number of ranked candidates to skip before selecting evidence.
+        This enables simple pagination by re-running the same query with a
+        higher offset.
+    :vartype offset: int
     :ivar max_total_characters: Optional maximum total characters across evidence text.
     :vartype max_total_characters: int or None
     :ivar max_items_per_source: Optional cap per source uniform resource identifier.
@@ -235,6 +243,7 @@ class QueryBudget(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     max_total_items: int = Field(ge=1)
+    offset: int = Field(default=0, ge=0)
     max_total_characters: Optional[int] = Field(default=None, ge=1)
     max_items_per_source: Optional[int] = Field(default=None, ge=1)
 

biblicus/retrieval.py

@@ -108,8 +108,13 @@ def apply_budget(evidence: Iterable[Evidence], budget: QueryBudget) -> List[Evid
     selected_evidence: List[Evidence] = []
     source_counts: Dict[str, int] = {}
     total_characters = 0
+    skipped = 0
 
     for candidate_evidence in evidence:
+        if skipped < budget.offset:
+            skipped += 1
+            continue
+
         if len(selected_evidence) >= budget.max_total_items:
             break