biblicus 0.15.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

biblicus/backends/{vector.py → tf_vector.py}

@@ -8,7 +8,7 @@ import math
 import re
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict
 
 from ..corpus import Corpus
 from ..frontmatter import parse_front_matter
@@ -24,23 +24,23 @@ from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifes
 from ..time import utc_now_iso
 
 
-class VectorRecipeConfig(BaseModel):
+class TfVectorRecipeConfig(BaseModel):
     """
-    Configuration for the vector retrieval backend.
+    Configuration for the term-frequency vector retrieval backend.
 
-    :ivar snippet_characters: Maximum characters to include in evidence snippets.
-    :vartype snippet_characters: int
     :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
     :vartype extraction_run: str or None
+    :ivar snippet_characters: Optional maximum character count for returned evidence text.
+    :vartype snippet_characters: int or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
-    snippet_characters: int = Field(default=400, ge=1)
     extraction_run: Optional[str] = None
+    snippet_characters: Optional[int] = None
 
 
-class VectorBackend:
+class TfVectorBackend:
     """
     Deterministic vector backend using term-frequency cosine similarity.
 
@@ -48,7 +48,7 @@ class VectorBackend:
     :vartype backend_id: str
     """
 
-    backend_id = "vector"
+    backend_id = "tf-vector"
 
     def build_run(
         self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
@@ -65,7 +65,7 @@ class VectorBackend:
         :return: Run manifest describing the build.
         :rtype: RetrievalRun
         """
-        recipe_config = VectorRecipeConfig.model_validate(config)
+        recipe_config = TfVectorRecipeConfig.model_validate(config)
         catalog = corpus.load_catalog()
         recipe = create_recipe_manifest(
             backend_id=self.backend_id,
@@ -102,7 +102,7 @@ class VectorBackend:
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-        recipe_config = VectorRecipeConfig.model_validate(run.recipe.config)
+        recipe_config = TfVectorRecipeConfig.model_validate(run.recipe.config)
         query_tokens = _tokenize_text(query_text)
         if not query_tokens:
             return RetrievalResult(
@@ -125,8 +125,8 @@ class VectorBackend:
             query_tokens=query_tokens,
             query_vector=query_vector,
             query_norm=query_norm,
-            snippet_characters=recipe_config.snippet_characters,
             extraction_reference=extraction_reference,
+            snippet_characters=recipe_config.snippet_characters,
         )
         sorted_candidates = sorted(
             scored_candidates,
@@ -157,7 +157,7 @@ class VectorBackend:
 
 
 def _resolve_extraction_reference(
-    corpus: Corpus, recipe_config: VectorRecipeConfig
+    corpus: Corpus, recipe_config: TfVectorRecipeConfig
 ) -> Optional[ExtractionRunReference]:
     """
     Resolve an extraction run reference from a recipe config.
@@ -165,7 +165,7 @@ def _resolve_extraction_reference(
     :param corpus: Corpus associated with the recipe.
     :type corpus: Corpus
     :param recipe_config: Parsed vector recipe configuration.
-    :type recipe_config: VectorRecipeConfig
+    :type recipe_config: TfVectorRecipeConfig
     :return: Parsed extraction reference or None.
     :rtype: ExtractionRunReference or None
     :raises FileNotFoundError: If an extraction run is referenced but not present.
@@ -183,7 +183,7 @@ def _resolve_extraction_reference(
 
 
 def _count_text_items(
-    corpus: Corpus, items: Iterable[object], recipe_config: VectorRecipeConfig
+    corpus: Corpus, items: Iterable[object], recipe_config: TfVectorRecipeConfig
 ) -> int:
     """
     Count catalog items that represent text content.
@@ -193,7 +193,7 @@ def _count_text_items(
     :param items: Catalog items to inspect.
     :type items: Iterable[object]
     :param recipe_config: Parsed vector recipe configuration.
-    :type recipe_config: VectorRecipeConfig
+    :type recipe_config: TfVectorRecipeConfig
     :return: Number of text items.
     :rtype: int
     """
@@ -359,21 +359,13 @@ def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]
     return best_start, best_end
 
 
-def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int) -> str:
-    """
-    Build a snippet around a match span, constrained by a character budget.
-
-    :param text: Source text to slice.
-    :type text: str
-    :param span: Match span to center on.
-    :type span: tuple[int, int] or None
-    :param max_chars: Maximum snippet length.
-    :type max_chars: int
-    :return: Snippet text.
-    :rtype: str
-    """
+def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: Optional[int]) -> str:
+    if max_chars is None:
+        return text
     if not text:
         return ""
+    if max_chars <= 0:
+        return ""
     if span is None:
         return text[:max_chars]
     span_start, span_end = span
@@ -390,8 +382,8 @@ def _score_items(
     query_tokens: List[str],
     query_vector: Dict[str, float],
     query_norm: float,
-    snippet_characters: int,
     extraction_reference: Optional[ExtractionRunReference],
+    snippet_characters: Optional[int],
 ) -> List[Evidence]:
     """
     Score catalog items and return evidence candidates.
@@ -406,10 +398,10 @@ def _score_items(
     :type query_vector: dict[str, float]
     :param query_norm: Query vector norm.
     :type query_norm: float
-    :param snippet_characters: Snippet length budget.
-    :type snippet_characters: int
     :param extraction_reference: Optional extraction run reference.
     :type extraction_reference: ExtractionRunReference or None
+    :param snippet_characters: Optional maximum character count for returned evidence text.
+    :type snippet_characters: int or None
     :return: Evidence candidates with provisional ranks.
     :rtype: list[Evidence]
     """
@@ -437,9 +429,9 @@ def _score_items(
         if similarity <= 0:
             continue
         span = _find_first_match(item_text, query_tokens)
-        snippet = _build_snippet(item_text, span, max_chars=snippet_characters)
         span_start = span[0] if span else None
         span_end = span[1] if span else None
+        evidence_text = _build_snippet(item_text, span, max_chars=snippet_characters)
         evidence_items.append(
             Evidence(
                 item_id=str(getattr(catalog_item, "id")),
@@ -447,14 +439,15 @@ def _score_items(
                 media_type=str(media_type),
                 score=float(similarity),
                 rank=1,
-                text=snippet,
+                text=evidence_text,
                 content_ref=None,
                 span_start=span_start,
                 span_end=span_end,
-                stage="vector",
+                stage="tf-vector",
                 recipe_id="",
                 run_id="",
-                hash=hash_text(snippet),
+                metadata=getattr(catalog_item, "metadata", {}) or {},
+                hash=hash_text(evidence_text or ""),
             )
         )
     return evidence_items

biblicus/chunking.py

@@ -0,0 +1,396 @@
+"""
+Chunking primitives for text retrieval backends.
+
+Chunking converts a document-sized text string into smaller spans that can be embedded or indexed.
+The chunk span offsets are expressed as character positions into the original text string.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class TokenSpan(BaseModel):
+    """
+    A token with its character span in a source string.
+
+    :ivar token: Token text.
+    :vartype token: str
+    :ivar span_start: Inclusive start character offset.
+    :vartype span_start: int
+    :ivar span_end: Exclusive end character offset.
+    :vartype span_end: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    token: str = Field(min_length=1)
+    span_start: int = Field(ge=0)
+    span_end: int = Field(ge=0)
+
+    @model_validator(mode="after")
+    def _validate_span(self) -> "TokenSpan":
+        if self.span_end <= self.span_start:
+            raise ValueError("token span_end must be greater than span_start")
+        return self
+
+
+class Tokenizer(ABC):
+    """
+    Interface for producing token spans from text.
+
+    Tokenizers are used by token-window chunking strategies to convert token indices into
+    stable character spans.
+
+    :ivar tokenizer_id: Tokenizer identifier.
+    :vartype tokenizer_id: str
+    """
+
+    tokenizer_id: str
+
+    @abstractmethod
+    def tokenize(self, text: str) -> List[TokenSpan]:
+        """
+        Tokenize a string and return spans for each token.
+
+        :param text: Input text.
+        :type text: str
+        :return: Token spans.
+        :rtype: list[TokenSpan]
+        """
+        raise NotImplementedError
+
+
+class WhitespaceTokenizer(Tokenizer):
+    """
+    Tokenizer that treats runs of non-whitespace characters as tokens.
+    """
+
+    tokenizer_id = "whitespace"
+
+    def tokenize(self, text: str) -> List[TokenSpan]:
+        """
+        Tokenize a string by whitespace boundaries.
+
+        :param text: Input text.
+        :type text: str
+        :return: Token spans for each non-whitespace token.
+        :rtype: list[TokenSpan]
+        """
+        import re
+
+        spans: List[TokenSpan] = []
+        for match in re.finditer(r"\S+", text):
+            spans.append(
+                TokenSpan(
+                    token=match.group(0),
+                    span_start=int(match.start()),
+                    span_end=int(match.end()),
+                )
+            )
+        return spans
+
+
+class TextChunk(BaseModel):
+    """
+    A chunk extracted from a larger text string.
+
+    :ivar chunk_id: Stable chunk identifier within a build.
+    :vartype chunk_id: int
+    :ivar item_id: Source item identifier.
+    :vartype item_id: str
+    :ivar span_start: Inclusive start character offset.
+    :vartype span_start: int
+    :ivar span_end: Exclusive end character offset.
+    :vartype span_end: int
+    :ivar text: Chunk text.
+    :vartype text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    chunk_id: int = Field(ge=0)
+    item_id: str = Field(min_length=1)
+    span_start: int = Field(ge=0)
+    span_end: int = Field(ge=0)
+    text: str
+
+    @model_validator(mode="after")
+    def _validate_span(self) -> "TextChunk":
+        if self.span_end <= self.span_start:
+            raise ValueError("chunk span_end must be greater than span_start")
+        if not isinstance(self.text, str) or not self.text:
+            raise ValueError("chunk text must be non-empty")
+        return self
+
+
+class Chunker(ABC):
+    """
+    Interface for converting text into chunks.
+
+    :ivar chunker_id: Chunker identifier.
+    :vartype chunker_id: str
+    """
+
+    chunker_id: str
+
+    @abstractmethod
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Split a text string into chunks.
+
+        :param item_id: Item identifier that produced the text.
+        :type item_id: str
+        :param text: Full text to chunk.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        raise NotImplementedError
+
+
+class FixedCharWindowChunker(Chunker):
+    """
+    Chunker that produces overlapping fixed-size character windows.
+    """
+
+    chunker_id = "fixed-char-window"
+
+    def __init__(self, *, window_characters: int, overlap_characters: int) -> None:
+        self._window_characters = int(window_characters)
+        self._overlap_characters = int(overlap_characters)
+        if self._window_characters <= 0:
+            raise ValueError("window_characters must be greater than 0")
+        if self._overlap_characters < 0:
+            raise ValueError("overlap_characters must be greater than or equal to 0")
+        if self._overlap_characters >= self._window_characters:
+            raise ValueError("overlap_characters must be less than window_characters")
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string into fixed-size character windows.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        position = 0
+        stride = self._window_characters - self._overlap_characters
+        while position < len(text):
+            span_start = position
+            span_end = min(position + self._window_characters, len(text))
+            chunk_text = text[span_start:span_end]
+            if chunk_text.strip():
+                chunks.append(
+                    TextChunk(
+                        chunk_id=chunk_id,
+                        item_id=item_id,
+                        span_start=span_start,
+                        span_end=span_end,
+                        text=chunk_text,
+                    )
+                )
+                chunk_id += 1
+            if span_end >= len(text):
+                position = len(text)
+            else:
+                position += stride
+        return chunks
+
+
+class ParagraphChunker(Chunker):
+    """
+    Chunker that produces paragraph spans separated by blank lines.
+    """
+
+    chunker_id = "paragraph"
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string by paragraph boundaries.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        import re
+
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        for match in re.finditer(r"(?:[^\n]|\n(?!\n))+", text):
+            span_start = int(match.start())
+            span_end = int(match.end())
+            chunk_text = text[span_start:span_end]
+            if not chunk_text.strip():
+                continue
+            chunks.append(
+                TextChunk(
+                    chunk_id=chunk_id,
+                    item_id=item_id,
+                    span_start=span_start,
+                    span_end=span_end,
+                    text=chunk_text,
+                )
+            )
+            chunk_id += 1
+        return chunks
+
+
+class FixedTokenWindowChunker(Chunker):
+    """
+    Chunker that produces overlapping fixed-size token windows.
+    """
+
+    chunker_id = "fixed-token-window"
+
+    def __init__(self, *, window_tokens: int, overlap_tokens: int, tokenizer: Tokenizer) -> None:
+        self._window_tokens = int(window_tokens)
+        self._overlap_tokens = int(overlap_tokens)
+        self._tokenizer = tokenizer
+        if self._window_tokens <= 0:
+            raise ValueError("window_tokens must be greater than 0")
+        if self._overlap_tokens < 0:
+            raise ValueError("overlap_tokens must be greater than or equal to 0")
+        if self._overlap_tokens >= self._window_tokens:
+            raise ValueError("overlap_tokens must be less than window_tokens")
+
+    def chunk_text(self, *, item_id: str, text: str, starting_chunk_id: int) -> List[TextChunk]:
+        """
+        Chunk a text string into fixed-size token windows.
+
+        :param item_id: Item identifier.
+        :type item_id: str
+        :param text: Input text.
+        :type text: str
+        :param starting_chunk_id: Starting chunk identifier.
+        :type starting_chunk_id: int
+        :return: Chunk list.
+        :rtype: list[TextChunk]
+        """
+        token_spans = self._tokenizer.tokenize(text)
+        if not token_spans:
+            return []
+        chunks: List[TextChunk] = []
+        chunk_id = int(starting_chunk_id)
+        stride = self._window_tokens - self._overlap_tokens
+        token_index = 0
+        while token_index < len(token_spans):
+            window_end = min(token_index + self._window_tokens, len(token_spans))
+            span_start = token_spans[token_index].span_start
+            span_end = token_spans[window_end - 1].span_end
+            chunk_text = text[span_start:span_end]
+            chunks.append(
+                TextChunk(
+                    chunk_id=chunk_id,
+                    item_id=item_id,
+                    span_start=span_start,
+                    span_end=span_end,
+                    text=chunk_text,
+                )
+            )
+            chunk_id += 1
+            if window_end >= len(token_spans):
+                token_index = len(token_spans)
+            else:
+                token_index += stride
+        return chunks
+
+
+class TokenizerConfig(BaseModel):
+    """
+    Configuration for tokenizer selection.
+
+    :ivar tokenizer_id: Tokenizer identifier.
+    :vartype tokenizer_id: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    tokenizer_id: str = Field(min_length=1)
+
+    def build_tokenizer(self) -> Tokenizer:
+        """
+        Build a tokenizer instance from this configuration.
+
+        :return: Tokenizer instance.
+        :rtype: Tokenizer
+        :raises ValueError: If the tokenizer identifier is unknown.
+        """
+        if self.tokenizer_id == WhitespaceTokenizer.tokenizer_id:
+            return WhitespaceTokenizer()
+        raise ValueError(f"Unknown tokenizer_id: {self.tokenizer_id!r}")
+
+
+class ChunkerConfig(BaseModel):
+    """
+    Configuration for chunker selection.
+
+    :ivar chunker_id: Chunker identifier.
+    :vartype chunker_id: str
+    :ivar window_characters: Window size for fixed character chunking.
+    :vartype window_characters: int or None
+    :ivar overlap_characters: Overlap size for fixed character chunking.
+    :vartype overlap_characters: int or None
+    :ivar window_tokens: Window size for fixed token chunking.
+    :vartype window_tokens: int or None
+    :ivar overlap_tokens: Overlap size for fixed token chunking.
+    :vartype overlap_tokens: int or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    chunker_id: str = Field(min_length=1)
+    window_characters: Optional[int] = Field(default=None, ge=1)
+    overlap_characters: Optional[int] = Field(default=None, ge=0)
+    window_tokens: Optional[int] = Field(default=None, ge=1)
+    overlap_tokens: Optional[int] = Field(default=None, ge=0)
+
+    def build_chunker(self, *, tokenizer: Optional[Tokenizer]) -> Chunker:
+        """
+        Build a chunker instance from this configuration.
+
+        :param tokenizer: Tokenizer used by token-window chunking strategies.
+        :type tokenizer: Tokenizer or None
+        :return: Chunker instance.
+        :rtype: Chunker
+        :raises ValueError: If required configuration is missing or unknown.
+        """
+        if self.chunker_id == FixedCharWindowChunker.chunker_id:
+            if self.window_characters is None or self.overlap_characters is None:
+                raise ValueError(
+                    "fixed-char-window requires window_characters and overlap_characters"
+                )
+            return FixedCharWindowChunker(
+                window_characters=self.window_characters,
+                overlap_characters=self.overlap_characters,
+            )
+        if self.chunker_id == ParagraphChunker.chunker_id:
+            return ParagraphChunker()
+        if self.chunker_id == FixedTokenWindowChunker.chunker_id:
+            if self.window_tokens is None or self.overlap_tokens is None:
+                raise ValueError("fixed-token-window requires window_tokens and overlap_tokens")
+            if tokenizer is None:
+                raise ValueError("tokenizer configuration is required for fixed-token-window")
+            return FixedTokenWindowChunker(
+                window_tokens=self.window_tokens,
+                overlap_tokens=self.overlap_tokens,
+                tokenizer=tokenizer,
+            )
+        raise ValueError(f"Unknown chunker_id: {self.chunker_id!r}")