biblicus 0.16.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

biblicus/__init__.py

@@ -2,6 +2,17 @@
 Biblicus public package interface.
 """
 
+from .context_engine import (
+    ContextAssembler,
+    ContextBudgetSpec,
+    ContextDeclaration,
+    ContextExpansionSpec,
+    ContextPackBudgetSpec,
+    ContextPackSpec,
+    ContextPolicySpec,
+    ContextRetrieverRequest,
+    retrieve_context_pack,
+)
 from .corpus import Corpus
 from .knowledge_base import KnowledgeBase
 from .models import (
@@ -16,6 +27,15 @@ from .models import (
 
 __all__ = [
     "__version__",
+    "ContextAssembler",
+    "ContextBudgetSpec",
+    "ContextDeclaration",
+    "ContextExpansionSpec",
+    "ContextPackBudgetSpec",
+    "ContextPackSpec",
+    "ContextPolicySpec",
+    "ContextRetrieverRequest",
+    "retrieve_context_pack",
     "Corpus",
     "CorpusConfig",
     "Evidence",
@@ -27,4 +47,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.16.0"
+__version__ = "1.0.0"

biblicus/backends/embedding_index_common.py

@@ -47,8 +47,6 @@ class EmbeddingIndexRecipeConfig(BaseModel):
     """
     Configuration for embedding-index retrieval backends.
 
-    :ivar snippet_characters: Maximum characters to include in evidence snippets.
-    :vartype snippet_characters: int
     :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
     :vartype extraction_run: str or None
     :ivar chunker: Chunker configuration.
@@ -57,17 +55,52 @@ class EmbeddingIndexRecipeConfig(BaseModel):
     :vartype tokenizer: biblicus.chunking.TokenizerConfig or None
     :ivar embedding_provider: Embedding provider configuration.
     :vartype embedding_provider: biblicus.embedding_providers.EmbeddingProviderConfig
+    :ivar snippet_characters: Optional maximum character count for returned evidence text.
+    :vartype snippet_characters: int or None
+    :ivar maximum_cache_total_items: Optional maximum number of vectors cached per scan batch.
+    :vartype maximum_cache_total_items: int or None
+    :ivar maximum_cache_total_characters: Optional maximum characters cached per scan batch.
+    :vartype maximum_cache_total_characters: int or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
-    snippet_characters: int = Field(default=400, ge=1)
+    snippet_characters: Optional[int] = Field(default=None, ge=1)
+    maximum_cache_total_items: Optional[int] = Field(default=None, ge=1)
+    maximum_cache_total_characters: Optional[int] = Field(default=None, ge=1)
     extraction_run: Optional[str] = None
     chunker: ChunkerConfig = Field(default_factory=lambda: ChunkerConfig(chunker_id="paragraph"))
     tokenizer: Optional[TokenizerConfig] = None
     embedding_provider: EmbeddingProviderConfig
 
 
+def _extract_span_text(text: Optional[str], span: Tuple[int, int]) -> Optional[str]:
+    if not isinstance(text, str):
+        return None
+    span_start, span_end = span
+    if span_start < 0 or span_end <= span_start:
+        return text
+    return text[span_start:span_end]
+
+
+def _build_snippet(
+    text: Optional[str], span: Tuple[int, int], max_chars: Optional[int]
+) -> Optional[str]:
+    if not isinstance(text, str):
+        return None
+    if max_chars is None:
+        return _extract_span_text(text, span)
+    if max_chars <= 0:
+        return ""
+    span_start, span_end = span
+    if span_start < 0 or span_end <= span_start:
+        return text[:max_chars]
+    half_window = max_chars // 2
+    snippet_start = max(span_start - half_window, 0)
+    snippet_end = min(span_end + half_window, len(text))
+    return text[snippet_start:snippet_end]
+
+
 def resolve_extraction_reference(
     corpus: Corpus, recipe_config: EmbeddingIndexRecipeConfig
 ) -> Optional[ExtractionRunReference]:

biblicus/backends/embedding_index_file.py

@@ -16,6 +16,8 @@ from ..time import utc_now_iso
 from .embedding_index_common import (
     ChunkRecord,
     EmbeddingIndexRecipeConfig,
+    _build_snippet,
+    _extract_span_text,
     artifact_paths_for_run,
     chunks_to_records,
     collect_chunks,
@@ -26,7 +28,6 @@ from .embedding_index_common import (
     write_chunks_jsonl,
     write_embeddings,
 )
-from .scan import _build_snippet
 
 
 class EmbeddingIndexFileBackend:
@@ -132,10 +133,12 @@ class EmbeddingIndexFileBackend:
         if query_embedding.shape[0] != 1:
             raise ValueError("Embedding provider returned an invalid query embedding shape")
 
+        batch_rows = recipe_config.maximum_cache_total_items or 4096
         candidates = _top_indices_batched(
             embeddings=embeddings,
             query_vector=query_embedding[0],
             limit=_candidate_limit(budget.max_total_items + budget.offset),
+            batch_rows=batch_rows,
         )
         evidence_items = _build_evidence(
             corpus,
@@ -222,9 +225,11 @@ def _build_evidence(
             media_type=str(getattr(catalog_item, "media_type")),
             extraction_reference=extraction_reference,
         )
-        snippet = _build_snippet(
-            text, (record.span_start, record.span_end), max_chars=recipe_config.snippet_characters
+        span_text = _build_snippet(
+            text, (record.span_start, record.span_end), recipe_config.snippet_characters
         )
+        if span_text is None:
+            span_text = _extract_span_text(text, (record.span_start, record.span_end))
         score = float(cosine_similarity_scores(embeddings[idx : idx + 1], query_vector)[0])
         evidence_items.append(
             Evidence(
@@ -233,7 +238,7 @@ def _build_evidence(
                 media_type=str(getattr(catalog_item, "media_type")),
                 score=score,
                 rank=1,
-                text=snippet,
+                text=span_text,
                 content_ref=None,
                 span_start=record.span_start,
                 span_end=record.span_end,
@@ -241,7 +246,8 @@ def _build_evidence(
                 stage_scores=None,
                 recipe_id=run.recipe.recipe_id,
                 run_id=run.run_id,
-                hash=hash_text(snippet),
+                metadata=getattr(catalog_item, "metadata", {}) or {},
+                hash=hash_text(span_text or ""),
             )
         )
     return evidence_items

biblicus/backends/embedding_index_inmemory.py

@@ -16,6 +16,8 @@ from ..time import utc_now_iso
 from .embedding_index_common import (
     ChunkRecord,
     EmbeddingIndexRecipeConfig,
+    _build_snippet,
+    _extract_span_text,
     artifact_paths_for_run,
     chunks_to_records,
     collect_chunks,
@@ -26,20 +28,19 @@ from .embedding_index_common import (
     write_chunks_jsonl,
     write_embeddings,
 )
-from .scan import _build_snippet
 
 
 class EmbeddingIndexInMemoryRecipeConfig(EmbeddingIndexRecipeConfig):
     """
     Configuration for embedding-index-inmemory retrieval.
 
-    :ivar max_chunks: Maximum chunks allowed for in-memory query loading.
-    :vartype max_chunks: int
+    :ivar maximum_cache_total_items: Maximum chunks allowed for in-memory query loading.
+    :vartype maximum_cache_total_items: int
     """
 
     model_config = ConfigDict(extra="forbid")
 
-    max_chunks: int = Field(default=25000, ge=1)
+    maximum_cache_total_items: int = Field(default=25000, ge=1)
 
 
 class EmbeddingIndexInMemoryBackend:
@@ -66,10 +67,10 @@ class EmbeddingIndexInMemoryBackend:
         """
         recipe_config = EmbeddingIndexInMemoryRecipeConfig.model_validate(config)
         chunks, text_items = collect_chunks(corpus, recipe_config=recipe_config)
-        if len(chunks) > recipe_config.max_chunks:
+        if len(chunks) > recipe_config.maximum_cache_total_items:
             raise ValueError(
-                "embedding-index-inmemory exceeded max_chunks. "
-                "Use embedding-index-file or increase max_chunks."
+                "embedding-index-inmemory exceeded maximum_cache_total_items. "
+                "Use embedding-index-file or increase maximum_cache_total_items."
             )
 
         provider = recipe_config.embedding_provider.build_provider()
@@ -225,9 +226,9 @@ def _build_evidence(
             media_type=media_type,
             extraction_reference=extraction_reference,
         )
-        snippet = _build_snippet(
-            text, (span_start, span_end), max_chars=recipe_config.snippet_characters
-        )
+        span_text = _build_snippet(text, (span_start, span_end), recipe_config.snippet_characters)
+        if span_text is None:
+            span_text = _extract_span_text(text, (span_start, span_end))
         evidence_items.append(
             Evidence(
                 item_id=item_id,
@@ -235,7 +236,7 @@ def _build_evidence(
                 media_type=media_type,
                 score=float(scores[idx]),
                 rank=1,
-                text=snippet,
+                text=span_text,
                 content_ref=None,
                 span_start=span_start,
                 span_end=span_end,
@@ -243,7 +244,8 @@ def _build_evidence(
                 stage_scores=None,
                 recipe_id=run.recipe.recipe_id,
                 run_id=run.run_id,
-                hash=hash_text(snippet),
+                metadata=getattr(catalog_item, "metadata", {}) or {},
+                hash=hash_text(span_text or ""),
             )
         )
     return evidence_items

biblicus/backends/hybrid.py

@@ -217,9 +217,9 @@ def _expand_component_budget(budget: QueryBudget, *, multiplier: int = 5) -> Que
     :return: Expanded budget for component backends.
     :rtype: QueryBudget
     """
-    max_total_characters = budget.max_total_characters
+    maximum_total_characters = budget.maximum_total_characters
     expanded_characters = (
-        max_total_characters * multiplier if max_total_characters is not None else None
+        maximum_total_characters * multiplier if maximum_total_characters is not None else None
     )
     expanded_max_items_per_source = (
         budget.max_items_per_source * multiplier
@@ -230,7 +230,7 @@ def _expand_component_budget(budget: QueryBudget, *, multiplier: int = 5) -> Que
     return QueryBudget(
         max_total_items=requested_items * multiplier,
         offset=0,
-        max_total_characters=expanded_characters,
+        maximum_total_characters=expanded_characters,
         max_items_per_source=expanded_max_items_per_source,
     )
 
@@ -285,6 +285,7 @@ def _fuse_evidence(
                 stage_scores={"lexical": lexical_score, "embedding": embedding_score},
                 recipe_id="",
                 run_id="",
+                metadata=base_evidence.metadata,
                 hash=base_evidence.hash,
             )
         )

biblicus/backends/scan.py

@@ -368,6 +368,7 @@ def _score_items(
                 stage="scan",
                 recipe_id="",
                 run_id="",
+                metadata=getattr(catalog_item, "metadata", {}) or {},
                 hash=hash_text(snippet),
             )
         )

biblicus/backends/tf_vector.py

@@ -8,7 +8,7 @@ import math
 import re
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict
 
 from ..corpus import Corpus
 from ..frontmatter import parse_front_matter
@@ -28,16 +28,16 @@ class TfVectorRecipeConfig(BaseModel):
     """
     Configuration for the term-frequency vector retrieval backend.
 
-    :ivar snippet_characters: Maximum characters to include in evidence snippets.
-    :vartype snippet_characters: int
     :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
     :vartype extraction_run: str or None
+    :ivar snippet_characters: Optional maximum character count for returned evidence text.
+    :vartype snippet_characters: int or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
-    snippet_characters: int = Field(default=400, ge=1)
     extraction_run: Optional[str] = None
+    snippet_characters: Optional[int] = None
 
 
 class TfVectorBackend:
@@ -125,8 +125,8 @@ class TfVectorBackend:
             query_tokens=query_tokens,
             query_vector=query_vector,
             query_norm=query_norm,
-            snippet_characters=recipe_config.snippet_characters,
             extraction_reference=extraction_reference,
+            snippet_characters=recipe_config.snippet_characters,
         )
         sorted_candidates = sorted(
             scored_candidates,
@@ -359,21 +359,13 @@ def _find_first_match(text: str, tokens: List[str]) -> Optional[Tuple[int, int]]
     return best_start, best_end
 
 
-def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: int) -> str:
-    """
-    Build a snippet around a match span, constrained by a character budget.
-
-    :param text: Source text to slice.
-    :type text: str
-    :param span: Match span to center on.
-    :type span: tuple[int, int] or None
-    :param max_chars: Maximum snippet length.
-    :type max_chars: int
-    :return: Snippet text.
-    :rtype: str
-    """
+def _build_snippet(text: str, span: Optional[Tuple[int, int]], *, max_chars: Optional[int]) -> str:
+    if max_chars is None:
+        return text
     if not text:
         return ""
+    if max_chars <= 0:
+        return ""
     if span is None:
         return text[:max_chars]
     span_start, span_end = span
@@ -390,8 +382,8 @@ def _score_items(
     query_tokens: List[str],
     query_vector: Dict[str, float],
     query_norm: float,
-    snippet_characters: int,
     extraction_reference: Optional[ExtractionRunReference],
+    snippet_characters: Optional[int],
 ) -> List[Evidence]:
     """
     Score catalog items and return evidence candidates.
@@ -406,10 +398,10 @@ def _score_items(
     :type query_vector: dict[str, float]
     :param query_norm: Query vector norm.
     :type query_norm: float
-    :param snippet_characters: Snippet length budget.
-    :type snippet_characters: int
     :param extraction_reference: Optional extraction run reference.
     :type extraction_reference: ExtractionRunReference or None
+    :param snippet_characters: Optional maximum character count for returned evidence text.
+    :type snippet_characters: int or None
     :return: Evidence candidates with provisional ranks.
     :rtype: list[Evidence]
     """
@@ -437,9 +429,9 @@ def _score_items(
         if similarity <= 0:
             continue
         span = _find_first_match(item_text, query_tokens)
-        snippet = _build_snippet(item_text, span, max_chars=snippet_characters)
         span_start = span[0] if span else None
         span_end = span[1] if span else None
+        evidence_text = _build_snippet(item_text, span, max_chars=snippet_characters)
         evidence_items.append(
             Evidence(
                 item_id=str(getattr(catalog_item, "id")),
@@ -447,14 +439,15 @@ def _score_items(
                 media_type=str(media_type),
                 score=float(similarity),
                 rank=1,
-                text=snippet,
+                text=evidence_text,
                 content_ref=None,
                 span_start=span_start,
                 span_end=span_end,
                 stage="tf-vector",
                 recipe_id="",
                 run_id="",
-                hash=hash_text(snippet),
+                metadata=getattr(catalog_item, "metadata", {}) or {},
+                hash=hash_text(evidence_text or ""),
             )
         )
     return evidence_items

biblicus/cli.py

@@ -24,7 +24,7 @@ from .context import (
 )
 from .corpus import Corpus
 from .crawl import CrawlRequest, crawl_into_corpus
-from .errors import ExtractionRunFatalError
+from .errors import ExtractionRunFatalError, IngestCollisionError
 from .evaluation import evaluate_run, load_dataset
 from .evidence_processing import apply_evidence_filter, apply_evidence_reranker
 from .extraction import build_extraction_run
@@ -117,18 +117,28 @@ def cmd_ingest(arguments: argparse.Namespace) -> int:
 
     results = []
 
-    if arguments.note is not None or arguments.stdin:
-        text = arguments.note if arguments.note is not None else sys.stdin.read()
-        ingest_result = corpus.ingest_note(
-            text,
-            title=arguments.title,
-            tags=tags,
-            source_uri="stdin" if arguments.stdin else "text",
+    try:
+        if arguments.note is not None or arguments.stdin:
+            text = arguments.note if arguments.note is not None else sys.stdin.read()
+            ingest_result = corpus.ingest_note(
+                text,
+                title=arguments.title,
+                tags=tags,
+                source_uri=None if arguments.stdin else None,
+            )
+            results.append(ingest_result)
+
+        for source_path in arguments.files or []:
+            results.append(corpus.ingest_source(source_path, tags=tags))
+    except IngestCollisionError as error:
+        print(
+            "Ingest failed: source already ingested\n"
+            f"source_uri: {error.source_uri}\n"
+            f"existing_item_id: {error.existing_item_id}\n"
+            f"existing_relpath: {error.existing_relpath}",
+            file=sys.stderr,
         )
-        results.append(ingest_result)
-
-    for source_path in arguments.files or []:
-        results.append(corpus.ingest_source(source_path, tags=tags))
+        return 3
 
     if not results:
         print("Nothing to ingest: provide file paths, --note, or --stdin", file=sys.stderr)
@@ -374,7 +384,7 @@ def _budget_from_args(arguments: argparse.Namespace) -> QueryBudget:
     return QueryBudget(
         max_total_items=arguments.max_total_items,
         offset=getattr(arguments, "offset", 0),
-        max_total_characters=arguments.max_total_characters,
+        maximum_total_characters=arguments.maximum_total_characters,
         max_items_per_source=arguments.max_items_per_source,
     )
 
@@ -1071,7 +1081,7 @@ def build_parser() -> argparse.ArgumentParser:
         help="Skip this many ranked candidates before selecting evidence (pagination).",
     )
     p_query.add_argument("--max-total-items", type=int, default=5)
-    p_query.add_argument("--max-total-characters", type=int, default=2000)
+    p_query.add_argument("--maximum-total-characters", type=int, default=2000)
     p_query.add_argument("--max-items-per-source", type=int, default=5)
     p_query.add_argument(
         "--reranker-id",
@@ -1131,7 +1141,7 @@ def build_parser() -> argparse.ArgumentParser:
         help="Path to dataset JavaScript Object Notation file.",
     )
     p_eval.add_argument("--max-total-items", type=int, default=5)
-    p_eval.add_argument("--max-total-characters", type=int, default=2000)
+    p_eval.add_argument("--maximum-total-characters", type=int, default=2000)
     p_eval.add_argument("--max-items-per-source", type=int, default=5)
     p_eval.set_defaults(func=cmd_eval)
 

biblicus/context.py

@@ -25,6 +25,8 @@ class ContextPackPolicy(BaseModel):
     :vartype ordering: str
     :ivar include_metadata: Whether to include evidence metadata lines in each block.
     :vartype include_metadata: bool
+    :ivar metadata_fields: Optional evidence metadata fields to include.
+    :vartype metadata_fields: list[str] or None
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -32,6 +34,7 @@ class ContextPackPolicy(BaseModel):
     join_with: str = Field(default="\n\n")
     ordering: str = Field(default="rank", min_length=1)
     include_metadata: bool = Field(default=False)
+    metadata_fields: Optional[List[str]] = None
 
 
 class ContextPack(BaseModel):
@@ -132,7 +135,9 @@ def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) ->
         trimmed_text = evidence.text.strip()
         if not trimmed_text:
             continue
-        metadata = _metadata_for_evidence(evidence) if policy.include_metadata else None
+        metadata = (
+            _metadata_for_evidence(evidence, policy=policy) if policy.include_metadata else None
+        )
         block_text = _format_block_text(trimmed_text, metadata=metadata)
         selected_blocks.append(
             ContextPackBlock(
@@ -276,7 +281,11 @@ def _order_evidence(
     raise ValueError(f"Unknown context pack ordering: {policy.ordering}")
 
 
-def _metadata_for_evidence(evidence: Evidence) -> Dict[str, object]:
+def _metadata_for_evidence(
+    evidence: Evidence,
+    *,
+    policy: ContextPackPolicy,
+) -> Dict[str, object]:
     """
     Build metadata for a context pack block.
 
@@ -285,12 +294,19 @@ def _metadata_for_evidence(evidence: Evidence) -> Dict[str, object]:
     :return: Metadata mapping.
     :rtype: dict[str, object]
     """
-    return {
+    metadata = {
         "item_id": evidence.item_id,
         "source_uri": evidence.source_uri or "none",
         "score": evidence.score,
         "stage": evidence.stage,
     }
+    extra = evidence.metadata or {}
+    if policy.metadata_fields is not None:
+        extra = {key: extra.get(key) for key in policy.metadata_fields if key in extra}
+    for key, value in extra.items():
+        if key not in metadata:
+            metadata[key] = value
+    return metadata
 
 
 def _format_block_text(text: str, *, metadata: Optional[Dict[str, object]]) -> str:
@@ -306,12 +322,11 @@ def _format_block_text(text: str, *, metadata: Optional[Dict[str, object]]) -> s
     """
     if not metadata:
         return text
-    metadata_lines = "\n".join(
-        [
-            f"item_id: {metadata['item_id']}",
-            f"source_uri: {metadata['source_uri']}",
-            f"score: {metadata['score']}",
-            f"stage: {metadata['stage']}",
-        ]
-    )
-    return f"{metadata_lines}\n{text}"
+    ordered_keys = ["item_id", "source_uri", "score", "stage"]
+    metadata_lines = [f"{key}: {metadata[key]}" for key in ordered_keys if key in metadata]
+    for key in sorted(metadata.keys()):
+        if key in ordered_keys:
+            continue
+        metadata_lines.append(f"{key}: {metadata[key]}")
+    metadata_text = "\n".join(metadata_lines)
+    return f"{metadata_text}\n{text}"

biblicus/context_engine/__init__.py

@@ -0,0 +1,53 @@
+"""
+Public interface for the Biblicus Context Engine.
+"""
+
+from .assembler import ContextAssembler, ContextAssemblyResult
+from .compaction import BaseCompactor, CompactionRequest, SummaryCompactor, TruncateCompactor
+from .models import (
+    AssistantMessageSpec,
+    CompactorDeclaration,
+    ContextBudgetSpec,
+    ContextDeclaration,
+    ContextExpansionSpec,
+    ContextInsertSpec,
+    ContextMessageSpec,
+    ContextPackBudgetSpec,
+    ContextPackSpec,
+    ContextPolicySpec,
+    ContextRetrieverRequest,
+    ContextTemplateSpec,
+    CorpusDeclaration,
+    HistoryInsertSpec,
+    RetrieverDeclaration,
+    SystemMessageSpec,
+    UserMessageSpec,
+)
+from .retrieval import retrieve_context_pack
+
+__all__ = [
+    "ContextAssembler",
+    "ContextAssemblyResult",
+    "BaseCompactor",
+    "CompactionRequest",
+    "SummaryCompactor",
+    "TruncateCompactor",
+    "ContextBudgetSpec",
+    "ContextDeclaration",
+    "ContextExpansionSpec",
+    "ContextInsertSpec",
+    "ContextMessageSpec",
+    "ContextPackBudgetSpec",
+    "ContextPackSpec",
+    "ContextPolicySpec",
+    "ContextRetrieverRequest",
+    "ContextTemplateSpec",
+    "CorpusDeclaration",
+    "RetrieverDeclaration",
+    "CompactorDeclaration",
+    "HistoryInsertSpec",
+    "SystemMessageSpec",
+    "UserMessageSpec",
+    "AssistantMessageSpec",
+    "retrieve_context_pack",
+]