biblicus 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

biblicus/__init__.py

@@ -25,4 +25,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"

biblicus/cli.py

@@ -13,12 +13,19 @@ from typing import Dict, List, Optional
 from pydantic import ValidationError
 
 from .backends import get_backend
+from .context import (
+    ContextPackPolicy,
+    TokenBudget,
+    build_context_pack,
+    fit_context_pack_to_token_budget,
+)
 from .corpus import Corpus
 from .crawl import CrawlRequest, crawl_into_corpus
 from .errors import ExtractionRunFatalError
 from .evaluation import evaluate_run, load_dataset
+from .evidence_processing import apply_evidence_filter, apply_evidence_reranker
 from .extraction import build_extraction_run
-from .models import QueryBudget, parse_extraction_run_reference
+from .models import QueryBudget, RetrievalResult, parse_extraction_run_reference
 from .uris import corpus_ref_to_path
 
 
@@ -449,10 +456,62 @@ def cmd_query(arguments: argparse.Namespace) -> int:
     query_text = arguments.query if arguments.query is not None else sys.stdin.read()
     budget = _budget_from_args(arguments)
     result = backend.query(corpus, run=run, query_text=query_text, budget=budget)
+    processed_evidence = result.evidence
+    if getattr(arguments, "reranker_id", None):
+        processed_evidence = apply_evidence_reranker(
+            reranker_id=arguments.reranker_id,
+            query_text=result.query_text,
+            evidence=processed_evidence,
+        )
+    if getattr(arguments, "minimum_score", None) is not None:
+        processed_evidence = apply_evidence_filter(
+            filter_id="filter-minimum-score",
+            query_text=result.query_text,
+            evidence=processed_evidence,
+            config={"minimum_score": float(arguments.minimum_score)},
+        )
+    if processed_evidence is not result.evidence:
+        result = result.model_copy(update={"evidence": processed_evidence})
     print(result.model_dump_json(indent=2))
     return 0
 
 
+def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
+    """
+    Build a context pack from a retrieval result.
+
+    The retrieval result is read from standard input as JavaScript Object Notation.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    input_text = sys.stdin.read()
+    if not input_text.strip():
+        raise ValueError("Context pack build requires a retrieval result JavaScript Object Notation on standard input")
+    retrieval_result = RetrievalResult.model_validate_json(input_text)
+    join_with = bytes(arguments.join_with, "utf-8").decode("unicode_escape")
+    policy = ContextPackPolicy(join_with=join_with)
+    context_pack = build_context_pack(retrieval_result, policy=policy)
+    if arguments.max_tokens is not None:
+        context_pack = fit_context_pack_to_token_budget(
+            context_pack,
+            policy=policy,
+            token_budget=TokenBudget(max_tokens=int(arguments.max_tokens)),
+        )
+    print(
+        json.dumps(
+            {
+                "policy": policy.model_dump(),
+                "context_pack": context_pack.model_dump(),
+            },
+            indent=2,
+        )
+    )
+    return 0
+
+
 def cmd_eval(arguments: argparse.Namespace) -> int:
     """
     Evaluate a retrieval run against a dataset.
@@ -657,8 +716,38 @@ def build_parser() -> argparse.ArgumentParser:
     p_query.add_argument("--max-total-items", type=int, default=5)
     p_query.add_argument("--max-total-characters", type=int, default=2000)
     p_query.add_argument("--max-items-per-source", type=int, default=5)
+    p_query.add_argument(
+        "--reranker-id",
+        default=None,
+        help="Optional reranker identifier to apply after retrieval (for example: rerank-longest-text).",
+    )
+    p_query.add_argument(
+        "--minimum-score",
+        type=float,
+        default=None,
+        help="Optional minimum score threshold to filter evidence after retrieval.",
+    )
     p_query.set_defaults(func=cmd_query)
 
+    p_context_pack = sub.add_parser("context-pack", help="Build context pack text from evidence.")
+    context_pack_sub = p_context_pack.add_subparsers(dest="context_pack_command", required=True)
+
+    p_context_pack_build = context_pack_sub.add_parser(
+        "build", help="Build a context pack from a retrieval result JavaScript Object Notation."
+    )
+    p_context_pack_build.add_argument(
+        "--join-with",
+        default="\\n\\n",
+        help="Separator between evidence blocks (escape sequences supported, default is two newlines).",
+    )
+    p_context_pack_build.add_argument(
+        "--max-tokens",
+        default=None,
+        type=int,
+        help="Optional token budget for the final context pack using the naive-whitespace tokenizer.",
+    )
+    p_context_pack_build.set_defaults(func=cmd_context_pack_build)
+
     p_eval = sub.add_parser("eval", help="Evaluate a run against a dataset.")
     _add_common_corpus_arg(p_eval)
     p_eval.add_argument("--run", default=None, help="Run identifier (defaults to latest run).")

biblicus/context.py

@@ -0,0 +1,183 @@
+"""
+Context pack building for Biblicus.
+
+A context pack is the text that your application sends to a large language model.
+Biblicus produces a context pack from structured retrieval results so that evidence remains a
+stable contract while context formatting remains an explicit policy surface.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .models import RetrievalResult
+
+
+class ContextPackPolicy(BaseModel):
+    """
+    Policy that controls how evidence becomes context pack text.
+
+    :ivar join_with: Separator inserted between evidence text blocks.
+    :vartype join_with: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    join_with: str = Field(default="\n\n")
+
+
+class ContextPack(BaseModel):
+    """
+    Context pack derived from retrieval evidence.
+
+    :ivar text: Context pack text suitable for inclusion in a model call.
+    :vartype text: str
+    :ivar evidence_count: Number of evidence blocks included in the context pack.
+    :vartype evidence_count: int
+    :ivar blocks: Structured blocks that produced the context pack.
+    :vartype blocks: list[ContextPackBlock]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    evidence_count: int = Field(ge=0)
+    blocks: List["ContextPackBlock"] = Field(default_factory=list)
+
+
+class ContextPackBlock(BaseModel):
+    """
+    A single context pack block derived from one evidence item.
+
+    :ivar evidence_item_id: Item identifier that produced this block.
+    :vartype evidence_item_id: str
+    :ivar text: Text included in this block.
+    :vartype text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    evidence_item_id: str = Field(min_length=1)
+    text: str = Field(min_length=1)
+
+
+class TokenCounter(BaseModel):
+    """
+    Token counter configuration for token budget fitting.
+
+    This is a lightweight model wrapper so token fitting remains explicit and testable even when
+    the underlying tokenizer is provided by an optional dependency.
+
+    :ivar tokenizer_id: Tokenizer identifier (for example, naive-whitespace).
+    :vartype tokenizer_id: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    tokenizer_id: str = Field(default="naive-whitespace", min_length=1)
+
+
+class TokenBudget(BaseModel):
+    """
+    Token budget for a context pack.
+
+    :ivar max_tokens: Maximum tokens permitted for the final context pack text.
+    :vartype max_tokens: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_tokens: int = Field(ge=1)
+
+
+def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) -> ContextPack:
+    """
+    Build a context pack from a retrieval result using an explicit policy.
+
+    :param result: Retrieval result containing ranked evidence.
+    :type result: RetrievalResult
+    :param policy: Policy controlling how evidence text is joined.
+    :type policy: ContextPackPolicy
+    :return: Context pack containing concatenated evidence text.
+    :rtype: ContextPack
+    """
+    selected_blocks: List[ContextPackBlock] = []
+    for evidence in result.evidence:
+        if not isinstance(evidence.text, str):
+            continue
+        trimmed_text = evidence.text.strip()
+        if not trimmed_text:
+            continue
+        selected_blocks.append(
+            ContextPackBlock(evidence_item_id=evidence.item_id, text=trimmed_text)
+        )
+
+    return ContextPack(
+        text=policy.join_with.join([block.text for block in selected_blocks]),
+        evidence_count=len(selected_blocks),
+        blocks=selected_blocks,
+    )
+
+
+def count_tokens(text: str, *, tokenizer_id: str) -> int:
+    """
+    Count tokens in a text using a tokenizer identifier.
+
+    The default tokenizer is naive-whitespace, which counts whitespace-separated tokens.
+
+    :param text: Text payload to count.
+    :type text: str
+    :param tokenizer_id: Tokenizer identifier.
+    :type tokenizer_id: str
+    :return: Token count.
+    :rtype: int
+    :raises KeyError: If the tokenizer identifier is unknown.
+    """
+    tokenizers = {
+        "naive-whitespace": lambda value: len([token for token in value.split() if token]),
+    }
+    tokenizer = tokenizers[tokenizer_id]
+    return int(tokenizer(text))
+
+
+def fit_context_pack_to_token_budget(
+    context_pack: ContextPack,
+    *,
+    policy: ContextPackPolicy,
+    token_budget: TokenBudget,
+    token_counter: Optional[TokenCounter] = None,
+) -> ContextPack:
+    """
+    Fit a context pack to a token budget by dropping trailing blocks.
+
+    This function is deterministic. It never rewrites block text. It only removes blocks from the
+    end of the block list until the token budget is met.
+
+    :param context_pack: Context pack to fit.
+    :type context_pack: ContextPack
+    :param policy: Policy controlling how blocks are joined into text.
+    :type policy: ContextPackPolicy
+    :param token_budget: Token budget to enforce.
+    :type token_budget: TokenBudget
+    :param token_counter: Optional token counter configuration.
+    :type token_counter: TokenCounter or None
+    :return: Fitted context pack.
+    :rtype: ContextPack
+    """
+    token_counter = token_counter or TokenCounter()
+    remaining_blocks: List[ContextPackBlock] = list(context_pack.blocks)
+
+    while remaining_blocks:
+        candidate_text = policy.join_with.join([block.text for block in remaining_blocks])
+        candidate_tokens = count_tokens(candidate_text, tokenizer_id=token_counter.tokenizer_id)
+        if candidate_tokens <= token_budget.max_tokens:
+            return ContextPack(
+                text=candidate_text,
+                evidence_count=len(remaining_blocks),
+                blocks=remaining_blocks,
+            )
+        remaining_blocks = remaining_blocks[:-1]
+
+    return ContextPack(text="", evidence_count=0, blocks=[])

biblicus/evidence_processing.py

@@ -0,0 +1,201 @@
+"""
+Evidence processing stages for Biblicus.
+
+Retrieval backends return ranked evidence. Additional stages can be applied without changing the
+backend implementation:
+
+- Rerank: reorder evidence.
+- Filter: remove evidence.
+
+These stages are explicit so they can be configured, tested, and evaluated independently from the
+retrieval backend.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .models import Evidence
+
+
+class EvidenceReranker(ABC):
+    """
+    Evidence reranker interface.
+
+    :param reranker_id: Stable identifier for this reranker implementation.
+    :type reranker_id: str
+    """
+
+    reranker_id: str
+
+    @abstractmethod
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Reranked evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceFilter(ABC):
+    """
+    Evidence filter interface.
+
+    :param filter_id: Stable identifier for this filter implementation.
+    :type filter_id: str
+    """
+
+    filter_id: str
+
+    @abstractmethod
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter-specific configuration values.
+        :type config: dict[str, Any]
+        :return: Filtered evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceRerankLongestText(EvidenceReranker):
+    """
+    Reranker that prioritizes evidence with longer text.
+
+    This is a deterministic policy that is useful when a downstream context pack is limited by a
+    character or token budget and longer evidence is preferred.
+
+    :ivar reranker_id: Stable reranker identifier.
+    :vartype reranker_id: str
+    """
+
+    reranker_id = "rerank-longest-text"
+
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence by descending text length.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Evidence list ordered by text length.
+        :rtype: list[Evidence]
+        """
+        return sorted(
+            evidence,
+            key=lambda evidence_item: (-len((evidence_item.text or "").strip()), evidence_item.item_id),
+        )
+
+
+class EvidenceFilterMinimumScoreConfig(BaseModel):
+    """
+    Configuration for the minimum score evidence filter.
+
+    :ivar minimum_score: Evidence with score below this threshold is removed.
+    :vartype minimum_score: float
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    minimum_score: float = Field(ge=0.0)
+
+
+class EvidenceFilterMinimumScore(EvidenceFilter):
+    """
+    Filter that removes evidence below a minimum score threshold.
+
+    :ivar filter_id: Stable filter identifier.
+    :vartype filter_id: str
+    """
+
+    filter_id = "filter-minimum-score"
+
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence by score threshold.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter configuration values.
+        :type config: dict[str, Any]
+        :return: Evidence list with low-score items removed.
+        :rtype: list[Evidence]
+        """
+        parsed_config = EvidenceFilterMinimumScoreConfig.model_validate(config)
+        return [
+            evidence_item
+            for evidence_item in evidence
+            if float(evidence_item.score) >= parsed_config.minimum_score
+        ]
+
+
+_EVIDENCE_RERANKERS: Dict[str, EvidenceReranker] = {
+    EvidenceRerankLongestText.reranker_id: EvidenceRerankLongestText(),
+}
+
+_EVIDENCE_FILTERS: Dict[str, EvidenceFilter] = {
+    EvidenceFilterMinimumScore.filter_id: EvidenceFilterMinimumScore(),
+}
+
+
+def apply_evidence_reranker(
+    *, reranker_id: str, query_text: str, evidence: List[Evidence]
+) -> List[Evidence]:
+    """
+    Apply a reranker to evidence by identifier.
+
+    :param reranker_id: Reranker identifier.
+    :type reranker_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to rerank.
+    :type evidence: list[Evidence]
+    :return: Reranked evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the reranker identifier is unknown.
+    """
+    reranker = _EVIDENCE_RERANKERS[reranker_id]
+    return reranker.rerank(query_text=query_text, evidence=evidence)
+
+
+def apply_evidence_filter(
+    *, filter_id: str, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+) -> List[Evidence]:
+    """
+    Apply a filter to evidence by identifier.
+
+    :param filter_id: Filter identifier.
+    :type filter_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to filter.
+    :type evidence: list[Evidence]
+    :param config: Filter-specific configuration values.
+    :type config: dict[str, Any]
+    :return: Filtered evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the filter identifier is unknown.
+    """
+    evidence_filter = _EVIDENCE_FILTERS[filter_id]
+    return evidence_filter.filter(query_text=query_text, evidence=evidence, config=config)
+

{biblicus-0.4.0.dist-info → biblicus-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.4.0
+Version: 0.5.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -41,11 +41,11 @@ The first practical problem is not retrieval. It is collection and care. You nee
 
 This library gives you a corpus, which is a normal folder on disk. It stores each ingested item as a file, with optional metadata stored next to it. You can open and inspect the raw files directly. Any derived catalog or index can be rebuilt from the raw corpus.
 
-It can be used alongside LangChain, Tactus, Pydantic AI, or the agent development kit. Use it from Python or from the command line interface.
+It can be used alongside LangGraph, Tactus, Pydantic AI, any agent framework, or your own setup. Use it from Python or from the command line interface.
 
 See [retrieval augmented generation overview] for a short introduction to the idea.
 
-## A beginner friendly mental model
+## A simple mental model
 
 Think in three stages.
 
@@ -63,94 +63,30 @@ If you learn a few project words, the rest of the system becomes predictable.
 - Run is a recorded retrieval build for a corpus.
 - Evidence is what retrieval returns, with identifiers and source information.
 
-## Diagram
+## Where it fits in an assistant
 
-This diagram shows how a corpus becomes evidence for an assistant.
-Extraction is introduced here as a separate stage so you can swap extraction approaches without changing the raw corpus.
-The legend shows what the block styles mean.
-Your code is where you decide how to turn evidence into context and how to call a model.
+Biblicus does not answer user questions. It is not a language model. It helps your assistant answer them by retrieving relevant material and returning it as structured evidence. Your code decides how to turn evidence into a context pack for the model call, which is then passed to a model you choose.
 
-```mermaid
-%%{init: {"flowchart": {"useMaxWidth": true, "nodeSpacing": 18, "rankSpacing": 22}}}%%
-flowchart LR
-  subgraph Legend[Legend]
-    direction LR
-    LegendArtifact[Stored artifact or evidence]
-    LegendStep[Step]
-    LegendArtifact --- LegendStep
-  end
-
-  subgraph Main[" "]
-    direction TB
+In a coding assistant, retrieval is often triggered by what the user is doing right now. For example: you are about to propose a user interface change, so you retrieve the user's stated preferences, then you include that as context for the model call.
 
-    subgraph StableCore[Stable core]
-      direction TB
-      Source[Source items] --> Ingest[Ingest]
-      Ingest --> Raw[Raw item files]
-      Raw --> Catalog[Catalog file]
-    end
-
-    subgraph PluggableExtractionPipeline[Pluggable: extraction pipeline]
-      direction TB
-      Catalog --> Extract[Extract pipeline]
-      Extract --> ExtractedText[Extracted text artifacts]
-      ExtractedText --> ExtractionRun[Extraction run manifest]
-    end
-
-    subgraph PluggableRetrievalBackend[Pluggable: retrieval backend]
-      direction LR
-
-      subgraph BackendIngestionIndexing[Ingestion and indexing]
-        direction TB
-        ExtractionRun --> Build[Build run]
-        Build --> BackendIndex[Backend index]
-        BackendIndex --> Run[Run manifest]
-      end
+This diagram shows two sequential Biblicus calls. They are shown separately to make the boundaries explicit: retrieval returns evidence, and context pack building consumes evidence.
 
-      subgraph BackendRetrievalGeneration[Retrieval and generation]
-        direction TB
-        Run --> Query[Query]
-        Query --> Evidence[Evidence]
-      end
-    end
-
-    Evidence --> Context
-
-    subgraph YourCode[Your code]
-      direction TB
-      Context[Assistant context] --> Model[Large language model call]
-      Model --> Answer[Answer]
-    end
-
-    style StableCore fill:#ffffff,stroke:#8e24aa,stroke-width:2px,color:#111111
-    style PluggableExtractionPipeline fill:#ffffff,stroke:#5e35b1,stroke-dasharray:6 3,stroke-width:2px,color:#111111
-    style PluggableRetrievalBackend fill:#ffffff,stroke:#1e88e5,stroke-dasharray:6 3,stroke-width:2px,color:#111111
-    style YourCode fill:#ffffff,stroke:#d81b60,stroke-width:2px,color:#111111
-    style BackendIngestionIndexing fill:#ffffff,stroke:#cfd8dc,color:#111111
-    style BackendRetrievalGeneration fill:#ffffff,stroke:#cfd8dc,color:#111111
-
-    style Raw fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Catalog fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style ExtractedText fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style ExtractionRun fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style BackendIndex fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Run fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Evidence fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Context fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Answer fill:#f3e5f5,stroke:#8e24aa,color:#111111
-    style Source fill:#f3e5f5,stroke:#8e24aa,color:#111111
-
-    style Ingest fill:#eceff1,stroke:#90a4ae,color:#111111
-    style Extract fill:#eceff1,stroke:#90a4ae,color:#111111
-    style Build fill:#eceff1,stroke:#90a4ae,color:#111111
-    style Query fill:#eceff1,stroke:#90a4ae,color:#111111
-    style Model fill:#eceff1,stroke:#90a4ae,color:#111111
-  end
-
-  style Legend fill:#ffffff,stroke:#ffffff,color:#111111
-  style Main fill:#ffffff,stroke:#ffffff,color:#111111
-  style LegendArtifact fill:#f3e5f5,stroke:#8e24aa,color:#111111
-  style LegendStep fill:#eceff1,stroke:#90a4ae,color:#111111
+```mermaid
+%%{init: {"theme": "base", "themeVariables": {"primaryColor": "#f3e5f5", "primaryTextColor": "#111111", "primaryBorderColor": "#8e24aa", "lineColor": "#90a4ae", "secondaryColor": "#eceff1", "tertiaryColor": "#ffffff", "noteBkgColor": "#ffffff", "noteTextColor": "#111111", "actorBkg": "#f3e5f5", "actorBorder": "#8e24aa", "actorTextColor": "#111111"}}}%%
+sequenceDiagram
+  participant User
+  participant App as Your assistant code
+  participant Bib as Biblicus
+  participant LLM as Large language model
+
+  User->>App: request
+  App->>Bib: query retrieval
+  Bib-->>App: retrieval result evidence JSON
+  App->>Bib: build context pack from evidence
+  Bib-->>App: context pack text
+  App->>LLM: context pack plus prompt
+  LLM-->>App: response draft
+  App-->>User: response
 ```
 
 ## Practical value
@@ -217,6 +153,216 @@ biblicus crawl --corpus corpora/example \\
   --tag crawled
 ```
 
+## End-to-end example: evidence to assistant context
+
+The command-line interface returns JavaScript Object Notation by default. This makes it easy to use Biblicus in scripts and to treat retrieval as a deterministic, testable step.
+
+Start with a few short “memories” from a chat system. Each memory is stored as a normal item in the corpus.
+
+```python
+from biblicus.backends import get_backend
+from biblicus.context import ContextPackPolicy, TokenBudget, build_context_pack, fit_context_pack_to_token_budget
+from biblicus.corpus import Corpus
+from biblicus.models import QueryBudget
+
+
+corpus = Corpus.init("corpora/story")
+
+notes = [
+    ("User name", "The user's name is Tactus Maximus."),
+    ("Button style preference", "Primary button style preference: the user's favorite color is magenta."),
+    ("Style preference", "The user prefers concise answers."),
+    ("Language preference", "The user dislikes idioms and abbreviations."),
+    ("Engineering preference", "The user likes code that is over-documented and behavior-driven."),
+]
+for note_title, note_text in notes:
+    corpus.ingest_note(note_text, title=note_title, tags=["memory"])
+
+backend = get_backend("scan")
+run = backend.build_run(corpus, recipe_name="Story demo", config={})
+budget = QueryBudget(max_total_items=5, max_total_characters=2000, max_items_per_source=None)
+result = backend.query(
+    corpus,
+    run=run,
+    query_text="Primary button style preference",
+    budget=budget,
+)
+
+policy = ContextPackPolicy(join_with="\n\n")
+context_pack = build_context_pack(result, policy=policy)
+context_pack = fit_context_pack_to_token_budget(
+    context_pack,
+    policy=policy,
+    token_budget=TokenBudget(max_tokens=60),
+)
+print(context_pack.text)
+```
+
+If you want a runnable version of this story, use the script at `scripts/readme_end_to_end_demo.py`.
+
+If you prefer the command-line interface, here is the same flow in compressed form:
+
+```
+biblicus init corpora/story
+biblicus ingest --corpus corpora/story --stdin --title "User name" --tag memory <<< "The user's name is Tactus Maximus."
+biblicus ingest --corpus corpora/story --stdin --title "Button style preference" --tag memory <<< "Primary button style preference: the user's favorite color is magenta."
+biblicus ingest --corpus corpora/story --stdin --title "Style preference" --tag memory <<< "The user prefers concise answers."
+biblicus ingest --corpus corpora/story --stdin --title "Language preference" --tag memory <<< "The user dislikes idioms and abbreviations."
+biblicus ingest --corpus corpora/story --stdin --title "Engineering preference" --tag memory <<< "The user likes code that is over-documented and behavior-driven."
+biblicus build --corpus corpora/story --backend scan
+biblicus query --corpus corpora/story --query "Primary button style preference"
+```
+
+Example output:
+
+```json
+{
+  "query_text": "Primary button style preference",
+  "budget": {
+    "max_total_items": 5,
+    "max_total_characters": 2000,
+    "max_items_per_source": null
+  },
+  "run_id": "RUN_ID",
+  "recipe_id": "RECIPE_ID",
+  "backend_id": "scan",
+  "generated_at": "2026-01-29T00:00:00.000000Z",
+  "evidence": [
+    {
+      "item_id": "ITEM_ID",
+      "source_uri": "text",
+      "media_type": "text/markdown",
+      "score": 1.0,
+      "rank": 1,
+      "text": "Primary button style preference: the user's favorite color is magenta.",
+      "content_ref": null,
+      "span_start": null,
+      "span_end": null,
+      "stage": "scan",
+      "recipe_id": "RECIPE_ID",
+      "run_id": "RUN_ID",
+      "hash": null
+    }
+  ],
+  "stats": {}
+}
+```
+
+Evidence is the output contract. Your code decides how to convert evidence into assistant context.
+
+### Turn evidence into a context pack
+
+A context pack is a readable text block you send to a model. There is no single correct format. Treat it as a policy surface you can iterate on.
+
+Here is a minimal example that builds a context pack from evidence:
+
+```python
+from biblicus.context import ContextPackPolicy, build_context_pack
+
+
+policy = ContextPackPolicy(
+    join_with="\n\n",
+)
+context_pack = build_context_pack(result, policy=policy)
+print(context_pack.text)
+```
+
+Example context pack output:
+
+```text
+Primary button style preference: the user's favorite color is magenta.
+```
+
+You can also build a context pack from the command-line interface by piping the retrieval result:
+
+```
+biblicus query --corpus corpora/story --query "Primary button style preference" \\
+  | biblicus context-pack build
+```
+
+Most production systems also apply a budget when building context. If you want a precise token budget, the budgeting logic needs a specific tokenizer and should be treated as its own stage.
+
+## Pipeline diagram
+
+This diagram shows how a corpus becomes evidence for your assistant. Your code decides how to turn evidence into context and how to call a model.
+
+```mermaid
+%%{init: {"theme": "base", "themeVariables": {"primaryColor": "#f3e5f5", "primaryTextColor": "#111111", "primaryBorderColor": "#8e24aa", "lineColor": "#90a4ae", "secondaryColor": "#eceff1", "tertiaryColor": "#ffffff"}, "flowchart": {"useMaxWidth": true, "nodeSpacing": 18, "rankSpacing": 22}}}%%
+flowchart TB
+  subgraph Legend[Legend]
+    direction LR
+    LegendArtifact[Stored artifact or evidence]
+    LegendStep[Step]
+    LegendArtifact --- LegendStep
+  end
+
+  subgraph Main[" "]
+    direction TB
+
+    subgraph Pipeline[" "]
+      direction TB
+
+      subgraph RowStable[Stable core]
+        direction TB
+        Source[Source items] --> Ingest[Ingest] --> Raw[Raw item files] --> Catalog[Catalog file]
+      end
+
+      subgraph RowExtraction[Pluggable: extraction pipeline]
+        direction TB
+        Catalog --> Extract[Extract pipeline] --> ExtractedText[Extracted text artifacts] --> ExtractionRun[Extraction run manifest]
+      end
+
+      subgraph RowRetrieval[Pluggable: retrieval backend]
+        direction TB
+        ExtractionRun --> Build[Build run] --> BackendIndex[Backend index] --> Run[Run manifest] --> Retrieve[Retrieve] --> Rerank[Rerank optional] --> Filter[Filter optional] --> Evidence[Evidence]
+      end
+
+      subgraph RowContext[Context]
+        direction TB
+        Evidence --> ContextPack[Context pack] --> FitTokens[Fit tokens optional] --> Context[Assistant context]
+      end
+
+      subgraph RowYourCode[Your code]
+        direction TB
+        Context --> Model[Large language model call] --> Answer[Answer]
+      end
+    end
+
+    style RowStable fill:#ffffff,stroke:#8e24aa,stroke-width:2px,color:#111111
+    style RowExtraction fill:#ffffff,stroke:#5e35b1,stroke-dasharray:6 3,stroke-width:2px,color:#111111
+    style RowRetrieval fill:#ffffff,stroke:#1e88e5,stroke-dasharray:6 3,stroke-width:2px,color:#111111
+    style RowContext fill:#ffffff,stroke:#7b1fa2,stroke-width:2px,color:#111111
+    style RowYourCode fill:#ffffff,stroke:#d81b60,stroke-width:2px,color:#111111
+
+    style Raw fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Catalog fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style ExtractedText fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style ExtractionRun fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style BackendIndex fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Run fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Evidence fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style ContextPack fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Context fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Answer fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Source fill:#f3e5f5,stroke:#8e24aa,color:#111111
+
+    style Ingest fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Extract fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Build fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Retrieve fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Rerank fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Filter fill:#eceff1,stroke:#90a4ae,color:#111111
+    style FitTokens fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Model fill:#eceff1,stroke:#90a4ae,color:#111111
+  end
+
+  style Legend fill:#ffffff,stroke:#ffffff,color:#111111
+  style Main fill:#ffffff,stroke:#ffffff,color:#111111
+  style Pipeline fill:#ffffff,stroke:#ffffff,color:#111111
+  style LegendArtifact fill:#f3e5f5,stroke:#8e24aa,color:#111111
+  style LegendStep fill:#eceff1,stroke:#90a4ae,color:#111111
+```
+
 ## Python usage
 
 From Python, the same flow is available through the Corpus class and backend interfaces. The public surface area is small on purpose.
@@ -229,30 +375,28 @@ From Python, the same flow is available through the Corpus class and backend int
 - Query a run with `backend.query`.
 - Evaluate with `evaluate_run`.
 
-## How it fits into an assistant
-
-In an assistant system, retrieval usually produces context for a model call. This library treats evidence as the primary output so you can decide how to use it.
-
-- Use a corpus as the source of truth for raw items.
-- Use a backend run to build any derived artifacts needed for retrieval.
-- Use queries to obtain evidence objects.
-- Convert evidence into the format your framework expects, such as message content, tool output, or citations.
-
 ## Learn more
 
 Full documentation is published on GitHub Pages: https://anthusai.github.io/Biblicus/
 
-The documents below are written to be read in order.
+The documents below follow the pipeline from raw items to model context:
 
-- [Architecture][architecture]
-- [Roadmap][roadmap]
-- [Feature index][feature-index]
 - [Corpus][corpus]
 - [Text extraction][text-extraction]
-- [User configuration][user-configuration]
 - [Backends][backends]
+- [Context packs][context-packs]
+- [Testing and evaluation][testing]
+
+Reference:
+
 - [Demos][demos]
-- [Testing][testing]
+- [User configuration][user-configuration]
+
+Design and implementation map:
+
+- [Feature index][feature-index]
+- [Roadmap][roadmap]
+- [Architecture][architecture]
 
 ## Metadata and catalog
 
@@ -344,6 +488,7 @@ License terms are in `LICENSE`.
 [text-extraction]: docs/EXTRACTION.md
 [user-configuration]: docs/USER_CONFIGURATION.md
 [backends]: docs/BACKENDS.md
+[context-packs]: docs/CONTEXT_PACK.md
 [demos]: docs/DEMOS.md
 [testing]: docs/TESTING.md
 

{biblicus-0.4.0.dist-info → biblicus-0.5.0.dist-info}/RECORD

@@ -1,11 +1,13 @@
-biblicus/__init__.py,sha256=6TFpzDiMJlFyBfVrHfS6xnGd8P7Zybj6DxpWkCJqyf4,432
+biblicus/__init__.py,sha256=9YH3nGunYPrO2wrwwya94mgHqWXnGOiIwDCB1THgGqo,432
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=MtYTkJh0lVOTrbwY3u6V8ti5WZUsb96fLatqh23cUYg,24289
+biblicus/cli.py,sha256=hBau464XNdSGdWeOCE2Q7dm0P8I4sR0W-NgVT0wPmh4,27724
 biblicus/constants.py,sha256=R6fZDoLVMCwgKvTaxEx7G0CstwHGaUTlW9MsmNLDZ44,269
+biblicus/context.py,sha256=qnT9CH7_ldoPcg-rxnUOtRhheOmpDAbF8uqhf8OdjC4,5832
 biblicus/corpus.py,sha256=gF1RNl6fdz7wplzpHEIkEBkhYxHgKTKguBR_kD9IgUw,54109
 biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
 biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
 biblicus/evaluation.py,sha256=5xWpb-8f49Osh9aHzo1ab3AXOmls3Imc5rdnEC0pN-8,8143
+biblicus/evidence_processing.py,sha256=EMv1AkV_Eufk-poBz9nRR1dZgC-QewvI-NrULBUGVGA,6074
 biblicus/extraction.py,sha256=VEjBjIpaBboftGgEcpDj7z7um41e5uDZpP_7acQg7fw,19448
 biblicus/frontmatter.py,sha256=JOGjIDzbbOkebQw2RzA-3WDVMAMtJta2INjS4e7-LMg,2463
 biblicus/hook_logging.py,sha256=IMvde-JhVWrx9tNz3eDJ1CY_rr5Sj7DZ2YNomYCZbz0,5366
@@ -37,9 +39,9 @@ biblicus/extractors/rapidocr_text.py,sha256=OMAuZealLSSTFVVmBalT-AFJy2pEpHyyvpuW
 biblicus/extractors/select_longest_text.py,sha256=wRveXAfYLdj7CpGuo4RoD7zE6SIfylRCbv40z2azO0k,3702
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.4.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.4.0.dist-info/METADATA,sha256=JMZjfIOMEmbWFHgzjUHsQDUUg11jdxudtJdRK8Iu29U,13586
-biblicus-0.4.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.4.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.4.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.4.0.dist-info/RECORD,,
+biblicus-0.5.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.5.0.dist-info/METADATA,sha256=SHMtWua4egS09DGjX-YZviQOXojtkVvgrisgPmnlSnk,19666
+biblicus-0.5.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.5.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.5.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.5.0.dist-info/RECORD,,