biblicus 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

biblicus/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.11.0"
+__version__ = "0.12.0"

biblicus/cli.py

@@ -15,9 +15,11 @@ from pydantic import ValidationError
 from .analysis import get_analysis_backend
 from .backends import get_backend
 from .context import (
+    CharacterBudget,
     ContextPackPolicy,
     TokenBudget,
     build_context_pack,
+    fit_context_pack_to_character_budget,
     fit_context_pack_to_token_budget,
 )
 from .corpus import Corpus
@@ -568,7 +570,11 @@ def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
         )
     retrieval_result = RetrievalResult.model_validate_json(input_text)
     join_with = bytes(arguments.join_with, "utf-8").decode("unicode_escape")
-    policy = ContextPackPolicy(join_with=join_with)
+    policy = ContextPackPolicy(
+        join_with=join_with,
+        ordering=arguments.ordering,
+        include_metadata=arguments.include_metadata,
+    )
     context_pack = build_context_pack(retrieval_result, policy=policy)
     if arguments.max_tokens is not None:
         context_pack = fit_context_pack_to_token_budget(
@@ -576,6 +582,12 @@ def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
             policy=policy,
             token_budget=TokenBudget(max_tokens=int(arguments.max_tokens)),
         )
+    if arguments.max_characters is not None:
+        context_pack = fit_context_pack_to_character_budget(
+            context_pack,
+            policy=policy,
+            character_budget=CharacterBudget(max_characters=int(arguments.max_characters)),
+        )
     print(
         json.dumps(
             {
@@ -921,12 +933,29 @@ def build_parser() -> argparse.ArgumentParser:
         default="\\n\\n",
         help="Separator between evidence blocks (escape sequences supported, default is two newlines).",
     )
+    p_context_pack_build.add_argument(
+        "--ordering",
+        choices=["rank", "score", "source"],
+        default="rank",
+        help="Evidence ordering policy (rank, score, source).",
+    )
+    p_context_pack_build.add_argument(
+        "--include-metadata",
+        action="store_true",
+        help="Include evidence metadata in each context pack block.",
+    )
     p_context_pack_build.add_argument(
         "--max-tokens",
         default=None,
         type=int,
         help="Optional token budget for the final context pack using the naive-whitespace tokenizer.",
     )
+    p_context_pack_build.add_argument(
+        "--max-characters",
+        default=None,
+        type=int,
+        help="Optional character budget for the final context pack.",
+    )
     p_context_pack_build.set_defaults(func=cmd_context_pack_build)
 
     p_eval = sub.add_parser("eval", help="Evaluate a run against a dataset.")

biblicus/context.py

@@ -8,11 +8,11 @@ stable contract while context formatting remains an explicit policy surface.
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Literal, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from .models import RetrievalResult
+from .models import Evidence, RetrievalResult
 
 
 class ContextPackPolicy(BaseModel):
@@ -21,11 +21,17 @@ class ContextPackPolicy(BaseModel):
 
     :ivar join_with: Separator inserted between evidence text blocks.
     :vartype join_with: str
+    :ivar ordering: Evidence ordering policy (rank, score, or source).
+    :vartype ordering: str
+    :ivar include_metadata: Whether to include evidence metadata lines in each block.
+    :vartype include_metadata: bool
     """
 
     model_config = ConfigDict(extra="forbid")
 
     join_with: str = Field(default="\n\n")
+    ordering: Literal["rank", "score", "source"] = Field(default="rank")
+    include_metadata: bool = Field(default=False)
 
 
 class ContextPack(BaseModel):
@@ -55,12 +61,15 @@ class ContextPackBlock(BaseModel):
     :vartype evidence_item_id: str
     :ivar text: Text included in this block.
     :vartype text: str
+    :ivar metadata: Optional metadata included with the block.
+    :vartype metadata: dict[str, object] or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
     evidence_item_id: str = Field(min_length=1)
     text: str = Field(min_length=1)
+    metadata: Optional[Dict[str, object]] = None
 
 
 class TokenCounter(BaseModel):
@@ -92,6 +101,19 @@ class TokenBudget(BaseModel):
     max_tokens: int = Field(ge=1)
 
 
+class CharacterBudget(BaseModel):
+    """
+    Character budget for a context pack.
+
+    :ivar max_characters: Maximum characters permitted for the final context pack text.
+    :vartype max_characters: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_characters: int = Field(ge=1)
+
+
 def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) -> ContextPack:
     """
     Build a context pack from a retrieval result using an explicit policy.
@@ -104,14 +126,20 @@ def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) ->
     :rtype: ContextPack
     """
     selected_blocks: List[ContextPackBlock] = []
-    for evidence in result.evidence:
+    for evidence in _order_evidence(result.evidence, policy=policy):
         if not isinstance(evidence.text, str):
             continue
         trimmed_text = evidence.text.strip()
         if not trimmed_text:
             continue
+        metadata = _metadata_for_evidence(evidence) if policy.include_metadata else None
+        block_text = _format_block_text(trimmed_text, metadata=metadata)
         selected_blocks.append(
-            ContextPackBlock(evidence_item_id=evidence.item_id, text=trimmed_text)
+            ContextPackBlock(
+                evidence_item_id=evidence.item_id,
+                text=block_text,
+                metadata=metadata,
+            )
         )
 
     return ContextPack(
@@ -181,3 +209,109 @@ def fit_context_pack_to_token_budget(
         remaining_blocks = remaining_blocks[:-1]
 
     return ContextPack(text="", evidence_count=0, blocks=[])
+
+
+def fit_context_pack_to_character_budget(
+    context_pack: ContextPack,
+    *,
+    policy: ContextPackPolicy,
+    character_budget: CharacterBudget,
+) -> ContextPack:
+    """
+    Fit a context pack to a character budget by dropping trailing blocks.
+
+    :param context_pack: Context pack to fit.
+    :type context_pack: ContextPack
+    :param policy: Policy controlling how blocks are joined into text.
+    :type policy: ContextPackPolicy
+    :param character_budget: Character budget to enforce.
+    :type character_budget: CharacterBudget
+    :return: Fitted context pack.
+    :rtype: ContextPack
+    """
+    remaining_blocks: List[ContextPackBlock] = list(context_pack.blocks)
+    max_characters = character_budget.max_characters
+
+    while remaining_blocks:
+        candidate_text = policy.join_with.join([block.text for block in remaining_blocks])
+        if len(candidate_text) <= max_characters:
+            return ContextPack(
+                text=candidate_text,
+                evidence_count=len(remaining_blocks),
+                blocks=remaining_blocks,
+            )
+        remaining_blocks = remaining_blocks[:-1]
+
+    return ContextPack(text="", evidence_count=0, blocks=[])
+
+
+def _order_evidence(
+    evidence: List[Evidence],
+    *,
+    policy: ContextPackPolicy,
+) -> List[Evidence]:
+    """
+    Order evidence items according to the context pack policy.
+
+    :param evidence: Evidence list to order.
+    :type evidence: list[Evidence]
+    :param policy: Context pack policy.
+    :type policy: ContextPackPolicy
+    :return: Ordered evidence list.
+    :rtype: list[Evidence]
+    """
+    if policy.ordering == "rank":
+        return sorted(evidence, key=lambda item: (item.rank, item.item_id))
+    if policy.ordering == "score":
+        return sorted(evidence, key=lambda item: (-item.score, item.item_id))
+    if policy.ordering == "source":
+        return sorted(
+            evidence,
+            key=lambda item: (
+                item.source_uri or item.item_id,
+                -item.score,
+                item.item_id,
+            ),
+        )
+    raise ValueError(f"Unknown context pack ordering: {policy.ordering}")
+
+
+def _metadata_for_evidence(evidence: Evidence) -> Dict[str, object]:
+    """
+    Build metadata for a context pack block.
+
+    :param evidence: Evidence item to describe.
+    :type evidence: Evidence
+    :return: Metadata mapping.
+    :rtype: dict[str, object]
+    """
+    return {
+        "item_id": evidence.item_id,
+        "source_uri": evidence.source_uri or "none",
+        "score": evidence.score,
+        "stage": evidence.stage,
+    }
+
+
+def _format_block_text(text: str, *, metadata: Optional[Dict[str, object]]) -> str:
+    """
+    Format a context pack block text with optional metadata.
+
+    :param text: Evidence text.
+    :type text: str
+    :param metadata: Optional metadata mapping.
+    :type metadata: dict[str, object] or None
+    :return: Formatted block text.
+    :rtype: str
+    """
+    if not metadata:
+        return text
+    metadata_lines = "\n".join(
+        [
+            f"item_id: {metadata['item_id']}",
+            f"source_uri: {metadata['source_uri']}",
+            f"score: {metadata['score']}",
+            f"stage: {metadata['stage']}",
+        ]
+    )
+    return f"{metadata_lines}\n{text}"

{biblicus-0.11.0.dist-info → biblicus-0.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.11.0
+Version: 0.12.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9

{biblicus-0.11.0.dist-info → biblicus-0.12.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-biblicus/__init__.py,sha256=sT0PFc3DRGFRcN7Zx4Yooc8OzmLvaj1-ZjbvFHce8lU,496
+biblicus/__init__.py,sha256=okAXmTSud_hQzaGEURDqX95I66SlcvTERCrWbUZA5ko,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=aH3plnednnYgcPnSoYQf200nboKc6N-tuc3FuLPQEcU,35132
+biblicus/cli.py,sha256=bZV-ZxeWskRL4CFCGzyVpcaFC8KOb0xmxx3bnMqP-1I,36118
 biblicus/constants.py,sha256=-JaHI3Dngte2drawx93cGWxFVobbgIuaVhmjUJpf4GI,333
-biblicus/context.py,sha256=qnT9CH7_ldoPcg-rxnUOtRhheOmpDAbF8uqhf8OdjC4,5832
+biblicus/context.py,sha256=U7qkOwMdqNgYnqaC9hgQY0kv0R-6qcjV6bhXQl2WUkE,10215
 biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
 biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
 biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
@@ -57,9 +57,9 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.11.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.11.0.dist-info/METADATA,sha256=zrJESYGfGLu7Iq1I--GPIkEY9gXDb9szBIuenlWor7I,27765
-biblicus-0.11.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.11.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.11.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.11.0.dist-info/RECORD,,
+biblicus-0.12.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.12.0.dist-info/METADATA,sha256=fhWcCcczfuLn2mZ_Moqe2zMKJ1-Q7KxZtR_x9YaiFO8,27765
+biblicus-0.12.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.12.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.12.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.12.0.dist-info/RECORD,,