biblicus 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

biblicus/context.py

@@ -8,11 +8,11 @@ stable contract while context formatting remains an explicit policy surface.
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Literal, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from .models import RetrievalResult
+from .models import Evidence, RetrievalResult
 
 
 class ContextPackPolicy(BaseModel):
@@ -21,11 +21,17 @@ class ContextPackPolicy(BaseModel):
 
     :ivar join_with: Separator inserted between evidence text blocks.
     :vartype join_with: str
+    :ivar ordering: Evidence ordering policy (rank, score, or source).
+    :vartype ordering: str
+    :ivar include_metadata: Whether to include evidence metadata lines in each block.
+    :vartype include_metadata: bool
     """
 
     model_config = ConfigDict(extra="forbid")
 
     join_with: str = Field(default="\n\n")
+    ordering: Literal["rank", "score", "source"] = Field(default="rank")
+    include_metadata: bool = Field(default=False)
 
 
 class ContextPack(BaseModel):
@@ -55,12 +61,15 @@ class ContextPackBlock(BaseModel):
     :vartype evidence_item_id: str
     :ivar text: Text included in this block.
     :vartype text: str
+    :ivar metadata: Optional metadata included with the block.
+    :vartype metadata: dict[str, object] or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
     evidence_item_id: str = Field(min_length=1)
     text: str = Field(min_length=1)
+    metadata: Optional[Dict[str, object]] = None
 
 
 class TokenCounter(BaseModel):
@@ -92,6 +101,19 @@ class TokenBudget(BaseModel):
     max_tokens: int = Field(ge=1)
 
 
+class CharacterBudget(BaseModel):
+    """
+    Character budget for a context pack.
+
+    :ivar max_characters: Maximum characters permitted for the final context pack text.
+    :vartype max_characters: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_characters: int = Field(ge=1)
+
+
 def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) -> ContextPack:
     """
     Build a context pack from a retrieval result using an explicit policy.
@@ -104,14 +126,20 @@ def build_context_pack(result: RetrievalResult, *, policy: ContextPackPolicy) ->
     :rtype: ContextPack
     """
     selected_blocks: List[ContextPackBlock] = []
-    for evidence in result.evidence:
+    for evidence in _order_evidence(result.evidence, policy=policy):
         if not isinstance(evidence.text, str):
             continue
         trimmed_text = evidence.text.strip()
         if not trimmed_text:
             continue
+        metadata = _metadata_for_evidence(evidence) if policy.include_metadata else None
+        block_text = _format_block_text(trimmed_text, metadata=metadata)
         selected_blocks.append(
-            ContextPackBlock(evidence_item_id=evidence.item_id, text=trimmed_text)
+            ContextPackBlock(
+                evidence_item_id=evidence.item_id,
+                text=block_text,
+                metadata=metadata,
+            )
         )
 
     return ContextPack(
@@ -181,3 +209,109 @@ def fit_context_pack_to_token_budget(
         remaining_blocks = remaining_blocks[:-1]
 
     return ContextPack(text="", evidence_count=0, blocks=[])
+
+
+def fit_context_pack_to_character_budget(
+    context_pack: ContextPack,
+    *,
+    policy: ContextPackPolicy,
+    character_budget: CharacterBudget,
+) -> ContextPack:
+    """
+    Fit a context pack to a character budget by dropping trailing blocks.
+
+    :param context_pack: Context pack to fit.
+    :type context_pack: ContextPack
+    :param policy: Policy controlling how blocks are joined into text.
+    :type policy: ContextPackPolicy
+    :param character_budget: Character budget to enforce.
+    :type character_budget: CharacterBudget
+    :return: Fitted context pack.
+    :rtype: ContextPack
+    """
+    remaining_blocks: List[ContextPackBlock] = list(context_pack.blocks)
+    max_characters = character_budget.max_characters
+
+    while remaining_blocks:
+        candidate_text = policy.join_with.join([block.text for block in remaining_blocks])
+        if len(candidate_text) <= max_characters:
+            return ContextPack(
+                text=candidate_text,
+                evidence_count=len(remaining_blocks),
+                blocks=remaining_blocks,
+            )
+        remaining_blocks = remaining_blocks[:-1]
+
+    return ContextPack(text="", evidence_count=0, blocks=[])
+
+
+def _order_evidence(
+    evidence: List[Evidence],
+    *,
+    policy: ContextPackPolicy,
+) -> List[Evidence]:
+    """
+    Order evidence items according to the context pack policy.
+
+    :param evidence: Evidence list to order.
+    :type evidence: list[Evidence]
+    :param policy: Context pack policy.
+    :type policy: ContextPackPolicy
+    :return: Ordered evidence list.
+    :rtype: list[Evidence]
+    """
+    if policy.ordering == "rank":
+        return sorted(evidence, key=lambda item: (item.rank, item.item_id))
+    if policy.ordering == "score":
+        return sorted(evidence, key=lambda item: (-item.score, item.item_id))
+    if policy.ordering == "source":
+        return sorted(
+            evidence,
+            key=lambda item: (
+                item.source_uri or item.item_id,
+                -item.score,
+                item.item_id,
+            ),
+        )
+    raise ValueError(f"Unknown context pack ordering: {policy.ordering}")
+
+
+def _metadata_for_evidence(evidence: Evidence) -> Dict[str, object]:
+    """
+    Build metadata for a context pack block.
+
+    :param evidence: Evidence item to describe.
+    :type evidence: Evidence
+    :return: Metadata mapping.
+    :rtype: dict[str, object]
+    """
+    return {
+        "item_id": evidence.item_id,
+        "source_uri": evidence.source_uri or "none",
+        "score": evidence.score,
+        "stage": evidence.stage,
+    }
+
+
+def _format_block_text(text: str, *, metadata: Optional[Dict[str, object]]) -> str:
+    """
+    Format a context pack block text with optional metadata.
+
+    :param text: Evidence text.
+    :type text: str
+    :param metadata: Optional metadata mapping.
+    :type metadata: dict[str, object] or None
+    :return: Formatted block text.
+    :rtype: str
+    """
+    if not metadata:
+        return text
+    metadata_lines = "\n".join(
+        [
+            f"item_id: {metadata['item_id']}",
+            f"source_uri: {metadata['source_uri']}",
+            f"score: {metadata['score']}",
+            f"stage: {metadata['stage']}",
+        ]
+    )
+    return f"{metadata_lines}\n{text}"

biblicus/models.py

@@ -263,6 +263,8 @@ class Evidence(BaseModel):
     :vartype span_end: int or None
     :ivar stage: Retrieval stage label (for example, scan, full-text search, rerank).
     :vartype stage: str
+    :ivar stage_scores: Optional per-stage scores for multi-stage retrieval.
+    :vartype stage_scores: dict[str, float] or None
     :ivar recipe_id: Recipe identifier used to create the run.
     :vartype recipe_id: str
     :ivar run_id: Retrieval run identifier.
@@ -283,6 +285,7 @@ class Evidence(BaseModel):
     span_start: Optional[int] = None
     span_end: Optional[int] = None
     stage: str
+    stage_scores: Optional[Dict[str, float]] = None
     recipe_id: str
     run_id: str
     hash: Optional[str] = None

{biblicus-0.10.0.dist-info → biblicus-0.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.10.0
+Version: 0.12.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -493,6 +493,12 @@ Two backends are included.
 
 For detailed documentation including configuration options, performance characteristics, and usage examples, see the [Backend Reference][backend-reference].
 
+## Retrieval documentation
+
+For the retrieval pipeline overview and run artifacts, see `docs/RETRIEVAL.md`. For retrieval quality upgrades
+(tuned lexical baseline, reranking, hybrid retrieval), see `docs/RETRIEVAL_QUALITY.md`. For evaluation workflows
+and dataset formats, see `docs/RETRIEVAL_EVALUATION.md`.
+
 ## Extraction backends
 
 These extractors are built in. Optional ones require extra dependencies. See [text extraction documentation][text-extraction] for details.

{biblicus-0.10.0.dist-info → biblicus-0.12.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-biblicus/__init__.py,sha256=BejOPHIlCnT74pu9fNuLm14HsmWjGqCIwpfD9hDOqSo,496
+biblicus/__init__.py,sha256=okAXmTSud_hQzaGEURDqX95I66SlcvTERCrWbUZA5ko,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=aH3plnednnYgcPnSoYQf200nboKc6N-tuc3FuLPQEcU,35132
+biblicus/cli.py,sha256=bZV-ZxeWskRL4CFCGzyVpcaFC8KOb0xmxx3bnMqP-1I,36118
 biblicus/constants.py,sha256=-JaHI3Dngte2drawx93cGWxFVobbgIuaVhmjUJpf4GI,333
-biblicus/context.py,sha256=qnT9CH7_ldoPcg-rxnUOtRhheOmpDAbF8uqhf8OdjC4,5832
+biblicus/context.py,sha256=U7qkOwMdqNgYnqaC9hgQY0kv0R-6qcjV6bhXQl2WUkE,10215
 biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
 biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
 biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
@@ -16,7 +16,7 @@ biblicus/hooks.py,sha256=OHQOmOi7rUcQqYWVeod4oPe8nVLepD7F_SlN7O_-BsE,7863
 biblicus/ignore.py,sha256=fyjt34E6tWNNrm1FseOhgH2MgryyVBQVzxhKL5s4aio,1800
 biblicus/inference.py,sha256=_k00AIPoXD2lruiTB-JUagtY4f_WKcdzA3axwiq1tck,3512
 biblicus/knowledge_base.py,sha256=JmlJw8WD_fgstuq1PyWVzU9kzvVzyv7_xOvhS70xwUw,6654
-biblicus/models.py,sha256=vlvPP7AOZGtnHSq47-s9YW-fqLwjgYR6NBcSfeC8YKk,15665
+biblicus/models.py,sha256=r28O6cg3d1bjJnKqpLieVLTgtXTfzb_60wMORvVuDN0,15846
 biblicus/retrieval.py,sha256=A1SI4WK5cX-WbtN6FJ0QQxqlEOtQhddLrL0LZIuoTC4,4180
 biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
 biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
@@ -30,13 +30,15 @@ biblicus/analysis/__init__.py,sha256=Z4Wb4d-EoUuGHkcfRm9ILuZ8vr9FBqRxC0u1i6Fp_0w
 biblicus/analysis/base.py,sha256=gB4ilvyMpiWU1m_ydy2dIHGP96ZFIFvVUL9iVDZKPJM,1265
 biblicus/analysis/llm.py,sha256=VjkZDKauHCDfj-TP-bTbI6a9WAXEIDe8bEiwErPx9xc,3309
 biblicus/analysis/models.py,sha256=LuR52w27JRzV-Mr-WAOduZrBOCTrp5uYkMc46QHTRrI,27300
-biblicus/analysis/profiling.py,sha256=z4w14LVJrTEXcQ3PBNwwb_61KuuwQgXw4-EiAaxOQ4Y,10672
+biblicus/analysis/profiling.py,sha256=v2B4Tn9WiXRRP_wIADBPRQVKkMc92KXCas7OBa7n0LU,10670
 biblicus/analysis/schema.py,sha256=MCiAQJmijVk8iM8rOUYbzyaDwsMR-Oo86iZU5NCbDMM,435
 biblicus/analysis/topic_modeling.py,sha256=ZGXvm2MyU6plxz2FE1RQU-3bra6QZ-t8EJj8kG1TW0M,19438
-biblicus/backends/__init__.py,sha256=wLXIumV51l6ZIKzjoKKeU7AgIxGOryG7T7ls3a_Fv98,1212
+biblicus/backends/__init__.py,sha256=3HJY0oMm8pFFVGC4Z-dlPRHhIPVDdUzsa4IMjKP_9dI,1378
 biblicus/backends/base.py,sha256=Erfj9dXg0nkRKnEcNjHR9_0Ddb2B1NvbmRksVm_g1dU,1776
+biblicus/backends/hybrid.py,sha256=CXh6QrlE0RsTJjSlZRdtomLlILfkglBDQG3YVa8RpFU,10589
 biblicus/backends/scan.py,sha256=hdNnQWqi5IH6j95w30BZHxLJ0W9PTaOkqfWJuxCCEMI,12478
-biblicus/backends/sqlite_full_text_search.py,sha256=XFuIbEHYWMD9JkjgRZcgYH3kP3b4hRnJ3PwP8rSFjUU,16502
+biblicus/backends/sqlite_full_text_search.py,sha256=VAn4fDdfiaS1Rn6zHlYz3E10_3vMU9P94QU8cL0l8Mk,24466
+biblicus/backends/vector.py,sha256=3RdxSBPb1kOX4Sfd4d1qXFW9ecuiRvGpOHadLCbeh1g,15183
 biblicus/extractors/__init__.py,sha256=ci3oldbdQZ8meAfHccM48CqQtZsPSRg3HkPrBSZF15M,2673
 biblicus/extractors/base.py,sha256=ka-nz_1zHPr4TS9sU4JfOoY-PJh7lbHPBOEBrbQFGSc,2171
 biblicus/extractors/deepgram_stt.py,sha256=VI71i4lbE-EFHcvpNcCPRpT8z7A5IuaSrT1UaPyZ8UY,6323
@@ -55,9 +57,9 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.10.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.10.0.dist-info/METADATA,sha256=xZ7scJLdlKHRtm0EU5Ravq5ih2mS2KNfMbbLXNqZ8Ek,27455
-biblicus-0.10.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.10.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.10.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.10.0.dist-info/RECORD,,
+biblicus-0.12.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.12.0.dist-info/METADATA,sha256=fhWcCcczfuLn2mZ_Moqe2zMKJ1-Q7KxZtR_x9YaiFO8,27765
+biblicus-0.12.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.12.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.12.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.12.0.dist-info/RECORD,,