biblicus 0.16.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

biblicus/backends/hybrid.py

@@ -1,291 +0,0 @@
-"""
-Hybrid retrieval backend combining lexical and vector results.
-"""
-
-from __future__ import annotations
-
-from typing import Dict, List, Optional
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from ..corpus import Corpus
-from ..models import Evidence, QueryBudget, RetrievalResult, RetrievalRun
-from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest
-from ..time import utc_now_iso
-
-
-class HybridRecipeConfig(BaseModel):
-    """
-    Configuration for hybrid retrieval fusion.
-
-    :ivar lexical_backend: Backend identifier for lexical retrieval.
-    :vartype lexical_backend: str
-    :ivar embedding_backend: Backend identifier for embedding retrieval.
-    :vartype embedding_backend: str
-    :ivar lexical_weight: Weight for lexical scores.
-    :vartype lexical_weight: float
-    :ivar embedding_weight: Weight for embedding scores.
-    :vartype embedding_weight: float
-    :ivar lexical_config: Optional lexical backend configuration.
-    :vartype lexical_config: dict[str, object]
-    :ivar embedding_config: Optional embedding backend configuration.
-    :vartype embedding_config: dict[str, object]
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    lexical_backend: str = Field(default="sqlite-full-text-search", min_length=1)
-    embedding_backend: str = Field(default="tf-vector", min_length=1)
-    lexical_weight: float = Field(default=0.5, ge=0, le=1)
-    embedding_weight: float = Field(default=0.5, ge=0, le=1)
-    lexical_config: Dict[str, object] = Field(default_factory=dict)
-    embedding_config: Dict[str, object] = Field(default_factory=dict)
-
-    @model_validator(mode="after")
-    def _validate_weights(self) -> "HybridRecipeConfig":
-        if abs((self.lexical_weight + self.embedding_weight) - 1.0) > 1e-6:
-            raise ValueError("weights must sum to 1")
-        return self
-
-
-class HybridBackend:
-    """
-    Hybrid backend that fuses lexical and embedding retrieval.
-
-    :ivar backend_id: Backend identifier.
-    :vartype backend_id: str
-    """
-
-    backend_id = "hybrid"
-
-    def build_run(
-        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
-    ) -> RetrievalRun:
-        """
-        Build or register a hybrid retrieval run.
-
-        :param corpus: Corpus to build against.
-        :type corpus: Corpus
-        :param recipe_name: Human-readable recipe name.
-        :type recipe_name: str
-        :param config: Backend-specific configuration values.
-        :type config: dict[str, object]
-        :return: Run manifest describing the build.
-        :rtype: RetrievalRun
-        """
-        recipe_config = HybridRecipeConfig.model_validate(config)
-        _ensure_backend_supported(recipe_config)
-        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
-        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
-        lexical_run = lexical_backend.build_run(
-            corpus, recipe_name=f"{recipe_name}-lexical", config=recipe_config.lexical_config
-        )
-        embedding_run = embedding_backend.build_run(
-            corpus, recipe_name=f"{recipe_name}-embedding", config=recipe_config.embedding_config
-        )
-        recipe = create_recipe_manifest(
-            backend_id=self.backend_id,
-            name=recipe_name,
-            config=recipe_config.model_dump(),
-        )
-        stats = {
-            "lexical_run_id": lexical_run.run_id,
-            "embedding_run_id": embedding_run.run_id,
-        }
-        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
-        corpus.write_run(run)
-        return run
-
-    def query(
-        self,
-        corpus: Corpus,
-        *,
-        run: RetrievalRun,
-        query_text: str,
-        budget: QueryBudget,
-    ) -> RetrievalResult:
-        """
-        Query using both lexical and embedding backends and fuse scores.
-
-        :param corpus: Corpus associated with the run.
-        :type corpus: Corpus
-        :param run: Run manifest to use for querying.
-        :type run: RetrievalRun
-        :param query_text: Query text to execute.
-        :type query_text: str
-        :param budget: Evidence selection budget.
-        :type budget: QueryBudget
-        :return: Retrieval results containing evidence.
-        :rtype: RetrievalResult
-        """
-        recipe_config = HybridRecipeConfig.model_validate(run.recipe.config)
-        _ensure_backend_supported(recipe_config)
-        lexical_backend = _resolve_backend(recipe_config.lexical_backend)
-        embedding_backend = _resolve_backend(recipe_config.embedding_backend)
-        lexical_run_id = run.stats.get("lexical_run_id")
-        embedding_run_id = run.stats.get("embedding_run_id")
-        if not lexical_run_id or not embedding_run_id:
-            raise ValueError("Hybrid run missing lexical or embedding run identifiers")
-        lexical_run = corpus.load_run(str(lexical_run_id))
-        embedding_run = corpus.load_run(str(embedding_run_id))
-        component_budget = _expand_component_budget(budget)
-        lexical_result = lexical_backend.query(
-            corpus, run=lexical_run, query_text=query_text, budget=component_budget
-        )
-        embedding_result = embedding_backend.query(
-            corpus, run=embedding_run, query_text=query_text, budget=component_budget
-        )
-        candidates = _fuse_evidence(
-            lexical_result.evidence,
-            embedding_result.evidence,
-            lexical_weight=recipe_config.lexical_weight,
-            embedding_weight=recipe_config.embedding_weight,
-        )
-        sorted_candidates = sorted(
-            candidates,
-            key=lambda evidence_item: (-evidence_item.score, evidence_item.item_id),
-        )
-        ranked = [
-            evidence_item.model_copy(
-                update={
-                    "rank": index,
-                    "recipe_id": run.recipe.recipe_id,
-                    "run_id": run.run_id,
-                }
-            )
-            for index, evidence_item in enumerate(sorted_candidates, start=1)
-        ]
-        evidence = apply_budget(ranked, budget)
-        stats = {
-            "candidates": len(sorted_candidates),
-            "returned": len(evidence),
-            "fusion_weights": {
-                "lexical": recipe_config.lexical_weight,
-                "embedding": recipe_config.embedding_weight,
-            },
-        }
-        return RetrievalResult(
-            query_text=query_text,
-            budget=budget,
-            run_id=run.run_id,
-            recipe_id=run.recipe.recipe_id,
-            backend_id=self.backend_id,
-            generated_at=utc_now_iso(),
-            evidence=evidence,
-            stats=stats,
-        )
-
-
-def _ensure_backend_supported(recipe_config: HybridRecipeConfig) -> None:
-    """
-    Validate that hybrid backends do not reference the hybrid backend itself.
-
-    :param recipe_config: Parsed hybrid recipe configuration.
-    :type recipe_config: HybridRecipeConfig
-    :return: None.
-    :rtype: None
-    :raises ValueError: If hybrid is used as a component backend.
-    """
-    if recipe_config.lexical_backend == HybridBackend.backend_id:
-        raise ValueError("Hybrid backend cannot use itself as the lexical backend")
-    if recipe_config.embedding_backend == HybridBackend.backend_id:
-        raise ValueError("Hybrid backend cannot use itself as the embedding backend")
-
-
-def _resolve_backend(backend_id: str):
-    """
-    Resolve a backend by identifier.
-
-    :param backend_id: Backend identifier.
-    :type backend_id: str
-    :return: Backend instance.
-    :rtype: object
-    """
-    from . import get_backend
-
-    return get_backend(backend_id)
-
-
-def _expand_component_budget(budget: QueryBudget, *, multiplier: int = 5) -> QueryBudget:
-    """
-    Expand a final budget to collect more candidates for fusion.
-
-    :param budget: Final evidence budget.
-    :type budget: QueryBudget
-    :param multiplier: Candidate expansion multiplier.
-    :type multiplier: int
-    :return: Expanded budget for component backends.
-    :rtype: QueryBudget
-    """
-    max_total_characters = budget.max_total_characters
-    expanded_characters = (
-        max_total_characters * multiplier if max_total_characters is not None else None
-    )
-    expanded_max_items_per_source = (
-        budget.max_items_per_source * multiplier
-        if budget.max_items_per_source is not None
-        else None
-    )
-    requested_items = budget.max_total_items + budget.offset
-    return QueryBudget(
-        max_total_items=requested_items * multiplier,
-        offset=0,
-        max_total_characters=expanded_characters,
-        max_items_per_source=expanded_max_items_per_source,
-    )
-
-
-def _fuse_evidence(
-    lexical: List[Evidence],
-    embedding: List[Evidence],
-    *,
-    lexical_weight: float,
-    embedding_weight: float,
-) -> List[Evidence]:
-    """
-    Fuse lexical and embedding evidence lists into hybrid candidates.
-
-    :param lexical: Lexical evidence list.
-    :type lexical: list[Evidence]
-    :param embedding: Embedding evidence list.
-    :type embedding: list[Evidence]
-    :param lexical_weight: Lexical score weight.
-    :type lexical_weight: float
-    :param embedding_weight: Embedding score weight.
-    :type embedding_weight: float
-    :return: Hybrid evidence list.
-    :rtype: list[Evidence]
-    """
-    merged: Dict[str, Dict[str, Optional[Evidence]]] = {}
-    for evidence_item in lexical:
-        merged.setdefault(evidence_item.item_id, {})["lexical"] = evidence_item
-    for evidence_item in embedding:
-        merged.setdefault(evidence_item.item_id, {})["embedding"] = evidence_item
-
-    candidates: List[Evidence] = []
-    for item_id, sources in merged.items():
-        lexical_evidence = sources.get("lexical")
-        embedding_evidence = sources.get("embedding")
-        lexical_score = lexical_evidence.score if lexical_evidence else 0.0
-        embedding_score = embedding_evidence.score if embedding_evidence else 0.0
-        combined_score = (lexical_score * lexical_weight) + (embedding_score * embedding_weight)
-        base_evidence = lexical_evidence or embedding_evidence
-        candidates.append(
-            Evidence(
-                item_id=item_id,
-                source_uri=base_evidence.source_uri,
-                media_type=base_evidence.media_type,
-                score=combined_score,
-                rank=1,
-                text=base_evidence.text,
-                content_ref=base_evidence.content_ref,
-                span_start=base_evidence.span_start,
-                span_end=base_evidence.span_end,
-                stage="hybrid",
-                stage_scores={"lexical": lexical_score, "embedding": embedding_score},
-                recipe_id="",
-                run_id="",
-                hash=base_evidence.hash,
-            )
-        )
-    return candidates

biblicus-0.16.0.dist-info/RECORD

@@ -1,86 +0,0 @@
-biblicus/__init__.py,sha256=VK1nvxxf1NI1u5Ad94yyMpC5Xc_HJ7-3F_TnJdhItbA,496
-biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/chunking.py,sha256=GdJr0skAAI0Su99mr7dXqCgR7eJ0sJu8n2XesVGyddY,13206
-biblicus/cli.py,sha256=x3bbtg_nzvIZlHmiPp-4L2EtV6wugTMueFTkXQy9y1s,43372
-biblicus/constants.py,sha256=gAlEVJhxdFj-eWWJrlYbP7H1X3c5gwhrIBq9NQ1Vq_E,371
-biblicus/context.py,sha256=iXRFGpf_5YDPsDsm_iTK6nCvtUWDoYVI7op-l2QU3uA,10189
-biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
-biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
-biblicus/embedding_providers.py,sha256=phWEsq1vryyTFRRs6uZ0sx9FhrqWIkDsS3I52I64zqM,3839
-biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
-biblicus/evaluation.py,sha256=5xWpb-8f49Osh9aHzo1ab3AXOmls3Imc5rdnEC0pN-8,8143
-biblicus/evidence_processing.py,sha256=sJe6T1nLxvU0xs9yMH8JZZS19zHXMR-Fpr5lWi5ndUM,6120
-biblicus/extraction.py,sha256=qvrsq6zSz2Kg-cap-18HPHC9pQlqEGo7pyID2uKCyBo,19760
-biblicus/extraction_evaluation.py,sha256=cBC2B1nQCtXmOcVWUhHyO2NJRX8QSDuqhVjEc8PXrOA,10400
-biblicus/frontmatter.py,sha256=uFC4iIrgpnTDiP1gvAnT_CbFYdNuUVtETX7tZ3a9g-Y,2517
-biblicus/hook_logging.py,sha256=IMvde-JhVWrx9tNz3eDJ1CY_rr5Sj7DZ2YNomYCZbz0,5366
-biblicus/hook_manager.py,sha256=ZCAkE5wLvn4lnQz8jho_o0HGEC9KdQd9qitkAEUQRcw,6997
-biblicus/hooks.py,sha256=OHQOmOi7rUcQqYWVeod4oPe8nVLepD7F_SlN7O_-BsE,7863
-biblicus/ignore.py,sha256=fyjt34E6tWNNrm1FseOhgH2MgryyVBQVzxhKL5s4aio,1800
-biblicus/inference.py,sha256=_k00AIPoXD2lruiTB-JUagtY4f_WKcdzA3axwiq1tck,3512
-biblicus/knowledge_base.py,sha256=JmlJw8WD_fgstuq1PyWVzU9kzvVzyv7_xOvhS70xwUw,6654
-biblicus/models.py,sha256=5AQ6oXK_KJyU0Kyv5ff8yD8nevNKb_6Hjr2_vlRSlK0,16297
-biblicus/recipes.py,sha256=rqU66QnjOup6O8Y9Yq7XszmpoM0Pyrjw3RrfdnlVqgE,4210
-biblicus/retrieval.py,sha256=GXYT_3RPdqZEYdBQ4F4lIXDOhWw0nfL9bd781bgrn_4,4279
-biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
-biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
-biblicus/uris.py,sha256=xXD77lqsT9NxbyzI1spX9Y5a3-U6sLYMnpeSAV7g-nM,2013
-biblicus/user_config.py,sha256=UXUYBNUN4FR37ggZGJG1wv3K8XzsMR8pXW1T18lrivw,6495
-biblicus/_vendor/dotyaml/__init__.py,sha256=OVv6IsuCvsjaUznLzuit4UbSLVg4TiTVm9cOPY1Y2Cs,409
-biblicus/_vendor/dotyaml/interpolation.py,sha256=FVUkdQr_KbXjoFPvGTv6I5v0X5iZkJe5yhZtYKRbYzI,1991
-biblicus/_vendor/dotyaml/loader.py,sha256=vFfnhbvHYYyOKzl5iq2FH97GSHH2GvEHmGiPnE0g0kA,6954
-biblicus/_vendor/dotyaml/transformer.py,sha256=RWNrm_KAsanG409HEIWquTH9i_jz-ZFK9fM86emXeF4,3724
-biblicus/ai/__init__.py,sha256=HY8PKhqRLIDYJYlL9A2JjqKxQaujITNLYgIytNUhnrU,1161
-biblicus/ai/embeddings.py,sha256=n2xlonZOHcmDrP1XMhGcja5Hzr8r87PF-IecH-Yhu98,3703
-biblicus/ai/llm.py,sha256=g724_UAxmicB_W-Z7Uu9SRsI9-aVNZUlYIjvnlE17VE,4712
-biblicus/ai/models.py,sha256=6newnT0NJf3uf9FvWXVC-9Gkk5xRB-PjXDZpeBHA04Y,7857
-biblicus/analysis/__init__.py,sha256=I4LqxfKPKF3DEVmAyagQ8J1RN-ia3fyfKJ9frCllZQE,1385
-biblicus/analysis/base.py,sha256=gB4ilvyMpiWU1m_ydy2dIHGP96ZFIFvVUL9iVDZKPJM,1265
-biblicus/analysis/markov.py,sha256=1OESAJYUkk1F3l3DUryrV8Hx1aa6OTPiXCDdgfAg6pU,63168
-biblicus/analysis/models.py,sha256=FnUAO6n1yjagYlhe2ocRtfb3IZT3W03rrEZ9LdCx7Kc,56214
-biblicus/analysis/profiling.py,sha256=v2B4Tn9WiXRRP_wIADBPRQVKkMc92KXCas7OBa7n0LU,10670
-biblicus/analysis/schema.py,sha256=MCiAQJmijVk8iM8rOUYbzyaDwsMR-Oo86iZU5NCbDMM,435
-biblicus/analysis/topic_modeling.py,sha256=mNBiRMpY5Jtyz8Aj-WXYY8guEghx9jozTfgveinJLoc,22135
-biblicus/backends/__init__.py,sha256=WJSvXc6boEj8PeFr__AC6l_0lfBPJpaVgMbVq30vtZU,1669
-biblicus/backends/base.py,sha256=Erfj9dXg0nkRKnEcNjHR9_0Ddb2B1NvbmRksVm_g1dU,1776
-biblicus/backends/embedding_index_common.py,sha256=WdmnOr8-QaLZH9b7TNmiWg_5auIqDYviXc-twOd7WsM,10216
-biblicus/backends/embedding_index_file.py,sha256=igwlWHuqQD72qyJ3OZ9uyBGeha7AMg6uA3Uu9QD_2_M,9552
-biblicus/backends/embedding_index_inmemory.py,sha256=oZCNS1kMajEj8u9C5iOjs_dAwhZ8ajU_br0F_8onRLY,9457
-biblicus/backends/hybrid.py,sha256=_kqpvD9V6oFdFSXE7K_rTN8qqH-vDIigPDX0uQxFhaM,10828
-biblicus/backends/scan.py,sha256=hdNnQWqi5IH6j95w30BZHxLJ0W9PTaOkqfWJuxCCEMI,12478
-biblicus/backends/sqlite_full_text_search.py,sha256=tkFYdKwH6WvAF3En1fvGN_03Ud0_Z1igGxhUW4meCbA,24496
-biblicus/backends/tf_vector.py,sha256=HFV2aj1i4uxFYBd4GOv2M665ZcaQNgHWA4qmLbpIc4k,15220
-biblicus/extractors/__init__.py,sha256=ci3oldbdQZ8meAfHccM48CqQtZsPSRg3HkPrBSZF15M,2673
-biblicus/extractors/base.py,sha256=ka-nz_1zHPr4TS9sU4JfOoY-PJh7lbHPBOEBrbQFGSc,2171
-biblicus/extractors/deepgram_stt.py,sha256=VI71i4lbE-EFHcvpNcCPRpT8z7A5IuaSrT1UaPyZ8UY,6323
-biblicus/extractors/docling_granite_text.py,sha256=aFNx-HubvaMmVJHbNqk3CR_ilSwN96-phkaENT6E2B0,6879
-biblicus/extractors/docling_smol_text.py,sha256=cSbQcT4O47MMcM6_pmQCvqgC5ferLvaxJnm3v9EQd0A,6811
-biblicus/extractors/markitdown_text.py,sha256=ZvN2TFh65icTTdzCe7L-ZB8zTPP2mxQ4MhOOqSc81Z0,4547
-biblicus/extractors/metadata_text.py,sha256=7FbEPp0K1mXc7FH1_c0KhPhPexF9U6eLd3TVY1vTp1s,3537
-biblicus/extractors/openai_stt.py,sha256=fggErIu6YN6tXbleNTuROhfYi7zDgMd2vD_ecXZ7eXs,7162
-biblicus/extractors/paddleocr_vl_text.py,sha256=59csxihkqK0lELpAtK2YLcfbSUvNGiuOw7CwPa_0l_c,11692
-biblicus/extractors/pass_through_text.py,sha256=DNxkCwpH2bbXjPGPEQwsx8kfqXi6rIxXNY_n3TU2-WI,2777
-biblicus/extractors/pdf_text.py,sha256=YtUphgLVxyWJXew6ZsJ8wBRh67Y5ri4ZTRlMmq3g1Bk,3255
-biblicus/extractors/pipeline.py,sha256=LY6eM3ypw50MDB2cPEQqZrjxkhVvIc6sv4UEhHdNDrE,3208
-biblicus/extractors/rapidocr_text.py,sha256=StvizEha5BkEG7i5KJmnOUtji89p5pghF4w8iQ-WwFk,4776
-biblicus/extractors/select_longest_text.py,sha256=wRveXAfYLdj7CpGuo4RoD7zE6SIfylRCbv40z2azO0k,3702
-biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_mvjehiSec,4014
-biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
-biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
-biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus/text/__init__.py,sha256=MiaGAY7xWlUCeBzDzNz6pJnSMiU_Ge5EmlSiEzhqTRo,947
-biblicus/text/annotate.py,sha256=asmpj3_s_t8hl6stEg99apmqxAhDTkoPzHhZNggYE3Y,8355
-biblicus/text/extract.py,sha256=pdnUiZWtfCUj7kZK5zhd-tjqokgmhYYheWhyN3iShRU,7669
-biblicus/text/link.py,sha256=Xl0yxD1rvbRJRVdWG_ZP6zgmbpgWSJYcUcNM06-OUWU,20077
-biblicus/text/markup.py,sha256=8jj9aX03HiZTOWdPs_VC4JLpQ7TlPHgGuXj_QUQIHVw,6265
-biblicus/text/models.py,sha256=REp6RowUWFdV-6y437JENP7XtGKt57BOvVtF91KmUqI,10853
-biblicus/text/prompts.py,sha256=Z5fSsy1Xzr0rCI0WZ3djiEQlbRDncyNBQ7_ZoWVPL4g,6704
-biblicus/text/redact.py,sha256=tkDRmA0VvOZwMryEmBPLEHf3Z6VHJkkaWjBaNIMyGZ0,8415
-biblicus/text/slice.py,sha256=dlHxGO8c5P8BszXGwlNQoQ-cyWjJf6PfS1LUBJXXGEE,5762
-biblicus/text/tool_loop.py,sha256=w1PGLBvIemOdi6l0ArdYDVL7zgx-RC76bBOO0PKqpt0,11831
-biblicus-0.16.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.16.0.dist-info/METADATA,sha256=D_qXRDdM_7LnOD-GBtnqwldEWOKjt_uV-TfqvnL8MAU,30933
-biblicus-0.16.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.16.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.16.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.16.0.dist-info/RECORD,,