biblicus 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

biblicus/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.12.0"
+__version__ = "0.13.0"

biblicus/cli.py

@@ -28,6 +28,11 @@ from .errors import ExtractionRunFatalError
 from .evaluation import evaluate_run, load_dataset
 from .evidence_processing import apply_evidence_filter, apply_evidence_reranker
 from .extraction import build_extraction_run
+from .extraction_evaluation import (
+    evaluate_extraction_run,
+    load_extraction_dataset,
+    write_extraction_evaluation_result,
+)
 from .models import QueryBudget, RetrievalResult, parse_extraction_run_reference
 from .uris import corpus_ref_to_path
 
@@ -506,6 +511,54 @@ def cmd_extract_delete(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_extract_evaluate(arguments: argparse.Namespace) -> int:
+    """
+    Evaluate an extraction run against a dataset.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+    if arguments.run:
+        run_ref = parse_extraction_run_reference(arguments.run)
+    else:
+        run_ref = corpus.latest_extraction_run_reference()
+        if run_ref is None:
+            raise ValueError("Extraction evaluation requires an extraction run")
+        print(
+            "Warning: using latest extraction run; pass --run for reproducibility.",
+            file=sys.stderr,
+        )
+
+    dataset_path = Path(arguments.dataset)
+    if not dataset_path.is_file():
+        raise FileNotFoundError(f"Dataset file not found: {dataset_path}")
+    try:
+        dataset = load_extraction_dataset(dataset_path)
+    except ValidationError as exc:
+        raise ValueError(f"Invalid extraction dataset: {exc}") from exc
+
+    run = corpus.load_extraction_run_manifest(
+        extractor_id=run_ref.extractor_id,
+        run_id=run_ref.run_id,
+    )
+    result = evaluate_extraction_run(
+        corpus=corpus,
+        run=run,
+        extractor_id=run_ref.extractor_id,
+        dataset=dataset,
+    )
+    write_extraction_evaluation_result(corpus=corpus, run_id=run.run_id, result=result)
+    print(result.model_dump_json(indent=2))
+    return 0
+
+
 def cmd_query(arguments: argparse.Namespace) -> int:
     """
     Execute a retrieval query.
@@ -901,6 +954,22 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_extract_delete.set_defaults(func=cmd_extract_delete)
 
+    p_extract_evaluate = extract_sub.add_parser(
+        "evaluate", help="Evaluate an extraction run against a dataset."
+    )
+    _add_common_corpus_arg(p_extract_evaluate)
+    p_extract_evaluate.add_argument(
+        "--run",
+        default=None,
+        help="Extraction run reference in the form extractor_id:run_id (defaults to latest run).",
+    )
+    p_extract_evaluate.add_argument(
+        "--dataset",
+        required=True,
+        help="Path to the extraction evaluation dataset JSON file.",
+    )
+    p_extract_evaluate.set_defaults(func=cmd_extract_evaluate)
+
     p_query = sub.add_parser("query", help="Run a retrieval query.")
     _add_common_corpus_arg(p_query)
     p_query.add_argument("--run", default=None, help="Run identifier (defaults to latest run).")

biblicus/constants.py

@@ -4,6 +4,7 @@ Shared constants for Biblicus.
 
 SCHEMA_VERSION = 2
 DATASET_SCHEMA_VERSION = 1
+EXTRACTION_DATASET_SCHEMA_VERSION = 1
 ANALYSIS_SCHEMA_VERSION = 1
 CORPUS_DIR_NAME = ".biblicus"
 DEFAULT_RAW_DIR = "raw"

biblicus/extraction_evaluation.py

@@ -0,0 +1,312 @@
+"""
+Extraction evaluation utilities for Biblicus.
+"""
+
+from __future__ import annotations
+
+import json
+from difflib import SequenceMatcher
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from .constants import EXTRACTION_DATASET_SCHEMA_VERSION
+from .corpus import Corpus
+from .extraction import ExtractionRunManifest
+from .models import CatalogItem
+from .time import utc_now_iso
+
+
+class ExtractionEvaluationItem(BaseModel):
+    """
+    Dataset item for extraction evaluation.
+
+    :ivar item_id: Optional item identifier.
+    :vartype item_id: str or None
+    :ivar source_uri: Optional source uniform resource identifier.
+    :vartype source_uri: str or None
+    :ivar expected_text: Expected extracted text.
+    :vartype expected_text: str
+    :ivar kind: Label kind (gold or synthetic).
+    :vartype kind: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    item_id: Optional[str] = None
+    source_uri: Optional[str] = None
+    expected_text: str
+    kind: str = Field(default="gold")
+
+    @model_validator(mode="after")
+    def _require_locator(self) -> "ExtractionEvaluationItem":
+        if not self.item_id and not self.source_uri:
+            raise ValueError("Evaluation items must include item_id or source_uri")
+        return self
+
+
+class ExtractionEvaluationDataset(BaseModel):
+    """
+    Dataset for extraction evaluation.
+
+    :ivar schema_version: Dataset schema version.
+    :vartype schema_version: int
+    :ivar name: Dataset name.
+    :vartype name: str
+    :ivar description: Optional description.
+    :vartype description: str or None
+    :ivar items: Labeled evaluation items.
+    :vartype items: list[ExtractionEvaluationItem]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    schema_version: int = Field(ge=1)
+    name: str
+    description: Optional[str] = None
+    items: List[ExtractionEvaluationItem] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _enforce_schema_version(self) -> "ExtractionEvaluationDataset":
+        if self.schema_version != EXTRACTION_DATASET_SCHEMA_VERSION:
+            raise ValueError(
+                f"Unsupported extraction dataset schema version: {self.schema_version}"
+            )
+        return self
+
+
+class ExtractionEvaluationItemReport(BaseModel):
+    """
+    Per-item report for extraction evaluation.
+
+    :ivar item_id: Item identifier.
+    :vartype item_id: str
+    :ivar source_uri: Source uniform resource identifier.
+    :vartype source_uri: str
+    :ivar expected_text: Expected text from the dataset.
+    :vartype expected_text: str
+    :ivar extracted_text: Extracted text when available.
+    :vartype extracted_text: str or None
+    :ivar coverage_status: Coverage status (present, empty, missing).
+    :vartype coverage_status: str
+    :ivar extraction_status: Extraction status from the run (extracted, skipped, errored, missing).
+    :vartype extraction_status: str
+    :ivar similarity_score: Similarity score between expected and extracted text.
+    :vartype similarity_score: float
+    :ivar kind: Label kind from the dataset.
+    :vartype kind: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    item_id: str
+    source_uri: str
+    expected_text: str
+    extracted_text: Optional[str] = None
+    coverage_status: str
+    extraction_status: str
+    similarity_score: float
+    kind: str
+
+
+class ExtractionEvaluationResult(BaseModel):
+    """
+    Result bundle for an extraction evaluation.
+
+    :ivar dataset: Dataset metadata.
+    :vartype dataset: dict[str, object]
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    :ivar run_id: Extraction run identifier.
+    :vartype run_id: str
+    :ivar recipe_id: Extraction recipe identifier.
+    :vartype recipe_id: str
+    :ivar recipe_name: Extraction recipe name.
+    :vartype recipe_name: str
+    :ivar evaluated_at: International Organization for Standardization 8601 timestamp.
+    :vartype evaluated_at: str
+    :ivar metrics: Evaluation metrics for coverage and accuracy.
+    :vartype metrics: dict[str, float]
+    :ivar items: Per-item evaluation reports.
+    :vartype items: list[ExtractionEvaluationItemReport]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    dataset: Dict[str, object]
+    extractor_id: str
+    run_id: str
+    recipe_id: str
+    recipe_name: str
+    evaluated_at: str
+    metrics: Dict[str, float]
+    items: List[ExtractionEvaluationItemReport]
+
+
+def load_extraction_dataset(path: Path) -> ExtractionEvaluationDataset:
+    """
+    Load an extraction evaluation dataset from JavaScript Object Notation.
+
+    :param path: Path to the dataset file.
+    :type path: Path
+    :return: Parsed extraction evaluation dataset.
+    :rtype: ExtractionEvaluationDataset
+    """
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError("Invalid extraction dataset") from exc
+    return ExtractionEvaluationDataset.model_validate(data)
+
+
+def evaluate_extraction_run(
+    *,
+    corpus: Corpus,
+    run: ExtractionRunManifest,
+    extractor_id: str,
+    dataset: ExtractionEvaluationDataset,
+) -> ExtractionEvaluationResult:
+    """
+    Evaluate an extraction run against a dataset.
+
+    :param corpus: Corpus associated with the run.
+    :type corpus: Corpus
+    :param run: Extraction run manifest.
+    :type run: ExtractionRunManifest
+    :param extractor_id: Extractor identifier for the run.
+    :type extractor_id: str
+    :param dataset: Extraction evaluation dataset.
+    :type dataset: ExtractionEvaluationDataset
+    :return: Extraction evaluation result bundle.
+    :rtype: ExtractionEvaluationResult
+    """
+    catalog = corpus.load_catalog()
+    item_index = {item.item_id: item for item in run.items}
+    coverage_present = 0
+    coverage_empty = 0
+    coverage_missing = 0
+    processable = 0
+    similarity_scores: List[float] = []
+    item_reports: List[ExtractionEvaluationItemReport] = []
+
+    for dataset_item in dataset.items:
+        item_id = _resolve_item_id(dataset_item, catalog_items=catalog.items)
+        catalog_item = catalog.items.get(item_id)
+        if catalog_item is None:
+            raise ValueError(f"Unknown item identifier: {item_id}")
+        extraction_item = item_index.get(item_id)
+        extraction_status = extraction_item.status if extraction_item else "missing"
+        if extraction_status != "errored" and extraction_status != "missing":
+            processable += 1
+
+        extracted_text = corpus.read_extracted_text(
+            extractor_id=extractor_id, run_id=run.run_id, item_id=item_id
+        )
+        coverage_status = _coverage_status(extracted_text)
+        if coverage_status == "present":
+            coverage_present += 1
+        elif coverage_status == "empty":
+            coverage_empty += 1
+        else:
+            coverage_missing += 1
+
+        similarity_score = _similarity_score(
+            expected_text=dataset_item.expected_text, extracted_text=extracted_text
+        )
+        similarity_scores.append(similarity_score)
+        item_reports.append(
+            ExtractionEvaluationItemReport(
+                item_id=item_id,
+                source_uri=catalog_item.source_uri,
+                expected_text=dataset_item.expected_text,
+                extracted_text=extracted_text,
+                coverage_status=coverage_status,
+                extraction_status=extraction_status,
+                similarity_score=similarity_score,
+                kind=dataset_item.kind,
+            )
+        )
+
+    total_items = max(len(dataset.items), 1)
+    average_similarity = sum(similarity_scores) / total_items if similarity_scores else 0.0
+    metrics = {
+        "coverage_present": float(coverage_present),
+        "coverage_empty": float(coverage_empty),
+        "coverage_missing": float(coverage_missing),
+        "processable_fraction": processable / total_items,
+        "average_similarity": average_similarity,
+    }
+    dataset_meta = {
+        "name": dataset.name,
+        "description": dataset.description,
+        "items": len(dataset.items),
+    }
+    return ExtractionEvaluationResult(
+        dataset=dataset_meta,
+        extractor_id=extractor_id,
+        run_id=run.run_id,
+        recipe_id=run.recipe.recipe_id,
+        recipe_name=run.recipe.name,
+        evaluated_at=utc_now_iso(),
+        metrics=metrics,
+        items=item_reports,
+    )
+
+
+def write_extraction_evaluation_result(
+    *, corpus: Corpus, run_id: str, result: ExtractionEvaluationResult
+) -> Path:
+    """
+    Persist extraction evaluation output under the corpus.
+
+    :param corpus: Corpus associated with the evaluation.
+    :type corpus: Corpus
+    :param run_id: Extraction run identifier.
+    :type run_id: str
+    :param result: Evaluation result to write.
+    :type result: ExtractionEvaluationResult
+    :return: Output path.
+    :rtype: Path
+    """
+    output_dir = corpus.runs_dir / "evaluation" / "extraction" / run_id
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path = output_dir / "output.json"
+    output_path.write_text(result.model_dump_json(indent=2) + "\n", encoding="utf-8")
+    return output_path
+
+
+def _resolve_item_id(
+    dataset_item: ExtractionEvaluationItem, *, catalog_items: Dict[str, CatalogItem]
+) -> str:
+    if dataset_item.item_id:
+        return dataset_item.item_id
+    source_uri = dataset_item.source_uri
+    if not source_uri:
+        raise ValueError("Evaluation item is missing item_id and source_uri")
+    for item_id, catalog_item in catalog_items.items():
+        if getattr(catalog_item, "source_uri", None) == source_uri:
+            return item_id
+    raise ValueError(f"Unknown source uniform resource identifier: {source_uri}")
+
+
+def _coverage_status(extracted_text: Optional[str]) -> str:
+    if extracted_text is None:
+        return "missing"
+    if extracted_text.strip():
+        return "present"
+    return "empty"
+
+
+def _normalize_text(text: str) -> str:
+    return " ".join(text.lower().split())
+
+
+def _similarity_score(*, expected_text: str, extracted_text: Optional[str]) -> float:
+    if extracted_text is None:
+        return 0.0
+    expected = _normalize_text(expected_text)
+    actual = _normalize_text(extracted_text)
+    if not expected and not actual:
+        return 1.0
+    return SequenceMatcher(None, expected, actual).ratio()

{biblicus-0.12.0.dist-info → biblicus-0.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.12.0
+Version: 0.13.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -486,10 +486,11 @@ corpus/
 
 ## Retrieval backends
 
-Two backends are included.
+Three backends are included.
 
 - `scan` is a minimal baseline that scans raw items directly.
 - `sqlite-full-text-search` is a practical baseline that builds a full text search index in SQLite.
+- `vector` is a deterministic term-frequency vector baseline with cosine similarity scoring.
 
 For detailed documentation including configuration options, performance characteristics, and usage examples, see the [Backend Reference][backend-reference].
 
@@ -535,6 +536,9 @@ These extractors are built in. Optional ones require extra dependencies. See [te
 
 For detailed documentation on all extractors, see the [Extractor Reference][extractor-reference].
 
+For extraction evaluation workflows, dataset formats, and report interpretation, see
+`docs/EXTRACTION_EVALUATION.md`.
+
 ## Topic modeling analysis
 
 Biblicus can run analysis pipelines on extracted text without changing the raw corpus. Profiling and topic modeling

{biblicus-0.12.0.dist-info → biblicus-0.13.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-biblicus/__init__.py,sha256=okAXmTSud_hQzaGEURDqX95I66SlcvTERCrWbUZA5ko,496
+biblicus/__init__.py,sha256=pD55sYei6AGGLcN1AWnpUY6-zPIPq1WxOp-sexOOlT0,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=bZV-ZxeWskRL4CFCGzyVpcaFC8KOb0xmxx3bnMqP-1I,36118
-biblicus/constants.py,sha256=-JaHI3Dngte2drawx93cGWxFVobbgIuaVhmjUJpf4GI,333
+biblicus/cli.py,sha256=cMoirLFPhTwftNuqaadajCcRUEz_FBaLkupjVxpAxO8,38403
+biblicus/constants.py,sha256=gAlEVJhxdFj-eWWJrlYbP7H1X3c5gwhrIBq9NQ1Vq_E,371
 biblicus/context.py,sha256=U7qkOwMdqNgYnqaC9hgQY0kv0R-6qcjV6bhXQl2WUkE,10215
 biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
 biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
@@ -9,6 +9,7 @@ biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
 biblicus/evaluation.py,sha256=5xWpb-8f49Osh9aHzo1ab3AXOmls3Imc5rdnEC0pN-8,8143
 biblicus/evidence_processing.py,sha256=sJe6T1nLxvU0xs9yMH8JZZS19zHXMR-Fpr5lWi5ndUM,6120
 biblicus/extraction.py,sha256=qvrsq6zSz2Kg-cap-18HPHC9pQlqEGo7pyID2uKCyBo,19760
+biblicus/extraction_evaluation.py,sha256=cBC2B1nQCtXmOcVWUhHyO2NJRX8QSDuqhVjEc8PXrOA,10400
 biblicus/frontmatter.py,sha256=JOGjIDzbbOkebQw2RzA-3WDVMAMtJta2INjS4e7-LMg,2463
 biblicus/hook_logging.py,sha256=IMvde-JhVWrx9tNz3eDJ1CY_rr5Sj7DZ2YNomYCZbz0,5366
 biblicus/hook_manager.py,sha256=ZCAkE5wLvn4lnQz8jho_o0HGEC9KdQd9qitkAEUQRcw,6997
@@ -57,9 +58,9 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.12.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.12.0.dist-info/METADATA,sha256=fhWcCcczfuLn2mZ_Moqe2zMKJ1-Q7KxZtR_x9YaiFO8,27765
-biblicus-0.12.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.12.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.12.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.12.0.dist-info/RECORD,,
+biblicus-0.13.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.13.0.dist-info/METADATA,sha256=Ae0gttdvOggyE1vQVab4IOSmbx-JklxzvBZJ_3UyxIA,27979
+biblicus-0.13.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.13.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.13.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.13.0.dist-info/RECORD,,