sci-fi-parser 0.1.0__py3-none-any.whl

sci_fi_parser/__init__.py

@@ -0,0 +1,5 @@
+from .api import parse_folder
+
+__all__ = [
+    "parse_folder",
+]

sci_fi_parser/api.py

@@ -0,0 +1,184 @@
+"""Public parsing API for running the Sci-Fi-Parser pipeline.
+
+This module exposes a convenience function for parsing a folder of PDFs and a
+result object for accessing the parsed image and PDF sets, saving outputs, and
+loading benchmark-friendly data frames.
+"""
+
+from importlib.resources import files
+from pathlib import Path
+
+import pandas as pd
+
+from sci_fi_parser.classifier.classifier_pipeline import start_classification
+from sci_fi_parser.image_extraction.extraction_pipeline import start_extraction
+from sci_fi_parser.object_detection.detection_pipeline import start_ocr
+from sci_fi_parser.schema import ImageSet, PdfSet
+from sci_fi_parser.storage.writer import save_image_set
+from sci_fi_parser.vlm.vlm_pipeline import start_vlm
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+
+DEFAULT_EXTRACTED_IMAGE_DIR = PROJECT_ROOT / "temp" / "extracted_images"
+DEFAULT_VLM_CONFIG = files("sci_fi_parser.config").joinpath("vlm.toml")
+
+
+class ParseResult:
+    """Represents the result of one parsing run."""
+
+    def __init__(
+        self,
+        image_set: ImageSet,
+        pdf_set: PdfSet,
+        output_dir: str | Path | None = None,
+    ):
+        self._image_set = image_set
+        self._pdf_set = pdf_set
+        self._output_dir = Path(output_dir) if output_dir else None
+
+    @property
+    def images(self) -> ImageSet:
+        """Return the parsed images and associated metadata."""
+
+        return self._image_set
+
+    @property
+    def pdfs(self) -> PdfSet:
+        """Return the parsed PDFs and associated metadata."""
+
+        return self._pdf_set
+
+    @property
+    def output_dir(self) -> Path | None:
+        """Return the saved output directory, if one has been set."""
+
+        return self._output_dir
+
+    def summary(self) -> dict:
+        """Return a compact summary of the parsed dataset.
+
+        Returns:
+            A dictionary containing PDF and image counts.
+        """
+
+        return {
+            "pdf_count": len(self._pdf_set),
+            "image_count": len(self._image_set),
+        }
+
+    def save(self, output_dir: str | Path) -> None:
+        """Save the parsed dataset as JSONL and Parquet.
+
+        Args:
+            output_dir: Directory where the parsed dataset will be written.
+        """
+
+        output_dir = Path(output_dir)
+
+        save_image_set(
+            image_set=self._image_set,
+            output_dir=output_dir,
+        )
+
+        self._output_dir = output_dir
+
+    def _require_saved_output(self) -> Path:
+        if self._output_dir is None:
+            raise ValueError(
+                "No output directory available. Pass output_dir to parse_folder() or call result.save(...)."
+            )
+
+        return self._output_dir
+
+    def charts_dataframe(self) -> pd.DataFrame:
+        """Load ``charts.parquet`` as a DataFrame.
+
+        Returns:
+            A DataFrame loaded from ``tables/charts.parquet``.
+        """
+
+        output_dir = self._require_saved_output()
+
+        return pd.read_parquet(output_dir / "tables" / "charts.parquet")
+
+    def series_dataframe(self) -> pd.DataFrame:
+        """Load ``series.parquet`` as a DataFrame.
+
+        Returns:
+            A DataFrame loaded from ``tables/series.parquet``.
+        """
+
+        output_dir = self._require_saved_output()
+
+        return pd.read_parquet(output_dir / "tables" / "series.parquet")
+
+    def points_dataframe(self) -> pd.DataFrame:
+        """Load ``points.parquet`` as a DataFrame.
+
+        Returns:
+            A DataFrame loaded from ``tables/points.parquet``.
+        """
+
+        output_dir = self._require_saved_output()
+
+        return pd.read_parquet(output_dir / "tables" / "points.parquet")
+
+
+def parse_folder(
+    input_dir: str | Path,
+    *,
+    output_dir: str | Path | None = None,
+    extracted_image_dir: str | Path = DEFAULT_EXTRACTED_IMAGE_DIR,
+    vlm_config: str | Path = DEFAULT_VLM_CONFIG,
+    classify: bool = True,
+    ocr: bool = True,
+    vlm: bool = True,
+) -> ParseResult:
+    """Parse all PDFs inside a folder.
+
+    Args:
+        input_dir: Folder containing PDF files.
+        output_dir: If provided, save JSONL and Parquet outputs.
+        extracted_image_dir: Temporary folder used during image extraction.
+        vlm_config: Path to the VLM configuration file.
+        classify: Whether to run chart classification.
+        ocr: Whether to run OCR.
+        vlm: Whether to run VLM extraction.
+
+    Returns:
+        A parse result containing the populated image and PDF sets.
+    """
+
+    image_set = ImageSet()
+    pdf_set = PdfSet()
+
+    start_extraction(
+        Path(input_dir),
+        image_set,
+        pdf_set,
+        Path(extracted_image_dir),
+    )
+
+    if classify:
+        start_classification(image_set)
+
+    if ocr:
+        start_ocr(image_set)
+
+    if vlm:
+        start_vlm(
+            image_set,
+            Path(vlm_config),
+        )
+
+    if output_dir is not None:
+        save_image_set(
+            image_set=image_set,
+            output_dir=Path(output_dir),
+        )
+
+    return ParseResult(
+        image_set=image_set,
+        pdf_set=pdf_set,
+        output_dir=output_dir,
+    )

sci_fi_parser/benchmark/README.md

@@ -0,0 +1,130 @@
+# `sci_fi_parser.accuracy` — measurement layer
+
+Tools for generating chart datasets with **ground-truth labels by construction**
+and scoring any extractor against them. Two CLI commands and a pluggable
+`Extractor` protocol.
+
+The CLIs (`synthetic-bars`, `benchmark`) are installed by `uv sync` via
+`[project.scripts]` — no `python -m …` prefix needed.
+
+## `synthetic-bars` — generate ground-truth chart datasets
+
+Charts whose data is known *by construction* (no model-labelled images, no
+circularity risk). Default mode is a *controlled density series*: per enabled
+type the style is fixed once and only the bar count varies across
+`density_steps`; each density is rendered as a matched **labels-off / labels-on
+pair** sharing identical data. Labels are auto-fitted so they never overlap.
+
+```bash
+synthetic-bars --config config/synthetic_bars.toml --overlay --sqlite
+```
+
+Pick types, density ladder, and resolutions in
+[../../../config/synthetic_bars.toml](../../../config/synthetic_bars.toml):
+`output_types`, `density_steps`, `resolutions`. Run with `--preview` first to
+render a small sample of every catalog type.
+
+| Flag | Effect |
+|---|---|
+| `--preview` | one sample per catalog type into `preview/` |
+| `--random N` | random mode instead: N fully-random charts |
+| `--augment` | add JPEG/noise/blur realism (labels stay exact) |
+| `--overlay` | write `_debug/overlay_*.png` to eyeball label accuracy |
+| `--sqlite` | also write `dataset.sqlite3` (mirrors the real schema + geometry) |
+| `--refresh` | delete prior outputs in `--out` before generating |
+| `--seed N` | reproducibility |
+
+Outputs in `--out`:
+- `images/<type>_s<k>_d<NN>_{off,on}_<res>.png` (series) or `random_NNNNN_<res>.png`
+- `labels.jsonl` — per chart: `label1` (chart type), `label2` (axes +
+  per-series points + `value_range`), `geometry` (bboxes + tick positions, or
+  `null`), `meta`
+- `_debug/` overlays (with `--overlay`)
+- `dataset.sqlite3` (with `--sqlite`)
+
+Tuning: every knob is documented inline in `GenConfig`
+([synthetic/config.py](synthetic/config.py)) and mirrored with comments in the
+TOML.
+
+## `benchmark` — score any extractor against ground truth
+
+```bash
+benchmark --data train_data/synthetic --out reports/run1
+```
+
+The default extractor is `noisy-oracle` (a test double that perturbs truth) so
+the harness runs without a model server. For a real model, install
+[ollama](https://ollama.com) locally, `ollama pull qwen2.5vl:7b`, then use
+`--extractor vlm` to pick up the model declared in
+[../../../config/vlm.toml](../../../config/vlm.toml) — or override per-run
+with `--extractor vlm:<tag>`. The extractor talks to any OpenAI-compatible
+chat-completions endpoint (`base_url` in the profile); ollama's `/v1` compat
+layer is the default.
+
+**Metrics**
+- Error = `|predicted − true|` as **% of the true value**. Hover any card in
+  the HTML report for the exact definition.
+- **Recall** = bars found / true bars. **Precision** = correct
+  (series, category) bars / predicted.
+- **Type acc** = fraction of charts where the extractor's `chart_type` matched
+  truth. **Mean conf** = average VLM self-reported confidence (not necessarily
+  calibrated).
+
+**Outputs** in `--out`:
+- `report.html` — self-contained: summary cards (hover for tooltips), breakdowns
+  by preset / density / labels, ranked best/worst charts with thumbnails,
+  sortable per-chart table
+- `results.json` — machine-readable per-chart rows for CI / trend tracking
+
+## Configuring the VLM extractor
+
+The active model, prompt, and endpoint live in
+[../../../config/vlm.toml](../../../config/vlm.toml). Edit that file to swap
+models permanently, or override on the CLI for one-offs:
+
+```bash
+benchmark --data ... --extractor vlm:qwen2.5vl:7b-q8_0   # one-shot
+benchmark --data ... --vlm-config config/vlm_q8.toml      # A/B test
+```
+
+Resolution order: CLI flag > `config/vlm.toml` in cwd > built-in defaults.
+
+## Adding a new extractor
+
+Anything implementing the `Extractor` protocol from
+[../schema.py](../schema.py) plugs in:
+
+```python
+from pathlib import Path
+from sci_fi_parser.schema import ChartData
+
+class MyExtractor:
+    name = "my-extractor"
+    def extract(self, image_path: Path) -> ChartData: ...
+```
+
+Register it in `build_extractor` ([benchmark.py](benchmark.py)). `ChartData`
+is the single canonical schema every extractor must return: the VLM is
+constrained to emit it (via `response_format: json_schema`), CV+OCR pipelines
+are mapped into it.
+
+> Note: pure OCR is *not* a standalone extractor (it reads text, not data
+> points) — pair it with CV.
+
+## Layout
+
+```
+accuracy/
+  __init__.py
+  benchmark.py      # `benchmark` CLI: runs an extractor, writes report
+  vlm_compare.py    # `benchmark-compare` CLI: runs N models, builds leaderboard
+  synthetic/        # ground-truth chart generator
+```
+
+The VLM extractor itself (`ChatCompletionsVLM` + `VLMProfile` + `load_profile`) lives
+in the sibling package [../vlm/](../vlm/) — separated from this package so
+non-benchmark callers (e.g. the runtime pipeline) can import the model
+client without pulling in the measurement machinery.
+
+Config files live at repo-root [config/](../../../config/), separate from the
+package source so users edit data, not code.

sci_fi_parser/benchmark/__init__.py

@@ -0,0 +1,12 @@
+"""Accuracy-metrics layer.
+
+Two cooperating pieces, coupled by a data contract (``labels.jsonl``), not by code:
+
+* :mod:`sci_fi_parser.accuracy.synthetic` -- generates charts whose values are known
+  *by construction*, so they carry exact ground truth.
+* :mod:`sci_fi_parser.accuracy.benchmark` -- scores any extractor against that ground
+  truth and turns the per-chart errors into an empirical error margin.
+
+Import the subpackage you need directly; this package intentionally pulls in neither,
+so e.g. importing ``benchmark`` does not drag in matplotlib.
+"""

sci_fi_parser/benchmark/bench_pipeline.py

@@ -0,0 +1,234 @@
+"""MVP staged benchmark pipeline.
+
+This module only orchestrates stages. Scoring, aggregation, extractor building,
+and report formatting are reused from the legacy benchmark where possible.
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+
+from sci_fi_parser.benchmark import benchmark
+from sci_fi_parser.benchmark.report_adapter import (
+    value_results_to_breakdowns,
+    value_results_to_draw_lap_charts,
+)
+from sci_fi_parser.benchmark.scoring import (
+    aggregate_value_results,
+    score_vlm_outputs,
+)
+from sci_fi_parser.benchmark.truth import (
+    ChartTruth,
+    load_benetech_truth,
+    load_synthetic_truth,
+)
+from sci_fi_parser.image_extraction.image_loader import load_images_from_folder
+from sci_fi_parser.schema import ImageSet
+from sci_fi_parser.vlm.vlm_config import VLMProfile
+from sci_fi_parser.vlm.vlm_schema import ChartData
+
+
+@dataclass(slots=True)
+class PipelineInputs:
+    image_set: ImageSet
+    truth_by_image_id: dict[str, ChartTruth]
+    image_dir: Path
+
+
+DatasetKind = Literal["synthetic", "benetech"]
+
+
+def load_inputs(
+    data: Path,
+    limit: int | None = None,
+    dataset: DatasetKind = "synthetic",
+) -> PipelineInputs:
+    def truth_key(path: Path) -> str:
+        if dataset == "synthetic":
+            return path.name
+        if dataset == "benetech":
+            return path.stem
+
+    if dataset == "synthetic":
+        truth_by_key, metadata_by_key = load_synthetic_truth(data)
+    elif dataset == "benetech":
+        truth_by_key, metadata_by_key = load_benetech_truth(data)
+    else:
+        raise ValueError(f"unknown benchmark dataset: {dataset}")
+
+    img_dir = data / "images"
+
+    image_set = ImageSet()
+    load_images_from_folder(img_dir, image_set)
+
+    image_ids = [
+        image_id
+        for image_id, _ in sorted(image_set.items(), key=lambda item: image_set.get_image_path(item[0]).name)
+        if truth_key(image_set.get_image_path(image_id)) in truth_by_key
+    ]
+    if limit:
+        image_ids = image_ids[:limit]
+
+    truth_by_image_id: dict[str, ChartTruth] = {}
+    for image_id in image_ids:
+        key = truth_key(image_set.get_image_path(image_id))
+        truth_by_image_id[image_id] = truth_by_key[key]
+        image_set.get(image_id)["metadata"]["benchmark"] = metadata_by_key.get(key, {})
+
+    return PipelineInputs(
+        image_set=image_set,
+        truth_by_image_id=truth_by_image_id,
+        image_dir=img_dir,
+    )
+
+
+def run_classification_stage(inputs: PipelineInputs) -> None:
+    from sci_fi_parser.classifier.classifier_pipeline import start_classification
+
+    start_classification(inputs.image_set)
+
+
+def run_ocr_cv_stage(inputs: PipelineInputs) -> None:
+    from sci_fi_parser.object_detection.detection_pipeline import start_ocr
+
+    scoped = ImageSet()
+    for image_id in inputs.truth_by_image_id:
+        scoped.add(image_id, inputs.image_set.get(image_id))
+    start_ocr(scoped)
+
+
+def _prompt_suffix(inputs: PipelineInputs, image_id: str) -> str:
+    value = inputs.image_set.get(image_id).get("ocrcv", {}).get("result", "")
+    return f"OCR/CV context:\n{value}" if isinstance(value, str) and value else ""
+
+
+def run_vlm_stage(inputs: PipelineInputs, extractor: benchmark.Extractor) -> None:
+    from tqdm import tqdm
+
+    for image_id in tqdm(inputs.truth_by_image_id):
+        record = inputs.image_set.get(image_id)
+        path = inputs.image_set.get_image_path(image_id)
+        t0 = time.perf_counter()
+        try:
+            parsed, raw = extractor.extract(
+                path,
+                prompt_suffix=_prompt_suffix(inputs, image_id),
+            )
+            parsed_payload = ChartData.model_validate(parsed).model_dump()
+        except Exception as exc:
+            print(f"  ! {path.name}: {type(exc).__name__}: {exc}")
+            parsed_payload = ChartData(chart_type="none", series=[], log_scale=False).model_dump()
+            raw = {"error": f"{type(exc).__name__}: {exc}"}
+        inputs.image_set.add_vlm_result(image_id, parsed_payload)
+        inputs.image_set.add_vlm_result_raw(image_id, raw)
+        record["metadata"]["vlm"]["seconds"] = time.perf_counter() - t0
+
+
+def write_outputs(
+    out: Path,
+    extractor: benchmark.Extractor,
+    agg: dict,
+    results: list[benchmark.ChartResult],
+    img_dir: Path,
+) -> None:
+    out.mkdir(parents=True, exist_ok=True)
+    benchmark._write_results_json(out, extractor, agg, results)
+    from sci_fi_parser.benchmark import draw_lap
+
+    draw_lap.write_html(
+        out / "report.html",
+        extractor._model,
+        agg,
+        value_results_to_draw_lap_charts(results),
+        value_results_to_breakdowns(results),
+        img_dir,
+    )
+
+
+def run_pipeline(
+    *,
+    data: Path,
+    out: Path,
+    extractor_name: str = "noisy-oracle",
+    profile: VLMProfile | None = None,
+    seed: int = 0,
+    limit: int | None = None,
+    dataset: DatasetKind = "synthetic",
+    classification: bool = False,
+    ocr_cv: bool = False,
+    print_summary: bool = True,
+) -> dict:
+    inputs = load_inputs(data, limit, dataset)
+    if classification:
+        run_classification_stage(inputs)
+    if ocr_cv:
+        run_ocr_cv_stage(inputs)
+
+    extractor = benchmark.build_extractor(
+        extractor_name,
+        {
+            inputs.image_set.get_image_path(image_id).name: truth
+            for image_id, truth in inputs.truth_by_image_id.items()
+        },
+        np.random.default_rng(seed),
+        profile=profile,
+    )
+    run_vlm_stage(inputs, extractor)
+
+    results = score_vlm_outputs(
+        inputs.image_set,
+        inputs.truth_by_image_id,
+    )
+    agg = aggregate_value_results(results)
+    write_outputs(out, extractor, agg, results, inputs.image_dir)
+    if print_summary:
+        benchmark._print_summary(extractor, agg, results, out)
+    return agg
+
+
+def _parse_args() -> argparse.Namespace:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument(
+        "--data",
+        type=Path,
+        required=True,
+        help="dataset dir containing images/ plus truth.jsonl or annotations/",
+    )
+    ap.add_argument("--dataset", choices=("synthetic", "benetech"), default="synthetic")
+    ap.add_argument("--out", type=Path, default=Path("reports/pipeline"))
+    ap.add_argument(
+        "--extractor",
+        default="noisy-oracle",
+        help="noisy-oracle | vlm | vlm:<model>",
+    )
+    ap.add_argument("--vlm-config", type=Path, default=None, help="VLM profile TOML")
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--limit", type=int, default=None, help="only first N charts")
+    ap.add_argument("--classification", action="store_true", help="run classification stage")
+    ap.add_argument("--ocr-cv", action="store_true", help="run OCR/CV stage")
+    return ap.parse_args()
+
+
+def main() -> None:
+    args = _parse_args()
+    run_pipeline(
+        data=args.data,
+        out=args.out,
+        extractor_name=args.extractor,
+        profile=benchmark._resolve_profile(args.vlm_config),
+        seed=args.seed,
+        limit=args.limit,
+        dataset=args.dataset,
+        classification=args.classification,
+        ocr_cv=args.ocr_cv,
+    )
+
+
+if __name__ == "__main__":
+    main()