maskflow-bench 0.1.0__tar.gz

maskflow_bench-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+.env
+node_modules/
+dist/
+build/
+*.egg-info/
+.DS_Store
+.pytest_cache/
+.idea/
+.coverage
+.mypy_cache/
+.ruff_cache/
+bench/indiapii/quality/.cache/

maskflow_bench-0.1.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.5
+Name: maskflow-bench
+Version: 0.1.0
+Summary: MaskFlow benchmark scoring core: load labelled PII data, score detections under strict/partial-overlap precision-recall-F1, write JSON/Markdown reports. Powers `maskflow bench --my-data`.
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: maskflow-core<0.9,>=0.8.0
+Provides-Extra: dev
+Requires-Dist: maskflow-pack-india<0.6,>=0.5.1; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# maskflow-bench
+
+The scoring core behind MaskFlow's accuracy measurements: load a labelled
+JSONL corpus, canonicalize its entity taxonomy, score detections under
+strict-span and partial-overlap precision/recall/F1, write `results.json`
+and `results.md`.
+
+Most people reach this through `maskflow bench --my-data <path>`
+(`maskflow-cli`) — see [`docs/bench.md`](../../docs/bench.md) for the
+labelled-data schema and command reference.
+
+This package intentionally ships **one** adapter — MaskFlow itself
+(`adapters.maskflow_adapter.MaskflowAdapter`) — with no dependency beyond
+`maskflow-core`. The multi-adapter comparison against Presidio,
+mask-privacy, a naive-regex baseline, and an LLM judge that produced the
+published IndiaPII-Bench tables lives in `bench/indiapii/harness/` (repo
+dev tooling, not published), and imports this package for the parts that
+don't change per adapter.

maskflow_bench-0.1.0/README.md

@@ -0,0 +1,18 @@
+# maskflow-bench
+
+The scoring core behind MaskFlow's accuracy measurements: load a labelled
+JSONL corpus, canonicalize its entity taxonomy, score detections under
+strict-span and partial-overlap precision/recall/F1, write `results.json`
+and `results.md`.
+
+Most people reach this through `maskflow bench --my-data <path>`
+(`maskflow-cli`) — see [`docs/bench.md`](../../docs/bench.md) for the
+labelled-data schema and command reference.
+
+This package intentionally ships **one** adapter — MaskFlow itself
+(`adapters.maskflow_adapter.MaskflowAdapter`) — with no dependency beyond
+`maskflow-core`. The multi-adapter comparison against Presidio,
+mask-privacy, a naive-regex baseline, and an LLM judge that produced the
+published IndiaPII-Bench tables lives in `bench/indiapii/harness/` (repo
+dev tooling, not published), and imports this package for the parts that
+don't change per adapter.

maskflow_bench-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "maskflow-bench"
+version = "0.1.0"
+description = "MaskFlow benchmark scoring core: load labelled PII data, score detections under strict/partial-overlap precision-recall-F1, write JSON/Markdown reports. Powers `maskflow bench --my-data`."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+dependencies = [
+    # Only adapters.maskflow_adapter needs this, to call
+    # maskflow_core.detection.detect(). No presidio/mask-privacy/anthropic
+    # here -- those five competitor adapters stay dev-only in
+    # bench/indiapii/harness/adapters/, never a dependency of this package.
+    "maskflow-core>=0.8.0,<0.9",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    # test_scorer.py's end-to-end test needs a real recognizer registered
+    # to detect something -- this package's own runtime code never imports
+    # a pack (see module docstring), only its test suite does.
+    "maskflow-pack-india>=0.5.1,<0.6",
+]
+
+[tool.uv.sources]
+maskflow-core = { workspace = true }
+maskflow-pack-india = { workspace = true }
+
+[tool.uv]
+package = true
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/maskflow_bench"]

maskflow_bench-0.1.0/src/maskflow_bench/__init__.py

@@ -0,0 +1,25 @@
+"""maskflow-bench: the corpus-agnostic PII-detection scoring core.
+
+Loads a labelled JSONL corpus into `Document` objects (corpus.py),
+canonicalizes its entity taxonomy (labels.py), scores an adapter's raw
+(start, end, label) predictions against it under strict-span and
+partial-overlap matching (matching.py), times and profiles that run
+(profiling.py, runner.py), and writes `results.json`/`results.md`
+(report.py).
+
+This package has exactly one built-in adapter, `adapters.maskflow_adapter.
+MaskflowAdapter` (wraps `maskflow_core.detection.detect`, zero extra
+third-party dependencies) -- the `loader`/`scorer` modules on top of it are
+what `maskflow bench --my-data <path>` (packages/maskflow-cli) uses to
+score MaskFlow against a user's own labelled documents.
+
+It is also the shared implementation behind bench/indiapii/harness's,
+bench/intlpii/harness's, and bench/scanbench/harness's own dev-only,
+multi-adapter comparisons (Presidio, mask-privacy, naive regex, an LLM
+judge) used to produce the published IndiaPII-Bench-style figures -- those
+adapters, and the harness CLIs that run all of them together, stay in
+bench/ since they pull heavy, optional third-party dependencies this
+package never needs.
+"""
+
+from __future__ import annotations

maskflow_bench-0.1.0/src/maskflow_bench/adapters/__init__.py

@@ -0,0 +1,15 @@
+"""Just the one adapter this package owns (`maskflow_adapter.MaskflowAdapter`)
+plus the `Adapter` protocol (`base.Adapter`) it implements. The other five
+competitor adapters (Presidio, mask-privacy, naive regex, an LLM judge)
+stay in bench/indiapii/harness/adapters/ -- they pull heavy, optional
+third-party dependencies (presidio-analyzer, mask-privacy, anthropic) this
+package never needs, and they are only ever used by the dev-only
+multi-adapter comparison harness, never by `maskflow bench --my-data`.
+"""
+
+from __future__ import annotations
+
+from .base import Adapter, AdapterEntry
+from .maskflow_adapter import MaskflowAdapter
+
+__all__ = ["Adapter", "AdapterEntry", "MaskflowAdapter"]

maskflow_bench-0.1.0/src/maskflow_bench/adapters/base.py

@@ -0,0 +1,31 @@
+"""The Adapter protocol every competitor detector implements.
+
+`available()` is checked once per harness run, before any `detect()` call:
+an adapter missing an optional dependency (presidio, mask-privacy) or an
+API key (the LLM adapter) reports `(False, reason)` and is skipped, never
+crashing the run. `detect()` itself is called once per document by
+runner.py, which wraps each call in its own bounded try/except so one bad
+document can't take out the rest of an adapter's run either.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class Adapter(Protocol):
+    name: str
+
+    def available(self) -> tuple[bool, str]:
+        """Returns (True, "") if this adapter can run, else (False, reason)."""
+        ...
+
+    def detect(self, text: str) -> list[tuple[int, int, str]]:
+        """Returns (start, end, raw_label) spans in this adapter's own
+        label vocabulary -- translated to the corpus's canonical labels by
+        labels.py, not here."""
+        ...
+
+
+# adapter instance, label_map -- the pair runner.py needs for each entry.
+AdapterEntry = tuple[Adapter, dict[str, str]]

maskflow_bench-0.1.0/src/maskflow_bench/adapters/maskflow_adapter.py

@@ -0,0 +1,27 @@
+"""Adapter 1: MaskFlow itself, both bundled packs.
+
+Importing maskflow_pack_intl and maskflow_pack_india registers every
+recognizer against maskflow-core's global registry as a side effect (same
+mechanism bench/indiapii/report.py already relies on for its own,
+unrelated L1-L3 accuracy report) -- there is no separate "activate" step.
+PIIType values already equal the corpus's own label vocabulary (the corpus
+was generated from this pack's own types), so no label translation table
+is needed here; labels.py's identity_map() is used by the caller.
+"""
+
+from __future__ import annotations
+
+import maskflow_pack_india  # noqa: F401  (import-time registration side effect)
+import maskflow_pack_intl  # noqa: F401  (import-time registration side effect)
+from maskflow_core.detection import detect
+
+
+class MaskflowAdapter:
+    name = "maskflow"
+
+    def available(self) -> tuple[bool, str]:
+        return True, ""
+
+    def detect(self, text: str) -> list[tuple[int, int, str]]:
+        spans = detect(text)
+        return [(s.start, s.end, str(s.entity_type)) for s in spans]

maskflow_bench-0.1.0/src/maskflow_bench/corpus.py

@@ -0,0 +1,73 @@
+"""Loads bench/indiapii/data/*.jsonl into Document objects.
+
+Each corpus line has an `entities` list mixing `value_class: "positive"`
+(real gold spans) and `value_class: "hard_negative"` (decoy spans, shaped
+like real PII but never gold -- see generator/hard_negatives.py). Document
+keeps these separate: `gold` is the only thing matching.py scores recall
+against; `decoys` exist purely so a prediction landing on one still counts
+as a false positive for whatever type it claimed (see matching.py).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Document:
+    id: str
+    text: str
+    domain: str
+    lang: str
+    gold: tuple[tuple[int, int, str], ...]
+    decoys: tuple[tuple[int, int, str], ...]
+
+    @property
+    def size_bytes(self) -> int:
+        return len(self.text.encode("utf-8"))
+
+
+def _parse_line(line: str) -> Document:
+    row = json.loads(line)
+    gold = []
+    decoys = []
+    for e in row["entities"]:
+        span = (e["start"], e["end"], e["label"])
+        if e["value_class"] == "positive":
+            gold.append(span)
+        else:
+            decoys.append(span)
+    return Document(
+        id=row["id"],
+        text=row["text"],
+        domain=row["domain"],
+        lang=row["lang"],
+        gold=tuple(gold),
+        decoys=tuple(decoys),
+    )
+
+
+def iter_corpus(path: Path) -> Iterator[Document]:
+    with path.open(encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                yield _parse_line(line)
+
+
+def load_corpus(path: Path, limit: int | None = None) -> list[Document]:
+    """Loads `path` in file order (the corpus's own doc-id order), which is
+    already deterministic per-seed at generation time -- `limit` therefore
+    always selects the same first-N documents run to run, which is what
+    the CI regression subset (see harness/tests/test_ci_regression.py)
+    relies on for a stable baseline.
+    """
+    docs = []
+    for i, doc in enumerate(iter_corpus(path)):
+        if limit is not None and i >= limit:
+            break
+        docs.append(doc)
+    return docs

maskflow_bench-0.1.0/src/maskflow_bench/labels.py

@@ -0,0 +1,97 @@
+"""Canonical entity taxonomy (the corpus's own positive labels) and each
+adapter's raw-label -> canonical-label map.
+
+The canonical taxonomy is never a new ontology invented here -- it's
+whatever `value_class: "positive"` labels actually appear in the loaded
+corpus, so a future corpus version's label set is picked up automatically
+rather than drifting out of sync with a hardcoded list here.
+
+Every LABEL_MAP below is intentionally partial: a raw label an adapter
+emits with no entry here is dropped before scoring rather than counted as
+a false positive against some unrelated canonical type -- see matching.py
+and the harness plan's "Label mapping" section for why (mirrors the
+`target_types`-restricted convention already used by
+bench/indiapii/metrics.py).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from .corpus import Document
+
+# One-line definitions handed to the LLM adapter's prompt -- the taxonomy
+# itself always comes from canonical_labels(docs) below; this dict is just
+# human-readable glosses for whichever of those labels happen to have one.
+LABEL_DESCRIPTIONS: dict[str, str] = {
+    "AADHAAR": "12-digit Indian national ID (Aadhaar), may be spaced/hyphenated in groups of 4",
+    "AADHAAR_MASKED": "Aadhaar number with first 8 digits masked, only last 4 digits visible",
+    "ABHA_ADDRESS": "Ayushman Bharat Health Account address, email-shaped (e.g. name@abdm)",
+    "ABHA_NUMBER": "14-digit Ayushman Bharat Health Account number",
+    "BANK_ACCOUNT_IN": "Indian bank account number, 9-18 digits",
+    "DRIVING_LICENCE": "Indian driving licence number (state code + digits)",
+    "GSTIN": "15-character Goods and Services Tax Identification Number",
+    "IFSC": "11-character bank branch code (4 letters + 0 + 6 alphanumeric)",
+    "INDIAN_ADDRESS": "a residential/postal address in India (street, locality, city, state)",
+    "INDIAN_MOBILE": "Indian mobile phone number, 10 digits, optionally with +91 prefix",
+    "INDIAN_PASSPORT": "Indian passport number (1 letter + 7 digits) or its MRZ block",
+    "PAN": "10-character Permanent Account Number (5 letters + 4 digits + 1 letter)",
+    "PERSON_NAME": "a person's full name",
+    "PIN_CODE": "6-digit Indian postal PIN code",
+    "UPI_VPA": "UPI Virtual Payment Address, looks like username@bank-handle",
+    "VEHICLE_REG": "Indian vehicle registration number (state code + district + series + digits)",
+    "VOTER_ID": "Voter ID / EPIC number, 3 letters + 7 digits",
+}
+
+
+def canonical_labels(docs: Iterable[Document]) -> tuple[str, ...]:
+    labels: set[str] = set()
+    for doc in docs:
+        for _start, _end, label in doc.gold:
+            labels.add(label)
+    return tuple(sorted(labels))
+
+
+# maskflow's own PIIType values are literally the corpus's label vocabulary
+# (the corpus was generated from maskflow-pack-india's own types) -- no
+# translation needed, but adapters.base.Adapter still calls through this
+# module for a uniform code path, so this is an identity function.
+def identity_map(raw_labels: Iterable[str]) -> dict[str, str]:
+    return {label: label for label in raw_labels}
+
+
+PRESIDIO_LABEL_MAP: dict[str, str] = {
+    "PERSON": "PERSON_NAME",
+    "PHONE_NUMBER": "INDIAN_MOBILE",
+    "LOCATION": "INDIAN_ADDRESS",
+}
+
+# Same as PRESIDIO_LABEL_MAP, plus the two custom pattern recognizers
+# presidio_custom_adapter.py registers directly under these names.
+PRESIDIO_CUSTOM_LABEL_MAP: dict[str, str] = {
+    **PRESIDIO_LABEL_MAP,
+    "IN_AADHAAR": "AADHAAR",
+    "IN_PAN": "PAN",
+}
+
+# mask-privacy's DLP registry (core/dlp/registry.py) has zero India-specific
+# entity types (confirmed by inspecting the installed package -- 38 raw
+# types, all US/EU/generic: BANK_ACCT_NUM, PHONE_NUM, EMAIL_ADDR, US_SSN,
+# VEHICLE_PLATE, ...). Its Tier-2 NLP tier is Presidio underneath, reusing
+# the same spaCy PERSON/LOCATION labels as PRESIDIO_LABEL_MAP.
+MASK_PRIVACY_LABEL_MAP: dict[str, str] = {
+    "PERSON": "PERSON_NAME",
+    "LOCATION": "INDIAN_ADDRESS",
+    "PHONE_NUM": "INDIAN_MOBILE",
+    "PHONE_NUM_INTL": "INDIAN_MOBILE",
+    "BANK_ACCT_NUM": "BANK_ACCOUNT_IN",
+    "VEHICLE_PLATE": "VEHICLE_REG",
+}
+
+# naive_regex_adapter.py's own made-up label names, mapped to canonical.
+NAIVE_REGEX_LABEL_MAP: dict[str, str] = {
+    "PHONE_SHAPED": "INDIAN_MOBILE",
+    "AADHAAR_SHAPED": "AADHAAR",
+    "PAN_SHAPED": "PAN",
+    "PINCODE_SHAPED": "PIN_CODE",
+}

maskflow_bench-0.1.0/src/maskflow_bench/loader.py

@@ -0,0 +1,105 @@
+"""Lenient JSONL loader for a user's own labelled data (`maskflow bench
+--my-data <path>`, packages/maskflow-cli), as opposed to corpus.py's
+strict parser for the bundled benchmark corpora.
+
+A user checking "is it accurate on my documents?" has no reason to know
+about `domain`/`lang`/`value_class` -- those are benchmark-corpus
+bookkeeping, not concepts a newcomer's own labelled file would naturally
+carry. Only `text` and `entities[].{start,end,label}` are required; every
+other field gets a sensible default. The result is the exact same
+`Document` dataclass corpus.py produces, so every downstream function
+(labels.canonical_labels, matching.evaluate, runner.run_adapter,
+report.write_report) is used unmodified.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from .corpus import Document
+
+_REQUIRED_ENTITY_FIELDS = ("start", "end", "label")
+
+
+@dataclass
+class LoaderError(Exception):
+    """Raised on one malformed line, with the 1-based line number so the
+    CLI can report exactly where a user's file needs fixing -- never a raw
+    KeyError/TypeError/JSONDecodeError with no context."""
+
+    line_no: int
+    message: str
+
+    def __str__(self) -> str:
+        return f"line {self.line_no}: {self.message}"
+
+
+def _parse_line(line_no: int, line: str) -> Document:
+    try:
+        row = json.loads(line)
+    except json.JSONDecodeError as e:
+        raise LoaderError(line_no, f"invalid JSON ({e})") from e
+    if not isinstance(row, dict):
+        raise LoaderError(line_no, "each line must be a JSON object")
+
+    if "text" not in row:
+        raise LoaderError(line_no, "missing required field 'text'")
+    text = row["text"]
+    if not isinstance(text, str):
+        raise LoaderError(line_no, "'text' must be a string")
+
+    if "entities" not in row:
+        raise LoaderError(line_no, "missing required field 'entities'")
+    raw_entities = row["entities"]
+    if not isinstance(raw_entities, list):
+        raise LoaderError(line_no, "'entities' must be a list")
+
+    gold = []
+    decoys = []
+    for i, entity in enumerate(raw_entities):
+        if not isinstance(entity, dict):
+            raise LoaderError(line_no, f"entities[{i}] must be an object")
+        missing = [f for f in _REQUIRED_ENTITY_FIELDS if f not in entity]
+        if missing:
+            raise LoaderError(line_no, f"entities[{i}] missing field(s): {', '.join(missing)}")
+        start, end, label = entity["start"], entity["end"], entity["label"]
+        if not isinstance(start, int) or not isinstance(end, int):
+            raise LoaderError(line_no, f"entities[{i}]: 'start'/'end' must be integers")
+        if not isinstance(label, str):
+            raise LoaderError(line_no, f"entities[{i}]: 'label' must be a string")
+        span = (start, end, label)
+        if entity.get("value_class", "positive") == "positive":
+            gold.append(span)
+        else:
+            decoys.append(span)
+
+    return Document(
+        id=str(row.get("id", f"line-{line_no}")),
+        text=text,
+        domain=str(row.get("domain", "user_data")),
+        lang=str(row.get("lang", "en")),
+        gold=tuple(gold),
+        decoys=tuple(decoys),
+    )
+
+
+def iter_my_data(path: Path) -> Iterator[Document]:
+    with path.open(encoding="utf-8") as f:
+        for line_no, line in enumerate(f, start=1):
+            line = line.strip()
+            if line:
+                yield _parse_line(line_no, line)
+
+
+def load_my_data(path: Path, limit: int | None = None) -> list[Document]:
+    """Loads `path` in file order. `limit` scores only the first N lines --
+    a quick smoke run on a large file, same convention as corpus.py's."""
+    docs = []
+    for i, doc in enumerate(iter_my_data(path)):
+        if limit is not None and i >= limit:
+            break
+        docs.append(doc)
+    return docs

maskflow_bench-0.1.0/src/maskflow_bench/matching.py

@@ -0,0 +1,130 @@
+"""Strict-span and partial-overlap precision/recall/F1 scoring.
+
+Gold spans for a Document are `Document.gold` only (`value_class ==
+"positive"`) -- `Document.decoys` (hard-negative spans) are never gold, but
+a prediction landing on one is still scored: after label-mapping (see
+labels.py), any predicted span left unmatched to a gold span -- whether it
+overlaps background text or a decoy -- counts as a false positive for its
+mapped canonical type. That's what makes the corpus's hard negatives (e.g.
+a naive-regex hit on PAN_SHAPED_INVOICE_NO) actually cost precision.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from .corpus import Document
+
+RawSpan = tuple[int, int, str]  # (start, end, raw_label) as an adapter emits it
+GoldSpan = tuple[int, int, str]  # (start, end, canonical_label)
+
+
+class MatchMode(str, Enum):
+    STRICT = "strict"
+    PARTIAL = "partial"
+
+
+@dataclass
+class PRFResult:
+    entity_type: str
+    tp: int = 0
+    fp: int = 0
+    fn: int = 0
+
+    @property
+    def precision(self) -> float | None:
+        denom = self.tp + self.fp
+        return self.tp / denom if denom else None
+
+    @property
+    def recall(self) -> float | None:
+        denom = self.tp + self.fn
+        return self.tp / denom if denom else None
+
+    @property
+    def f1(self) -> float | None:
+        """None only when there was nothing to score on one side (precision
+        or recall has no denominator -- no predictions AND/OR no gold). A
+        detector that fired on this type but matched no gold span (tp=0,
+        fp>0, fn>0) has a *measured* F1 of 0.0, not "not applicable" -- the
+        report renders that as `0.0%`, distinct from `—`."""
+        p, r = self.precision, self.recall
+        if p is None or r is None:
+            return None
+        if p + r == 0:
+            return 0.0
+        return 2 * p * r / (p + r)
+
+
+def _mapped_predictions(
+    predictions: list[RawSpan], label_map: dict[str, str]
+) -> list[tuple[int, int, str]]:
+    """Drops any prediction whose raw label has no canonical mapping --
+    never scored as a false positive against an unrelated type."""
+    out = []
+    for start, end, raw_label in predictions:
+        canonical = label_map.get(raw_label)
+        if canonical is not None:
+            out.append((start, end, canonical))
+    return out
+
+
+def _score_document(
+    gold: tuple[GoldSpan, ...],
+    predictions: list[tuple[int, int, str]],
+    mode: MatchMode,
+    results: dict[str, PRFResult],
+) -> None:
+    unclaimed_pred = list(range(len(predictions)))
+
+    def overlaps(a: tuple[int, int, str], b: tuple[int, int, str]) -> bool:
+        return max(a[0], b[0]) < min(a[1], b[1])
+
+    for g_start, g_end, g_label in gold:
+        gold_span = (g_start, g_end, g_label)
+        match_idx = None
+        for idx in unclaimed_pred:
+            p_start, p_end, p_label = predictions[idx]
+            if p_label != g_label:
+                continue
+            if mode is MatchMode.STRICT:
+                hit = (p_start, p_end) == (g_start, g_end)
+            else:
+                hit = overlaps((p_start, p_end, p_label), gold_span)
+            if hit:
+                match_idx = idx
+                break
+        if match_idx is not None:
+            results[g_label].tp += 1
+            unclaimed_pred.remove(match_idx)
+        else:
+            results[g_label].fn += 1
+
+    for idx in unclaimed_pred:
+        _p_start, _p_end, p_label = predictions[idx]
+        if p_label in results:
+            results[p_label].fp += 1
+
+
+def evaluate(
+    docs: list[Document],
+    predictions_by_doc: list[list[RawSpan]],
+    label_map: dict[str, str],
+    canonical_labels: tuple[str, ...],
+    mode: MatchMode,
+) -> dict[str, PRFResult]:
+    """Scores each doc's already-computed raw predictions (see runner.py,
+    which owns calling detect() so it can time/guard each call once and
+    reuse the same predictions for both matching modes) against
+    `doc.gold`, after mapping raw labels through `label_map`. Restricted to
+    `canonical_labels` so an adapter's detections of types outside this
+    corpus's taxonomy never enter the tally at all.
+    """
+    results: dict[str, PRFResult] = {t: PRFResult(entity_type=t) for t in canonical_labels}
+    for doc, raw_predictions in zip(docs, predictions_by_doc, strict=True):
+        predictions = _mapped_predictions(raw_predictions, label_map)
+        predictions = [p for p in predictions if p[2] in results]
+        gold = tuple(g for g in doc.gold if g[2] in results)
+        _score_document(gold, predictions, mode, results)
+    return results

maskflow_bench-0.1.0/src/maskflow_bench/offsets.py

@@ -0,0 +1,27 @@
+"""Recovers character offsets for adapters that only return matched
+substrings, not (start, end) positions -- mask-privacy's Tier-2 NLP
+entities and the LLM adapter both need this (see their modules' docstrings
+for why offsets aren't available directly from those APIs).
+"""
+
+from __future__ import annotations
+
+
+def locate_span(text: str, value: str, used_starts: set[int]) -> tuple[int, int] | None:
+    """Finds the first occurrence of `value` in `text` whose start index
+    isn't already in `used_starts`, claims it, and returns (start, end).
+    Returns None if `value` doesn't occur (or every occurrence is already
+    claimed) -- the caller drops that finding and counts it as unlocatable
+    rather than guessing.
+    """
+    if not value:
+        return None
+    cursor = 0
+    while True:
+        idx = text.find(value, cursor)
+        if idx == -1:
+            return None
+        if idx not in used_starts:
+            used_starts.add(idx)
+            return idx, idx + len(value)
+        cursor = idx + 1

maskflow_bench-0.1.0/src/maskflow_bench/profiling.py

@@ -0,0 +1,78 @@
+"""Latency (ms/KB) and peak-memory (RSS delta) measurement for one
+adapter's run over a corpus.
+
+Known limitation (documented, not hidden): adapters run in-process,
+sequentially, not subprocess-isolated -- ru_maxrss is a whole-process,
+monotonically-nondecreasing high-water mark, so peak_memory_mb here is an
+*attributable delta* across one adapter's calls, not a hard ceiling that
+adapter alone ever touched. Good enough for a first cut / relative
+comparison, not a substitute for real per-process isolation.
+
+A second limitation, specifically because `maskflow bench --my-data`
+(packages/maskflow-cli) makes this module reachable on Windows, unlike the
+dev-only multi-adapter harness that only ever ran on Linux/macOS CI:
+`resource` is POSIX-only. On a platform without it, peak_memory_mb is
+always reported as 0.0 rather than crashing -- timing (latency_ms_per_kb,
+median/p95 doc ms) is unaffected either way.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+
+try:
+    import resource
+except ImportError:  # Windows
+    resource = None  # type: ignore[assignment]
+
+
+def _current_maxrss_mb() -> float:
+    if resource is None:
+        return 0.0
+    raw = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    # macOS reports ru_maxrss in bytes; Linux reports it in KB.
+    return raw / (1024 * 1024) if sys.platform == "darwin" else raw / 1024
+
+
+@dataclass
+class ProfileResult:
+    latency_ms_per_kb: float
+    median_doc_ms: float
+    p95_doc_ms: float
+    peak_memory_mb: float
+    errors: int = 0
+
+
+@dataclass
+class Profiler:
+    _doc_ms: list[float] = field(default_factory=list)
+    _total_bytes: int = 0
+    _start_rss_mb: float = 0.0
+    _started: bool = False
+    errors: int = 0
+
+    def start(self) -> None:
+        self._start_rss_mb = _current_maxrss_mb()
+        self._started = True
+
+    def record(self, elapsed_seconds: float, size_bytes: int) -> None:
+        self._doc_ms.append(elapsed_seconds * 1000)
+        self._total_bytes += size_bytes
+
+    def finish(self) -> ProfileResult:
+        end_rss_mb = _current_maxrss_mb()
+        total_ms = sum(self._doc_ms)
+        total_kb = (self._total_bytes / 1024) or 1.0
+        sorted_ms = sorted(self._doc_ms)
+        median = sorted_ms[len(sorted_ms) // 2] if sorted_ms else 0.0
+        p95_idx = min(len(sorted_ms) - 1, int(len(sorted_ms) * 0.95)) if sorted_ms else 0
+        p95 = sorted_ms[p95_idx] if sorted_ms else 0.0
+        start_rss = self._start_rss_mb if self._started else end_rss_mb
+        return ProfileResult(
+            latency_ms_per_kb=total_ms / total_kb,
+            median_doc_ms=median,
+            p95_doc_ms=p95,
+            peak_memory_mb=max(0.0, end_rss_mb - start_rss),
+            errors=self.errors,
+        )

maskflow_bench-0.1.0/src/maskflow_bench/report.py

@@ -0,0 +1,149 @@
+"""Writes results.json (full structured data) and results.md (generated
+tables) from a run_all() result. Module path is
+`maskflow_bench.report`, distinct from the unrelated, pre-existing
+`bench.indiapii.report` (the pack-india L1-L3 dev accuracy report) -- see
+harness/__init__.py's docstring.
+"""
+
+from __future__ import annotations
+
+import json
+
+# datetime.UTC is 3.11+ only; this package's floor is requires-python
+# >=3.10 (it's now reachable from maskflow-cli, unlike when this lived in
+# the dev-only bench/indiapii/harness/ and never ran on 3.10 in CI).
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .matching import PRFResult
+from .runner import AdapterRunResult
+
+
+def _prf_to_dict(r: PRFResult) -> dict[str, float | int | None]:
+    return {
+        "precision": r.precision,
+        "recall": r.recall,
+        "f1": r.f1,
+        "tp": r.tp,
+        "fp": r.fp,
+        "fn": r.fn,
+    }
+
+
+def to_json_dict(
+    corpus_name: str,
+    num_docs: int,
+    canonical_labels: tuple[str, ...],
+    results: dict[str, AdapterRunResult],
+) -> dict:
+    adapters: dict[str, dict] = {}
+    for name, r in results.items():
+        entry: dict = {"available": r.available, "skipped_reason": r.skipped_reason}
+        if r.available:
+            entry["strict"] = {label: _prf_to_dict(v) for label, v in r.strict.items()}
+            entry["partial"] = {label: _prf_to_dict(v) for label, v in r.partial.items()}
+            if r.profile is not None:
+                entry["latency_ms_per_kb"] = r.profile.latency_ms_per_kb
+                entry["median_doc_ms"] = r.profile.median_doc_ms
+                entry["p95_doc_ms"] = r.profile.p95_doc_ms
+                entry["peak_memory_mb"] = r.profile.peak_memory_mb
+                entry["doc_errors"] = r.profile.errors
+        adapters[name] = entry
+    return {
+        "corpus": corpus_name,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "num_docs": num_docs,
+        "canonical_labels": list(canonical_labels),
+        "adapters": adapters,
+    }
+
+
+def write_json(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def _f1_cell(results: dict[str, AdapterRunResult], adapter_name: str, mode: str, label: str) -> str:
+    r = results[adapter_name]
+    if not r.available:
+        return "skipped"
+    prf = getattr(r, mode).get(label)
+    if prf is None or prf.f1 is None:
+        return "—"
+    return f"{prf.f1:.1%}"
+
+
+def _pivot_table(
+    results: dict[str, AdapterRunResult],
+    adapter_names: list[str],
+    canonical_labels: tuple[str, ...],
+    mode: str,
+    title: str,
+) -> str:
+    header = "| entity_type | " + " | ".join(adapter_names) + " |"
+    sep = "|---" * (len(adapter_names) + 1) + "|"
+    lines = [f"### {title}", "", header, sep]
+    for label in canonical_labels:
+        row = [_f1_cell(results, name, mode, label) for name in adapter_names]
+        lines.append(f"| {label} | " + " | ".join(row) + " |")
+    return "\n".join(lines)
+
+
+def _latency_table(results: dict[str, AdapterRunResult], adapter_names: list[str]) -> str:
+    header = "| adapter | ms/KB | median ms/doc | p95 ms/doc | peak memory (MB) | doc errors |"
+    sep = "|---|---|---|---|---|---|"
+    lines = ["### Latency & memory", "", header, sep]
+    for name in adapter_names:
+        r = results[name]
+        if not r.available:
+            lines.append(f"| {name} | skipped ({r.skipped_reason}) | | | | |")
+            continue
+        p = r.profile
+        assert p is not None
+        lines.append(
+            f"| {name} | {p.latency_ms_per_kb:.3f} | {p.median_doc_ms:.3f} | "
+            f"{p.p95_doc_ms:.3f} | {p.peak_memory_mb:.1f} | {p.errors} |"
+        )
+    return "\n".join(lines)
+
+
+def to_markdown(
+    corpus_name: str,
+    num_docs: int,
+    canonical_labels: tuple[str, ...],
+    results: dict[str, AdapterRunResult],
+) -> str:
+    adapter_names = list(results.keys())
+    parts = [
+        f"# {corpus_name} benchmark results",
+        "",
+        f"{num_docs} documents, {len(canonical_labels)} canonical entity types. "
+        'F1 shown per entity per adapter. "0.0%" is a *measured* zero -- the adapter '
+        'made predictions for this type but none matched a gold span. "—" means F1 '
+        "is undefined: the adapter made no prediction for this type at all (its "
+        "recognizer / label map doesn't cover it), or the corpus has no gold spans "
+        "for it. \"skipped\" means the adapter's dependency/API key wasn't available "
+        "in this environment.",
+        "",
+        _pivot_table(results, adapter_names, canonical_labels, "strict", "Strict-span F1"),
+        "",
+        _pivot_table(results, adapter_names, canonical_labels, "partial", "Partial-overlap F1"),
+        "",
+        _latency_table(results, adapter_names),
+        "",
+    ]
+    return "\n".join(parts)
+
+
+def write_report(
+    out_dir: Path,
+    corpus_name: str,
+    num_docs: int,
+    canonical_labels: tuple[str, ...],
+    results: dict[str, AdapterRunResult],
+) -> None:
+    data = to_json_dict(corpus_name, num_docs, canonical_labels, results)
+    write_json(out_dir / "results.json", data)
+    md = to_markdown(corpus_name, num_docs, canonical_labels, results)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "results.md").write_text(md, encoding="utf-8")

maskflow_bench-0.1.0/src/maskflow_bench/runner.py

@@ -0,0 +1,77 @@
+"""Orchestrates: for each adapter, check availability, run detect() once
+per document (timed, error-guarded), then score the same cached
+predictions under both matching modes -- no adapter's detect() is ever
+called twice for the same document.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+from .adapters import AdapterEntry
+from .corpus import Document
+from .matching import MatchMode, PRFResult, evaluate
+from .profiling import Profiler, ProfileResult
+
+
+@dataclass
+class AdapterRunResult:
+    name: str
+    available: bool
+    skipped_reason: str = ""
+    strict: dict[str, PRFResult] = field(default_factory=dict)
+    partial: dict[str, PRFResult] = field(default_factory=dict)
+    profile: ProfileResult | None = None
+
+
+def run_adapter(
+    entry: AdapterEntry, docs: list[Document], canonical_labels: tuple[str, ...]
+) -> AdapterRunResult:
+    adapter, label_map = entry
+    ok, reason = adapter.available()
+    if not ok:
+        return AdapterRunResult(name=adapter.name, available=False, skipped_reason=reason)
+
+    # One untimed warm-up call so a one-time cost every adapter pays somewhere
+    # (spaCy model load, first-call JIT/regex compilation, ...) lands the same
+    # way for all of them -- some adapters already pay it inside available()
+    # (e.g. presidio's engine construction), others pay it lazily on their
+    # first detect() call (maskflow's NER pass) -- without this, whichever
+    # adapter happens to defer its warm-up into the timed loop looks
+    # artificially slower for reasons that have nothing to do with steady-
+    # state per-document latency, which is what ms/KB is meant to measure.
+    try:
+        adapter.detect("warm-up, no PII here.")
+    except Exception:  # noqa: BLE001 -- warm-up failures don't affect timing/scoring
+        pass
+
+    profiler = Profiler()
+    profiler.start()
+    predictions_by_doc: list[list[tuple[int, int, str]]] = []
+    for doc in docs:
+        t0 = time.perf_counter()
+        try:
+            preds = adapter.detect(doc.text)
+        except Exception:  # noqa: BLE001 -- one bad doc must not sink the whole adapter run
+            preds = []
+            profiler.errors += 1
+        profiler.record(time.perf_counter() - t0, doc.size_bytes)
+        predictions_by_doc.append(preds)
+
+    strict = evaluate(docs, predictions_by_doc, label_map, canonical_labels, MatchMode.STRICT)
+    partial = evaluate(docs, predictions_by_doc, label_map, canonical_labels, MatchMode.PARTIAL)
+
+    return AdapterRunResult(
+        name=adapter.name,
+        available=True,
+        strict=strict,
+        partial=partial,
+        profile=profiler.finish(),
+    )
+
+
+def run_all(
+    entries: list[AdapterEntry], docs: list[Document], canonical_labels: tuple[str, ...]
+) -> dict[str, AdapterRunResult]:
+    return {entry[0].name: run_adapter(entry, docs, canonical_labels) for entry in entries}

maskflow_bench-0.1.0/src/maskflow_bench/scorer.py

@@ -0,0 +1,29 @@
+"""Scores MaskFlow itself against a user's own labelled data
+(`maskflow bench --my-data <path>`). Thin composition of loader.py +
+labels.py + runner.py + adapters.maskflow_adapter -- the same pieces
+bench/indiapii/harness/__main__.py assembles ad hoc for the bundled
+corpora, wired together here as one reusable call so the CLI (and anyone
+else) doesn't have to re-derive it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .adapters.maskflow_adapter import MaskflowAdapter
+from .labels import canonical_labels, identity_map
+from .loader import load_my_data
+from .runner import AdapterRunResult, run_adapter
+
+
+def score_my_data(
+    path: Path, limit: int | None = None
+) -> tuple[int, tuple[str, ...], AdapterRunResult]:
+    """Returns (num_docs, canonical_labels, result). Raises LoaderError
+    (loader.py) on a malformed line, FileNotFoundError if `path` doesn't
+    exist."""
+    docs = load_my_data(path, limit=limit)
+    labels = canonical_labels(docs)
+    entry = (MaskflowAdapter(), identity_map(labels))
+    result = run_adapter(entry, docs, labels)
+    return len(docs), labels, result

maskflow_bench-0.1.0/tests/test_corpus.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from maskflow_bench.corpus import load_corpus
+
+
+def _write_jsonl(path: Path, rows: list[dict]) -> None:
+    path.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+
+def test_load_corpus_splits_positive_and_hard_negative(tmp_path: Path) -> None:
+    row = {
+        "id": "doc-1",
+        "text": "PAN: ABCDE1234F, invoice INV-0001",
+        "entities": [
+            {"start": 5, "end": 15, "label": "PAN", "value_class": "positive"},
+            {
+                "start": 25,
+                "end": 33,
+                "label": "PAN_SHAPED_INVOICE_NO",
+                "value_class": "hard_negative",
+            },
+        ],
+        "domain": "kyc_form",
+        "lang": "en",
+    }
+    path = tmp_path / "corpus.jsonl"
+    _write_jsonl(path, [row])
+
+    docs = load_corpus(path)
+    assert len(docs) == 1
+    doc = docs[0]
+    assert doc.gold == ((5, 15, "PAN"),)
+    assert doc.decoys == ((25, 33, "PAN_SHAPED_INVOICE_NO"),)
+    assert doc.size_bytes == len(doc.text.encode("utf-8"))
+
+
+def test_load_corpus_limit_takes_first_n_in_file_order(tmp_path: Path) -> None:
+    rows = [
+        {"id": f"doc-{i}", "text": "t", "entities": [], "domain": "d", "lang": "en"}
+        for i in range(5)
+    ]
+    path = tmp_path / "corpus.jsonl"
+    _write_jsonl(path, rows)
+
+    docs = load_corpus(path, limit=2)
+    assert [d.id for d in docs] == ["doc-0", "doc-1"]

maskflow_bench-0.1.0/tests/test_labels.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from maskflow_bench.corpus import Document
+from maskflow_bench.labels import canonical_labels, identity_map
+
+
+def test_canonical_labels_derived_from_gold_only() -> None:
+    docs = [
+        Document(
+            id="d1",
+            text="x",
+            domain="t",
+            lang="en",
+            gold=((0, 1, "PAN"), (2, 3, "AADHAAR")),
+            decoys=((4, 5, "PAN_SHAPED_INVOICE_NO"),),
+        )
+    ]
+    labels = canonical_labels(docs)
+    assert labels == ("AADHAAR", "PAN")
+    assert "PAN_SHAPED_INVOICE_NO" not in labels
+
+
+def test_identity_map_is_reflexive() -> None:
+    m = identity_map(("AADHAAR", "PAN"))
+    assert m == {"AADHAAR": "AADHAAR", "PAN": "PAN"}

maskflow_bench-0.1.0/tests/test_loader.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from maskflow_bench.loader import LoaderError, load_my_data
+
+
+def _write(path: Path, lines: list[dict]) -> None:
+    path.write_text("\n".join(json.dumps(r) for r in lines) + "\n", encoding="utf-8")
+
+
+def test_minimal_line_needs_only_text_and_entities(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    # PAN below is structurally shaped but not a real allotted number --
+    # synthetic fixture data only.
+    _write(
+        path,
+        [{"text": "PAN: ABCDE1234F", "entities": [{"start": 5, "end": 15, "label": "PAN"}]}],
+    )
+    docs = load_my_data(path)
+    assert len(docs) == 1
+    doc = docs[0]
+    assert doc.gold == ((5, 15, "PAN"),)
+    assert doc.decoys == ()
+    # Defaults applied when the user's file doesn't supply them.
+    assert doc.domain == "user_data"
+    assert doc.lang == "en"
+    assert doc.id == "line-1"
+
+
+def test_explicit_id_domain_lang_are_preserved(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    _write(
+        path,
+        [
+            {
+                "id": "ticket-42",
+                "text": "x",
+                "entities": [],
+                "domain": "support_ticket",
+                "lang": "en-hi",
+            }
+        ],
+    )
+    doc = load_my_data(path)[0]
+    assert doc.id == "ticket-42"
+    assert doc.domain == "support_ticket"
+    assert doc.lang == "en-hi"
+
+
+def test_hard_negative_value_class_goes_to_decoys(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    _write(
+        path,
+        [
+            {
+                "text": "invoice INV-0001",
+                "entities": [
+                    {"start": 8, "end": 16, "label": "PAN_SHAPED", "value_class": "hard_negative"}
+                ],
+            }
+        ],
+    )
+    doc = load_my_data(path)[0]
+    assert doc.gold == ()
+    assert doc.decoys == ((8, 16, "PAN_SHAPED"),)
+
+
+def test_limit_takes_first_n_in_file_order(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    _write(path, [{"text": f"t{i}", "entities": []} for i in range(5)])
+    docs = load_my_data(path, limit=2)
+    assert [d.text for d in docs] == ["t0", "t1"]
+
+
+@pytest.mark.parametrize(
+    "row",
+    [
+        {"entities": []},  # missing text
+        {"text": "x"},  # missing entities
+        {"text": "x", "entities": [{"start": 0, "end": 1}]},  # entity missing label
+        {"text": "x", "entities": [{"start": 0, "label": "PAN"}]},  # entity missing end
+        {"text": "x", "entities": "not-a-list"},
+        {"text": 123, "entities": []},
+        {"text": "x", "entities": [{"start": "0", "end": 1, "label": "PAN"}]},
+    ],
+)
+def test_malformed_line_raises_loader_error_with_line_number(tmp_path: Path, row: dict) -> None:
+    path = tmp_path / "my.jsonl"
+    # Line 1 is valid, so the error must point at line 2, not line 1.
+    _write(path, [{"text": "ok", "entities": []}])
+    with path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(row) + "\n")
+
+    with pytest.raises(LoaderError) as exc_info:
+        load_my_data(path)
+    assert exc_info.value.line_no == 2
+
+
+def test_invalid_json_raises_loader_error(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    path.write_text("{not json\n", encoding="utf-8")
+    with pytest.raises(LoaderError) as exc_info:
+        load_my_data(path)
+    assert exc_info.value.line_no == 1

maskflow_bench-0.1.0/tests/test_matching.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from maskflow_bench.corpus import Document
+from maskflow_bench.matching import MatchMode, evaluate
+
+
+def _doc(gold: tuple, decoys: tuple = ()) -> Document:
+    return Document(id="d1", text="x" * 100, domain="test", lang="en", gold=gold, decoys=decoys)
+
+
+def test_strict_exact_match_is_tp() -> None:
+    docs = [_doc(gold=((0, 5, "PAN"),))]
+    preds = [[(0, 5, "PAN")]]
+    r = evaluate(docs, preds, {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].tp == 1
+    assert r["PAN"].fp == 0
+    assert r["PAN"].fn == 0
+
+
+def test_strict_offset_mismatch_is_fn_and_fp() -> None:
+    docs = [_doc(gold=((0, 5, "PAN"),))]
+    preds = [[(0, 6, "PAN")]]  # one char off
+    r = evaluate(docs, preds, {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].tp == 0
+    assert r["PAN"].fn == 1
+    assert r["PAN"].fp == 1
+    # fired + had gold + matched nothing -> a *measured* 0.0, not None.
+    assert r["PAN"].precision == 0.0
+    assert r["PAN"].recall == 0.0
+    assert r["PAN"].f1 == 0.0
+
+
+def test_partial_overlap_counts_as_tp() -> None:
+    docs = [_doc(gold=((10, 20, "AADHAAR"),))]
+    preds = [[(12, 18, "AADHAAR")]]  # contained, not exact
+    r = evaluate(docs, preds, {"AADHAAR": "AADHAAR"}, ("AADHAAR",), MatchMode.PARTIAL)
+    assert r["AADHAAR"].tp == 1
+    assert r["AADHAAR"].fp == 0
+    assert r["AADHAAR"].fn == 0
+
+
+def test_no_overlap_is_fn_and_fp() -> None:
+    docs = [_doc(gold=((10, 20, "AADHAAR"),))]
+    preds = [[(30, 40, "AADHAAR")]]
+    r = evaluate(docs, preds, {"AADHAAR": "AADHAAR"}, ("AADHAAR",), MatchMode.PARTIAL)
+    assert r["AADHAAR"].tp == 0
+    assert r["AADHAAR"].fn == 1
+    assert r["AADHAAR"].fp == 1
+
+
+def test_hard_negative_hit_counts_as_false_positive() -> None:
+    # A prediction landing on a hard-negative decoy span (never gold) is
+    # still a false positive for whatever canonical type it claimed.
+    docs = [_doc(gold=(), decoys=((0, 10, "PAN_SHAPED_INVOICE_NO"),))]
+    preds = [[(0, 10, "PAN")]]
+    r = evaluate(docs, preds, {"PAN": "PAN"}, ("PAN",), MatchMode.PARTIAL)
+    assert r["PAN"].tp == 0
+    assert r["PAN"].fp == 1
+    assert r["PAN"].fn == 0
+
+
+def test_unmapped_raw_label_is_dropped_not_scored() -> None:
+    # An adapter's raw label with no entry in label_map (e.g. Presidio's
+    # ORGANIZATION) must never count as a false positive against an
+    # unrelated canonical type.
+    docs = [_doc(gold=((0, 5, "PAN"),))]
+    preds = [[(50, 60, "ORGANIZATION")]]
+    r = evaluate(docs, preds, {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].tp == 0
+    assert r["PAN"].fp == 0
+    assert r["PAN"].fn == 1
+
+
+def test_precision_recall_f1_none_when_no_denominator() -> None:
+    docs = [_doc(gold=())]
+    preds = [[]]
+    r = evaluate(docs, preds, {}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].precision is None
+    assert r["PAN"].recall is None
+    assert r["PAN"].f1 is None
+
+
+def test_f1_is_zero_not_none_when_gold_exists_but_nothing_matched() -> None:
+    # Gold present, no predictions at all: recall has a denominator (0.0),
+    # precision does not -> still "nothing to score" on the precision side.
+    docs = [_doc(gold=((0, 5, "PAN"),))]
+    r = evaluate(docs, [[]], {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].recall == 0.0
+    assert r["PAN"].precision is None
+    assert r["PAN"].f1 is None  # can't compute F1 with no precision
+
+    # But once the detector fires (fp>0) it has both sides -> measured 0.0.
+    docs2 = [_doc(gold=((0, 5, "PAN"),))]
+    r2 = evaluate(docs2, [[(50, 60, "PAN")]], {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r2["PAN"].f1 == 0.0
+
+
+def test_multiple_gold_spans_claim_distinct_predictions() -> None:
+    # Two gold spans of the same type must each match a different
+    # prediction, not both greedily match the first candidate.
+    docs = [_doc(gold=((0, 5, "PAN"), (10, 15, "PAN")))]
+    preds = [[(0, 5, "PAN"), (10, 15, "PAN")]]
+    r = evaluate(docs, preds, {"PAN": "PAN"}, ("PAN",), MatchMode.STRICT)
+    assert r["PAN"].tp == 2
+    assert r["PAN"].fp == 0
+    assert r["PAN"].fn == 0

maskflow_bench-0.1.0/tests/test_scorer.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import maskflow_pack_india  # noqa: F401 -- registers AADHAAR/PAN/etc. recognizers
+from maskflow_bench.scorer import score_my_data
+
+
+def test_score_my_data_runs_maskflow_end_to_end(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    # Structurally valid (checksum-passing) but fabricated for this test --
+    # not a real allotted PAN.
+    row = {
+        "text": "Please update my PAN ABCDE1234F on file.",
+        "entities": [{"start": 22, "end": 32, "label": "PAN"}],
+    }
+    path.write_text(json.dumps(row) + "\n", encoding="utf-8")
+
+    num_docs, labels, result = score_my_data(path)
+
+    assert num_docs == 1
+    assert labels == ("PAN",)
+    assert result.name == "maskflow"
+    assert result.available is True
+    assert "PAN" in result.strict
+    assert "PAN" in result.partial
+
+
+def test_score_my_data_respects_limit(tmp_path: Path) -> None:
+    path = tmp_path / "my.jsonl"
+    rows = [{"text": f"doc {i}", "entities": []} for i in range(5)]
+    path.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+    num_docs, _labels, _result = score_my_data(path, limit=2)
+    assert num_docs == 2