maskflow-cli 0.7.0__tar.gz → 0.8.0__tar.gz

{maskflow_cli-0.7.0 → maskflow_cli-0.8.0}/PKG-INFO

@@ -1,16 +1,18 @@
 Metadata-Version: 2.5
 Name: maskflow-cli
-Version: 0.7.0
+Version: 0.8.0
 Summary: Command-line interface for MaskFlow: .maskflowrc config validation and inspection
 License: MIT
 Requires-Python: >=3.10
 Requires-Dist: httpx>=0.27
-Requires-Dist: maskflow-core[yaml]<0.8,>=0.6.0
+Requires-Dist: maskflow-core[yaml]<0.9,>=0.6.0
 Requires-Dist: maskflow-pack-india<0.6,>=0.1.0
 Requires-Dist: maskflow-pack-intl<0.4,>=0.3.0
 Requires-Dist: rich>=13.0
 Requires-Dist: tomli-w>=1.0
 Requires-Dist: typer>=0.12
+Provides-Extra: bench
+Requires-Dist: maskflow-bench<0.2,>=0.1.0; extra == 'bench'
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == 'dev'
 Provides-Extra: evidence

{maskflow_cli-0.7.0 → maskflow_cli-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "maskflow-cli"
-version = "0.7.0"
+version = "0.8.0"
 description = "Command-line interface for MaskFlow: .maskflowrc config validation and inspection"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -14,7 +14,7 @@ dependencies = [
     # -- pack-intl/pack-india now require it, so the floor moves with theirs
     # even though this package doesn't import maskflow_core.recognizer
     # directly.
-    "maskflow-core[yaml]>=0.6.0,<0.8",
+    "maskflow-core[yaml]>=0.6.0,<0.9",
     # maskflow_pack_intl is imported (side-effect only, to populate the
     # PIIType registry) for `maskflow config validate`'s soft entity-name
     # cross-check -- see app.py. pack-intl 0.1.0 itself requires core<0.3,
@@ -71,12 +71,24 @@ s3 = [
 postgres = [
     "psycopg[binary]>=3.1",
 ]
+# `maskflow bench --my-data` (issue #36): the corpus-agnostic scoring core
+# (JSONL loading, label canonicalization, strict/partial-overlap matching,
+# report writers). Same policy as [evidence] above -- opt-in so a bare CLI
+# install (and the standalone PyInstaller binary) doesn't require it;
+# bench_cmd.py imports it lazily and every other command runs without it.
+# maskflow-bench itself depends only on maskflow-core, not on the heavy
+# competitor-comparison adapters (presidio/mask-privacy/anthropic) -- those
+# stay dev-only in bench/, never installed here either way.
+bench = [
+    "maskflow-bench>=0.1.0,<0.2",
+]
 
 [tool.uv.sources]
 maskflow-core = { workspace = true }
 maskflow-pack-intl = { workspace = true }
 maskflow-pack-india = { workspace = true }
 maskflow-evidence = { workspace = true }
+maskflow-bench = { workspace = true }
 
 [tool.uv]
 package = true

{maskflow_cli-0.7.0 → maskflow_cli-0.8.0}/src/maskflow_cli/app.py

@@ -6,6 +6,7 @@ import maskflow_pack_india  # noqa: F401 -- import side effect registers pack-in
 import maskflow_pack_intl  # noqa: F401 -- import side effect registers pack-intl's entity types
 import typer
 
+from .commands.bench_cmd import bench
 from .commands.config_cmd import app as config_app
 from .commands.doctor_cmd import doctor
 from .commands.explain_cmd import explain
@@ -16,6 +17,7 @@ app.add_typer(config_app, name="config")
 app.command("doctor")(doctor)
 app.command("explain")(explain)
 app.command("scan")(scan)
+app.command("bench")(bench)
 
 
 def main() -> None:

maskflow_cli-0.8.0/src/maskflow_cli/bench_render.py

@@ -0,0 +1,88 @@
+"""Rich rendering for `maskflow bench --my-data`. Kept separate from
+bench_cmd.py so the command body stays thin, same split as
+doctor.py/doctor_render.py.
+
+Only type-checked against maskflow-bench (TYPE_CHECKING guard below,
+`from __future__ import annotations` keeps every annotation an
+unevaluated string at runtime) -- this module never constructs a
+PRFResult/AdapterRunResult itself, only duck-types over one handed in by
+bench_cmd.py, so it never needs the optional [bench] extra installed just
+to be imported (bench_cmd.py's own lazy import guards the actual need for
+it)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+if TYPE_CHECKING:
+    from maskflow_bench.matching import PRFResult
+    from maskflow_bench.runner import AdapterRunResult
+
+# Same rationale as doctor_render.py's: a fixed width so output doesn't
+# wrap unpredictably under a non-tty (CI logs, CliRunner in tests).
+CONSOLE_WIDTH = 100
+
+
+def _cell(prf: PRFResult | None, field: str) -> str:
+    """'—' when the metric is undefined (matching.PRFResult.f1's own
+    convention, see PR #97): no predictions and no gold for this type.
+    A real measured zero renders as "0.0%", not "—"."""
+    if prf is None:
+        return "—"
+    value = getattr(prf, field)
+    return "—" if value is None else f"{value:.1%}"
+
+
+def _results_table(canonical_labels: tuple[str, ...], result: AdapterRunResult) -> Table:
+    table = Table(box=box.SIMPLE_HEAD, show_edge=False, pad_edge=False)
+    table.add_column("Entity")
+    table.add_column("Strict P", justify="right")
+    table.add_column("Strict R", justify="right")
+    table.add_column("Strict F1", justify="right")
+    table.add_column("Partial P", justify="right")
+    table.add_column("Partial R", justify="right")
+    table.add_column("Partial F1", justify="right")
+    for label in canonical_labels:
+        strict = result.strict.get(label)
+        partial = result.partial.get(label)
+        table.add_row(
+            label,
+            _cell(strict, "precision"),
+            _cell(strict, "recall"),
+            _cell(strict, "f1"),
+            _cell(partial, "precision"),
+            _cell(partial, "recall"),
+            _cell(partial, "f1"),
+        )
+    return table
+
+
+def render_my_data_result(
+    console: Console,
+    num_docs: int,
+    canonical_labels: tuple[str, ...],
+    result: AdapterRunResult,
+) -> None:
+    console.print("[bold]MaskFlow bench --my-data[/bold]")
+    console.print(f"{num_docs} document(s), {len(canonical_labels)} entity type(s) in your labels.")
+    console.print()
+    if not canonical_labels:
+        console.print(
+            '[yellow]No gold (value_class="positive") entities found -- nothing to '
+            "score. Check your file against docs/bench.md's schema.[/yellow]"
+        )
+        return
+    console.print(_results_table(canonical_labels, result))
+    console.print()
+    console.print(
+        '[dim]"—" means undefined (no predictions and no gold for that type); '
+        '"0.0%" is a measured zero. Strict = exact span match, partial = any overlap.[/dim]'
+    )
+
+
+def make_console() -> Console:
+    return Console(width=CONSOLE_WIDTH)

maskflow_cli-0.8.0/src/maskflow_cli/commands/bench_cmd.py

@@ -0,0 +1,68 @@
+"""`maskflow bench --my-data <path>` -- scores MaskFlow's own detector
+against a user's own labelled documents and prints per-entity
+precision/recall/F1. Not a comparison tool (see docs/bench.md); the
+multi-adapter comparison against Presidio/mask-privacy/etc. that produced
+the published IndiaPII-Bench figures lives in bench/ (repo dev tooling).
+
+`maskflow-bench` is an **optional** dependency (`pip install
+'maskflow-cli[bench]'`) -- imported lazily here, same policy as
+`maskflow_cli.evidence`, so a bare CLI install (and the standalone
+PyInstaller binary) doesn't require it and every other command runs
+without it.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from ..bench_render import make_console, render_my_data_result
+
+_MISSING = (
+    "`maskflow bench` needs the bench extra -- install it with: pip install 'maskflow-cli[bench]'"
+)
+
+_MY_DATA_OPTION = typer.Option(
+    ..., "--my-data", exists=True, dir_okay=False, help="Path to your labelled JSONL file."
+)
+_OUT_OPTION = typer.Option(
+    None, "--out", help="Directory to also write results.json and results.md into."
+)
+_LIMIT_OPTION = typer.Option(
+    None, "--limit", help="Score only the first N documents (useful to smoke-test a large file)."
+)
+
+
+def bench(
+    my_data: Path = _MY_DATA_OPTION,
+    out: Path | None = _OUT_OPTION,
+    limit: int | None = _LIMIT_OPTION,
+) -> None:
+    """Runs MaskFlow's detector over your own labelled JSONL file and
+    reports per-entity precision/recall/F1 under both strict-span and
+    partial-overlap matching. See docs/bench.md for the file schema."""
+    try:
+        from maskflow_bench.loader import LoaderError
+        from maskflow_bench.report import write_report
+        from maskflow_bench.scorer import score_my_data
+    except ModuleNotFoundError as exc:
+        typer.echo(_MISSING, err=True)
+        raise typer.Exit(code=1) from exc
+
+    console = make_console()
+    try:
+        num_docs, labels, result = score_my_data(my_data, limit=limit)
+    except LoaderError as exc:
+        typer.echo(f"{my_data}:{exc}", err=True)
+        raise typer.Exit(code=1) from exc
+
+    if num_docs == 0:
+        typer.echo(f"{my_data}: no documents found -- is the file empty?", err=True)
+        raise typer.Exit(code=1)
+
+    render_my_data_result(console, num_docs, labels, result)
+
+    if out is not None:
+        write_report(out, "my-data", num_docs, labels, {"maskflow": result})
+        console.print(f"\nwrote {out / 'results.json'} and {out / 'results.md'}")

maskflow_cli-0.8.0/tests/test_cli_bench.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from maskflow_bench.scorer import score_my_data
+from maskflow_cli.app import app
+from typer.testing import CliRunner
+
+runner = CliRunner()
+
+
+def _write_fixture(path: Path) -> None:
+    # PAN below is structurally well-formed (5 letters + 4 digits + 1
+    # letter) but fabricated for this test, not a real allotted number --
+    # synthetic fixture data only, per project policy.
+    rows = [
+        {
+            "id": "fixture-1",
+            "text": "Please update my PAN ABCDE1234F on file.",
+            "entities": [{"start": 22, "end": 32, "label": "PAN"}],
+        },
+        {
+            "id": "fixture-2",
+            "text": "No PII in this line at all.",
+            "entities": [],
+        },
+    ]
+    path.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+
+def test_my_data_prints_a_per_entity_table(tmp_path: Path) -> None:
+    fixture = tmp_path / "my.jsonl"
+    _write_fixture(fixture)
+
+    result = runner.invoke(app, ["bench", "--my-data", str(fixture)])
+
+    assert result.exit_code == 0
+    assert "PAN" in result.stdout
+    assert "2 document(s)" in result.stdout
+
+
+def test_my_data_out_writes_results_matching_the_scorer_directly(tmp_path: Path) -> None:
+    fixture = tmp_path / "my.jsonl"
+    _write_fixture(fixture)
+    out_dir = tmp_path / "out"
+
+    result = runner.invoke(app, ["bench", "--my-data", str(fixture), "--out", str(out_dir)])
+
+    assert result.exit_code == 0
+    data = json.loads((out_dir / "results.json").read_text(encoding="utf-8"))
+    assert (out_dir / "results.md").exists()
+
+    num_docs, labels, expected = score_my_data(fixture)
+    assert data["num_docs"] == num_docs
+    assert data["canonical_labels"] == list(labels)
+    assert data["adapters"]["maskflow"]["available"] is True
+    assert data["adapters"]["maskflow"]["strict"]["PAN"]["tp"] == expected.strict["PAN"].tp
+
+
+def test_my_data_malformed_line_reports_line_number_and_exits_nonzero(tmp_path: Path) -> None:
+    fixture = tmp_path / "bad.jsonl"
+    fixture.write_text(
+        json.dumps({"text": "ok", "entities": []}) + "\n" + json.dumps({"text": "missing ents"}),
+        encoding="utf-8",
+    )
+
+    result = runner.invoke(app, ["bench", "--my-data", str(fixture)])
+
+    assert result.exit_code != 0
+    assert "line 2" in result.stdout + result.stderr
+
+
+def test_my_data_missing_file_exits_nonzero(tmp_path: Path) -> None:
+    result = runner.invoke(app, ["bench", "--my-data", str(tmp_path / "does-not-exist.jsonl")])
+    assert result.exit_code != 0