equiframe-benchmark 0.12.4__py3-none-any.whl

equiframe_benchmark/tables.py

@@ -0,0 +1,347 @@
+"""Regenerate exact CSV tables from the bound evaluation, never from edited totals."""
+
+import csv
+import hashlib
+import io
+from pathlib import Path
+from typing import Annotated
+
+from equiframe._path_compat import is_junction
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from equiframe_benchmark.evaluation import EvaluationReport, MetricValue
+from equiframe_benchmark.io import load_document, save_document
+from equiframe_benchmark.models import Digest, Document, Text, digest
+
+
+class EvaluationTable(Document):
+    filename: Text
+    sha256: Digest
+    size_bytes: Annotated[int, Field(ge=0)]
+    rows: Annotated[int, Field(ge=0)]
+
+    @model_validator(mode="after")
+    def filename_only(self) -> Self:
+        if Path(self.filename).name != self.filename or not self.filename.endswith(".csv"):
+            raise ValueError("a table must have one local CSV filename")
+        return self
+
+
+class TableExport(Document):
+    evaluation_sha256: Digest
+    scientific: bool
+    tables: tuple[EvaluationTable, ...]
+
+    @model_validator(mode="after")
+    def unique(self) -> Self:
+        if len({t.filename for t in self.tables}) != len(self.tables):
+            raise ValueError("duplicate table filenames")
+        if tuple(t.filename for t in self.tables) != (
+            "coverage.csv",
+            "metrics.csv",
+            "average_precision.csv",
+            "incremental.csv",
+            "bootstrap.csv",
+            "cases.csv",
+        ):
+            raise ValueError("the complete table inventory is required")
+        return self
+
+
+def _csv(header: tuple[str, ...], rows: list[list[object]]) -> bytes:
+    stream = io.StringIO(newline="")
+    writer = csv.writer(stream, lineterminator="\n")
+    writer.writerow(header)
+    writer.writerows(rows)
+    return stream.getvalue().encode("utf-8")
+
+
+def _value(metric: MetricValue) -> list[object]:
+    return [metric.numerator, metric.denominator, metric.value, metric.reason]
+
+
+def table_payloads(report: EvaluationReport) -> dict[str, tuple[bytes, int]]:
+    """Return deterministic bytes and data-row counts, including undefined/empty strata."""
+    return _validated_payloads(EvaluationReport.model_validate(report))
+
+
+def _validated_payloads(report: EvaluationReport) -> dict[str, tuple[bytes, int]]:
+    """Format a report already revalidated at the caller's public boundary.
+
+    Keeping formatting separate avoids a second full deterministic bootstrap
+    during write/read, without caching or trusting a caller-supplied instance.
+    """
+    coverage: list[list[object]] = []
+    metrics: list[list[object]] = []
+    ap: list[list[object]] = []
+    incremental: list[list[object]] = []
+    bootstrap: list[list[object]] = []
+    cases: list[list[object]] = []
+    scope_header = ("mode", "scope", "family", "migration")
+    ratio_header = ("numerator", "denominator", "value", "undefined_reason")
+    for stratum in report.strata:
+        scope: list[object] = [
+            report.config.mode,
+            stratum.kind,
+            stratum.family,
+            stratum.migration,
+        ]
+        summary = stratum.coverage
+        coverage.append(
+            [
+                *scope,
+                summary.total_cases,
+                summary.oracle_confirmed,
+                summary.oracle_pending,
+                summary.oracle_disagreement,
+                summary.oracle_not_evaluable,
+                len(summary.failed_case_ids),
+                len(summary.principal_case_ids),
+                len(summary.excluded_case_ids),
+                *_value(summary.principal_coverage),
+            ]
+        )
+        for method in stratum.methods:
+            for support_name, confusion in (
+                ("principal", method.principal),
+                ("own_support", method.own_support),
+            ):
+                for metric_name in ("precision", "recall", "f1", "false_positive_rate"):
+                    value = getattr(confusion, metric_name)
+                    metrics.append(
+                        [
+                            *scope,
+                            method.method,
+                            support_name,
+                            len(method.complete_case_ids),
+                            len(method.own_case_ids),
+                            len(method.partial_alert_case_ids),
+                            len(method.unavailable_case_ids),
+                            method.coverage.numerator,
+                            method.coverage.denominator,
+                            confusion.tp,
+                            confusion.fp,
+                            confusion.tn,
+                            confusion.fn,
+                            metric_name,
+                            *_value(value),
+                        ]
+                    )
+        for average in stratum.average_precision:
+            for support_name, value, ids in (
+                ("principal", average.principal, average.principal_case_ids),
+                ("own_support", average.own_support, average.own_case_ids),
+            ):
+                ap.append([*scope, average.method, support_name, len(ids), *_value(value)])
+        gain = stratum.incremental
+        for name, ids, value in (
+            (
+                "additional_true_positives_H_vs_R",
+                gain.additional_true_positive_case_ids,
+                gain.additional_true_positive_rate,
+            ),
+            (
+                "additional_false_positives_H_vs_R",
+                gain.additional_false_positive_case_ids,
+                gain.additional_false_positive_rate,
+            ),
+        ):
+            incremental.append([*scope, name, len(ids), *_value(value)])
+        interval = stratum.bootstrap
+        for estimate in interval.intervals:
+            bootstrap.append(
+                [
+                    *scope,
+                    estimate.metric,
+                    len(interval.group_ids),
+                    interval.attempted_replicates,
+                    estimate.defined_replicates,
+                    estimate.undefined_replicates,
+                    report.config.bootstrap.seed,
+                    report.config.bootstrap.confidence,
+                    interval.lower_rank,
+                    interval.upper_rank,
+                    estimate.lower,
+                    estimate.upper,
+                    estimate.reason,
+                ]
+            )
+    for record in report.records:
+        cases.append(
+            [
+                report.config.mode,
+                record.case_id,
+                record.scenario_id,
+                record.group_id,
+                record.family,
+                record.migration,
+                record.engine,
+                record.oracle_state,
+                record.oracle_label,
+                record.r.decision,
+                record.r.complete,
+                record.m.decision,
+                record.m.complete,
+                record.b.decision,
+                record.b.complete,
+                record.h.decision,
+                record.h.complete,
+                record.m_score,
+                record.b_score,
+                record.failure_reason,
+            ]
+        )
+    definitions = (
+        (
+            "coverage.csv",
+            (
+                *scope_header,
+                "scheduled",
+                "confirmed",
+                "pending",
+                "disagreement",
+                "oracle_not_evaluable",
+                "failed",
+                "principal",
+                "excluded",
+                *ratio_header,
+            ),
+            coverage,
+        ),
+        (
+            "metrics.csv",
+            (
+                *scope_header,
+                "method",
+                "support",
+                "complete",
+                "own",
+                "partial_alert",
+                "not_evaluable",
+                "coverage_numerator",
+                "coverage_denominator",
+                "tp",
+                "fp",
+                "tn",
+                "fn",
+                "metric",
+                *ratio_header,
+            ),
+            metrics,
+        ),
+        (
+            "average_precision.csv",
+            (*scope_header, "method", "support", "cases", *ratio_header),
+            ap,
+        ),
+        (
+            "incremental.csv",
+            (*scope_header, "statistic", "additional_cases", *ratio_header),
+            incremental,
+        ),
+        (
+            "bootstrap.csv",
+            (
+                *scope_header,
+                "statistic",
+                "groups",
+                "attempted",
+                "defined",
+                "undefined",
+                "seed",
+                "confidence",
+                "lower_rank",
+                "upper_rank",
+                "lower",
+                "upper",
+                "reason",
+            ),
+            bootstrap,
+        ),
+        (
+            "cases.csv",
+            (
+                "mode",
+                "case_id",
+                "scenario_id",
+                "group_id",
+                "family",
+                "migration",
+                "engine",
+                "oracle_state",
+                "label",
+                "R",
+                "R_complete",
+                "M",
+                "M_complete",
+                "B",
+                "B_complete",
+                "H",
+                "H_complete",
+                "M_score",
+                "B_score",
+                "failure_reason",
+            ),
+            cases,
+        ),
+    )
+    return {name: (_csv(header, rows), len(rows)) for name, header, rows in definitions}
+
+
+def write_tables(report: EvaluationReport, directory: Path) -> TableExport:
+    """Publish in a new directory; an interruption cannot leave a valid final receipt."""
+    report = EvaluationReport.model_validate(report)
+    payloads = _validated_payloads(report)
+    directory = directory.resolve()
+    directory.mkdir(parents=True, exist_ok=False)
+    tables: list[EvaluationTable] = []
+    for filename, (data, rows) in payloads.items():
+        with (directory / filename).open("xb") as stream:
+            stream.write(data)
+        tables.append(
+            EvaluationTable(
+                filename=filename,
+                sha256=hashlib.sha256(data).hexdigest(),
+                size_bytes=len(data),
+                rows=rows,
+            )
+        )
+    result = TableExport(
+        evaluation_sha256=digest(report), scientific=report.scientific, tables=tuple(tables)
+    )
+    save_document(result, directory / "tables.json")
+    return result
+
+
+def read_tables(directory: Path, *, report: EvaluationReport) -> TableExport:
+    """Verify every byte against regeneration and reject unlisted files or links."""
+    report = EvaluationReport.model_validate(report)
+    if directory.is_symlink() or is_junction(directory):
+        raise ValueError("the table directory cannot be a filesystem link")
+    directory = directory.resolve()
+    manifest = directory / "tables.json"
+    if manifest.is_symlink() or not manifest.is_file() or manifest.stat().st_size > 65536:
+        raise ValueError("table receipt must be a bounded regular file")
+    result = load_document(TableExport, manifest)
+    expected = _validated_payloads(report)
+    if result.evaluation_sha256 != digest(report) or result.scientific != report.scientific:
+        raise ValueError("tables belong to a different evaluation")
+    if {p.name for p in directory.iterdir()} != {*expected, "tables.json"}:
+        raise ValueError("table directory inventory is not closed")
+    for table in result.tables:
+        path = directory / table.filename
+        if path.is_symlink() or not path.is_file():
+            raise ValueError("tables must be regular local files")
+        payload, rows = expected[table.filename]
+        if path.stat().st_size != len(payload):
+            raise ValueError("table size differs from bound regeneration")
+        data = path.read_bytes()
+        if (
+            data != payload
+            or hashlib.sha256(data).hexdigest() != table.sha256
+            or len(data) != table.size_bytes
+            or rows != table.rows
+        ):
+            raise ValueError("table bytes or row counts differ from bound regeneration")
+    return result

equiframe_benchmark/tables_v2.py

@@ -0,0 +1,340 @@
+"""Exact V2 paired tables, with exclusions, denominators and common-support appendix."""
+
+import hashlib
+from pathlib import Path
+from typing import Literal
+
+from equiframe._path_compat import is_junction
+from pydantic import model_validator
+from typing_extensions import Self
+
+from equiframe_benchmark.evaluation_v2 import EvaluationReportV2, SignedMetricV2
+from equiframe_benchmark.io import load_document, save_document
+from equiframe_benchmark.models import Digest, digest
+from equiframe_benchmark.models_v2 import DocumentV2 as Document
+from equiframe_benchmark.tables import EvaluationTable, _csv
+
+TABLES_V2 = (
+    "coverage.csv",
+    "metrics.csv",
+    "bootstrap.csv",
+    "exclusions.csv",
+    "common_metrics.csv",
+    "common_bootstrap.csv",
+    "cases.csv",
+)
+
+
+class TableExportV2(Document):
+    schema_version: Literal["2.0"] = "2.0"
+    evaluation_sha256: Digest
+    scientific: bool
+    tables: tuple[EvaluationTable, ...]
+
+    @model_validator(mode="after")
+    def inventory(self) -> Self:
+        if tuple(table.filename for table in self.tables) != TABLES_V2:
+            raise ValueError("V2 requires its complete ordered seven-table inventory")
+        return self
+
+
+def _value(value: SignedMetricV2) -> list[object]:
+    return [value.numerator, value.denominator, value.value, value.reason]
+
+
+def _validated_payloads(report: EvaluationReportV2) -> dict[str, tuple[bytes, int]]:
+    rows: dict[str, list[list[object]]] = {name: [] for name in TABLES_V2}
+    scope_header = ("mode", "scope", "family", "migration")
+    pair_header = (*scope_header, "left_method", "right_method")
+    ratio = ("numerator", "denominator", "value", "undefined_reason")
+    for stratum in report.strata:
+        scope: list[object] = [
+            report.config.mode,
+            stratum.kind,
+            stratum.family,
+            stratum.migration,
+        ]
+        for contrast in stratum.principal:
+            pair = contrast.comparison
+            identity = [*scope, pair.left_method, pair.right_method]
+            rows["coverage.csv"].append(
+                [
+                    *identity,
+                    stratum.total_cases,
+                    stratum.oracle_confirmed,
+                    len(pair.support_case_ids),
+                    len(pair.group_ids),
+                    pair.divergent_cases,
+                    pair.normal_cases,
+                    pair.coverage_confirmed.numerator,
+                    pair.coverage_confirmed.denominator,
+                    pair.coverage_confirmed.value,
+                    pair.coverage_confirmed.reason,
+                    pair.coverage_scheduled.numerator,
+                    pair.coverage_scheduled.denominator,
+                    pair.coverage_scheduled.value,
+                    pair.coverage_scheduled.reason,
+                    len(pair.exclusions),
+                    len(pair.left_partial_alert_case_ids),
+                    len(pair.right_partial_alert_case_ids),
+                ]
+            )
+            for estimate in contrast.estimates:
+                rows["metrics.csv"].append(
+                    [
+                        *identity,
+                        estimate.name,
+                        *_value(estimate.estimate),
+                    ]
+                )
+            for interval in contrast.bootstrap.intervals:
+                rows["bootstrap.csv"].append(
+                    [
+                        *identity,
+                        interval.metric,
+                        len(contrast.bootstrap.group_ids),
+                        contrast.bootstrap.attempted_replicates,
+                        contrast.bootstrap.lower_rank,
+                        contrast.bootstrap.upper_rank,
+                        interval.lower,
+                        interval.upper,
+                        interval.defined_replicates,
+                        interval.undefined_replicates,
+                        interval.reason,
+                    ]
+                )
+            for exclusion in pair.exclusions:
+                for reason in exclusion.reasons:
+                    rows["exclusions.csv"].append(
+                        [
+                            *identity,
+                            exclusion.case_id,
+                            exclusion.oracle_state,
+                            reason,
+                        ]
+                    )
+        for method in stratum.common.methods:
+            confusion = method.metrics
+            values = [
+                (metric, getattr(confusion, metric))
+                for metric in ("precision", "recall", "f1", "false_positive_rate")
+            ]
+            if method.average_precision is not None:
+                values.append(("average_precision", method.average_precision))
+            for name, value in values:
+                rows["common_metrics.csv"].append(
+                    [
+                        *scope,
+                        stratum.common.role,
+                        len(stratum.common.case_ids),
+                        method.method,
+                        confusion.tp,
+                        confusion.fp,
+                        confusion.fn,
+                        confusion.tn,
+                        name,
+                        value.numerator,
+                        value.denominator,
+                        value.value,
+                        value.reason,
+                    ]
+                )
+        for interval in stratum.common.bootstrap.intervals:
+            bootstrap = stratum.common.bootstrap
+            rows["common_bootstrap.csv"].append(
+                [
+                    *scope,
+                    stratum.common.role,
+                    interval.metric,
+                    len(bootstrap.group_ids),
+                    bootstrap.attempted_replicates,
+                    bootstrap.lower_rank,
+                    bootstrap.upper_rank,
+                    interval.lower,
+                    interval.upper,
+                    interval.defined_replicates,
+                    interval.undefined_replicates,
+                    interval.reason,
+                ]
+            )
+    for record in report.records:
+        decisions: list[object] = []
+        for component in (record.r, record.m, record.b, record.h):
+            decisions.extend((component.decision, component.complete, component.reason))
+        rows["cases.csv"].append(
+            [
+                record.case_id,
+                record.scenario_id,
+                record.group_id,
+                record.family,
+                record.migration,
+                record.engine,
+                record.oracle_state,
+                record.oracle_label,
+                *decisions,
+                record.m_score,
+                record.b_score,
+                record.candidate_sha256,
+                record.pipeline_sha256,
+                record.oracle_sha256,
+                record.detector_bundle_sha256,
+                record.freeze_sha256,
+                record.failure_reason,
+            ]
+        )
+    headers = {
+        "coverage.csv": (
+            *pair_header,
+            "scheduled",
+            "confirmed",
+            "paired",
+            "groups",
+            "divergent",
+            "normal",
+            "confirmed_numerator",
+            "confirmed_denominator",
+            "confirmed_coverage",
+            "confirmed_undefined",
+            "scheduled_numerator",
+            "scheduled_denominator",
+            "scheduled_coverage",
+            "scheduled_undefined",
+            "excluded",
+            "left_partial_alerts",
+            "right_partial_alerts",
+        ),
+        "metrics.csv": (*pair_header, "metric", *ratio),
+        "bootstrap.csv": (
+            *pair_header,
+            "metric",
+            "groups",
+            "attempted",
+            "lower_rank",
+            "upper_rank",
+            "lower",
+            "upper",
+            "defined",
+            "undefined",
+            "reason",
+        ),
+        "exclusions.csv": (*pair_header, "case_id", "oracle_state", "reason"),
+        "common_metrics.csv": (
+            *scope_header,
+            "role",
+            "common_support",
+            "method",
+            "tp",
+            "fp",
+            "fn",
+            "tn",
+            "metric",
+            *ratio,
+        ),
+        "common_bootstrap.csv": (
+            *scope_header,
+            "role",
+            "metric",
+            "groups",
+            "attempted",
+            "lower_rank",
+            "upper_rank",
+            "lower",
+            "upper",
+            "defined",
+            "undefined",
+            "reason",
+        ),
+        "cases.csv": (
+            "case_id",
+            "scenario_id",
+            "group_id",
+            "family",
+            "migration",
+            "engine",
+            "oracle_state",
+            "oracle_label",
+            "R",
+            "R_complete",
+            "R_reason",
+            "M",
+            "M_complete",
+            "M_reason",
+            "B",
+            "B_complete",
+            "B_reason",
+            "H",
+            "H_complete",
+            "H_reason",
+            "M_score",
+            "B_score",
+            "candidate_sha256",
+            "pipeline_sha256",
+            "oracle_sha256",
+            "bundle_sha256",
+            "freeze_sha256",
+            "failure_reason",
+        ),
+    }
+    return {name: (_csv(headers[name], rows[name]), len(rows[name])) for name in TABLES_V2}
+
+
+def table_payloads_v2(report: EvaluationReportV2) -> dict[str, tuple[bytes, int]]:
+    return _validated_payloads(EvaluationReportV2.model_validate(report))
+
+
+def write_tables_v2(report: EvaluationReportV2, directory: Path) -> TableExportV2:
+    checked = EvaluationReportV2.model_validate(report)
+    payloads = _validated_payloads(checked)
+    directory = directory.resolve()
+    directory.mkdir(parents=True, exist_ok=False)
+    tables = []
+    for name, (payload, rows) in payloads.items():
+        with (directory / name).open("xb") as output:
+            output.write(payload)
+        tables.append(
+            EvaluationTable(
+                filename=name,
+                sha256=hashlib.sha256(payload).hexdigest(),
+                size_bytes=len(payload),
+                rows=rows,
+            )
+        )
+    receipt = TableExportV2(
+        evaluation_sha256=digest(checked),
+        scientific=checked.scientific,
+        tables=tuple(tables),
+    )
+    save_document(receipt, directory / "tables.json")
+    return receipt
+
+
+def read_tables_v2(directory: Path, *, report: EvaluationReportV2) -> TableExportV2:
+    checked = EvaluationReportV2.model_validate(report)
+    if directory.is_symlink() or is_junction(directory):
+        raise ValueError("V2 table directory must not be a filesystem link")
+    directory = directory.resolve()
+    path = directory / "tables.json"
+    if path.is_symlink() or not path.is_file() or path.stat().st_size > 65536:
+        raise ValueError("V2 table receipt must be a bounded regular file")
+    receipt = load_document(TableExportV2, path)
+    expected = _validated_payloads(checked)
+    if receipt.evaluation_sha256 != digest(checked) or receipt.scientific != checked.scientific:
+        raise ValueError("V2 tables belong to a different paired evaluation")
+    if {entry.name for entry in directory.iterdir()} != {*expected, "tables.json"}:
+        raise ValueError("V2 table inventory must be closed")
+    for table in receipt.tables:
+        target = directory / table.filename
+        if target.is_symlink() or is_junction(target) or not target.is_file():
+            raise ValueError("V2 tables must be regular local files")
+        payload, rows = expected[table.filename]
+        if target.stat().st_size != len(payload):
+            raise ValueError("V2 table size differs from regeneration")
+        data = target.read_bytes()
+        if (
+            data != payload
+            or hashlib.sha256(data).hexdigest() != table.sha256
+            or len(data) != table.size_bytes
+            or rows != table.rows
+        ):
+            raise ValueError("V2 table bytes or counts differ from captured decisions")
+    return receipt