ids2eval 0.2.2__py3-none-any.whl

ids2eval/modeling/preprocessing.py

@@ -0,0 +1,73 @@
+"""Scaling and class-balancing, driven by preprocessing.* config.
+
+Sampling is fit on the training split only and never applied to test:
+test sets stay at their real-world class distribution so metrics
+reflect real deployment, matching the IDS project's balancer.py design.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+import pandas as pd
+from imblearn.combine import SMOTEENN
+from imblearn.over_sampling import SMOTE
+from imblearn.under_sampling import EditedNearestNeighbours, RandomUnderSampler
+from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler
+
+from ..data import features
+
+logger = logging.getLogger(__name__)
+
+SCALERS = {"standard": StandardScaler, "minmax": MinMaxScaler, "robust": RobustScaler}
+SAMPLERS = {
+    "smote": lambda seed: SMOTE(random_state=seed),
+    "smoteenn": lambda seed: SMOTEENN(random_state=seed),
+    "enn": lambda seed: EditedNearestNeighbours(),  # deterministic, no randomness to seed
+    "random_undersample": lambda seed: RandomUnderSampler(random_state=seed),
+}
+
+
+def scale_features(
+    train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict
+) -> tuple[np.ndarray, np.ndarray, list[str]]:
+    cols = features.feature_columns(train_df, cfg)
+    x_train_raw, x_test_raw = features.encode_aligned(train_df, test_df, cols)
+
+    method = cfg["preprocessing"]["scaling"]
+    if isinstance(method, list):
+        raise ValueError(
+            f"preprocessing.scaling is a list ({method}) - scale_features takes a "
+            "single strategy; use run_benchmark for multi-strategy comparison"
+        )
+    if method == "none":
+        return x_train_raw.to_numpy(), x_test_raw.to_numpy(), cols
+
+    scaler = SCALERS[method]()
+    x_train = scaler.fit_transform(x_train_raw)
+    x_test = scaler.transform(x_test_raw)
+    return x_train, x_test, cols
+
+
+def apply_sampling(x_train: np.ndarray, y_train, cfg: dict, stage: str) -> tuple[np.ndarray, np.ndarray]:
+    algo = cfg["preprocessing"]["sampling"][stage]
+    if isinstance(algo, list):
+        raise ValueError(
+            f"preprocessing.sampling.{stage} is a list ({algo}) - apply_sampling takes a "
+            "single strategy; use run_benchmark for multi-strategy comparison"
+        )
+    y_train = np.asarray(y_train)
+    if algo == "none":
+        return x_train, y_train
+
+    sampler = SAMPLERS[algo](cfg["random_seed"])
+    try:
+        return sampler.fit_resample(x_train, y_train)
+    except ValueError as e:
+        logger.warning(
+            "Sampling '%s' failed for stage '%s' (%s), likely a class too small "
+            "for this sampler's neighbor count. Falling back to unsampled data.",
+            algo, stage, e,
+        )
+        return x_train, y_train

ids2eval/reporting/__init__.py

@@ -0,0 +1,2 @@
+"""Producing run outputs: scorecard, scorecard_plot, run_manager, drift.
+"""

ids2eval/reporting/cite.py

@@ -0,0 +1,57 @@
+"""Prints a citation block for a completed run's scorecard.
+
+Everything needed already lives in scorecard.json (dataset name, tool
+version and commit, schema version, verdict, generated timestamp, and
+the dataset_fingerprint content hashes): this reads that one file and
+reformats it, it computes nothing new.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from . import scorecard as _scorecard
+
+
+def _bibtex_key(dataset_name: str, train_content_hash: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", dataset_name.lower()).strip("_")
+    return f"ids2eval_{slug}_{train_content_hash[:8]}"
+
+
+def load_scorecard(run_dir: str | Path) -> dict:
+    path = Path(run_dir)
+    if path.is_dir():
+        path = path / "scorecard.json"
+    if not path.exists():
+        raise FileNotFoundError(f"No scorecard.json found at or under {run_dir}")
+    return _scorecard.upgrade(json.loads(path.read_text()))
+
+
+def citation_bibtex(scorecard: dict) -> str:
+    fp = scorecard["dataset_fingerprint"]
+    year = scorecard["generated_at"][:4]
+    key = _bibtex_key(scorecard["dataset_name"], fp["train_content_hash"])
+    commit = scorecard["ids2eval_git_commit"]
+    verdict = scorecard["overall_status"].replace("_", " ")
+
+    # commit is only known for a git checkout (version_info.py shells out to
+    # git); a pip install from a downloaded zip - no .git directory at all -
+    # has no commit to report, and "commit unknown" reads worse in a
+    # citation than just not mentioning a commit at all.
+    howpublished = f"IDS2Eval v{scorecard['ids2eval_version']} (scorecard schema {scorecard['scorecard_schema_version']})"
+    if commit:
+        howpublished += f", commit {commit}"
+
+    fields = {
+        "title": f"IDS2Eval scorecard for {scorecard['dataset_name']}",
+        "howpublished": howpublished,
+        "note": (
+            f"{verdict}; train content hash {fp['train_content_hash']}; "
+            f"test content hash {fp['test_content_hash']}; generated {scorecard['generated_at']}"
+        ),
+        "year": year,
+    }
+    body = ",\n".join(f"  {name} = {{{value}}}" for name, value in fields.items())
+    return f"@misc{{{key},\n{body}\n}}"

ids2eval/reporting/compare_datasets.py

@@ -0,0 +1,103 @@
+"""Two-dataset equivalence comparison.
+
+Answers a question no single-dataset audit can: are these two configs,
+both claiming to load "the same" dataset (an official release and a
+third-party mirror, say), actually equivalent? Loads both, computes
+each one's fingerprint (run_manager.compute_dataset_fingerprint), and
+compares schema, row counts, label distribution, and content.
+
+Not a run_audit check: this compares two DATASETS, not one dataset's
+train against its own test, and needs two configs rather than one, a
+different enough shape to be its own CLI subcommand
+(`ids2eval compare-datasets a.yaml b.yaml`) rather than an audit
+finding. Each dataset's own train/test split can use a different
+split_ratio or split_mode, so a train-vs-train, test-vs-test content
+hash comparison alone could report a mismatch that's really just a
+different split of identical underlying data; a combined (train+test
+concatenated) hash catches that case, alongside the split-specific
+hashes for detail.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ..config import load_config
+from ..data import dataset
+from . import run_manager
+
+
+def _combined_label_counts(fingerprint: dict) -> dict:
+    counts: dict[str, int] = {}
+    for source in ("train_class_distribution", "test_class_distribution"):
+        for label, count in fingerprint[source].items():
+            counts[label] = counts.get(label, 0) + count
+    return counts
+
+
+def compare(config_path_a: str, config_path_b: str) -> dict:
+    cfg_a = load_config(config_path_a)
+    cfg_b = load_config(config_path_b)
+    train_a, test_a = dataset.load_split(cfg_a)
+    train_b, test_b = dataset.load_split(cfg_b)
+
+    fp_a = run_manager.compute_dataset_fingerprint(train_a, test_a, cfg_a)
+    fp_b = run_manager.compute_dataset_fingerprint(train_b, test_b, cfg_b)
+
+    combined_hash_a = run_manager.content_hash_unordered(pd.concat([train_a, test_a], ignore_index=True))
+    combined_hash_b = run_manager.content_hash_unordered(pd.concat([train_b, test_b], ignore_index=True))
+
+    labels_a = _combined_label_counts(fp_a)
+    labels_b = _combined_label_counts(fp_b)
+
+    return {
+        "dataset_a": cfg_a["dataset"]["name"],
+        "dataset_b": cfg_b["dataset"]["name"],
+        "schema_match": set(fp_a["columns"]) == set(fp_b["columns"]),
+        "columns_only_in_a": sorted(set(fp_a["columns"]) - set(fp_b["columns"])),
+        "columns_only_in_b": sorted(set(fp_b["columns"]) - set(fp_a["columns"])),
+        "row_count_a": fp_a["train_rows"] + fp_a["test_rows"],
+        "row_count_b": fp_b["train_rows"] + fp_b["test_rows"],
+        "row_count_match": (fp_a["train_rows"] + fp_a["test_rows"]) == (fp_b["train_rows"] + fp_b["test_rows"]),
+        "label_distribution_a": labels_a,
+        "label_distribution_b": labels_b,
+        "label_distribution_match": labels_a == labels_b,
+        "combined_content_match": combined_hash_a == combined_hash_b,
+    }
+
+
+def render_report(result: dict) -> str:
+    def verdict(matched: bool, hard: bool = False) -> str:
+        return "PASS" if matched else ("FAIL" if hard else "WARN")
+
+    lines = [
+        f"Dataset A: {result['dataset_a']}",
+        f"Dataset B: {result['dataset_b']}",
+        "",
+        f"Schema match          {verdict(result['schema_match'], hard=True)}",
+        f"Row count match       {verdict(result['row_count_match'])}",
+        f"Label distribution    {verdict(result['label_distribution_match'])}",
+        f"Combined content      {verdict(result['combined_content_match'], hard=True)}",
+    ]
+
+    if not result["schema_match"]:
+        if result["columns_only_in_a"]:
+            lines.append(f"  columns only in A: {result['columns_only_in_a']}")
+        if result["columns_only_in_b"]:
+            lines.append(f"  columns only in B: {result['columns_only_in_b']}")
+    if not result["row_count_match"]:
+        lines.append(f"  A: {result['row_count_a']:,} rows, B: {result['row_count_b']:,} rows")
+    if not result["label_distribution_match"]:
+        lines.append(f"  A: {result['label_distribution_a']}")
+        lines.append(f"  B: {result['label_distribution_b']}")
+
+    # combined_content_match, when true, already means the two datasets' full row
+    # sets are identical, which implies matching row counts and label distribution
+    # too - those two fields stay useful for diagnosing WHERE a mismatch is when
+    # combined_content_match is false, not as a separate verdict branch here.
+    lines.append("")
+    if result["schema_match"] and result["combined_content_match"]:
+        lines.append("Equivalent: same schema, row counts, label distribution, and content.")
+    else:
+        lines.append("Likely non-equivalent dataset versions.")
+    return "\n".join(lines)

ids2eval/reporting/drift.py

@@ -0,0 +1,80 @@
+"""Compares this run's scorecard against the previous run in the same
+output.dir, so a dataset-quality change between two runs of the same
+config over time is visible without a person diffing two JSON files by
+hand.
+
+Every check here treats each run as fully independent: there is no
+mechanism to notice that a source file was quietly swapped, or that a
+dataset's duplication rate crept up between one export and the next.
+Deequ and whylogs (see general ML data-quality tooling) both track
+quality metrics across runs against a metrics repository rather than
+only within one; this is a minimal version of that idea, comparing
+against the immediately preceding run rather than a full history.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from . import run_manager
+from . import scorecard as _scorecard
+
+logger = logging.getLogger(__name__)
+
+
+def find_previous_run(output_dir: Path, current_run_dir: Path) -> Path | None:
+    """The most recent other run directory in output_dir that has a
+    scorecard.json, or None if there isn't one (a fresh output_dir, or
+    every prior run failed before the scorecard stage).
+    """
+    runs_dir = output_dir / run_manager.RUNS_DIRNAME
+    if not runs_dir.is_dir():
+        return None
+    candidates = sorted(
+        p for p in runs_dir.iterdir()
+        if p.is_dir() and p != current_run_dir and (p / "scorecard.json").is_file()
+    )
+    return candidates[-1] if candidates else None
+
+
+def _status_by_check(scorecard: dict) -> dict[str, str]:
+    if scorecard.get("checks"):
+        return {r["check"]: (r["after"] or r["before"])["status"] for r in scorecard["checks"]}
+    return {f["check"]: f["status"] for f in scorecard.get("findings", [])}
+
+
+def compare(scorecard: dict, previous_run_dir: Path) -> dict | None:
+    """None if the previous run's scorecard.json can't be read - a missing
+    comparison is reported as "no previous run" rather than crashing the
+    current run over an unrelated one's leftover state.
+    """
+    try:
+        previous = _scorecard.upgrade(json.loads((previous_run_dir / "scorecard.json").read_text()))
+    except (OSError, ValueError) as e:
+        logger.warning("Could not read previous run's scorecard at %s: %s", previous_run_dir, e)
+        return None
+
+    previous_status = _status_by_check(previous)
+    current_status = _status_by_check(scorecard)
+    changed_checks = [
+        {"check": check, "previous_status": previous_status[check], "current_status": status}
+        for check, status in current_status.items()
+        if check in previous_status and previous_status[check] != status
+    ]
+
+    previous_fp = previous.get("dataset_fingerprint", {})
+    current_fp = scorecard["dataset_fingerprint"]
+    fingerprint_changed = (
+        previous_fp.get("train_content_hash") != current_fp.get("train_content_hash")
+        or previous_fp.get("test_content_hash") != current_fp.get("test_content_hash")
+    )
+
+    return {
+        "previous_run": previous_run_dir.name,
+        "previous_generated_at": previous.get("generated_at"),
+        "previous_verdict": previous.get("overall_status"),
+        "fingerprint_changed": fingerprint_changed,
+        "changed_checks": changed_checks,
+    }

ids2eval/reporting/intervention_manifest.py

@@ -0,0 +1,83 @@
+"""Intervention manifest: reformats the checks that already ARE an
+intervention (they refit under an altered condition and compare, rather
+than measuring a static property of the data) into one structured record
+per intervention, machine-readable, matching this project's own falsification
+methodology directly rather than the ok/warning/flag scorecard's summary view.
+
+This is a re-projection of findings IDS²Eval already computes, not a new
+detection-and-selection engine: every check listed in INTERVENTION_CHECKS
+always runs (or doesn't) according to audit.<check_name> in the config, the
+same as every other check, rather than being dynamically triggered by a
+prior finding's specific value. Building that latter, fuller "detect a
+threat, then choose which intervention to run" rule engine is real,
+larger, separate work this module does not attempt; what it does do is give
+every intervention that DID run a citable, uniformly-shaped record.
+
+Each record's four sections follow the shape:
+    finding:      the check name, its status, and its own details dict as
+                  evidence (heterogeneous by check, e.g. resplit_falsification's
+                  random/grouped accuracy vs. feature_category_ablation_check's
+                  per-category drop; kept as-is rather than forced into one
+                  fixed evidence shape, which would lose information)
+    intervention: what was done (a fixed, human-readable label per check)
+                  and that it was performed (always true here, since the
+                  record only exists for a check that actually ran)
+    consequence:  the same details dict again, from the intervention's own
+                  perspective - the "before" and "after" of a resplit-style
+                  check are the same two numbers as its "evidence"
+    conclusion:   "material" / "investigated_not_material" / "inconclusive",
+                  reusing the check's own explicit `material` field
+                  (True/False/None) when it has one (repeated_seed_falsification_
+                  check), otherwise derived from its ok/warning/flag status
+"""
+
+from __future__ import annotations
+
+# check name -> human-readable intervention label.
+INTERVENTION_CHECKS = {
+    "resplit_falsification": "session/time-grouped resplit",
+    "repeated_seed_falsification_check": "session/time-grouped resplit, repeated across seeds",
+    "scenario_holdout_falsification": "held out one entire scenario from training",
+    "cross_capture_matrix_check": "trained on each scenario, tested on every other",
+    "feature_category_ablation_check": "removed one declared feature category and refit",
+    "cross_dataset_drift_check": "trained here, evaluated on an independent reference dataset",
+    "result_robustness_check": "refit under each buildable condition (split, dedup, identity columns)",
+    "artifact_sensitivity_check": "reshuffled one declared identity/timestamp column's values and re-predicted",
+}
+
+_STATUS_TO_CONCLUSION = {"flag": "material", "warning": "inconclusive", "ok": "investigated_not_material"}
+
+
+def _conclusion_status(finding: dict) -> str:
+    material = finding.get("details", {}).get("material")
+    if material is True:
+        return "material"
+    if material is False:
+        return "investigated_not_material"
+    if material is None and "material" in finding.get("details", {}):
+        return "inconclusive"  # explicit tri-state field said "can't tell", not just absent
+    return _STATUS_TO_CONCLUSION[finding["status"]]
+
+
+def build_manifest(findings: list[dict]) -> list[dict]:
+    """One record per finding in `findings` whose check is in INTERVENTION_CHECKS
+    AND actually ran, not a no-op. Every check here returns `"details": {}` for its
+    no-op case (schema.scenario_column/reference_dataset/etc. not configured, or no
+    candidate columns present) - confirmed real against a live run where
+    scenario_holdout_falsification's no-op ("no schema.scenario_column configured")
+    would otherwise have shown up as intervention.performed: true, a real bug this
+    empty-details check exists specifically to prevent from recurring. A
+    not_applicable finding is skipped the same way, whatever its details hold.
+    """
+    records = []
+    for finding in findings:
+        check = finding["check"]
+        if check not in INTERVENTION_CHECKS or finding["status"] == "not_applicable" or not finding.get("details"):
+            continue
+        records.append({
+            "finding": {"check": check, "status": finding["status"], "evidence": finding.get("details", {})},
+            "intervention": {"type": check, "description": INTERVENTION_CHECKS[check], "performed": True},
+            "consequence": finding.get("details", {}),
+            "conclusion": {"status": _conclusion_status(finding)},
+        })
+    return records