ids2eval 0.2.2__py3-none-any.whl

ids2eval/audit/near_duplicate_class.py

@@ -0,0 +1,87 @@
+"""Near-duplicate feature vectors across different classes.
+
+label_conflict_check catches identical features mapped to different
+labels. This extends that to features close enough in scaled distance
+to be effectively identical without matching exactly, the common case
+when the same traffic-generation script produced two nominally
+different attack labels, or two attack tools share almost all of their
+flow statistics. Runs on train only: this is about label confusability
+within the data a model actually learns from, not a train/test split
+property (that's homogeneity_test's job).
+
+Same nearest-neighbor machinery and near-zero distance threshold as
+homogeneity_test, applied across classes rather than across the split.
+A single NearestNeighbors index over all sampled rows, queried once,
+avoids an O(classes^2) pairwise comparison.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from sklearn.neighbors import NearestNeighbors
+from sklearn.preprocessing import StandardScaler
+
+from ..data import features
+from ._status import NOT_APPLICABLE
+
+SAMPLE_SIZE = 500
+MIN_CLASS_SIZE = 20
+NEAR_ZERO_DISTANCE = 1e-6
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    group_col = schema["attack_category_column"] or schema["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+
+    class_counts = train_df[group_col].value_counts()
+    eligible = class_counts[class_counts >= MIN_CLASS_SIZE].index
+    if len(eligible) < 2:
+        return {
+            "check": "near_duplicate_class_check", "status": NOT_APPLICABLE,
+            "summary": "fewer than two classes have enough rows to compare", "details": {},
+        }
+
+    rng = np.random.RandomState(0)
+    sample_parts = []
+    for cls in eligible:
+        idx = np.flatnonzero((train_df[group_col] == cls).to_numpy())
+        chosen = rng.choice(idx, size=min(SAMPLE_SIZE, len(idx)), replace=False)
+        sample_parts.append(train_df.iloc[chosen])
+    sample_df = pd.concat(sample_parts, ignore_index=True)
+
+    x_raw = features.encode_aligned(sample_df, sample_df, cols)[0]
+    x = StandardScaler().fit_transform(x_raw)
+    labels = sample_df[group_col].to_numpy()
+
+    nn = NearestNeighbors(n_neighbors=2, algorithm="auto", n_jobs=-1).fit(x)
+    dist, idx = nn.kneighbors(x)
+    nearest_dist, nearest_idx = dist[:, 1], idx[:, 1]  # excluding self
+
+    cross_label = labels != labels[nearest_idx]
+    near_zero = nearest_dist < NEAR_ZERO_DISTANCE
+    confused = cross_label & near_zero
+
+    if not confused.any():
+        return {
+            "check": "near_duplicate_class_check", "status": "ok",
+            "summary": f"no near-zero-distance feature vectors found across {len(eligible)} classes tested",
+            "details": {"classes_tested": len(eligible)},
+        }
+
+    pairs = pd.Series(
+        [tuple(sorted((a, b))) for a, b in zip(labels[confused], labels[nearest_idx[confused]], strict=True)]
+    ).value_counts()
+    top_pairs = {f"{a} / {b}": int(n) for (a, b), n in pairs.head(5).items()}
+    affected_rows = int(confused.sum())
+    pct = affected_rows / len(sample_df)
+
+    summary = (
+        f"{affected_rows:,} of {len(sample_df):,} sampled rows ({pct:.2%}) have a near-zero-distance "
+        f"neighbor under a different label. Most affected pairs: {top_pairs}"
+    )
+    return {
+        "check": "near_duplicate_class_check", "status": "flag", "summary": summary,
+        "details": {"classes_tested": len(eligible), "affected_rows": affected_rows, "top_pairs": top_pairs},
+    }

ids2eval/audit/one_rule.py

@@ -0,0 +1,82 @@
+"""One-rule ("one-liner") solvability check.
+
+Per Wu & Keogh 2021 (IEEE TKDE): most widely-used time-series anomaly
+benchmarks (Yahoo, NASA, Numenta) are largely solvable by a single line of
+code - one feature compared to one threshold - which means published
+comparisons of sophisticated algorithms on them are measuring nothing more
+than who found the same threshold. leakage_screen already reports that a
+feature carries most of a random forest's importance, but a share-of-
+importance number is abstract; this check names the winning rule in plain
+language ("Dst Port <= 1024.5"), which is far harder to argue with. A
+depth-1 decision tree (a "stump") is exactly a brute-force search over
+every feature and threshold for the single best split, done in one cheap
+fit rather than a manual loop.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.tree import DecisionTreeClassifier
+
+from ..data import features
+from ._status import NOT_APPLICABLE
+
+# Wu & Keogh don't give a numeric cutoff (their test is "can a human write this in
+# one line"); a one-rule accuracy this high is the tabular-data equivalent - a
+# threshold on a single feature is about as simple a rule as exists.
+ACCURACY_FLAG_THRESHOLD = 0.95
+# A rule must also remove at least this share of the errors made by always
+# predicting the training majority class: on a 99:1 dataset with no signal, any
+# stump scores ~99% by predicting the majority, which says nothing about shortcuts.
+MIN_ERROR_REDUCTION = 0.5
+MAX_FIT_ROWS = 200_000
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict, seed: int = 0) -> dict:
+    """seed varies the sample draw and the stump's own randomness. The
+    default single run is called with cfg["random_seed"] (run_audit's job);
+    seed_sensitivity_check separately sweeps its own fixed range of seeds to
+    test whether the flag/ok conclusion holds up regardless of that one value.
+    """
+    label_col = cfg["schema"]["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=seed)
+    x_train, x_test = features.encode_aligned(train_fit, test_df, cols, unseen="missing")
+    y_train, y_test = train_fit[label_col], test_df[label_col]
+
+    stump = DecisionTreeClassifier(max_depth=1, random_state=seed)
+    stump.fit(x_train, y_train)
+    tree = stump.tree_
+    root_feature = tree.feature[0]
+
+    if root_feature < 0:  # -2: the root is already a leaf - e.g. a single-class split
+        return {
+            "check": "one_rule_check", "status": NOT_APPLICABLE,
+            "summary": "no single-feature rule found (fewer than two classes in the fitted sample)",
+            "details": {},
+        }
+
+    rule = f"{x_train.columns[root_feature]} <= {tree.threshold[0]:.4g}"
+    train_acc = float(stump.score(x_train, y_train))
+    test_acc = float(stump.score(x_test, y_test))
+
+    majority = y_train.value_counts().idxmax()
+    baseline_acc = float((y_test == majority).mean())
+    error_reduction = (test_acc - baseline_acc) / (1 - baseline_acc) if baseline_acc < 1 else 0.0
+
+    flagged = test_acc > ACCURACY_FLAG_THRESHOLD and error_reduction >= MIN_ERROR_REDUCTION
+    status = "flag" if flagged else "ok"
+    summary = (
+        f"single rule '{rule}' reaches {test_acc:.1%} test accuracy ({train_acc:.1%} train), "
+        f"vs. {baseline_acc:.1%} from always predicting the majority class"
+    )
+    if flagged:
+        summary += ". This problem may be solvable without learning attack behavior"
+
+    return {
+        "check": "one_rule_check", "status": status, "summary": summary,
+        "details": {
+            "rule": rule, "train_accuracy": train_acc, "test_accuracy": test_acc,
+            "majority_baseline_accuracy": baseline_acc, "error_reduction_vs_baseline": error_reduction,
+        },
+    }

ids2eval/audit/port_protocol_shortcut.py

@@ -0,0 +1,75 @@
+"""Combined port+protocol shortcut check.
+
+identity_column_flag already tests each id_like_columns entry alone, but
+destination port and protocol together are a specific, well-known NIDS
+shortcut: a handful of (port, protocol) pairs (80/TCP, 443/TCP, 53/UDP,
+...) route almost all benign traffic, and attack tools often target one
+fixed port, so the pair can separate classes a single column understates,
+generalizing across CIC, UNSW, and ToN-IoT-style datasets alike (the same
+families where identity_column_flag already flags port alone). This
+check quantifies that combined signal specifically, rather than leaving
+it as an unmeasured gap between "one column alone" and "the full feature
+set."
+
+Needs no new config field: a port-like column is whichever entry in
+schema.id_like_columns has "port" in its name (case-insensitive), and a
+protocol-like column is whichever loaded column has "proto" in its name
+(case-insensitive) - protocol is ordinarily a plain feature, not
+id_like, so it is looked for across all columns rather than restricted
+to id_like_columns. No-ops if either is missing.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+
+from ._auc import robust_auc
+from ._status import NOT_APPLICABLE
+
+AUC_FLAG_THRESHOLD = 0.8
+
+
+def _find_column(candidates: list[str], needle: str) -> str | None:
+    for c in candidates:
+        if needle in c.lower():
+            return c
+    return None
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    id_cols = [c for c in cfg["schema"]["id_like_columns"] if c in train_df.columns]
+    port_col = _find_column(id_cols, "port")
+    proto_col = _find_column(list(train_df.columns), "proto")
+
+    if not port_col or not proto_col:
+        return {
+            "check": "port_protocol_shortcut_check", "status": NOT_APPLICABLE,
+            "summary": (
+                "no port-like entry in schema.id_like_columns and/or no proto-like "
+                "column found, this check needs both"
+            ),
+            "details": {"port_column": port_col, "protocol_column": proto_col},
+        }
+
+    label_col = cfg["schema"]["label_column"]
+    cols = [port_col, proto_col]
+    categories = {c: pd.Index(train_df[c].astype(str).unique()) for c in cols}
+    x_train = pd.DataFrame({c: categories[c].get_indexer(train_df[c].astype(str)) for c in cols})
+    x_test = pd.DataFrame({c: categories[c].get_indexer(test_df[c].astype(str)) for c in cols})
+
+    clf = RandomForestClassifier(n_estimators=50, random_state=cfg["random_seed"], n_jobs=-1)
+    clf.fit(x_train, train_df[label_col])
+    auc = robust_auc(test_df[label_col], clf.predict_proba(x_test), clf.classes_)
+
+    flagged = auc is not None and auc > AUC_FLAG_THRESHOLD
+    status = "flag" if flagged else "ok"
+    auc_text = "n/a" if auc is None else f"{auc:.3f}"
+    summary = f"standalone AUC of '{port_col}' + '{proto_col}' combined: {auc_text}"
+    if flagged:
+        summary += ". Port and protocol together may predict the label without attack behavior"
+
+    return {
+        "check": "port_protocol_shortcut_check", "status": status, "summary": summary,
+        "details": {"port_column": port_col, "protocol_column": proto_col, "standalone_auc": auc},
+    }

ids2eval/audit/repeated_seed_falsification.py

@@ -0,0 +1,182 @@
+"""Repeated-seed resplit falsification (v2, opt-in).
+
+resplit_falsification (Section 3.5) reports one random-split accuracy and
+one grouped-split accuracy, each from a single seed. That single comparison
+cannot itself distinguish "this gap is genuinely small" from "a different
+seed for either split would show a materially different gap purely from
+sampling variance" - exactly the limitation this project's own audit paper
+disclosed and left as its most important piece of undone follow-up work.
+
+This check closes that gap by repeating the same random-vs-grouped
+comparison across several independent seeds (each seed reshuffles both the
+random split and the StratifiedGroupKFold shuffle within the grouped split,
+and refits a freshly-seeded RandomForest for each), then reports the mean
+drop and a 95% confidence interval, using Student's t-distribution rather
+than a normal approximation since n_seeds is deliberately small (5-10, not
+hundreds - each additional seed is a full pair of RandomForest refits on
+the whole raw dataset).
+
+Materiality classification compares the CI against the SAME frozen
+threshold resplit_falsification itself uses (audit.materiality_thresholds.
+repeated_seed_falsification_check, defaulting to resplit_falsification's own
+default), via a three-way rule rather than a single point-estimate cutoff:
+
+    material       if the CI's lower bound is still above the threshold
+                   (even the most conservative estimate says it matters)
+    not material   if the CI's upper bound is below the threshold
+                   (even the most generous estimate says it doesn't)
+    inconclusive   if the CI straddles the threshold (n_seeds isn't enough
+                   to tell, one way or the other - reported as its own
+                   outcome, not silently folded into "ok", per this
+                   project's own statistical-analysis-plan rule: never
+                   infer "no effect" merely because a test isn't
+                   significant)
+
+Same preconditions as resplit_falsification: needs dataset.raw_files and
+dataset.group_columns, and reloads the raw data itself, so its result
+cannot depend on preprocessing.dedup either (see STRUCTURAL_CHECKS).
+
+If audit.parallel_checks is set, the remaining (not yet checkpointed) seeds
+run concurrently via a thread pool instead of one at a time - this check is
+n_seeds independent full refits, so it's the single biggest reason a config
+enabling it alongside only 1-2 other checks can still leave most of a
+multi-vCPU VM idle if only the OUTER per-check dispatch were parallelized.
+Never combined with checkpoint_path: per-seed checkpointing and concurrent
+seeds are mutually exclusive at the config level (audit.checkpoint and
+audit.parallel_checks can't both be true) for the same reason run_audit's
+outer dispatch keeps them apart - concurrent writers to one checkpoint file
+would race.
+
+Accepts an already-loaded `combined` DataFrame so run_audit can share one
+load across this, resplit_falsification, and result_robustness_check when
+several run together, instead of each reloading the full raw dataset from
+disk - a real cost at full scale (a real failure this was built to avoid:
+a 70M-row config running all three of these checks was reloading the same
+data three times sequentially, contributing to repeated VM-level
+instability on a long run). Still loads it itself when called standalone,
+so this is purely an optimization, not a behavior change.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+from statistics import mean, stdev
+
+import pandas as pd
+from scipy import stats
+
+from ..data import dataset
+from . import _checkpoint, _materiality
+from ._fit_score import fit_and_score
+from ._status import NOT_APPLICABLE
+from .resplit import MATERIAL_DROP_THRESHOLD
+
+DEFAULT_N_SEEDS = 10
+MIN_N_SEEDS = 3  # below this, a t-based CI is too unstable to report meaningfully
+_CHECK_NAME = "repeated_seed_falsification_check"
+
+
+def check(
+    cfg: dict, n_seeds: int | None = None, checkpoint_path: str | None = None,
+    combined: pd.DataFrame | None = None,
+) -> dict:
+    dataset_cfg = cfg["dataset"]
+    if n_seeds is None:
+        n_seeds = cfg["audit"]["repeated_seed_count"]
+    if n_seeds < MIN_N_SEEDS:
+        return {
+            "check": _CHECK_NAME, "status": NOT_APPLICABLE,
+            "summary": f"audit.repeated_seed_count ({n_seeds}) is below the minimum of "
+                       f"{MIN_N_SEEDS} needed for a meaningful confidence interval",
+            "details": {},
+        }
+
+    label_col = cfg["schema"]["label_column"]
+
+    # Resume from any seeds already checkpointed (e.g. a prior run of this same
+    # checkpoint_path that crashed partway through) rather than redoing them -
+    # this check alone can dominate a full-scale run's cost, so losing all
+    # progress to a mid-sweep crash is exactly the failure mode worth avoiding.
+    completed: dict[int, float] = (
+        _checkpoint.get_seed_progress(checkpoint_path, _CHECK_NAME) if checkpoint_path else {}
+    )
+    drops = [completed[s] for s in sorted(completed) if s < n_seeds]
+    start_seed = len(drops)
+
+    remaining_seeds = list(range(start_seed, n_seeds))
+    if combined is None and remaining_seeds:
+        combined = dataset.load_raw_combined(dataset_cfg, seed=cfg["random_seed"])
+
+    def run_seed(seed: int) -> float:
+        # Scored one split at a time, not both built up front - see resplit.check's
+        # own comment: holding combined plus both split pairs at once is the
+        # dominant memory cost at full scale, and this runs n_seeds times.
+        random_train, random_test = dataset._random_split(combined, label_col, dataset_cfg, seed=seed)
+        random_acc = fit_and_score(random_train, random_test, label_col, cfg, seed=seed)
+        del random_train, random_test
+
+        grouped_train, grouped_test = dataset._grouped_split(combined, label_col, dataset_cfg, seed=seed)
+        grouped_acc = fit_and_score(grouped_train, grouped_test, label_col, cfg, seed=seed)
+        del grouped_train, grouped_test
+        return random_acc - grouped_acc
+
+    if cfg["audit"]["parallel_checks"] and not checkpoint_path and len(remaining_seeds) > 1:
+        max_workers = min(cfg["audit"]["max_parallel_checks"], len(remaining_seeds))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+            # map preserves input order in its results regardless of completion order,
+            # so drops_by_seed stays seed-ordered the same as the sequential path.
+            drops.extend(pool.map(run_seed, remaining_seeds))
+    else:
+        for seed in remaining_seeds:
+            drop = run_seed(seed)
+            drops.append(drop)
+            if checkpoint_path:
+                _checkpoint.save_seed_progress(checkpoint_path, _CHECK_NAME, seed, drop)
+
+    mean_drop = mean(drops)
+    sample_std = stdev(drops)  # n_seeds >= MIN_N_SEEDS >= 2, safe for stdev's ddof=1
+    # Two-sided 95% CI for the mean via the t-distribution (n_seeds - 1 degrees of
+    # freedom), appropriate for this few independent repeats rather than a normal
+    # approximation that would understate the uncertainty at this sample size.
+    margin = stats.t.ppf(0.975, df=n_seeds - 1) * sample_std / (n_seeds ** 0.5)
+    ci_low, ci_high = mean_drop - margin, mean_drop + margin
+
+    threshold = _materiality.threshold(
+        cfg, "repeated_seed_falsification_check", MATERIAL_DROP_THRESHOLD,
+    )
+    if ci_low > threshold:
+        material, status = True, "flag"
+        verdict = (
+            f"material: even the most conservative estimate in the 95% CI ({ci_low:+.4f}) "
+            f"exceeds the {threshold:+.4f} materiality threshold"
+        )
+    elif ci_high < threshold:
+        material, status = False, "ok"
+        verdict = (
+            f"not material: even the most generous estimate in the 95% CI ({ci_high:+.4f}) "
+            f"stays below the {threshold:+.4f} materiality threshold"
+        )
+    else:
+        material, status = None, "warning"
+        verdict = (
+            f"inconclusive: the 95% CI [{ci_low:+.4f}, {ci_high:+.4f}] straddles the "
+            f"{threshold:+.4f} materiality threshold; {n_seeds} seeds aren't enough to "
+            "tell whether this gap is real or sampling noise"
+        )
+
+    return {
+        "check": _CHECK_NAME,
+        "status": status,
+        "summary": (
+            f"grouped-split accuracy drop across {n_seeds} seeds: mean={mean_drop:+.4f}, "
+            f"95% CI=[{ci_low:+.4f}, {ci_high:+.4f}]; {verdict}"
+        ),
+        "details": {
+            "n_seeds": n_seeds,
+            "drops_by_seed": drops,
+            "mean_drop": mean_drop,
+            "ci_95": [ci_low, ci_high],
+            "materiality_threshold": threshold,
+            "material": material,
+        },
+    }

ids2eval/audit/resplit.py

@@ -0,0 +1,92 @@
+"""Resplit falsification test (paper Section 3.5).
+
+The nearest-neighbor test alone can't distinguish "this class is
+inherently homogeneous" from "a different split would reveal real
+difficulty that random sampling happens to hide", both hypotheses look
+identical under random sampling. Constructing a non-random split along a
+dimension the random split ignores (session/group identity) and
+observing whether accuracy changes breaks that symmetry: if
+session-correlated leakage across the random split's boundary had been
+doing real work, the grouped split should cost measurable accuracy.
+
+Independent of whichever split_mode is configured for the main run:
+always builds both splits from the raw data to compare them directly.
+
+Accepts an already-loaded `combined` DataFrame so run_audit can share one
+load across this, scenario_holdout_falsification, result_robustness_check,
+and repeated_seed_falsification_check when several run together - each
+independently reloading the full raw dataset from disk is a real cost at
+full scale (a real failure this was built to avoid: a 70M-row config
+running all three of these checks was reloading the same data three times
+sequentially, contributing to repeated VM-level instability on a long run).
+Still loads it itself when called standalone (e.g. every existing direct
+unit test), so this is purely an optimization, not a behavior change.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ..data import dataset
+from . import _materiality
+from ._fit_score import fit_and_score
+
+# A grouped-split accuracy drop below this is read as "structurally
+# ruling out session-correlated leakage as the explanation," not proof
+# of zero leakage of any kind. Overridable via audit.materiality_thresholds.
+# resplit_falsification.
+MATERIAL_DROP_THRESHOLD = 0.01
+
+
+def check(cfg: dict, seed: int = 0, combined: pd.DataFrame | None = None) -> dict:
+    dataset_cfg = cfg["dataset"]
+    label_col = cfg["schema"]["label_column"]
+    if combined is None:
+        combined = dataset.load_raw_combined(dataset_cfg, seed=cfg["random_seed"])
+
+    # Scored one split at a time, not both built up front: at full scale (hundreds
+    # of millions of rows) holding combined plus both split pairs simultaneously -
+    # five full-size DataFrames at once - is itself the dominant memory cost, a
+    # real failure this was built to avoid, not a theoretical one. fit_and_score
+    # only needs the float back, so each split's train/test can be freed before
+    # the next is built.
+    random_train, random_test = dataset._random_split(combined, label_col, dataset_cfg, seed=seed)
+    random_acc = fit_and_score(random_train, random_test, label_col, cfg, seed=seed)
+    n_random = len(random_test)
+    del random_train, random_test
+
+    grouped_train, grouped_test = dataset._grouped_split(combined, label_col, dataset_cfg, seed=seed)
+    grouped_acc = fit_and_score(grouped_train, grouped_test, label_col, cfg, seed=seed)
+    n_grouped = len(grouped_test)
+    del grouped_train, grouped_test
+    drop = random_acc - grouped_acc
+
+    threshold = _materiality.threshold(cfg, "resplit_falsification", MATERIAL_DROP_THRESHOLD)
+    # One comparison on finite test sets: a drop only counts as material once it
+    # clears the threshold by more than its own sampling margin. A drop above the
+    # threshold but within that margin is reported as inconclusive (warning).
+    margin = _materiality.accuracy_gap_margin(random_acc, n_random, grouped_acc, n_grouped)
+    if drop - margin > threshold:
+        status, verdict = "flag", (
+            "grouped split costs measurable accuracy beyond test-set sampling noise, "
+            "session-correlated leakage in the random split may be doing real work"
+        )
+    elif drop > threshold:
+        status, verdict = "warning", (
+            f"the drop exceeds the {threshold:.2f} threshold but lies within its sampling margin "
+            f"(+/-{margin:.4f}) at this test size; inconclusive, see repeated_seed_falsification_check"
+        )
+    else:
+        status, verdict = "ok", (
+            "grouped split reproduces random-split accuracy: no evidence that information tied "
+            "to the grouping key drives the result (other grouping keys and shortcuts are not tested)"
+        )
+    summary = (
+        f"random-split accuracy={random_acc:.4f}, grouped-split accuracy={grouped_acc:.4f} "
+        f"(drop={drop:+.4f}); {verdict}"
+    )
+    return {
+        "check": "resplit_falsification", "status": status, "summary": summary,
+        "details": {"random_accuracy": random_acc, "grouped_accuracy": grouped_acc, "drop": drop,
+                    "sampling_margin": margin, "random_test_rows": n_random, "grouped_test_rows": n_grouped},
+    }

ids2eval/audit/result_robustness.py

@@ -0,0 +1,154 @@
+"""Result robustness matrix (v2, opt-in).
+
+Every other check here measures a property of the data. This measures
+something different: how much a benchmark's headline accuracy moves
+across a handful of methodology choices that are each individually
+reasonable, random split vs. a session-grouped split, with vs. without
+exact duplicates, with vs. without declared identity columns, rather
+than assuming a single reported number is the whole story. A result
+that barely moves across these conditions is robust to how it was
+produced; one that swings widely was measuring the methodology as much
+as the attack behavior.
+
+Deliberately reports the full accuracy-by-condition vector rather than
+collapsing it into one fragility score: which specific condition moved
+the result, and by how much, is the useful information, and a single
+number would hide exactly that (the same reasoning class_distribution_
+report and every per-class breakdown in this project already follow).
+
+Builds on infrastructure already in place rather than duplicating it:
+dataset._random_split/_grouped_split (resplit_falsification), dataset.
+dedup (dedup_check), and fit_and_score's column-subset override
+(feature_category_ablation_check). Needs dataset.raw_files, like every
+other check that builds its own comparison splits from the raw data;
+the grouped-split and identity-column conditions are each included only
+when the config actually supports them (dataset.group_columns,
+schema.id_like_columns), so a minimal config still gets at least the
+random-split-vs-dedup comparison.
+
+Every condition's (train, test) pair is built first - relatively cheap next
+to the fit itself - so the actual fit_and_score calls can be dispatched
+together, concurrently via a thread pool when audit.parallel_checks is set.
+
+Accepts an already-loaded `combined` DataFrame so run_audit can share one
+load across this, resplit_falsification, and repeated_seed_falsification_
+check when several run together, instead of each reloading the full raw
+dataset from disk - a real cost at full scale. Still loads it itself when
+called standalone, so this is purely an optimization.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+
+import pandas as pd
+
+from ..data import dataset, features
+from . import _materiality
+from ._fit_score import fit_and_score
+from ._status import NOT_APPLICABLE
+
+# Overridable via audit.materiality_thresholds.result_robustness_check ({"warning": .., "flag": ..}).
+SPREAD_FLAG_THRESHOLD = 0.15
+SPREAD_WARNING_THRESHOLD = 0.08
+
+
+def check(cfg: dict, combined: pd.DataFrame | None = None) -> dict:
+    dataset_cfg = cfg["dataset"]
+    if not dataset_cfg["raw_files"]:
+        return {
+            "check": "result_robustness_check", "status": NOT_APPLICABLE,
+            "summary": "requires dataset.raw_files, not applicable to a pre-split train_file/test_file pair",
+            "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    label_col = cfg["schema"]["label_column"]
+    if combined is None:
+        combined = dataset.load_raw_combined(dataset_cfg, seed=seed)
+
+    # name -> (train_df, test_df, cols_override); built up front so the actual fits
+    # (the expensive part) can all be dispatched together below.
+    pending: dict[str, tuple] = {}
+    random_train, random_test = dataset._random_split(combined, label_col, dataset_cfg, seed=seed)
+    pending["random_split"] = (random_train, random_test, None)
+
+    if dataset_cfg["group_columns"]:
+        try:
+            grouped_train, grouped_test = dataset._grouped_split(combined, label_col, dataset_cfg, seed=seed)
+            pending["grouped_split"] = (grouped_train, grouped_test, None)
+        except (ValueError, RuntimeError):
+            pass  # same graceful-skip as resplit_falsification when a grouped split can't be built
+
+    deduped_train, deduped_test, dedup_stats = dataset.dedup(random_train.copy(), random_test.copy(), cfg)
+    # Only a distinct condition if dedup actually removed something; otherwise it's
+    # an identical refit of the random_split condition, not a second data point.
+    if (
+        dedup_stats["train_rows_raw"] != dedup_stats["train_rows_deduped"]
+        or dedup_stats["test_rows_raw"] != dedup_stats["test_rows_deduped"]
+    ):
+        pending["deduplicated"] = (deduped_train, deduped_test, None)
+
+    id_cols = [c for c in cfg["schema"]["id_like_columns"] if c in random_train.columns]
+    if id_cols:
+        all_cols = features.feature_columns(random_train, cfg)
+        without_ids = [c for c in all_cols if c not in id_cols]
+        if without_ids:
+            pending["identity_columns_dropped"] = (random_train, random_test, without_ids)
+
+    if len(pending) < 2:
+        return {
+            "check": "result_robustness_check", "status": NOT_APPLICABLE,
+            "summary": "fewer than two conditions could be built to compare", "details": {},
+        }
+
+    test_rows = {name: len(test_df) for name, (_, test_df, _) in pending.items()}
+
+    def run(name: str) -> tuple[str, float]:
+        train_df, test_df, cols = pending[name]
+        return name, fit_and_score(train_df, test_df, label_col, cfg, cols=cols, seed=seed)
+
+    if cfg["audit"]["parallel_checks"]:
+        max_workers = min(cfg["audit"]["max_parallel_checks"], len(pending))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+            conditions = dict(pool.map(run, pending))
+    else:
+        conditions = dict(run(name) for name in pending)
+
+    worst_condition, worst_acc = min(conditions.items(), key=lambda kv: kv[1])
+    best_condition, best_acc = max(conditions.items(), key=lambda kv: kv[1])
+    spread = best_acc - worst_acc
+
+    tiers = _materiality.threshold(
+        cfg, "result_robustness_check",
+        {"warning": SPREAD_WARNING_THRESHOLD, "flag": SPREAD_FLAG_THRESHOLD},
+    )
+    # Tiers are judged on the spread minus its sampling margin at these test sizes,
+    # so test-set noise alone can't flag; a spread that crosses a tier only within
+    # that margin is reported as a warning (inconclusive) instead.
+    margin = _materiality.accuracy_gap_margin(
+        best_acc, test_rows[best_condition], worst_acc, test_rows[worst_condition],
+    )
+    if spread - margin > tiers["flag"]:
+        status = "flag"
+    elif spread > tiers["warning"]:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = (
+        f"accuracy ranges from {worst_acc:.4f} ({worst_condition}) to {best_acc:.4f} "
+        f"({best_condition}) across {len(conditions)} conditions, a spread of {spread:.4f} "
+        f"(sampling margin +/-{margin:.4f})"
+    )
+    if status != "ok":
+        summary += "; the headline result depends on methodology choices as much as the classifier"
+
+    return {
+        "check": "result_robustness_check", "status": status, "summary": summary,
+        "details": {
+            "accuracy_by_condition": conditions, "spread": spread,
+            "worst_condition": worst_condition, "best_condition": best_condition,
+            "sampling_margin": margin,
+        },
+    }