ids2eval 0.2.2__py3-none-any.whl

ids2eval/audit/homogeneity.py

@@ -0,0 +1,160 @@
+"""Nearest-neighbor class-homogeneity test (paper Section 3.4).
+
+For each class, compares test-to-train nearest-neighbor distance against
+a *control*, train-internal nearest-neighbor distance, excluding self, via
+the near-zero-distance rate each side shows (the share of rows whose
+nearest neighbor is an effectively-exact feature-space duplicate). If test
+rows are no more prone to this than train rows are to each other,
+near-duplication is a property of the class population itself (present
+within each split independently), not train/test boundary leakage. If
+test rows are notably more prone to it, that's a real leakage signature
+the resplit-falsification check can then corroborate.
+
+Reports a 95% CI on the rate difference (test minus control) against a
+pre-specified margin rather than a single p-value, since failing to detect
+a difference is not the same claim as having shown there is none. Leakage
+is one-sided, so a class is "leakage" when the CI's lower bound exceeds
++margin, cleared when its upper bound is below +margin ("equivalent" if
+the CI also sits inside +/-margin, "below_control" if test rows are
+materially less near-duplicated than train is with itself), and
+"inconclusive" when the CI straddles +margin.
+"""
+
+from __future__ import annotations
+
+import math
+
+import numpy as np
+import pandas as pd
+from sklearn.neighbors import NearestNeighbors
+from sklearn.preprocessing import StandardScaler
+
+from ..data import features
+from . import _materiality
+from ._status import NOT_APPLICABLE
+
+SAMPLE_SIZE = 500
+MIN_CLASS_SIZE = 20
+MAX_INDEX_ROWS = 200_000
+# Post-StandardScaler distance below which two rows are treated as an
+# effectively-exact feature-space duplicate.
+NEAR_ZERO_DISTANCE = 1e-6
+# Equivalence margin on the near-zero-distance *rate difference* (test minus
+# control), on the same 0-1 scale as the rates themselves: if test rows are
+# near-zero-distance to train no more than 5 percentage points more often
+# than train rows are to each other, that gap is read as noise around zero,
+# not a leakage signature - the same order of magnitude as this codebase's
+# other diagnostic materiality thresholds (e.g. resplit_falsification's 0.01
+# accuracy-drop, result_robustness_check's 0.08/0.15 spread tiers), chosen
+# for consistency rather than independently derived. Overridable via
+# audit.materiality_thresholds.homogeneity_test.
+EQUIVALENCE_MARGIN = 0.05
+
+
+_Z = 1.959964  # z_{0.975}
+
+
+def _wilson(p: float, n: int) -> tuple[float, float]:
+    centre = (p + _Z**2 / (2 * n)) / (1 + _Z**2 / n)
+    half = _Z * math.sqrt(p * (1 - p) / n + _Z**2 / (4 * n**2)) / (1 + _Z**2 / n)
+    return centre - half, centre + half
+
+
+def _rate_diff_ci(p_test: float, n_test: int, p_control: float, n_control: int) -> tuple[float, float]:
+    """95% Newcombe hybrid-score CI for the difference of two independent
+    proportions (test rate minus control rate). Unlike the Wald interval it
+    stays a proper interval when a rate is at 0 or 1, which match rates
+    often are here."""
+    diff = p_test - p_control
+    lo_t, hi_t = _wilson(p_test, n_test)
+    lo_c, hi_c = _wilson(p_control, n_control)
+    lower = diff - math.sqrt((p_test - lo_t) ** 2 + (hi_c - p_control) ** 2)
+    upper = diff + math.sqrt((hi_t - p_test) ** 2 + (p_control - lo_c) ** 2)
+    return lower, upper
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    group_col = schema["attack_category_column"] or schema["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+
+    index_df = train_df.sample(n=min(len(train_df), MAX_INDEX_ROWS), random_state=cfg["random_seed"])
+    x_index_raw, x_test_raw = features.encode_aligned(index_df, test_df, cols)
+    scaler = StandardScaler().fit(x_index_raw)
+    x_index = scaler.transform(x_index_raw)
+    x_test = scaler.transform(x_test_raw)
+
+    nn = NearestNeighbors(n_neighbors=2, algorithm="auto", n_jobs=-1).fit(x_index)
+    margin = _materiality.threshold(cfg, "homogeneity_test", EQUIVALENCE_MARGIN)
+
+    per_class = {}
+    flagged_classes = []
+    inconclusive_classes = []
+    for cls, count in train_df[group_col].value_counts().items():
+        test_mask = test_df[group_col] == cls
+        index_mask = (index_df[group_col] == cls).values
+        if count < MIN_CLASS_SIZE or test_mask.sum() < MIN_CLASS_SIZE or index_mask.sum() < MIN_CLASS_SIZE:
+            continue
+
+        test_sample_idx = np.random.RandomState(0).choice(
+            np.flatnonzero(test_mask.values), size=min(SAMPLE_SIZE, test_mask.sum()), replace=False
+        )
+        control_sample_idx = np.random.RandomState(0).choice(
+            np.flatnonzero(index_mask), size=min(SAMPLE_SIZE, index_mask.sum()), replace=False
+        )
+
+        test_dist, _ = nn.kneighbors(x_test[test_sample_idx], n_neighbors=1)
+        control_dist, _ = nn.kneighbors(x_index[control_sample_idx], n_neighbors=2)
+
+        test_dist = test_dist[:, 0]
+        control_dist = control_dist[:, 1]  # nearest neighbor excluding self
+
+        test_near_zero = float((test_dist < NEAR_ZERO_DISTANCE).mean())
+        control_near_zero = float((control_dist < NEAR_ZERO_DISTANCE).mean())
+        n_test, n_control = len(test_dist), len(control_dist)
+        ci_low, ci_high = _rate_diff_ci(test_near_zero, n_test, control_near_zero, n_control)
+
+        # Leakage is one-sided (test closer to train than the control), so the
+        # decision bound is +margin only. A CI entirely below -margin means the
+        # test rows are materially *less* near-duplicated than train is with
+        # itself: decisively not leakage, so it must not land in "inconclusive".
+        if ci_low > margin:
+            classification = "leakage"
+            flagged_classes.append(cls)
+        elif ci_high < margin:
+            classification = "equivalent" if ci_low > -margin else "below_control"
+        else:
+            classification = "inconclusive"
+            inconclusive_classes.append(cls)
+
+        per_class[cls] = {
+            "test_near_zero_rate": test_near_zero,
+            "control_near_zero_rate": control_near_zero,
+            "rate_diff_ci_95": [ci_low, ci_high],
+            "equivalence_margin": margin,
+            "classification": classification,
+        }
+
+    status = "flag" if flagged_classes else ("warning" if inconclusive_classes else "ok")
+    if not per_class:
+        status = NOT_APPLICABLE
+        detail = f"no class has {MIN_CLASS_SIZE} rows in train, test and the control sample, nothing to compare"
+    elif flagged_classes:
+        detail = f"leakage signature (95% CI lower bound exceeds the {margin:+.2f} margin) in: {flagged_classes}"
+    elif inconclusive_classes:
+        detail = (
+            f"inconclusive (95% CI straddles the +/-{margin:.2f} equivalence margin) for: "
+            f"{inconclusive_classes}; sample size isn't enough to tell for these classes"
+        )
+    else:
+        detail = (
+            f"for every class the 95% CI upper bound on (test-to-train minus train-internal) "
+            f"near-duplicate rate is below the +{margin:.2f} margin, so no class shows a "
+            f"train/test leakage signature"
+        )
+    summary = f"{len(per_class)} classes tested; {detail}"
+
+    return {
+        "check": "homogeneity_test", "status": status, "summary": summary,
+        "details": {"per_class": per_class, "equivalence_margin": margin},
+    }

ids2eval/audit/identity_columns.py

@@ -0,0 +1,171 @@
+"""Identity-column checks: standalone predictive power + low-cardinality risk.
+
+Per Sarhan et al. 2022 (NetFlow-standard feature set drops IP/port
+columns) and Kostas et al. 2024/2025 (packet-level IP/port models hit
+near-100% in-dataset, lose >90% cross-dataset): id_like_columns (IP,
+port, MAC) are checked both for how well they predict the label alone
+(identity_column_flag) and for how few unique values they take relative
+to dataset size (low_cardinality_warning generalizes N-BaIoT's
+per-device overfitting finding to CIC/UNSW's limited attacker/victim IP
+pool).
+
+identity_column_flag asks one specific question: on this split, how well
+does looking an identifier value up in the training data predict the
+test label? It builds, from the training partition only, a table of how
+often each identifier value occurs with each class, and predicts a test
+row's class distribution from its value's row of that table (additively
+smoothed toward the training class prior). A test value never seen in
+training gets the training class prior itself, i.e. no information. The
+method uses only equality of values, so it does not depend on any
+ordering or integer coding of the identifiers. It is a lookup, not a
+model fit, so it uses the whole training partition, not a row sample.
+
+Reported per column: seen_coverage (share of test rows whose value occurs
+in training), standalone_auc (weighted one-vs-rest AUC over all test
+rows, unseen ones scored with the prior) and seen_only_auc (the same AUC
+on the seen rows only). The flag is on standalone_auc, which is the
+shortcut's effect on this split as a whole; seen_only_auc shows how
+strong the lookup is where it applies. Whether a column is associated
+with the label for values never seen in training is a different question,
+answered by a split grouped by that column, not by this check.
+
+For a flagged column, details["by_class"] gives the one-vs-rest AUC of the
+same lookup for each class of the attack-category (or label) column, so a
+global AUC of 0.9 that is actually 0.99 for one attack family and 0.5 for
+the rest is visible rather than averaged away.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from . import _by_class
+from ._auc import robust_auc
+from ._status import NOT_APPLICABLE
+
+# Standalone AUC above this on an identity column alone is a specific,
+# checkable topology-shortcut signature, not just "IPs are informative."
+AUC_FLAG_THRESHOLD = 0.8
+# Fewer unique values than this (absolute) suggests the model could be
+# memorizing a handful of attacker/victim hosts rather than a behavior.
+LOW_CARDINALITY_THRESHOLD = 50
+# Past this many classes the per-class breakdown is left out of the report.
+MAX_CLASSES_FOR_BREAKDOWN = 25
+# Pseudo-count of the additive smoothing toward the training class prior: a
+# value seen once in training predicts half its own class, half the prior.
+SMOOTHING = 1.0
+
+
+def _keys(train_col: pd.Series, test_col: pd.Series) -> tuple[np.ndarray, np.ndarray]:
+    # Numeric on both sides: compare as float64, exact for integers such as
+    # ports. Otherwise compare as strings, so train and test still match on
+    # value when one side loaded a column as int and the other as object.
+    if pd.api.types.is_numeric_dtype(train_col) and pd.api.types.is_numeric_dtype(test_col):
+        return train_col.to_numpy(dtype=float), test_col.to_numpy(dtype=float)
+    return train_col.astype(str).to_numpy(), test_col.astype(str).to_numpy()
+
+
+def lookup_proba(train_keys: np.ndarray, train_y, test_keys: np.ndarray):
+    """Class probabilities for each test row from a training-only value table.
+
+    Returns (classes, proba, seen): proba has one column per training class,
+    in sorted class order; seen marks test rows whose value occurs in
+    training. Unseen rows get the training class prior.
+    """
+    key_codes, values = pd.factorize(train_keys, use_na_sentinel=False)
+    classes = np.asarray(sorted(pd.unique(np.asarray(train_y))))
+    y_codes = pd.Index(classes).get_indexer(np.asarray(train_y))
+    k = len(classes)
+    table = np.bincount(key_codes * k + y_codes, minlength=len(values) * k).reshape(len(values), k).astype(float)
+    prior = table.sum(axis=0) / table.sum()
+    smoothed = (table + SMOOTHING * prior) / (table.sum(axis=1, keepdims=True) + SMOOTHING)
+    idx = pd.Index(values).get_indexer(test_keys)
+    seen = idx >= 0
+    proba = np.empty((len(idx), k))
+    proba[seen] = smoothed[idx[seen]]
+    proba[~seen] = prior
+    return classes, proba, seen
+
+
+def check_predictive_power(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    id_cols = [c for c in cfg["schema"]["id_like_columns"] if c in train_df.columns]
+    label_col = cfg["schema"]["label_column"]
+    if not id_cols:
+        return {
+            "check": "identity_column_flag", "status": NOT_APPLICABLE,
+            "summary": "no id_like_columns configured", "details": {},
+        }
+
+    auc_all, auc_seen, coverage, flagged, keys = {}, {}, {}, [], {}
+    y_test = test_df[label_col].to_numpy()
+    for col in id_cols:
+        train_keys, test_keys = _keys(train_df[col], test_df[col])
+        keys[col] = (train_keys, test_keys)
+        classes, proba, seen = lookup_proba(train_keys, train_df[label_col], test_keys)
+        coverage[col] = float(seen.mean()) if len(seen) else 0.0
+        auc_all[col] = robust_auc(y_test, proba, classes)
+        auc_seen[col] = robust_auc(y_test[seen], proba[seen], classes) if seen.any() else None
+        if auc_all[col] is not None and auc_all[col] > AUC_FLAG_THRESHOLD:
+            flagged.append(col)
+
+    def fmt(a):
+        return None if a is None else round(a, 3)
+
+    status = "flag" if flagged else "ok"
+    summary = "standalone AUC by column (seen-value coverage, AUC on seen values): " + ", ".join(
+        f"{c}: {fmt(auc_all[c])} ({coverage[c]:.1%}, {fmt(auc_seen[c])})" for c in id_cols
+    )
+    if flagged:
+        summary += f". Suggest dropping: {flagged}"
+
+    details = {"standalone_auc": auc_all, "seen_only_auc": auc_seen, "seen_coverage": coverage,
+               "suggested_drop": flagged, "method": "training-only value lookup, unseen values scored with the prior"}
+    if flagged:
+        details["by_class"] = {col: _by_class_auc(train_df, test_df, *keys[col], cfg) for col in flagged}
+
+    return {"check": "identity_column_flag", "status": status, "summary": summary, "details": details}
+
+
+def _by_class_auc(train_df: pd.DataFrame, test_df: pd.DataFrame, train_keys, test_keys, cfg: dict) -> dict | str:
+    group_col = _by_class.group_column(cfg)
+    classes = _by_class.eligible_classes(train_df, group_col)
+    if len(classes) > MAX_CLASSES_FOR_BREAKDOWN:
+        return f"skipped, {len(classes)} eligible classes exceeds the {MAX_CLASSES_FOR_BREAKDOWN}-class cap"
+
+    table_classes, proba, _ = lookup_proba(train_keys, train_df[group_col], test_keys)
+    position = {c: i for i, c in enumerate(table_classes)}
+    y_test = test_df[group_col].to_numpy()
+    by_class = {}
+    for cls in classes:
+        y_binary = y_test == cls
+        if not y_binary.any() or y_binary.all():
+            continue
+        score = proba[:, position[cls]]
+        by_class[str(cls)] = robust_auc(y_binary, np.column_stack([1 - score, score]), np.array([False, True]))
+    return by_class
+
+
+def check_cardinality(train_df: pd.DataFrame, cfg: dict) -> dict:
+    id_cols = [c for c in cfg["schema"]["id_like_columns"] if c in train_df.columns]
+    if not id_cols:
+        return {
+            "check": "low_cardinality_warning", "status": NOT_APPLICABLE,
+            "summary": "no id_like_columns configured", "details": {},
+        }
+
+    cardinalities = {c: int(train_df[c].nunique()) for c in id_cols}
+    flagged = [c for c, n in cardinalities.items() if n < LOW_CARDINALITY_THRESHOLD]
+
+    status = "warning" if flagged else "ok"
+    summary = f"unique values by column: {cardinalities}"
+    if flagged:
+        summary += (
+            f". {flagged} have fewer than {LOW_CARDINALITY_THRESHOLD} unique values, "
+            "risk of memorizing specific hosts rather than learning attack behavior"
+        )
+
+    return {
+        "check": "low_cardinality_warning", "status": status, "summary": summary,
+        "details": {"cardinalities": cardinalities, "flagged": flagged},
+    }

ids2eval/audit/known_issues.py

@@ -0,0 +1,144 @@
+"""Known-issue lookup (v2, opt-in): curated per-dataset documented problems.
+
+Unlike schema_fingerprint_check (which matches the *extractor tool*'s
+column-naming signature regardless of dataset), this matches on
+dataset.name directly and surfaces problems specific to that dataset,
+curation work, not a new algorithm. Needs no reference_dataset.
+
+Extend KNOWN_ISSUES as more curated, citable findings are gathered.
+Matching is substring-based against a lowercased dataset.name so
+"cic-ids2018", "CIC_IDS2018", and "cic2018" all resolve the same way.
+Matching by name cannot tell one release of a dataset from another (an original
+CSV release from a corrected re-extraction of the same traffic), so an entry that
+documents one release says which in its "release" field, and the summary prints
+it; whether the issue applies to the audited files is for the reader to check.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+KNOWN_ISSUES = {
+    "cic-ids2018": [
+        {
+            "issue": "In the original release, 172 of the 362 flows labeled Brute Force-Web "
+                     "on 2018-02-23 are not part of the attack: 21 have malformed addresses "
+                     "and 151 (41.7%) are background traffic, 128 of them UDP (126 to port 500) "
+                     "(a ground-truth labeling error, not an extraction or pipeline bug).",
+            "citation": "Liu et al. 2022, IEEE CNS, companion documentation",
+            "release": "the original CSE-CIC-IDS2018 CSVs (UNB/AWS)",
+        },
+        {
+            "issue": "A full re-labeling of the original release found 7.53% of labels wrong, "
+                     "with the error rate above 75% for some attack classes.",
+            "citation": "Liu et al. 2022, IEEE CNS (rates as summarized by Cantone et al. 2024)",
+            "release": "the original CSE-CIC-IDS2018 CSVs (UNB/AWS)",
+        },
+    ],
+    "cic-ids2017": [
+        {
+            "issue": "The traffic capture has packet misorder and duplication, and some "
+                     "attacks that were actually launched are not correctly labeled as "
+                     "attack traffic in the released CSVs.",
+            "citation": "Engelen et al. 2021, IEEE S&P Workshops (WTMC)",
+            "release": "the original CIC-IDS2017 CSVs (UNB)",
+        },
+        {
+            "issue": "A full re-labeling of the original release found 6.67% of labels wrong, "
+                     "with the error rate above 75% for some attack classes.",
+            "citation": "Liu et al. 2022, IEEE CNS (rates as summarized by Cantone et al. 2024)",
+            "release": "the original CIC-IDS2017 CSVs (UNB)",
+        },
+        {
+            "issue": "Heartbleed has only 11 rows (about 0.0004% of the dataset), too few to "
+                     "evaluate reliably regardless of labeling accuracy.",
+            "citation": "Cantone et al. 2024, IEEE Access, Table 1",
+            "release": "the original CIC-IDS2017 CSVs and the corrected re-extraction alike",
+        },
+    ],
+    "nsl-kdd": [
+        {
+            "issue": "The test set intentionally includes attack types absent from "
+                     "the training set, by design, to test generalization to unknown "
+                     "attacks rather than memorization; this project's own audit of "
+                     "the official partition finds 17 such test-only attack types.",
+            "citation": "Tavallaee et al. 2009, IEEE CISDA",
+        },
+    ],
+    "bot-iot": [
+        {
+            "issue": "The NF-BoT-IoT-V2 NetFlow conversion is over 99.9% attack "
+                     "traffic (benign flows are 129,437 of 30,420,086 rows, 0.43%), "
+                     "an order of magnitude more skewed than most NIDS datasets; a "
+                     "uniform row-level sample may carry very few or zero benign "
+                     "rows, and a classifier's accuracy on this data mostly reflects "
+                     "its attack-vs-attack discrimination, not its ability to "
+                     "recognize normal traffic.",
+            "citation": "Koroniotis et al. 2019, Future Generation Computer Systems",
+        },
+    ],
+    "bot-iot-official": [
+        {
+            "issue": "The full original (non-NetFlow) release is over 99.9% attack "
+                     "traffic (benign flows are 477 of 3,668,522 rows, 0.013%), even "
+                     "more skewed than the NetFlow-V2 conversion of this dataset; a "
+                     "classifier's accuracy on this data mostly reflects its "
+                     "attack-vs-attack discrimination, not its ability to recognize "
+                     "normal traffic.",
+            "citation": "Koroniotis et al. 2019, Future Generation Computer Systems",
+        },
+    ],
+    "cic-ddos2019": [
+        {
+            "issue": "By design, the two collection days cover different attack sets: "
+                     "the testing day includes several DDoS types absent from the "
+                     "training day (e.g. DNS, NTP, SNMP, TFTP, WebDDoS reflection "
+                     "attacks), so a classifier trained only on the training day has "
+                     "never seen those attack types.",
+            "citation": "Sharafaldin et al. 2019, IEEE ICCST",
+        },
+        {
+            "issue": "The same attack type is labeled inconsistently between the two "
+                     "days (e.g. training's 'UDPLag'/'MSSQL' vs. testing's 'UDP-lag'/"
+                     "'DrDoS_MSSQL'), so label strings must be remapped before any "
+                     "train/test comparison, not matched literally.",
+            "citation": "Independently verified in this project's own audit methodology",
+        },
+    ],
+    "unsw-nb15": [
+        {
+            "issue": "Some redistributions invert the UNSW_NB15_training-set.csv/"
+                     "testing-set.csv file-name-to-content mapping relative to the "
+                     "dataset's documented convention (82,332 vs. 175,341 rows), "
+                     "verify against the published per-attack-category counts before "
+                     "trusting file names.",
+            "citation": "Independently verified in this project's own audit methodology",
+        },
+    ],
+}
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    name = (cfg["dataset"]["name"] or "").strip().lower()
+    matched_keys = [key for key in KNOWN_ISSUES if key in name]
+    # A key that is itself a substring of another matched key (e.g. "bot-iot" inside
+    # "bot-iot-official") is a less specific match for the same dataset name and is
+    # dropped in favor of the more specific one, so variant-specific curated numbers
+    # don't get mixed with a different variant's numbers under the same lookup.
+    matched_keys = [
+        key for key in matched_keys
+        if not any(key != other and key in other for other in matched_keys)
+    ]
+    matches = [issue for key in matched_keys for issue in KNOWN_ISSUES[key]]
+
+    status = "warning" if matches else "ok"
+    # The issue text itself, not just a count - a scorecard reader shouldn't have to
+    # go open audit_report_*.json to find out what the curated problem actually is.
+    summary = (
+        "; ".join(
+            f"{m['issue']} ({'applies to ' + m['release'] + '; ' if m.get('release') else ''}{m['citation']})"
+            for m in matches
+        ) if matches
+        else f"no curated known issues for dataset '{name}'"
+    )
+    return {"check": "known_issue_lookup", "status": status, "summary": summary, "details": {"matches": matches}}

ids2eval/audit/label_conflict.py

@@ -0,0 +1,92 @@
+"""Label-conflict check: identical features, contradictory labels.
+
+Independently motivated from three directions: Deepchecks' "Conflicting
+Labels" check (general ML), Northcutt et al. 2021's finding that label
+errors average 3.3% across major ML benchmarks, and Wu & Keogh 2021's
+"mislabeled ground truth" flaw in time-series anomaly benchmarks. A feature
+vector mapped to more than one label is a stronger, more specific claim
+than a duplicate. The ground truth contradicts itself, and no model
+can get both instances right regardless of how well it memorizes.
+
+Deliberately scoped like dedup_check: it must run on the RAW data, before
+preprocessing.dedup. dataset.dedup() already compares on features only
+(ignoring the label column, see dataset.dedup's own docstring), so a
+conflicting-label group is exactly the kind of "duplicate" it collapses to
+one arbitrarily-kept row - after dedup, every feature vector maps to
+exactly one label by construction, and this check will correctly report
+zero. That's not the conflict being fixed, just made unobservable; the
+"before" pass is what actually shows it existed.
+
+Also attaches a per-class breakdown (details["by_class"]) when flagged:
+the global conflict rate can be driven almost entirely by one attack
+category, invisible in the aggregate count alone.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from . import _by_class
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    label_col = schema["label_column"]
+    group_col = _by_class.group_column(cfg)
+    ignore = set(schema["drop_columns"]) | {label_col}
+    if schema["attack_category_column"]:
+        ignore.add(schema["attack_category_column"])
+    compare_cols = [c for c in train_df.columns if c not in ignore]
+    carry_cols = [label_col] if group_col == label_col else [label_col, group_col]
+
+    combined = pd.concat(
+        [
+            train_df[[*compare_cols, *carry_cols]].assign(_split="train"),
+            test_df[[*compare_cols, *carry_cols]].assign(_split="test"),
+        ],
+        ignore_index=True,
+    )
+    # Group on a single hashed key rather than all compare_cols directly: pandas'
+    # groupby builds an internal combined index proportional to the number of key
+    # columns, which turned out to actually exhaust memory on a real ~2.9M-row,
+    # 78-feature run (CIC-IDS2017) - a real OOM kill, not a hypothetical concern.
+    # Same hash function dataset.py's own content fingerprint already relies on for
+    # an equally big-stakes purpose (proving two runs used identical data), so a
+    # collision here is the same astronomically unlikely event, not a new risk.
+    combined["_key"] = pd.util.hash_pandas_object(combined[compare_cols], index=False).to_numpy()
+    combined["_n_labels"] = combined.groupby("_key", sort=False)[label_col].transform("nunique")
+    conflicting = combined[combined["_n_labels"] > 1]
+
+    conflicting_rows = len(conflicting)
+    if conflicting_rows == 0:
+        return {
+            "check": "label_conflict_check", "status": "ok",
+            "summary": "no feature vector maps to more than one label", "details": {},
+        }
+
+    conflicting_groups = int(conflicting["_key"].nunique())
+    cross_split = int((conflicting.groupby("_key", sort=False)["_split"].nunique() > 1).sum())
+    pct = conflicting_rows / len(combined)
+
+    summary = (
+        f"{conflicting_groups:,} feature vector(s) ({conflicting_rows:,} rows, {pct:.2%}) map to more than "
+        f"one label. The ground truth contradicts itself for these rows"
+    )
+    if cross_split:
+        summary += f"; {cross_split:,} of these span train and test"
+
+    conflicting_mask = (combined["_n_labels"] > 1).to_numpy()
+    by_class = {}
+    for cls in _by_class.eligible_classes(combined, group_col):
+        cls_mask = (combined[group_col] == cls).to_numpy()
+        by_class[str(cls)] = {"conflicting_rate": float(conflicting_mask[cls_mask].mean())}
+
+    return {
+        "check": "label_conflict_check", "status": "flag", "summary": summary,
+        "details": {
+            "conflicting_groups": conflicting_groups,
+            "conflicting_rows": conflicting_rows,
+            "cross_split_conflicting_groups": cross_split,
+            "by_class": by_class,
+        },
+    }

ids2eval/audit/leakage.py

@@ -0,0 +1,99 @@
+"""Feature-importance-driven leakage screen (paper Section 3.2).
+
+Fits a fast RandomForest, inspects its own feature_importances_ rather
+than guessing candidate leakage columns from domain knowledge, and flags
+a single feature or feature pair carrying a large majority of total
+split importance. Same method that found real leakage features in
+CIC-IDS2018 (Dst Port, Init Win Bytes, FIN Flag Count).
+
+When flagged, also attaches a per-class breakdown (details["by_class"]):
+one-vs-rest AUC of the flagged feature's raw values against each class,
+via roc_auc_score directly rather than fitting one classifier per class -
+a numeric feature's own values already are the score, no model needed.
+A class where the feature separates as well as the global fit and one
+where it doesn't tells a different story than the aggregate hides.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import roc_auc_score
+
+from ..data import features
+from . import _by_class
+from ._auc import robust_auc
+
+# A single feature carrying more than this share of total split
+# importance is flagged as a specific, checkable shortcut-learning signature.
+TOP1_SHARE_THRESHOLD = 0.5
+TOP2_SHARE_THRESHOLD = 0.7
+MAX_FIT_ROWS = 200_000
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict, seed: int = 0) -> dict:
+    """seed varies the sample draw and the classifier's own randomness. The
+    default single run is called with cfg["random_seed"] (run_audit's job);
+    seed_sensitivity_check separately sweeps its own fixed range of seeds to
+    test whether the flag/ok conclusion holds up regardless of that one value.
+    """
+    label_col = cfg["schema"]["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=seed)
+    test_fit = test_df
+
+    x_train, x_test = features.encode_aligned(train_fit, test_fit, cols, unseen="missing")
+    y_train, y_test = train_fit[label_col], test_fit[label_col]
+
+    clf = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1)
+    clf.fit(x_train, y_train)
+
+    importances = pd.Series(clf.feature_importances_, index=cols).sort_values(ascending=False)
+    top1_share = float(importances.iloc[0])
+    top2_share = float(importances.iloc[:2].sum())
+
+    flagged = top1_share > TOP1_SHARE_THRESHOLD or top2_share > TOP2_SHARE_THRESHOLD
+    details = {
+        "top_features": importances.head(5).to_dict(),
+        "top1_share": top1_share,
+        "top2_share": top2_share,
+    }
+
+    if flagged:
+        top_feature = importances.index[0]
+        standalone_auc = _standalone_auc(x_train[[top_feature]], y_train, x_test[[top_feature]], y_test, seed)
+        details["standalone_auc"] = {top_feature: standalone_auc}
+        details["by_class"] = _by_class_auc(train_fit, x_train[top_feature].to_numpy(), cfg)
+        auc_text = "n/a" if standalone_auc is None else f"{standalone_auc:.3f}"
+        summary = (
+            f"'{top_feature}' carries {top1_share:.1%} of total split importance "
+            f"(standalone AUC={auc_text}). Check whether it's a leakage "
+            f"artifact rather than attack-behavior signal"
+        )
+        status = "flag"
+    else:
+        summary = f"no single feature or pair dominates importance (top1={top1_share:.1%}, top2={top2_share:.1%})"
+        status = "ok"
+
+    return {"check": "leakage_screen", "status": status, "summary": summary, "details": details}
+
+
+def _standalone_auc(x_train_col, y_train, x_test_col, y_test, seed: int = 0) -> float | None:
+    clf = RandomForestClassifier(n_estimators=50, random_state=seed, n_jobs=-1)
+    clf.fit(x_train_col, y_train)
+    return robust_auc(y_test, clf.predict_proba(x_test_col), clf.classes_)
+
+
+def _by_class_auc(train_fit: pd.DataFrame, feature_values, cfg: dict) -> dict:
+    group_col = _by_class.group_column(cfg)
+    by_class = {}
+    for cls in _by_class.eligible_classes(train_fit, group_col):
+        y_binary = (train_fit[group_col] == cls).to_numpy()
+        if not y_binary.any() or y_binary.all():
+            continue
+        auc = roc_auc_score(y_binary, feature_values)
+        # A feature can separate a class either by high values or low values -
+        # max() with the inverse reports separability regardless of direction,
+        # same convention as a rule-based threshold could use either side.
+        by_class[str(cls)] = float(max(auc, 1 - auc))
+    return by_class