ids2eval 0.2.2__py3-none-any.whl

ids2eval/audit/class_distribution.py

@@ -0,0 +1,57 @@
+"""Class-distribution / imbalance report.
+
+Universal IDS-benchmark pattern (KDD99 U2R/R2L <1-2%, BoT-IoT 0.01%
+benign, TON_IoT 3.56% benign). Reported unconditionally, not only when
+it crosses some threshold, since even a "known" imbalance shifts which
+metrics (accuracy vs. F1/AUC) are meaningful for a given dataset.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+# A minority class below this share of a split is called out explicitly,
+# rather than just left to be read off the full per-class table.
+RARE_CLASS_THRESHOLD = 0.01
+
+# 100:1 sits at the boundary of what general ML data-quality literature calls
+# "moderate" imbalance (roughly 1:4-1:100), with "extreme" starting around
+# 1:1000-1:5000 (Zhou, Tu, Sha, Ding & Chen 2024, arXiv:2406.19614, Table II) -
+# not an arbitrary cutoff, though NIDS-specific imbalance can be far worse than
+# either range in practice (KDD99's U2R/R2L classes, BoT-IoT's 0.01% benign).
+
+
+def _distribution(series: pd.Series) -> dict:
+    counts = series.value_counts()
+    shares = (counts / counts.sum()).to_dict()
+    return {"counts": counts.to_dict(), "shares": shares}
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    label_col = cfg["schema"]["label_column"]
+    train_dist = _distribution(train_df[label_col])
+    test_dist = _distribution(test_df[label_col])
+
+    train_counts = train_dist["counts"]
+    imbalance_ratio = max(train_counts.values()) / max(min(train_counts.values()), 1)
+    rare_classes = [c for c, s in train_dist["shares"].items() if s < RARE_CLASS_THRESHOLD]
+
+    status = "warning" if (imbalance_ratio > 100 or rare_classes) else "ok"
+    summary = (
+        f"{len(train_counts)} classes, train imbalance ratio (majority:minority) "
+        f"= {imbalance_ratio:.0f}:1"
+    )
+    if rare_classes:
+        summary += f"; classes below {RARE_CLASS_THRESHOLD:.0%} of train: {rare_classes}"
+
+    return {
+        "check": "class_distribution_report",
+        "status": status,
+        "summary": summary,
+        "details": {
+            "train": train_dist,
+            "test": test_dist,
+            "imbalance_ratio": imbalance_ratio,
+            "rare_classes": rare_classes,
+        },
+    }

ids2eval/audit/cross_capture_matrix.py

@@ -0,0 +1,115 @@
+"""Cross-capture generalization matrix (v2, opt-in).
+
+scenario_holdout_falsification reports one number: accuracy when the
+single smallest scenario is held out entirely. That can't distinguish
+"generalization to any unseen scenario is genuinely hard here" from
+"this one specific pair of scenarios is asymmetric": training on
+scenario A and testing on B might collapse while the reverse does not,
+invisible in a single holdout figure. This trains on each declared
+scenario (schema.scenario_column) and tests on every other one,
+producing the full N x N accuracy matrix instead.
+
+Real, not negligible, compute: N*(N-1) RandomForest fits rather than
+scenario_holdout_falsification's one, so this is opt-in and capped at
+MAX_SCENARIOS scenarios, skipped rather than run past that cap. Same
+raw-data reload as resplit_falsification/scenario_holdout_falsification,
+so its result cannot depend on preprocessing.dedup either.
+"""
+
+from __future__ import annotations
+
+from ..data import dataset
+from . import _materiality
+from ._fit_score import fit_and_score
+from ._status import NOT_APPLICABLE
+
+MAX_SCENARIOS = 6
+MIN_SCENARIO_ROWS = 20
+# Overridable via audit.materiality_thresholds.cross_capture_matrix_check ({"warning": .., "flag": ..}).
+SPREAD_FLAG_THRESHOLD = 0.20
+SPREAD_WARNING_THRESHOLD = 0.10
+
+
+def check(cfg: dict) -> dict:
+    dataset_cfg = cfg["dataset"]
+    scenario_col = cfg["schema"]["scenario_column"]
+    if not scenario_col:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.scenario_column configured", "details": {},
+        }
+    if not dataset_cfg["raw_files"]:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": "requires dataset.raw_files, not applicable to a pre-split train_file/test_file pair",
+            "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    label_col = cfg["schema"]["label_column"]
+    combined = dataset.load_raw_combined(dataset_cfg, seed=seed)
+    if scenario_col not in combined.columns:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": f"'{scenario_col}' not found in the loaded data", "details": {},
+        }
+
+    counts = combined[scenario_col].value_counts()
+    eligible = counts[counts >= MIN_SCENARIO_ROWS].index.tolist()
+    if len(eligible) < 2:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": "fewer than two scenarios with enough rows to build a matrix", "details": {},
+        }
+    if len(eligible) > MAX_SCENARIOS:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": f"skipped, {len(eligible)} eligible scenarios exceeds the {MAX_SCENARIOS}-scenario cap",
+            "details": {},
+        }
+
+    matrix = {}
+    for train_scenario in eligible:
+        train_part = combined[combined[scenario_col] == train_scenario]
+        if train_part[label_col].nunique() < 2:
+            continue
+        for test_scenario in eligible:
+            if test_scenario == train_scenario:
+                continue
+            test_part = combined[combined[scenario_col] == test_scenario]
+            matrix[f"{train_scenario}->{test_scenario}"] = fit_and_score(
+                train_part, test_part, label_col, cfg, seed=seed
+            )
+
+    if not matrix:
+        return {
+            "check": "cross_capture_matrix_check", "status": NOT_APPLICABLE,
+            "summary": "no scenario pair had enough distinct labels to test", "details": {},
+        }
+
+    worst_pair, worst_acc = min(matrix.items(), key=lambda kv: kv[1])
+    best_pair, best_acc = max(matrix.items(), key=lambda kv: kv[1])
+    spread = best_acc - worst_acc
+
+    tiers = _materiality.threshold(
+        cfg, "cross_capture_matrix_check",
+        {"warning": SPREAD_WARNING_THRESHOLD, "flag": SPREAD_FLAG_THRESHOLD},
+    )
+    if spread > tiers["flag"]:
+        status = "flag"
+    elif spread > tiers["warning"]:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = (
+        f"cross-scenario accuracy ranges from {worst_acc:.4f} ({worst_pair}) to "
+        f"{best_acc:.4f} ({best_pair}), a spread of {spread:.4f} across {len(matrix)} scenario pairs"
+    )
+    if status != "ok":
+        summary += "; generalization across scenarios is uneven rather than uniformly hard or easy"
+
+    return {
+        "check": "cross_capture_matrix_check", "status": status, "summary": summary,
+        "details": {"matrix": matrix, "worst_pair": worst_pair, "best_pair": best_pair, "spread": spread},
+    }

ids2eval/audit/cross_dataset_drift.py

@@ -0,0 +1,74 @@
+"""Cross-dataset drift check (v2, opt-in).
+
+Trains on this dataset and evaluates on a reference dataset
+(audit.reference_dataset) in a unified feature space, comparing accuracy
+against the same model's in-dataset test accuracy. A large drop is a
+specific, checkable signature that the model learned dataset-specific
+artifacts rather than generalizable attack behavior, the pattern
+documented across BoT-IoT/TON_IoT/UNSW-NB15 cross-dataset evaluations.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+
+from ..data import chunked_io, features
+from . import _materiality
+
+MAX_FIT_ROWS = 200_000
+# An accuracy drop beyond this crossing datasets is read as evidence of
+# dataset-specific overfitting rather than ordinary generalization noise.
+# Overridable via audit.materiality_thresholds.cross_dataset_drift_check.
+MATERIAL_DROP_THRESHOLD = 0.10
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    dataset_cfg = cfg["dataset"]
+    ref_df = chunked_io.load_file(
+        cfg["audit"]["reference_dataset"], dataset_cfg["chunk_size"], dataset_cfg["max_rows"]
+    )
+
+    label_col = cfg["schema"]["label_column"]
+    if label_col not in ref_df.columns:
+        return {
+            "check": "cross_dataset_drift_check", "status": "warning",
+            "summary": f"reference_dataset has no '{label_col}' column, cannot compute cross-dataset accuracy",
+            "details": {},
+        }
+
+    cols = [c for c in features.feature_columns(train_df, cfg) if c in ref_df.columns]
+    if not cols:
+        return {
+            "check": "cross_dataset_drift_check", "status": "warning",
+            "summary": "no feature columns in common with reference_dataset", "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=seed)
+    x_train, (x_test, x_ref) = features.encode_multi(train_fit, [test_df, ref_df], cols, unseen="missing")
+
+    clf = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1)
+    clf.fit(x_train, train_fit[label_col])
+
+    in_dataset_acc = float(accuracy_score(test_df[label_col], clf.predict(x_test)))
+    cross_dataset_acc = float(accuracy_score(ref_df[label_col], clf.predict(x_ref)))
+    drop = in_dataset_acc - cross_dataset_acc
+
+    flagged = drop > _materiality.threshold(cfg, "cross_dataset_drift_check", MATERIAL_DROP_THRESHOLD)
+    status = "flag" if flagged else "ok"
+    summary = (
+        f"in-dataset accuracy={in_dataset_acc:.4f}, cross-dataset (reference_dataset) "
+        f"accuracy={cross_dataset_acc:.4f} (drop={drop:+.4f}); "
+        + (
+            "material drop crossing datasets, model likely learned dataset-specific "
+            "artifacts rather than generalizable attack behavior"
+            if flagged else
+            "no material drop crossing datasets"
+        )
+    )
+    return {
+        "check": "cross_dataset_drift_check", "status": status, "summary": summary,
+        "details": {"in_dataset_accuracy": in_dataset_acc, "cross_dataset_accuracy": cross_dataset_acc, "drop": drop},
+    }

ids2eval/audit/data_integrity.py

@@ -0,0 +1,67 @@
+"""Basic dataset sanity checks: missing labels, constant features, ±inf values.
+
+Complements leakage_screen/identity_column_flag (which catch features
+that are *too* informative) by catching the opposite class of problem:
+features carrying no information at all, or values that will silently
+corrupt downstream numeric coercion if left unreported. Structural
+problems severe enough to make the dataset unusable (duplicate column
+names, a train/test schema that shares no feature columns) are handled
+separately in dataset.validate_loaded() as hard failures, not here:
+this check only reports things worth a human's attention, not things
+that make the rest of the pipeline meaningless.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from ..data import features
+
+# A feature with fewer unique values than this (or zero variance for a
+# numeric column) is flagged as carrying little-to-no signal.
+CONSTANT_NUNIQUE_THRESHOLD = 1
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    label_col = schema["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+
+    missing_label = int(train_df[label_col].isna().sum())
+    missing_by_feature = {
+        c: int(n) for c in cols if (n := train_df[c].isna().sum()) > 0
+    }
+
+    inf_by_feature = {}
+    constant_features = []
+    for c in cols:
+        if pd.api.types.is_numeric_dtype(train_df[c]):
+            n_inf = int(np.isinf(train_df[c].to_numpy(dtype="float64", na_value=0.0)).sum())
+            if n_inf:
+                inf_by_feature[c] = n_inf
+        if train_df[c].nunique(dropna=True) <= CONSTANT_NUNIQUE_THRESHOLD:
+            constant_features.append(c)
+
+    issues = []
+    if missing_label:
+        issues.append(f"{missing_label} missing label value(s)")
+    if missing_by_feature:
+        issues.append(f"missing values in {len(missing_by_feature)} feature column(s)")
+    if inf_by_feature:
+        issues.append(f"+-inf values in {len(inf_by_feature)} feature column(s)")
+    if constant_features:
+        issues.append(f"{len(constant_features)} constant/near-constant feature(s)")
+
+    status = "flag" if missing_label else ("warning" if issues else "ok")
+    summary = "; ".join(issues) if issues else "no missing labels, constant features, or +-inf values found"
+
+    return {
+        "check": "data_integrity_check", "status": status, "summary": summary,
+        "details": {
+            "missing_label_count": missing_label,
+            "missing_by_feature": missing_by_feature,
+            "inf_by_feature": inf_by_feature,
+            "constant_features": constant_features,
+        },
+    }

ids2eval/audit/dedup.py

@@ -0,0 +1,67 @@
+"""Report exact + feature-space duplicate counts, without mutating the input.
+
+Runs even when preprocessing.dedup is false, so users see how much
+duplication is present before deciding whether to remove it.
+
+Also attaches a per-class breakdown (details["by_class"]): the global
+rate can look mild while one attack family carries almost all of it.
+Recomputes the same duplicate/leakage masks dataset.dedup() computes
+rather than calling it a second time, since dataset.dedup() drops rows
+and returns aggregate counts only, not a per-row mask to group by class.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ..data import dataset
+from . import _by_class
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    _, _, stats = dataset.dedup(train_df.copy(), test_df.copy(), cfg)
+
+    total_dropped = (
+        stats["train_duplicates_dropped"]
+        + stats["test_leakage_dropped"]
+        + stats["test_duplicates_dropped"]
+    )
+    status = "flag" if stats["test_leakage_dropped"] > 0 else (
+        "warning" if total_dropped > 0 else "ok"
+    )
+    summary = (
+        f"train duplicates: {stats['train_duplicates_dropped']:,} "
+        f"({stats['train_duplicates_dropped'] / max(stats['train_rows_raw'], 1):.2%}); "
+        f"test rows leaking a train feature-match: {stats['test_leakage_dropped']:,} "
+        f"({stats['test_leakage_dropped'] / max(stats['test_rows_raw'], 1):.2%}); "
+        f"test-internal duplicates: {stats['test_duplicates_dropped']:,}"
+    )
+    details = {**stats, "by_class": _by_class_rates(train_df, test_df, cfg)}
+    return {"check": "dedup_check", "status": status, "summary": summary, "details": details}
+
+
+def _by_class_rates(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    ignore = set(schema["drop_columns"]) | {schema["label_column"]}
+    if schema["attack_category_column"]:
+        ignore.add(schema["attack_category_column"])
+    compare_cols = [c for c in train_df.columns if c not in ignore]
+
+    train_dup_mask = train_df.duplicated(subset=compare_cols).to_numpy()
+    merged = test_df.merge(
+        train_df[compare_cols].drop_duplicates(), on=compare_cols, how="left", indicator=True
+    )
+    test_leak_mask = (merged["_merge"] == "both").to_numpy()
+
+    group_col = _by_class.group_column(cfg)
+    by_class = {}
+    for cls in _by_class.eligible_classes(train_df, group_col):
+        cls_train_mask = (train_df[group_col] == cls).to_numpy()
+        cls_test_mask = (test_df[group_col] == cls).to_numpy()
+        by_class[str(cls)] = {
+            "train_duplicate_rate": float(train_dup_mask[cls_train_mask].mean()),
+            "test_leak_rate": (
+                float(test_leak_mask[cls_test_mask].mean()) if cls_test_mask.any() else None
+            ),
+        }
+    return by_class

ids2eval/audit/feature_auc_ranking.py

@@ -0,0 +1,125 @@
+"""All-feature standalone AUC ranking.
+
+identity_column_flag, port_protocol_shortcut_check, and
+temporal_leakage_check each test one declared column (or pair) for
+standalone predictive power, via a model fit, since a small classifier
+is needed to turn an arbitrary categorical column into a score. This
+check ranks every NUMERIC feature the same way, but without fitting
+anything: a numeric column's own raw values already are a score, so
+its one-vs-rest AUC against each class is a direct roc_auc_score call.
+Categorical columns are not covered here (their category codes carry
+no meaningful order, unlike a numeric value), the model-based checks
+already cover those.
+
+Deliberately a higher flag threshold than identity_column_flag's 0.8:
+this ranks every behavioral feature too, ones expected to correlate
+with the label to some degree on their own, so only a near-perfect
+single feature is a specific, checkable oddity worth a flag.
+"""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+import pandas as pd
+from scipy.stats import rankdata
+
+from ..data import features
+from ._status import NOT_APPLICABLE
+
+AUC_FLAG_THRESHOLD = 0.95
+AUC_WARNING_THRESHOLD = 0.90
+TOP_N = 5
+
+
+def _best_auc(values, labels: pd.Series):
+    """The best one-vs-rest AUC this column reaches against any single
+    class, and which class, or None if no class has both members present.
+
+    Each class's AUC is the Mann-Whitney statistic from one ranking of the
+    column (average ranks for ties, which is how ROC AUC scores ties), so
+    the column is sorted once rather than once per class: equal to
+    roc_auc_score's value, at a fraction of the cost on tens of millions of
+    rows. A column holding +-inf is skipped, as roc_auc_score rejects it.
+    """
+    values = np.asarray(values, dtype=float)
+    if not np.isfinite(values).all():
+        return None, None
+    classes, codes = np.unique(labels.to_numpy(), return_inverse=True)
+    n = len(values)
+    n_pos = np.bincount(codes, minlength=len(classes)).astype(float)
+    rank_sum = np.bincount(codes, weights=rankdata(values), minlength=len(classes))
+    auc_by_class = {}
+    for i, cls in enumerate(classes):
+        if n_pos[i] == 0 or n_pos[i] == n:
+            continue
+        auc = (rank_sum[i] - n_pos[i] * (n_pos[i] + 1) / 2) / (n_pos[i] * (n - n_pos[i]))
+        # A feature can separate a class by high values or low values;
+        # max() with the inverse reports separability regardless of direction.
+        auc_by_class[cls] = max(auc, 1 - auc)
+    best_auc, best_class = None, None
+    for cls in labels.unique():  # same tie-break order as before: first class to reach the maximum
+        auc = auc_by_class.get(cls)
+        if auc is not None and (best_auc is None or auc > best_auc):
+            best_auc, best_class = float(auc), cls
+    return best_auc, best_class
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    label_col = cfg["schema"]["label_column"]
+    cols = features.feature_columns(train_df, cfg)
+    numeric_cols = [c for c in cols if pd.api.types.is_numeric_dtype(train_df[c])]
+    if not numeric_cols:
+        return {
+            "check": "feature_auc_ranking_check", "status": NOT_APPLICABLE,
+            "summary": "no numeric feature columns to rank", "details": {},
+        }
+
+    labels = train_df[label_col]
+
+    def score(col):
+        values = train_df[col].to_numpy()
+        valid = ~pd.isna(values)
+        if valid.sum() < 2:
+            return None, None
+        return _best_auc(values[valid], labels[valid])
+
+    # Columns are ranked in threads: numpy's sort releases the GIL, and at
+    # full scale ranking dozens of columns of tens of millions of rows one by
+    # one dominates the audit's run time.
+    with ThreadPoolExecutor(max_workers=min(len(numeric_cols), os.cpu_count() or 1, 16)) as pool:
+        scored = list(pool.map(score, numeric_cols))
+
+    best_per_feature = {}
+    top_feature, top_class, top_auc = None, None, 0.0
+    for col, (auc, cls) in zip(numeric_cols, scored, strict=True):
+        if auc is None:
+            continue
+        best_per_feature[col] = auc
+        if auc > top_auc:
+            top_feature, top_class, top_auc = col, cls, auc
+
+    if not best_per_feature:
+        return {
+            "check": "feature_auc_ranking_check", "status": NOT_APPLICABLE,
+            "summary": "no numeric feature had both classes present to rank", "details": {},
+        }
+
+    if top_auc > AUC_FLAG_THRESHOLD:
+        status = "flag"
+    elif top_auc > AUC_WARNING_THRESHOLD:
+        status = "warning"
+    else:
+        status = "ok"
+
+    top_ranked = dict(sorted(best_per_feature.items(), key=lambda kv: kv[1], reverse=True)[:TOP_N])
+    summary = f"'{top_feature}' alone reaches AUC {top_auc:.3f} identifying '{top_class}'"
+    if status != "ok":
+        summary += "; a single feature this separable is worth checking for a leakage artifact"
+
+    return {
+        "check": "feature_auc_ranking_check", "status": status, "summary": summary,
+        "details": {"top_features": top_ranked, "best_feature": str(top_feature), "best_class": str(top_class)},
+    }

ids2eval/audit/feature_category_ablation.py

@@ -0,0 +1,98 @@
+"""Feature-category ablation (v2, opt-in).
+
+Needs schema.feature_categories, a user-declared mapping from column
+name to a category label (e.g. "behavioural", "topology", "identity",
+"timing", any names the user chooses). Fits once on every feature
+(baseline), then once per declared category with that category's
+columns removed, and reports how much accuracy each category's removal
+costs. A category responsible for a large share of the model's
+accuracy on its own is worth a second look, especially if it is a
+category like identity or topology rather than behavioural, though
+this check reports the fact and leaves that judgment to the reader
+rather than hardcoding which category names should be considered
+suspicious.
+
+Deliberately opt-in, not a default check: a category label is exactly
+as good as the person who assigned it, unlike every other check here,
+which measures something rather than trusting a declared fact. Wrong
+labels produce a plausible-looking result that says nothing real.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ..data import features
+from . import _materiality
+from ._fit_score import fit_and_score
+from ._status import NOT_APPLICABLE
+
+# Overridable via audit.materiality_thresholds.feature_category_ablation_check
+# ({"warning": .., "flag": ..}).
+MATERIAL_DROP_THRESHOLD = 0.30
+WARNING_DROP_THRESHOLD = 0.15
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    categories_map = cfg["schema"]["feature_categories"]
+    if not categories_map:
+        return {
+            "check": "feature_category_ablation_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.feature_categories configured", "details": {},
+        }
+
+    label_col = cfg["schema"]["label_column"]
+    all_cols = features.feature_columns(train_df, cfg)
+
+    by_category: dict[str, list[str]] = {}
+    for col, category in categories_map.items():
+        if col in all_cols:
+            by_category.setdefault(category, []).append(col)
+    if not by_category:
+        return {
+            "check": "feature_category_ablation_check", "status": NOT_APPLICABLE,
+            "summary": "none of schema.feature_categories's columns are present in the loaded data",
+            "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    baseline_acc = fit_and_score(train_df, test_df, label_col, cfg, cols=all_cols, seed=seed)
+
+    drop_by_category = {}
+    for category, cols_in_category in by_category.items():
+        remaining = [c for c in all_cols if c not in cols_in_category]
+        if not remaining:
+            continue  # every feature falls under this one category, nothing to ablate against
+        ablated_acc = fit_and_score(train_df, test_df, label_col, cfg, cols=remaining, seed=seed)
+        drop_by_category[category] = baseline_acc - ablated_acc
+
+    if not drop_by_category:
+        return {
+            "check": "feature_category_ablation_check", "status": NOT_APPLICABLE,
+            "summary": "every declared category covers all features, nothing left to ablate against",
+            "details": {"baseline_accuracy": baseline_acc},
+        }
+
+    worst_category, worst_drop = max(drop_by_category.items(), key=lambda kv: kv[1])
+    tiers = _materiality.threshold(
+        cfg, "feature_category_ablation_check",
+        {"warning": WARNING_DROP_THRESHOLD, "flag": MATERIAL_DROP_THRESHOLD},
+    )
+    if worst_drop > tiers["flag"]:
+        status = "flag"
+    elif worst_drop > tiers["warning"]:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = (
+        f"baseline accuracy={baseline_acc:.4f}; removing '{worst_category}' features costs "
+        f"{worst_drop:+.4f} accuracy, the most of any declared category"
+    )
+    if status != "ok":
+        summary += "; a category this load-bearing is worth checking, especially if it isn't behavioural"
+
+    return {
+        "check": "feature_category_ablation_check", "status": status, "summary": summary,
+        "details": {"baseline_accuracy": baseline_acc, "drop_by_category": drop_by_category},
+    }

ids2eval/audit/flow_group_leakage.py

@@ -0,0 +1,66 @@
+"""Flow-identity leakage check: same flow tuple on both sides of the split.
+
+Direct counterpart to resplit_falsification, which asks "would a
+session-grouped split cost accuracy" without ever naming which specific
+rows would move. This check names them: given schema.flow_id_columns
+(typically a 5-tuple, e.g. source/destination IP, source/destination
+port, protocol, or whatever subset survives in a given release), it
+finds identity values that appear in both train and test. Many NIDS
+papers still split flow-level data at random rather than by session,
+letting the same connection contribute a row to both sides, so a model
+can partly recognize the connection rather than the attack behavior.
+
+Unlike label_conflict_check (which compares on every feature column),
+this compares on only the columns named in schema.flow_id_columns, so
+two different flows that happen to share a source IP and port (against
+different destinations, say) are correctly treated as different
+identities, not merged into one.
+
+No-ops if schema.flow_id_columns is empty or none of its columns survive
+in the loaded data (many public releases already strip IP columns before
+distribution, e.g. the NetFlow-V2 mirrors this project audits).
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ._status import NOT_APPLICABLE
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    flow_cols = [c for c in cfg["schema"]["flow_id_columns"] if c in train_df.columns]
+    if not flow_cols:
+        return {
+            "check": "flow_group_leakage_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.flow_id_columns configured", "details": {},
+        }
+
+    train_keys = pd.util.hash_pandas_object(train_df[flow_cols], index=False)
+    test_keys = pd.util.hash_pandas_object(test_df[flow_cols], index=False)
+    overlap = set(train_keys.to_numpy()) & set(test_keys.to_numpy())
+
+    if not overlap:
+        return {
+            "check": "flow_group_leakage_check", "status": "ok",
+            "summary": f"no flow identity in {flow_cols} appears on both sides of the split",
+            "details": {"flow_id_columns": flow_cols},
+        }
+
+    train_rows = int(train_keys.isin(overlap).sum())
+    test_rows = int(test_keys.isin(overlap).sum())
+    summary = (
+        f"{len(overlap):,} flow identit{'y' if len(overlap) == 1 else 'ies'} in {flow_cols} "
+        f"appear on both sides of the split, touching {train_rows:,} train row(s) and "
+        f"{test_rows:,} test row(s); a model can partly recognize the connection instead "
+        f"of the attack behavior on these rows"
+    )
+    return {
+        "check": "flow_group_leakage_check", "status": "flag", "summary": summary,
+        "details": {
+            "flow_id_columns": flow_cols,
+            "overlapping_identities": len(overlap),
+            "train_rows_affected": train_rows,
+            "test_rows_affected": test_rows,
+        },
+    }