ids2eval 0.2.2__py3-none-any.whl

ids2eval/audit/row_order_leakage.py

@@ -0,0 +1,86 @@
+"""Row-order (sequence) leakage check.
+
+Datasets assembled by concatenating scenario-specific collection blocks
+(benign traffic captured first, then one attack type, then the next)
+leave a detectable signature in raw row order: adjacent rows share a
+label far more often than a randomly shuffled arrangement of the same
+label distribution would. Measured via the number of label transitions
+between adjacent rows (a Wald-Wolfowitz runs statistic), compared
+against the closed-form expectation for a shuffled multiset (the
+Simpson-diversity complement, 1 - sum(p_i^2), the exact probability two
+independently drawn labels differ), rather than fitting and evaluating
+a model.
+
+A standalone-AUC check, the method identity_column_flag and
+temporal_leakage_check use, does not fit this signal: that method needs
+the same feature to be directly comparable between train and test (a
+shared timestamp, a shared IP space). Row position within one dataframe
+has no such correspondence to row position within another, so train
+and test are each tested against their own row order independently,
+not fit on one and evaluated on the other.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from ._status import NOT_APPLICABLE
+
+# Below this share of the shuffled-baseline transition rate, adjacent rows
+# share a label often enough to be a specific, checkable block-ordering
+# signature, not incidental clustering.
+CONTIGUITY_FLAG_THRESHOLD = 0.3
+CONTIGUITY_WARNING_THRESHOLD = 0.6
+
+
+def _contiguity_ratio(labels: pd.Series) -> float | None:
+    n = len(labels)
+    if n < 2:
+        return None
+    values = labels.to_numpy()
+    observed_transitions = int((values[1:] != values[:-1]).sum())
+    shares = labels.value_counts(normalize=True).to_numpy()
+    expected_rate = 1.0 - float((shares**2).sum())
+    if expected_rate <= 0:
+        return None  # a single class: no transition rate to compare against
+    observed_rate = observed_transitions / (n - 1)
+    return observed_rate / expected_rate
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    label_col = cfg["schema"]["label_column"]
+    ratios = {
+        "train": _contiguity_ratio(train_df[label_col]),
+        "test": _contiguity_ratio(test_df[label_col]),
+    }
+    valid = {split: ratio for split, ratio in ratios.items() if ratio is not None}
+    if not valid:
+        return {
+            "check": "row_order_leakage_check", "status": NOT_APPLICABLE,
+            "summary": "not enough rows or classes in either split to test row order",
+            "details": {},
+        }
+
+    worst_split, worst_ratio = min(valid.items(), key=lambda kv: kv[1])
+    if worst_ratio < CONTIGUITY_FLAG_THRESHOLD:
+        status = "flag"
+    elif worst_ratio < CONTIGUITY_WARNING_THRESHOLD:
+        status = "warning"
+    else:
+        status = "ok"
+
+    ratio_text = ", ".join(f"{split}={ratio:.2f}" for split, ratio in valid.items())
+    summary = (
+        f"adjacent-row label-transition rate ({ratio_text}) as a share of what a "
+        f"randomly shuffled ordering would produce"
+    )
+    if status != "ok":
+        summary += (
+            f"; {worst_split} is the most contiguous, consistent with rows still in "
+            f"collection order rather than shuffled before this split"
+        )
+
+    return {
+        "check": "row_order_leakage_check", "status": status, "summary": summary,
+        "details": {"contiguity_ratio": ratios},
+    }

ids2eval/audit/scenario_holdout.py

@@ -0,0 +1,115 @@
+"""Scenario-holdout falsification.
+
+resplit_falsification asks whether session-correlated leakage crosses
+a random split's boundary; this asks a different, complementary
+question: does high random-split accuracy survive never having seen an
+entire collection scenario (a capture day, an attacker machine, a
+scenario label, whatever schema.scenario_column names) during training
+at all? Compares random-split accuracy against holding out the single
+scenario value with the fewest rows entirely as test, training on
+every other scenario. A model that only memorized capture-environment
+artifacts (Monday's specific traffic mix, one attacker host's quirks)
+rather than attack behavior in general should collapse on a scenario
+it never trained on; a model that learned the actual behavior should
+not.
+
+Some accuracy drop under a genuinely novel scenario is expected even
+from a clean dataset, generalizing to conditions never seen in training
+is a harder problem than a random split of the same conditions, so the
+material-drop threshold here is deliberately looser than
+resplit_falsification's: this test is falsifying "the model learned
+attack behavior," not "there is zero session-correlated leakage."
+
+Needs dataset.raw_files (like resplit_falsification, it builds its own
+comparison split from the raw data) and schema.scenario_column; no-ops
+otherwise, since scenario_column has no universal name to detect
+automatically.
+"""
+
+from __future__ import annotations
+
+from ..data import dataset
+from . import _materiality
+from ._fit_score import fit_and_score
+from ._status import NOT_APPLICABLE
+
+# Overridable via audit.materiality_thresholds.scenario_holdout_falsification
+# ({"warning": .., "flag": ..}).
+MATERIAL_DROP_THRESHOLD = 0.10
+WARNING_DROP_THRESHOLD = 0.05
+MIN_HOLDOUT_ROWS = 20
+
+
+def check(cfg: dict) -> dict:
+    dataset_cfg = cfg["dataset"]
+    scenario_col = cfg["schema"]["scenario_column"]
+    if not scenario_col:
+        return {
+            "check": "scenario_holdout_falsification", "status": NOT_APPLICABLE,
+            "summary": "no schema.scenario_column configured", "details": {},
+        }
+    if not dataset_cfg["raw_files"]:
+        return {
+            "check": "scenario_holdout_falsification", "status": NOT_APPLICABLE,
+            "summary": "requires dataset.raw_files, not applicable to a pre-split train_file/test_file pair",
+            "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    label_col = cfg["schema"]["label_column"]
+    combined = dataset.load_raw_combined(dataset_cfg, seed=seed)
+    if scenario_col not in combined.columns:
+        return {
+            "check": "scenario_holdout_falsification", "status": NOT_APPLICABLE,
+            "summary": f"'{scenario_col}' not found in the loaded data", "details": {},
+        }
+
+    scenario_counts = combined[scenario_col].value_counts()
+    if len(scenario_counts) < 2:
+        return {
+            "check": "scenario_holdout_falsification", "status": NOT_APPLICABLE,
+            "summary": "fewer than two distinct scenarios to hold one out against", "details": {},
+        }
+
+    held_out = scenario_counts.index[-1]  # the smallest scenario, cheapest to hold out
+    held_out_mask = combined[scenario_col] == held_out
+    scenario_test = combined[held_out_mask]
+    scenario_train = combined[~held_out_mask]
+    if len(scenario_test) < MIN_HOLDOUT_ROWS or scenario_train[label_col].nunique() < 2:
+        return {
+            "check": "scenario_holdout_falsification", "status": NOT_APPLICABLE,
+            "summary": f"scenario '{held_out}' has too few rows or labels to test", "details": {},
+        }
+
+    random_train, random_test = dataset._random_split(combined, label_col, dataset_cfg, seed=seed)
+    random_acc = fit_and_score(random_train, random_test, label_col, cfg, seed=seed)
+    scenario_acc = fit_and_score(scenario_train, scenario_test, label_col, cfg, seed=seed)
+    drop = random_acc - scenario_acc
+
+    tiers = _materiality.threshold(
+        cfg, "scenario_holdout_falsification",
+        {"warning": WARNING_DROP_THRESHOLD, "flag": MATERIAL_DROP_THRESHOLD},
+    )
+    if drop > tiers["flag"]:
+        status = "flag"
+    elif drop > tiers["warning"]:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = (
+        f"random-split accuracy={random_acc:.4f}, holding out scenario '{held_out}' entirely: "
+        f"accuracy={scenario_acc:.4f} (drop={drop:+.4f})"
+    )
+    if status == "flag":
+        summary += "; accuracy collapses on a scenario never seen in training"
+    elif status == "warning":
+        summary += "; a real but moderate drop under a novel scenario"
+
+    return {
+        "check": "scenario_holdout_falsification", "status": status, "summary": summary,
+        "details": {
+            "held_out_scenario": str(held_out), "random_accuracy": random_acc,
+            "scenario_accuracy": scenario_acc, "drop": drop,
+        },
+    }

ids2eval/audit/schema_fingerprint.py

@@ -0,0 +1,52 @@
+"""Known-extractor-bug warning via column-name schema fingerprinting.
+
+Not a per-dataset curated issue list (that's the v2 known_issue_lookup
+check), this matches the *extractor tool*'s column-naming signature
+(e.g. CICFlowMeter) and surfaces that tool's documented bugs generally,
+regardless of which specific dataset the columns came from.
+
+Extend KNOWN_EXTRACTORS as more extractor signatures get curated.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+KNOWN_EXTRACTORS = {
+    "CICFlowMeter": {
+        # A handful of column names distinctive enough to fingerprint the
+        # tool without requiring every one of its ~80 columns present.
+        "signature_columns": {
+            "Flow Duration", "Fwd Packet Length Mean", "Bwd Packet Length Mean",
+            "Flow Bytes/s", "Flow Packets/s", "Fwd IAT Mean", "Bwd IAT Mean",
+            "Init_Win_bytes_forward", "Init_Win_bytes_backward",
+        },
+        "match_threshold": 0.6,
+        "warning": (
+            "CICFlowMeter is known to have miscalculated ~34 features in early "
+            "releases (Rosay et al. 2021), motivating the corrected LYCOS-IDS2017 "
+            "release. Verify which extractor version produced this data before "
+            "trusting derived-rate/duration features."
+        ),
+    },
+}
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    columns = set(train_df.columns)
+    matches = {}
+    for name, spec in KNOWN_EXTRACTORS.items():
+        overlap = spec["signature_columns"] & columns
+        match_share = len(overlap) / len(spec["signature_columns"])
+        if match_share >= spec["match_threshold"]:
+            matches[name] = {"match_share": match_share, "warning": spec["warning"]}
+
+    status = "warning" if matches else "ok"
+    summary = (
+        f"matched extractor signature(s): {list(matches)}" if matches
+        else "no known extractor signature matched"
+    )
+    return {
+        "check": "schema_fingerprint_check", "status": status, "summary": summary,
+        "details": matches,
+    }

ids2eval/audit/seed_sensitivity.py

@@ -0,0 +1,69 @@
+"""Seed-sensitivity check (v2, opt-in).
+
+Per D'Amour et al. 2022 (JMLR, "Underspecification Presents Challenges
+for Credibility in Modern Machine Learning"): predictors with identical
+training-domain performance can behave very differently under stress
+tests, purely from iid-performance-preserving perturbations like random
+seed choice. leakage_screen and one_rule_check each fit exactly one
+seeded classifier and report its result as fact; a borderline result
+(say, leakage_screen's top1 share at 0.51 against a 0.5 threshold) could
+flip with a different seed and nobody would know. This check re-fits
+both across a handful of seeds and reports whether their flag/ok
+conclusion is stable, not just their point estimate.
+
+Opt-in rather than v1: it refits two classifiers per seed on top of the
+single-seed runs those checks already do, meaningful extra compute on a
+large dataset.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from . import leakage, one_rule
+
+SEEDS = (0, 1, 2, 3, 4)
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    leakage_results = [leakage.check(train_df, test_df, cfg, seed=s) for s in SEEDS]
+    one_rule_results = [one_rule.check(train_df, test_df, cfg, seed=s) for s in SEEDS]
+
+    leakage_statuses = sorted({r["status"] for r in leakage_results})
+    one_rule_statuses = sorted({r["status"] for r in one_rule_results})
+    leakage_top1 = [r["details"]["top1_share"] for r in leakage_results if "top1_share" in r["details"]]
+    one_rule_acc = [r["details"]["test_accuracy"] for r in one_rule_results if "test_accuracy" in r["details"]]
+
+    unstable = []
+    if len(leakage_statuses) > 1:
+        unstable.append("leakage_screen")
+    if len(one_rule_statuses) > 1:
+        unstable.append("one_rule_check")
+
+    parts = []
+    if leakage_top1:
+        parts.append(
+            f"leakage_screen top1 share ranged {min(leakage_top1):.1%}-{max(leakage_top1):.1%} "
+            f"across {len(SEEDS)} seeds"
+        )
+    if one_rule_acc:
+        parts.append(
+            f"one_rule_check test accuracy ranged {min(one_rule_acc):.1%}-{max(one_rule_acc):.1%} "
+            f"across {len(SEEDS)} seeds"
+        )
+    summary = "; ".join(parts) if parts else "no comparable metric produced by either check across seeds"
+    if unstable:
+        summary += f". Verdict changed across seeds for: {', '.join(unstable)}"
+
+    return {
+        "check": "seed_sensitivity_check",
+        "status": "warning" if unstable else "ok",
+        "summary": summary,
+        "details": {
+            "seeds": list(SEEDS),
+            "leakage_screen_statuses": leakage_statuses,
+            "one_rule_check_statuses": one_rule_statuses,
+            "leakage_screen_top1_share_range": [min(leakage_top1), max(leakage_top1)] if leakage_top1 else None,
+            "one_rule_check_accuracy_range": [min(one_rule_acc), max(one_rule_acc)] if one_rule_acc else None,
+        },
+    }

ids2eval/audit/synthetic_realism.py

@@ -0,0 +1,73 @@
+"""Synthetic-vs-real distributional realism check (v2, opt-in).
+
+Per Layeghy et al. 2021 and Catillo et al. 2021: lab-generated/synthetic
+IDS traffic can diverge from real network traffic in ways that make a
+dataset's near-perfect classifier accuracy an artifact of unrepresentative
+data rather than a real capability. Needs a reference real-traffic sample
+(audit.reference_dataset) to compare against: there's no way to detect
+"unrealistic compared to what" without something to compare to.
+
+Method: a domain classifier trained to distinguish this dataset's rows
+from the reference sample's rows. High separability (AUC) means the two
+are easily distinguishable, a specific, checkable signature of
+distributional divergence, not merely "different datasets differ."
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from scipy.stats import ks_2samp
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import roc_auc_score
+from sklearn.model_selection import train_test_split
+
+from ..data import chunked_io, features
+
+DOMAIN_AUC_FLAG_THRESHOLD = 0.9
+MAX_ROWS_PER_SIDE = 50_000
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    dataset_cfg = cfg["dataset"]
+    ref_df = chunked_io.load_file(
+        cfg["audit"]["reference_dataset"], dataset_cfg["chunk_size"], dataset_cfg["max_rows"]
+    )
+
+    cols = [c for c in features.feature_columns(train_df, cfg) if c in ref_df.columns]
+    if not cols:
+        return {
+            "check": "synthetic_realism_check", "status": "warning",
+            "summary": "no feature columns in common with reference_dataset", "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    ours = train_df.sample(n=min(len(train_df), MAX_ROWS_PER_SIDE), random_state=seed)
+    ref = ref_df.sample(n=min(len(ref_df), MAX_ROWS_PER_SIDE), random_state=seed)
+
+    ours_enc, (ref_enc,) = features.encode_multi(ours, [ref], cols)
+    x = pd.concat([ours_enc, ref_enc], ignore_index=True)
+    y = pd.Series([0] * len(ours_enc) + [1] * len(ref_enc))
+
+    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=seed, stratify=y)
+    clf = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1)
+    clf.fit(x_train, y_train)
+    domain_auc = float(roc_auc_score(y_test, clf.predict_proba(x_test)[:, 1]))
+
+    ks_scores = {}
+    for col in cols:
+        if pd.api.types.is_numeric_dtype(ours[col]):
+            stat, _ = ks_2samp(ours[col].dropna(), ref[col].dropna())
+            ks_scores[col] = float(stat)
+    top_divergent = dict(sorted(ks_scores.items(), key=lambda kv: -kv[1])[:5])
+
+    flagged = domain_auc > DOMAIN_AUC_FLAG_THRESHOLD
+    status = "flag" if flagged else "ok"
+    summary = (
+        f"domain classifier AUC={domain_auc:.3f} distinguishing this dataset from "
+        f"reference_dataset (>{DOMAIN_AUC_FLAG_THRESHOLD} flagged as easily separable); "
+        f"most divergent features (KS statistic): {top_divergent}"
+    )
+    return {
+        "check": "synthetic_realism_check", "status": status, "summary": summary,
+        "details": {"domain_auc": domain_auc, "ks_by_feature": ks_scores},
+    }

ids2eval/audit/temporal_leakage.py

@@ -0,0 +1,58 @@
+"""Timestamp-only leakage check.
+
+Per Wu & Keogh 2021's "run-to-failure bias" (IEEE TKDE, time-series
+anomaly detection): several widely-used benchmarks were collected such
+that anomalies cluster near the end of the recording, so a model can win
+by learning "predict positive late" rather than the anomaly itself. NIDS
+datasets built as a sequence of scenario-specific time windows (attack X
+launched 2-3pm, attack Y launched 3-4pm, ...) have the same structure -
+independently confirmed against this project's own real CIC-IDS2018 run,
+where a RandomForest given nothing but the Timestamp column reached
+AUC 0.93 predicting benign vs. attack. Same method as identity_column_flag
+(a standalone classifier on one column), scoped to whichever column
+schema.timestamp_column names, since there's no universal column name to
+detect automatically the way id_like_columns are user-declared too.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+
+from ._auc import robust_auc
+from ._status import NOT_APPLICABLE
+from ._time import to_numeric_time
+
+AUC_FLAG_THRESHOLD = 0.8
+MAX_FIT_ROWS = 200_000
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    col = cfg["schema"]["timestamp_column"]
+    label_col = cfg["schema"]["label_column"]
+    if not col or col not in train_df.columns:
+        return {
+            "check": "temporal_leakage_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.timestamp_column configured", "details": {},
+        }
+
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=cfg["random_seed"])
+    x_train = to_numeric_time(train_fit[col]).fillna(0).to_numpy().reshape(-1, 1)
+    x_test = to_numeric_time(test_df[col]).fillna(0).to_numpy().reshape(-1, 1)
+    y_train, y_test = train_fit[label_col], test_df[label_col]
+
+    clf = RandomForestClassifier(n_estimators=50, random_state=cfg["random_seed"], n_jobs=-1)
+    clf.fit(x_train, y_train)
+    auc = robust_auc(y_test, clf.predict_proba(x_test), clf.classes_)
+
+    flagged = auc is not None and auc > AUC_FLAG_THRESHOLD
+    status = "flag" if flagged else "ok"
+    auc_text = "n/a" if auc is None else f"{auc:.3f}"
+    summary = f"standalone AUC of '{col}' alone: {auc_text}"
+    if flagged:
+        summary += ". Timestamp alone may predict the label without learning attack behavior"
+
+    return {
+        "check": "temporal_leakage_check", "status": status, "summary": summary,
+        "details": {"column": col, "standalone_auc": auc},
+    }

ids2eval/audit/temporal_realism.py

@@ -0,0 +1,81 @@
+"""Temporal-window realism check.
+
+Complements temporal_leakage_check's single aggregate AUC number with a
+per-class breakdown: how much of the full capture's time span does
+each attack type's own traffic actually cover? A class confined to a
+narrow burst window relative to the whole capture (attack X launched
+for ten minutes out of a five-day capture) is a specific, checkable
+scenario-window artifact. Visible here even when the aggregate
+standalone-timestamp AUC isn't dramatically high, because most other
+classes are well spread across the full period and only one narrow
+class is actually driving the risk.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from . import _by_class
+from ._status import NOT_APPLICABLE
+from ._time import to_numeric_time
+
+# Below this share of the full capture's time span, a class's own traffic is
+# confined to a narrow enough window that timestamp alone could stand in for
+# it, a specific, checkable scenario-window artifact.
+BURST_FLAG_THRESHOLD = 0.05
+BURST_WARNING_THRESHOLD = 0.15
+
+
+def check(train_df: pd.DataFrame, cfg: dict) -> dict:
+    col = cfg["schema"]["timestamp_column"]
+    if not col or col not in train_df.columns:
+        return {
+            "check": "temporal_realism_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.timestamp_column configured", "details": {},
+        }
+
+    times = to_numeric_time(train_df[col])
+    valid = times.notna()
+    if not valid.any():
+        return {
+            "check": "temporal_realism_check", "status": NOT_APPLICABLE,
+            "summary": "no usable timestamp values to test", "details": {},
+        }
+    overall_span = times[valid].max() - times[valid].min()
+    if not overall_span or overall_span <= 0:
+        return {
+            "check": "temporal_realism_check", "status": NOT_APPLICABLE,
+            "summary": "timestamp column has no usable time span to test", "details": {},
+        }
+
+    group_col = _by_class.group_column(cfg)
+    span_by_class = {}
+    for cls in _by_class.eligible_classes(train_df, group_col):
+        mask = (train_df[group_col] == cls) & valid
+        if mask.sum() < _by_class.MIN_CLASS_SIZE:
+            continue
+        cls_span = times[mask].max() - times[mask].min()
+        span_by_class[str(cls)] = float(cls_span / overall_span)
+
+    if not span_by_class:
+        return {
+            "check": "temporal_realism_check", "status": NOT_APPLICABLE,
+            "summary": "not enough valid timestamps per class to test", "details": {},
+        }
+
+    worst_cls, worst_ratio = min(span_by_class.items(), key=lambda kv: kv[1])
+    if worst_ratio < BURST_FLAG_THRESHOLD:
+        status = "flag"
+    elif worst_ratio < BURST_WARNING_THRESHOLD:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = f"narrowest class's own time span: '{worst_cls}' covers {worst_ratio:.1%} of the full capture window"
+    if status != "ok":
+        summary += "; timestamp alone may act as a proxy for this class rather than its actual traffic behavior"
+
+    return {
+        "check": "temporal_realism_check", "status": status, "summary": summary,
+        "details": {"span_ratio_by_class": span_by_class},
+    }