ids2eval 0.2.2__py3-none-any.whl

ids2eval/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

ids2eval/_json_utils.py

@@ -0,0 +1,24 @@
+"""Shared json.dumps(default=...) handler for numpy scalar/array types.
+
+Findings can carry numpy bool/int/float scalars or arrays in their `details`
+(most checks compute over pandas/numpy data), which the stdlib json module
+doesn't know how to serialize on its own. Shared rather than duplicated
+since both cli.py's output-writing and audit/_checkpoint.py's checkpoint
+writing need the exact same conversion.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def json_default(obj):
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    if isinstance(obj, np.integer):
+        return int(obj)
+    if isinstance(obj, np.floating):
+        return float(obj)
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")

ids2eval/audit/__init__.py

@@ -0,0 +1,239 @@
+"""Data-quality audit checks: Paper 1's methodology as reusable code.
+
+v1 checks need no external reference data; v2 checks (synthetic_realism_check,
+cross_dataset_drift_check) need audit.reference_dataset, and known_issue_lookup
+is a curated per-dataset lookup table needing neither.
+
+Each check function returns a Finding dict: {check, status, summary, details}.
+status is one of "ok" (nothing notable), "warning" (worth a human look),
+"flag" (a specific, checkable signature of a real data-quality problem), or
+"not_applicable" (the check had no input to test; see _status.py).
+
+STRUCTURAL_CHECKS are checks whose result cannot depend on
+preprocessing.dedup: known_issue_lookup looks only at dataset.name,
+schema_fingerprint_check only at column names (dedup removes rows, never
+columns), and resplit_falsification/scenario_holdout_falsification/
+cross_capture_matrix_check/result_robustness_check all reload the raw
+data themselves and never look at the train_df/test_df they're passed.
+Run once, on the raw data; run_audit's caller (cli.py)
+reuses that result instead of recomputing an answer that is guaranteed
+identical the second time.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import time
+
+import pandas as pd
+
+from ..data import dataset
+from . import (
+    _checkpoint,
+    artifact_sensitivity,
+    class_distribution,
+    cross_capture_matrix,
+    cross_dataset_drift,
+    data_integrity,
+    dedup,
+    feature_auc_ranking,
+    feature_category_ablation,
+    flow_group_leakage,
+    homogeneity,
+    identity_columns,
+    known_issues,
+    label_conflict,
+    leakage,
+    near_duplicate_class,
+    one_rule,
+    port_protocol_shortcut,
+    repeated_seed_falsification,
+    resplit,
+    result_robustness,
+    row_order_leakage,
+    scenario_holdout,
+    schema_fingerprint,
+    seed_sensitivity,
+    synthetic_realism,
+    temporal_leakage,
+    temporal_realism,
+)
+
+STRUCTURAL_CHECKS = frozenset({
+    "known_issue_lookup", "schema_fingerprint_check", "resplit_falsification",
+    "scenario_holdout_falsification", "cross_capture_matrix_check", "result_robustness_check",
+    "repeated_seed_falsification_check",
+})
+
+# A strict subset of STRUCTURAL_CHECKS: checks that report a documented, curated fact
+# about the dataset or the tool that extracted it (a published labelling error, a
+# known-buggy extractor), rather than measuring something about this run's actual
+# data. resplit_falsification is structural too, but it measures this run's own split
+# methodology and IS actionable (switch dataset.split_mode to "grouped"), so it stays
+# out of this set and is reported alongside the other audit checks, not here.
+KNOWN_ISSUE_CHECKS = frozenset({"known_issue_lookup", "schema_fingerprint_check"})
+
+# Evidence basis for a check's result, fixed per check name, not computed per
+# run: a citation needs a consistent answer to "how sure is this" for the
+# same check every time, not a judgment that could vary run to run.
+# "documented": a citation lookup against dataset.name or column names, not
+# computed from this run's data at all.
+# "direct-experiment": an actual counterfactual refit and comparison.
+# "statistical": a threshold or hypothesis test against this run's data,
+# with no independent corroboration. The default for anything not listed.
+# A fourth level, "cross-corroborated", is not a per-check label: it applies
+# only when homogeneity_test and resplit_falsification both flag on the
+# same run, and is computed once at scorecard-build time from those two
+# already-computed statuses (see ids2eval.reporting.scorecard).
+EVIDENCE_LEVEL = {
+    "known_issue_lookup": "documented",
+    "schema_fingerprint_check": "documented",
+    "resplit_falsification": "direct-experiment",
+    "scenario_holdout_falsification": "direct-experiment",
+    "cross_capture_matrix_check": "direct-experiment",
+    "feature_category_ablation_check": "direct-experiment",
+    "artifact_sensitivity_check": "direct-experiment",
+    "result_robustness_check": "direct-experiment",
+    "repeated_seed_falsification_check": "direct-experiment",
+}
+DEFAULT_EVIDENCE_LEVEL = "statistical"
+
+logger = logging.getLogger(__name__)
+
+
+def run_audit(
+    train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict, skip: frozenset[str] = frozenset(),
+    checkpoint_path: str | None = None,
+) -> list[dict]:
+    """Run every audit check enabled in cfg['audit'] and not in `skip`, in schema order.
+
+    Call this on the split BEFORE preprocessing.dedup is applied.
+    dedup_check reports duplication already present in train_df/test_df,
+    if the caller already deduplicated, it will always report zero.
+
+    `skip` exists for the caller to omit STRUCTURAL_CHECKS on a second call
+    with the same cfg (e.g. cli.py's after-dedup pass) - their answer can't
+    change, so recomputing them (including resplit_falsification's two
+    RandomForest fits) would be pure wasted work.
+
+    Runs sequentially by default. If audit.parallel_checks is set, enabled
+    checks instead run concurrently (capped at audit.max_parallel_checks) via
+    a thread pool - safe because no check mutates train_df/test_df, only
+    reads them. The returned list is always in the same schema order either
+    way; only execution order, not the caller-visible result, changes.
+    Off by default because several checks running at once means several may
+    hold their own encoded feature matrix/RandomForest fit in memory
+    simultaneously rather than one at a time, raising peak memory versus the
+    sequential default.
+
+    resplit_falsification, result_robustness_check, and repeated_seed_
+    falsification_check each need the full raw dataset (not just train_df/
+    test_df) and would otherwise each reload it independently - loaded once
+    here and shared across whichever of the three are enabled, since that
+    reload is a real, serious cost at full scale (see each check's own
+    docstring for the real failure this was built to avoid).
+
+    checkpoint_path, if given, persists each check's finding to a local JSON
+    file as soon as it's computed and skips recomputing a check whose result
+    is already there - so a crash partway through a long run only redoes the
+    remaining checks on the next call with the same path. Mutually exclusive
+    with parallel_checks (validated in config.py): concurrent checks writing
+    to one checkpoint file would race. repeated_seed_falsification_check
+    additionally checkpoints its own per-seed progress via this same path,
+    since it alone can dominate a full-scale run's cost.
+    """
+    audit_cfg = cfg["audit"]
+    seed = cfg["random_seed"]
+    dataset_cfg = cfg["dataset"]
+
+    # resplit_falsification, result_robustness_check, and repeated_seed_falsification_check
+    # each independently reload the full raw dataset when called standalone - fine at small
+    # scale, but a real, serious cost at full scale: a real failure hit running all three
+    # together against a 70M-row dataset, where each ~45-70 minute reload compounded into
+    # repeated VM-level instability on a run that redundantly reloaded the same data three
+    # times sequentially. Loaded once here and passed to whichever of the three are enabled.
+    _raw_reload_checks = ("resplit_falsification", "result_robustness_check", "repeated_seed_falsification_check")
+    shared_combined = None
+    if dataset_cfg["raw_files"] and any(
+        audit_cfg[name] and name not in skip for name in _raw_reload_checks
+    ):
+        shared_combined = dataset.load_raw_combined(dataset_cfg, seed=seed)
+
+    # Each entry is (check_name, thunk). Building the full ordered list up front - rather
+    # than appending each finding immediately - lets the dispatch loop below run these
+    # independently (they only read train_df/test_df, never mutate them) either in
+    # schema order (default) or concurrently, while the OUTPUT list always comes back in
+    # this same schema order regardless of which mode ran or which thunk finished first.
+    pending: list[tuple[str, object]] = []
+
+    def add(name: str, thunk) -> None:
+        if audit_cfg[name] and name not in skip:
+            pending.append((name, thunk))
+
+    add("dedup_check", lambda: dedup.check(train_df, test_df, cfg))
+    add("label_conflict_check", lambda: label_conflict.check(train_df, test_df, cfg))
+    add("near_duplicate_class_check", lambda: near_duplicate_class.check(train_df, cfg))
+    add("leakage_screen", lambda: leakage.check(train_df, test_df, cfg, seed=seed))
+    add("one_rule_check", lambda: one_rule.check(train_df, test_df, cfg, seed=seed))
+    add("feature_auc_ranking_check", lambda: feature_auc_ranking.check(train_df, cfg))
+    add("identity_column_flag", lambda: identity_columns.check_predictive_power(train_df, test_df, cfg))
+    add("port_protocol_shortcut_check", lambda: port_protocol_shortcut.check(train_df, test_df, cfg))
+    add("temporal_leakage_check", lambda: temporal_leakage.check(train_df, test_df, cfg))
+    add("temporal_realism_check", lambda: temporal_realism.check(train_df, cfg))
+    add("flow_group_leakage_check", lambda: flow_group_leakage.check(train_df, test_df, cfg))
+    add("row_order_leakage_check", lambda: row_order_leakage.check(train_df, test_df, cfg))
+    add("homogeneity_test", lambda: homogeneity.check(train_df, test_df, cfg))
+    add("resplit_falsification", lambda: resplit.check(cfg, seed=seed, combined=shared_combined))
+    add("scenario_holdout_falsification", lambda: scenario_holdout.check(cfg))
+    add("class_distribution_report", lambda: class_distribution.check(train_df, test_df, cfg))
+    add("low_cardinality_warning", lambda: identity_columns.check_cardinality(train_df, cfg))
+    add("schema_fingerprint_check", lambda: schema_fingerprint.check(train_df, cfg))
+    add("data_integrity_check", lambda: data_integrity.check(train_df, cfg))
+    add("synthetic_realism_check", lambda: synthetic_realism.check(train_df, cfg))
+    add("cross_dataset_drift_check", lambda: cross_dataset_drift.check(train_df, test_df, cfg))
+    add("cross_capture_matrix_check", lambda: cross_capture_matrix.check(cfg))
+    add("feature_category_ablation_check", lambda: feature_category_ablation.check(train_df, test_df, cfg))
+    add("artifact_sensitivity_check", lambda: artifact_sensitivity.check(train_df, test_df, cfg))
+    add("result_robustness_check", lambda: result_robustness.check(cfg, combined=shared_combined))
+    add("known_issue_lookup", lambda: known_issues.check(train_df, cfg))
+    add("seed_sensitivity_check", lambda: seed_sensitivity.check(train_df, test_df, cfg))
+    add(
+        "repeated_seed_falsification_check",
+        lambda: repeated_seed_falsification.check(
+            cfg, checkpoint_path=checkpoint_path, combined=shared_combined,
+        ),
+    )
+
+    # One log line per check: some take minutes on a large dataset, and
+    # without these a run (and the dashboard's live log) looks frozen.
+    total = len(pending)
+
+    def done(i: int, name: str, result: dict, started: float, note: str = "") -> None:
+        logger.info("Audit check %d/%d done: %s -> %s (%.1fs%s)", i, total, name,
+                    (result or {}).get("status", "?"), time.monotonic() - started, note)
+
+    if not audit_cfg["parallel_checks"]:
+        results = []
+        for i, (name, thunk) in enumerate(pending, 1):
+            started = time.monotonic()
+            cached = _checkpoint.get_check(checkpoint_path, name) if checkpoint_path else None
+            if cached is None:
+                logger.info("Audit check %d/%d: %s ...", i, total, name)
+            result = cached if cached is not None else thunk()
+            if checkpoint_path and cached is None:
+                _checkpoint.save_check(checkpoint_path, name, result)
+            done(i, name, result, started, ", from checkpoint" if cached is not None else "")
+            results.append(result)
+        return results
+
+    max_workers = min(audit_cfg["max_parallel_checks"], len(pending)) or 1
+    logger.info("Audit: running %d checks, up to %d at a time", total, max_workers)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        started = time.monotonic()
+        futures = [pool.submit(thunk) for _, thunk in pending]
+        names = {f: name for f, (name, _) in zip(futures, pending, strict=True)}
+        for i, f in enumerate(concurrent.futures.as_completed(futures), 1):
+            if f.exception() is None:
+                done(i, names[f], f.result(), started, " since the batch started")
+        return [f.result() for f in futures]

ids2eval/audit/_auc.py

@@ -0,0 +1,48 @@
+"""ROC AUC that survives a class-set mismatch between train and test.
+
+Shared by the checks that score a single-column classifier (leakage_screen's
+standalone AUC, identity_column_flag). A plain roc_auc_score call assumes the
+model's classes and the test labels are the same set, which real splits
+break both ways: a rare class can land entirely in train (CIC-IDS2018's
+SQL Injection, ~3 rows in a 500K reservoir sample - a real crash), and a
+dataset can hold test-only classes by design (NSL-KDD's novel attacks).
+"""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+logger = logging.getLogger(__name__)
+
+
+def robust_auc(y_true, proba: np.ndarray, classes) -> float | None:
+    """Weighted one-vs-rest AUC over the classes present on both sides.
+
+    Test rows whose class the model never saw are skipped (there's no
+    score column to rank them by); probability columns for classes absent
+    from the test labels are dropped and each row renormalized. Returns
+    None when fewer than two classes remain - AUC isn't defined there.
+    With identical class sets this equals plain roc_auc_score.
+    """
+    y_true = pd.Series(y_true).to_numpy()
+    classes = np.asarray(classes)
+
+    seen = np.isin(y_true, classes)
+    if not seen.all():
+        logger.warning("AUC: skipped %d test row(s) whose class never appeared in train", int((~seen).sum()))
+    y_true, proba = y_true[seen], proba[seen]
+
+    present = np.isin(classes, np.unique(y_true))
+    if present.sum() < 2:
+        return None
+    classes, proba = classes[present], proba[:, present]
+
+    row_sums = proba.sum(axis=1, keepdims=True)
+    proba = np.divide(proba, row_sums, out=np.full_like(proba, 1.0 / len(classes)), where=row_sums > 0)
+    if len(classes) == 2:
+        return float(roc_auc_score(y_true == classes[1], proba[:, 1]))
+    return float(roc_auc_score(y_true, proba, multi_class="ovr", average="weighted", labels=classes))

ids2eval/audit/_by_class.py

@@ -0,0 +1,29 @@
+"""Shared per-class breakdown machinery.
+
+A dataset can look fine in aggregate (8% duplication, say) while one
+attack family is 61% duplicated and another is untouched, invisible in
+a single global number. The checks that support a breakdown attach a
+"by_class" entry to their details, keyed by whichever column groups
+attack types: schema.attack_category_column if set, else the label
+column itself.
+
+Same MIN_CLASS_SIZE guard homogeneity_test and near_duplicate_class_check
+already use: a class with too few rows to say anything meaningful about
+is left out rather than reported on a handful of rows.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+MIN_CLASS_SIZE = 20
+
+
+def group_column(cfg: dict) -> str:
+    schema = cfg["schema"]
+    return schema["attack_category_column"] or schema["label_column"]
+
+
+def eligible_classes(df: pd.DataFrame, group_col: str, min_size: int = MIN_CLASS_SIZE) -> list:
+    counts = df[group_col].value_counts()
+    return counts[counts >= min_size].index.tolist()

ids2eval/audit/_checkpoint.py

@@ -0,0 +1,74 @@
+"""JSON-file checkpoint for resuming a long audit run after a crash.
+
+Exists for full-scale runs on Cloud Batch, where a preemption/retry starts a
+FRESH VM instance - anything checkpointed only to that VM's local disk is
+lost regardless of in-process logic. This module only handles the local
+read/write side; surviving an actual VM-level retry additionally requires
+the caller (the batch runner script) to sync the checkpoint directory
+to/from a durable location (GCS) around the run - this module doesn't know
+about that, it just needs a stable local path across separate process runs.
+
+Two granularities:
+- Whole-check: run_audit() skips recomputing a check entirely if its finding
+  is already checkpointed.
+- Per-seed, repeated_seed_falsification_check only: that check alone can
+  dominate a full-scale run's cost (it's N independent full refits), so a
+  crash partway through its seed loop should only cost the remaining seeds,
+  not the whole check.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from .._json_utils import json_default
+
+
+def load(path: str) -> dict:
+    p = Path(path)
+    if not p.exists():
+        return {}
+    try:
+        return json.loads(p.read_text())
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _atomic_write(path: str, data: dict) -> None:
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    tmp = p.with_suffix(p.suffix + ".tmp")
+    tmp.write_text(json.dumps(data, default=json_default))
+    os.replace(tmp, p)
+
+
+def get_check(path: str, check_name: str) -> dict | None:
+    return load(path).get("checks", {}).get(check_name)
+
+
+def save_check(path: str, check_name: str, finding: dict) -> None:
+    data = load(path)
+    data.setdefault("checks", {})[check_name] = finding
+    _atomic_write(path, data)
+
+
+def get_seed_progress(path: str, check_name: str) -> dict[int, float]:
+    raw = load(path).get("partial", {}).get(check_name, {}).get("drops_by_seed", {})
+    return {int(k): v for k, v in raw.items()}
+
+
+def save_seed_progress(path: str, check_name: str, seed: int, drop: float) -> None:
+    data = load(path)
+    partial = data.setdefault("partial", {}).setdefault(check_name, {"drops_by_seed": {}})
+    partial["drops_by_seed"][str(seed)] = drop
+    _atomic_write(path, data)
+
+
+def clear(path: str) -> None:
+    """Remove the checkpoint after a run completes successfully, so a later
+    fresh run of the same config doesn't silently resume stale progress."""
+    p = Path(path)
+    if p.exists():
+        p.unlink()

ids2eval/audit/_fit_score.py

@@ -0,0 +1,40 @@
+"""Shared fit-a-RandomForest-and-score-accuracy helper.
+
+Used by any check that compares accuracy across two different ways of
+splitting the same raw data (resplit_falsification: random vs. grouped;
+scenario_holdout_falsification: random vs. one scenario held out
+entirely), rather than measuring a property of a single given split.
+Also used by feature_category_ablation_check, which instead compares
+two different FEATURE subsets of the same split, via the optional
+`cols` override.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+
+from ..data import features
+
+MAX_FIT_ROWS = 200_000
+
+
+def fit_and_score(
+    train_df: pd.DataFrame, test_df: pd.DataFrame, label_col: str, cfg: dict,
+    cols: list[str] | None = None, seed: int = 0,
+) -> float:
+    """seed controls both which MAX_FIT_ROWS rows are subsampled and the
+    RandomForest's own random_state. Callers that want the run's single
+    baseline result pass cfg["random_seed"] (the default 0 only applies if a
+    caller omits it entirely); repeated_seed_falsification_check is the one
+    caller that instead sweeps its own independent range of seeds, to get a
+    distribution rather than one baseline point.
+    """
+    if cols is None:
+        cols = features.feature_columns(train_df, cfg)
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=seed)
+    x_train, x_test = features.encode_aligned(train_fit, test_df, cols, unseen="missing")
+    clf = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1)
+    clf.fit(x_train, train_fit[label_col])
+    return float(accuracy_score(test_df[label_col], clf.predict(x_test)))

ids2eval/audit/_materiality.py

@@ -0,0 +1,28 @@
+"""Shared lookup for audit.materiality_thresholds overrides.
+
+Each check that compares a metric against a materiality threshold keeps its
+own default as a module-level constant (see that check's own module for why
+its specific number was chosen); this only looks up whether the config
+overrides it. Validated in config.py's validate_config, not here - by the
+time a check calls this, the config has already passed validation, so this
+stays a trusting, unconditional lookup rather than repeating those checks.
+"""
+
+from __future__ import annotations
+
+
+def threshold(cfg: dict, check: str, default):
+    """Return cfg['audit']['materiality_thresholds'][check] if set, else default."""
+    return cfg["audit"]["materiality_thresholds"].get(check, default)
+
+
+def accuracy_gap_margin(acc_a: float, n_a: int, acc_b: float, n_b: int) -> float:
+    """Half-width of a 95% normal-approximation interval on the difference
+    between two accuracies measured on test sets of n_a and n_b rows.
+
+    Treats the two test sets as independent, which is conservative when they
+    share rows (a paired difference has smaller variance), so a gap that clears
+    a threshold by more than this margin is not just test-set sampling noise.
+    """
+    var = acc_a * (1 - acc_a) / max(n_a, 1) + acc_b * (1 - acc_b) / max(n_b, 1)
+    return 1.959964 * var ** 0.5

ids2eval/audit/_status.py

@@ -0,0 +1,15 @@
+"""Finding statuses.
+
+"ok", "warning" and "flag" are results: the check ran on its input and measured
+something. NOT_APPLICABLE means the check could not produce a result on this
+input (no configured column, a pre-split dataset where raw files are needed, too
+few classes or rows to test), so it says nothing about the dataset either way.
+It never raises the overall verdict and is counted separately in the scorecard.
+"""
+
+OK = "ok"
+WARNING = "warning"
+FLAG = "flag"
+NOT_APPLICABLE = "not_applicable"
+
+STATUSES = (OK, WARNING, FLAG, NOT_APPLICABLE)

ids2eval/audit/_time.py

@@ -0,0 +1,14 @@
+"""Shared timestamp-to-numeric parsing, used by any check scoped to
+schema.timestamp_column.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+
+def to_numeric_time(series: pd.Series) -> pd.Series:
+    parsed = pd.to_datetime(series, errors="coerce")
+    if parsed.notna().mean() > 0.5:
+        return parsed.astype("int64") // 10**9
+    return pd.to_numeric(series, errors="coerce")

ids2eval/audit/artifact_sensitivity.py

@@ -0,0 +1,88 @@
+"""Counterfactual artifact-sensitivity test (v2, opt-in).
+
+Fits one RandomForest on every feature and predicts on test, then, for
+each column already declared as a potential shortcut elsewhere
+(schema.id_like_columns, schema.timestamp_column), permutes just that
+column's own values across the test rows (a real, in-distribution
+reshuffle, not a synthetic replacement) and predicts again with the
+same already-fitted model. The share of predictions that flip is a
+direct measurement of how much the model's decision for a given row
+depends on that one column rather than the rest of the row's features:
+
+    AS(column) = P(model's prediction changes | only this column reshuffled)
+
+High sensitivity to an identity or timestamp column specifically is a
+concrete, checkable sign the model may be keying off which host or
+which moment produced a flow rather than what the flow's behavior was,
+complementing identity_column_flag/temporal_leakage_check's standalone
+predictive-power tests with the model's actual dependence on the
+column inside the full feature set, not the column in isolation.
+
+Deliberately opt-in: which classifier is fit and how a specific column
+is perturbed both shape the result, unlike a check that measures a
+fixed statistic of the data itself. No-ops unless at least one
+declared identity or timestamp column is actually present as a feature.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+
+from ..data import features
+from ._status import NOT_APPLICABLE
+
+MAX_FIT_ROWS = 200_000
+FLIP_RATE_FLAG_THRESHOLD = 0.30
+FLIP_RATE_WARNING_THRESHOLD = 0.15
+
+
+def check(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> dict:
+    schema = cfg["schema"]
+    candidate_cols = list(schema["id_like_columns"])
+    if schema["timestamp_column"]:
+        candidate_cols.append(schema["timestamp_column"])
+
+    cols = features.feature_columns(train_df, cfg)
+    suspicious_cols = [c for c in dict.fromkeys(candidate_cols) if c in cols and c in test_df.columns]
+    if not suspicious_cols:
+        return {
+            "check": "artifact_sensitivity_check", "status": NOT_APPLICABLE,
+            "summary": "no schema.id_like_columns or schema.timestamp_column present as a feature",
+            "details": {},
+        }
+
+    seed = cfg["random_seed"]
+    label_col = schema["label_column"]
+    train_fit = train_df.sample(n=min(len(train_df), MAX_FIT_ROWS), random_state=seed)
+    x_train, x_test = features.encode_aligned(train_fit, test_df, cols, unseen="missing")
+    clf = RandomForestClassifier(n_estimators=100, random_state=seed, n_jobs=-1)
+    clf.fit(x_train, train_fit[label_col])
+    baseline_pred = clf.predict(x_test)
+
+    rng = np.random.RandomState(seed)
+    flip_rate_by_column = {}
+    for col in suspicious_cols:
+        perturbed_test = test_df.copy()
+        perturbed_test[col] = rng.permutation(perturbed_test[col].to_numpy())
+        _, x_test_perturbed = features.encode_aligned(train_fit, perturbed_test, cols, unseen="missing")
+        perturbed_pred = clf.predict(x_test_perturbed)
+        flip_rate_by_column[col] = float((perturbed_pred != baseline_pred).mean())
+
+    worst_col, worst_rate = max(flip_rate_by_column.items(), key=lambda kv: kv[1])
+    if worst_rate > FLIP_RATE_FLAG_THRESHOLD:
+        status = "flag"
+    elif worst_rate > FLIP_RATE_WARNING_THRESHOLD:
+        status = "warning"
+    else:
+        status = "ok"
+
+    summary = f"reshuffling '{worst_col}' alone flips {worst_rate:.1%} of the model's predictions"
+    if status != "ok":
+        summary += "; the model's decisions depend on this column more than its own behavior may warrant"
+
+    return {
+        "check": "artifact_sensitivity_check", "status": status, "summary": summary,
+        "details": {"flip_rate_by_column": flip_rate_by_column},
+    }