ids2eval 0.2.2__py3-none-any.whl

ids2eval/data/chunked_io.py

@@ -0,0 +1,387 @@
+"""Chunked CSV/Parquet reading with optional reservoir sampling.
+
+dataset.raw_files/train_file/test_file accept, per file: plain CSV,
+CSV with a compression suffix pandas infers from the name (.csv.gz,
+.csv.bz2, .csv.xz, and .csv.zip provided the zip holds exactly one
+member - pandas' own read_csv handles all of these natively, chunked
+reading included, nothing in this module is compression-aware), or
+.parquet (dispatched to pyarrow, already a hard dependency).
+
+A multi-member zip (several files in one archive, the common shape of
+a real dataset's official download - a features/schema reference, a
+pre-split train/test pair, several raw per-scenario capture files,
+an event log, ...) has no single obvious member to read, so naming
+one explicitly is required: "archive.zip::member_name.csv" (a literal
+"::" separates the archive path from the member's name inside it,
+picked because neither a real filesystem path nor a zip member name
+uses "::"). CSV members stream chunk by chunk exactly like a plain
+file; Parquet members are read whole into memory first, since
+pyarrow's reader needs a seekable source and a zip member isn't one -
+fine for a single reference file, not a substitute for chunk_size on
+something too big to hold in RAM at all.
+
+Not supported: .rar, or a zip member that is itself a nested archive.
+A rar reader needs an external unrar/7z binary on PATH with no
+reliable cross-platform story; extract it yourself first and point at
+the CSV/Parquet file(s) inside, or name a member with "::" if it's
+zipped instead.
+
+A CSV with no header row at all (a real, recurring shape for an
+official download - NSL-KDD's original KDDTrain+.txt/KDDTest+.txt have
+none) is handled by dataset.column_names: when set, every CSV read
+treats row 0 as data and assigns these names positionally, instead of
+inferring a header from the first row. Ignored for Parquet, which
+already carries its own column names in the file. A column_names list
+built for the wrong file raises a clear error naming the mismatch
+rather than pandas' own cryptic "Expected N fields... saw M" - a real,
+reproduced case: UNSW-NB15's four raw capture files have 49 columns,
+while its separate pre-split training-set/testing-set files have 45.
+
+Two independent knobs (dataset.chunk_size / dataset.max_rows):
+
+- chunk_size bounds memory during the read/downcast phase only, by
+  reading each file chunk_size rows at a time and downcasting float64
+  columns to float32 immediately (same rationale as the IDS project's
+  loader: CICFlowMeter-style features are counts/durations/rates that
+  don't need float64 precision). On its own this does NOT prevent OOM
+  on a dataset bigger than RAM, since every downstream step (audit
+  checks, sklearn/imblearn fitting) still needs one consolidated
+  in-memory array.
+
+- max_rows is what actually bounds the final in-memory size: a uniform
+  reservoir sample (Algorithm R) across the whole stream of rows, so
+  the retained dataset never exceeds max_rows regardless of source
+  file size. This is a statistical sample, not the exact full dataset.
+  It operates at the row level: if dataset.split_mode is "grouped",
+  sampling may fragment session groups across the sample boundary;
+  that's a documented limitation, not a bug fix candidate for v1.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def _downcast_chunk(df: pd.DataFrame) -> pd.DataFrame:
+    float_cols = df.select_dtypes(include=["float64"]).columns
+    if len(float_cols):
+        df[float_cols] = df[float_cols].astype("float32")
+    return df
+
+
+def _drop_embedded_header_rows(df: pd.DataFrame) -> pd.DataFrame:
+    """Drop rows that are the column header duplicated as literal data.
+
+    A known, real quirk in multi-file exports (e.g. some CIC-IDS2018 raw
+    day-files): the header row gets embedded partway through the file as
+    an ordinary data row. Left in, it makes some chunks infer a
+    different (object) dtype for an otherwise-numeric column than other
+    chunks do, which crashes reservoir_sample's cross-chunk assignment
+    with a LossySetitemError - a real failure hit against real data, not
+    a hypothetical. Schema-agnostic: flags a row only when EVERY column's
+    value equals that column's own name, so it can't false-positive on
+    a legitimate row that happens to match one header string.
+    """
+    mask = (df.astype(str) == df.columns.astype(str)).all(axis=1)
+    if mask.any():
+        logger.warning("Dropped %d embedded header row(s) found as literal data", int(mask.sum()))
+        df = df.loc[~mask]
+    return df
+
+
+def _drop_blank_rows(df: pd.DataFrame) -> pd.DataFrame:
+    """Drop rows where every single column is missing.
+
+    A real quirk hit against CIC-IDS2017's official GeneratedLabelledFlows
+    release: its Thursday-WorkingHours-Morning-WebAttacks file has 170,366
+    real flow rows followed by 288,602 entirely blank rows (verified by
+    direct inspection of the raw bytes: literal comma-only lines with no
+    values at all) appended to the end of the file, not truncated data or a
+    header/schema issue. A row this empty carries no label and no features,
+    so it can't be used for anything downstream; left in, it crashes the
+    stratified train/test split with "Input contains NaN" before any audit
+    check gets a chance to report on it. Only drops a row that is blank in
+    EVERY column, not merely missing its label, since a row with real
+    feature values but no label is a different, more debatable case that
+    this function deliberately leaves for data_integrity_check to report
+    rather than silently discarding.
+    """
+    mask = df.isna().all(axis=1)
+    if mask.any():
+        logger.warning("Dropped %d fully blank row(s) (every column missing)", int(mask.sum()))
+        df = df.loc[~mask]
+    return df
+
+
+def _is_parquet(name: str) -> bool:
+    return str(name).lower().endswith(".parquet")
+
+
+def _iter_parquet_chunks(source, chunk_size: int | None, column_names: list[str] | None = None) -> Iterator[pd.DataFrame]:
+    """source is a path (str) or a seekable file-like object (an in-memory
+    zip member). pyarrow's reader needs random access for the footer, so a
+    zip member is read fully into a BytesIO buffer by the caller first.
+
+    column_names is ignored here: Parquet always carries its own column
+    names in the file itself, unlike a headerless CSV. Logged once so a
+    dataset.column_names set for a mixed CSV+Parquet raw_files list isn't
+    silently dropped without a trace.
+    """
+    import pyarrow.parquet as pq
+
+    if column_names:
+        logger.warning("dataset.column_names is set but %s is Parquet, which already has its "
+                        "own column names; ignoring dataset.column_names for this file", source)
+    parquet_file = pq.ParquetFile(source)
+    if chunk_size:
+        for batch in parquet_file.iter_batches(batch_size=chunk_size):
+            yield batch.to_pandas()
+    else:
+        yield parquet_file.read().to_pandas()
+
+
+def _iter_csv_chunks(source, chunk_size: int | None, column_names: list[str] | None = None) -> Iterator[pd.DataFrame]:
+    """source is a path (str) or any readable file-like object (a zip
+    member's stream, read sequentially - pandas never seeks it).
+
+    column_names, if given, means the file itself has no header row: every
+    row is data, and these names are assigned positionally (dataset.
+    column_names in the config, for a CSV export with the header stripped
+    or never included, a real, recurring shape of an official NIDS
+    download - NSL-KDD's own original KDDTrain+.txt/KDDTest+.txt are
+    exactly this).
+
+    Tries UTF-8 first and falls back to Windows-1252 for a path (not a zip
+    member stream) that isn't valid UTF-8: a real, reproduced case is
+    CIC-IDS2017's official GeneratedLabelledFlows release, whose Thursday
+    WebAttacks file has a Windows-1252 en-dash byte (0x96) in a "Web Attack
+    - Brute Force"-style label, otherwise plain ASCII. Each attempt is fully
+    materialized into a list before anything is yielded, specifically so a
+    decode error partway through a chunked read can't leave earlier chunks
+    already handed to the caller when the file is re-read from the start
+    under the fallback encoding, which would otherwise silently duplicate
+    those rows. A zip member's stream can't be safely re-read from the
+    start once partially consumed, so only a plain path gets the retry.
+    """
+    kwargs = {"header": None, "names": column_names} if column_names else {}
+    encodings = ("utf-8", "cp1252") if isinstance(source, str) else ("utf-8",)
+    for attempt, encoding in enumerate(encodings):
+        try:
+            read_kwargs = {**kwargs, "encoding": encoding}
+            reader = pd.read_csv(source, chunksize=chunk_size, low_memory=False, **read_kwargs) if chunk_size \
+                else [pd.read_csv(source, low_memory=False, **read_kwargs)]
+            chunks = list(reader)
+        except UnicodeDecodeError:
+            if attempt + 1 < len(encodings):
+                continue
+            raise
+        except pd.errors.ParserError as e:
+            if not column_names:
+                raise
+            # pandas' own message ("Expected N fields... saw M") already names both
+            # counts, but gives no hint that dataset.column_names is the likely
+            # cause - a real, reproduced case: UNSW-NB15's four raw capture files
+            # have 49 columns, while its separate pre-split training-set/
+            # testing-set files have 45; pointing column_names built for one at
+            # the other produces exactly this error.
+            raise ValueError(
+                f"{source}: dataset.column_names has {len(column_names)} names, but this file's "
+                f"rows don't have that many fields ({e}). Check that column_names lists every "
+                "column in this exact file's own order - a multi-file raw release sometimes has a "
+                "different column count per file."
+            ) from e
+        else:
+            if attempt > 0:
+                logger.warning(
+                    "%s: not valid UTF-8, read successfully as %s instead (a common source "
+                    "encoding for these datasets' Windows-based extraction tools)",
+                    source, encoding,
+                )
+            yield from chunks
+            return
+
+
+def _iter_zip_member_chunks(
+    archive_path: str, member: str, chunk_size: int | None, column_names: list[str] | None = None
+) -> Iterator[pd.DataFrame]:
+    import io
+    import zipfile
+
+    with zipfile.ZipFile(archive_path) as archive:
+        if _is_parquet(member):
+            yield from _iter_parquet_chunks(io.BytesIO(archive.read(member)), chunk_size, column_names)
+            return
+        with archive.open(member) as member_file:
+            yield from _iter_csv_chunks(member_file, chunk_size, column_names)
+
+
+def _iter_raw_chunks(path: str, chunk_size: int | None, column_names: list[str] | None = None) -> Iterator[pd.DataFrame]:
+    """One file's (or one zip member's) chunks, dispatched by extension.
+
+    "archive.zip::member.csv" reads just that member; see this module's
+    docstring for why one has to be named explicitly. Otherwise: CSV
+    (plain or with a compression suffix pandas already infers from the
+    name - .gz, .bz2, .zip [single member], .xz) via pd.read_csv,
+    chunksize supported natively. Parquet via pyarrow's own batched
+    reader, since pd.read_parquet has no chunksize equivalent; already
+    a hard dependency (used for output.format: parquet), so this adds
+    no new package to install.
+    """
+    if "::" in path:
+        archive_path, _, member = path.partition("::")
+        yield from _iter_zip_member_chunks(archive_path, member, chunk_size, column_names)
+    elif _is_parquet(path):
+        yield from _iter_parquet_chunks(path, chunk_size, column_names)
+    else:
+        yield from _iter_csv_chunks(path, chunk_size, column_names)
+
+
+def iter_chunks(
+    paths: list[str], chunk_size: int | None, column_names: list[str] | None = None
+) -> Iterator[pd.DataFrame]:
+    """Yield chunks with a dtype schema stable across the whole stream.
+
+    pandas infers dtype independently per chunk of a chunked CSV read -
+    a column that is int64 in one chunk can come out object in another
+    purely from chunk-local quirks (a stray NaN, an unusual value only
+    present in that chunk), with no embedded-header-row or other
+    "real" data-quality issue involved. Real failure hit against
+    CIC-IDS2018: this crashed reservoir_sample's cross-chunk assignment
+    with a LossySetitemError. Fix: lock in which columns are numeric
+    from the first chunk seen, then force every later chunk (including
+    chunks from later files, for a multi-file raw_files list) through
+    pd.to_numeric(errors="coerce") on exactly those columns, so the
+    dtype is stable across the entire stream regardless of per-chunk
+    inference quirks. Matches the IDS project's own loader, which does
+    the same coercion for the same reason. Known limitation: trusts the
+    first chunk's inference: a column that's genuinely numeric but
+    happens to look non-numeric in just the first chunk would be
+    (wrongly) treated as categorical for the rest of the stream.
+
+    Columns locked as non-numeric from the first chunk get the mirror-image
+    treatment: forced to string dtype on every later chunk. Real failure hit
+    against CICDDoS2019: a column mostly empty/placeholder values (inferred
+    object dtype in the first chunk) had a later chunk where every value
+    happened to parse as a plain integer, so the concatenated column ended
+    up with genuinely mixed str/int Python objects in one array - pyarrow's
+    to_parquet (via cache.save) rejects that outright ("Expected bytes, got
+    a 'int' object"). Without this, only numeric-locked columns were immune
+    to the equivalent problem in the other direction.
+    """
+    numeric_columns = None
+    object_columns = None
+    reference_columns = None
+    for path in paths:
+        for chunk in _iter_raw_chunks(path, chunk_size, column_names):
+            chunk.columns = chunk.columns.str.strip()
+            chunk = _drop_embedded_header_rows(chunk)
+            chunk = _drop_blank_rows(chunk)
+
+            if reference_columns is None:
+                reference_columns = chunk.columns
+            elif not chunk.columns.equals(reference_columns):
+                # Real quirk hit against CIC-IDS2018: one raw day-file out of
+                # ten carries 4 extra columns (Flow ID/Src IP/Src Port/Dst IP)
+                # no other file has. Reservoir_sample's positional cross-chunk
+                # assignment requires every chunk to share one fixed column
+                # set, so align to whichever schema was seen first - extra
+                # columns are dropped, missing ones filled NaN - rather than
+                # crashing or silently misaligning columns by position.
+                extra = set(chunk.columns) - set(reference_columns)
+                missing = set(reference_columns) - set(chunk.columns)
+                logger.warning(
+                    "%s: column schema differs from the first file read (extra: %s, "
+                    "missing: %s) - aligning to the first file's schema",
+                    path, sorted(extra), sorted(missing),
+                )
+                chunk = chunk.reindex(columns=reference_columns)
+
+            if numeric_columns is None:
+                numeric_columns = chunk.select_dtypes(include=["number"]).columns
+                object_columns = chunk.columns.difference(numeric_columns)
+            for col in numeric_columns:
+                if col in chunk.columns:
+                    # Always land on float64 here, never int64: an int64
+                    # column in one chunk (e.g. a 0/1 flag column with no
+                    # NaN in that particular chunk) and a float64 version
+                    # of the same column in another (NaN introduced by
+                    # coercion, or genuinely fractional values) hits the
+                    # same LossySetitemError this whole schema-locking
+                    # exists to prevent - real failure, CIC-IDS2018.
+                    chunk[col] = pd.to_numeric(chunk[col], errors="coerce").astype("float64")
+            for col in object_columns:
+                if col in chunk.columns and not pd.api.types.is_object_dtype(chunk[col]):
+                    # Mirror image of the numeric case above: this column looked
+                    # non-numeric in the first chunk, but this later chunk's values
+                    # all happen to parse as a different dtype (e.g. every value here
+                    # is a plain integer) - force back to string, preserving real NaN
+                    # rather than stringifying it to the literal text "nan".
+                    chunk[col] = chunk[col].where(chunk[col].isna(), chunk[col].astype(str))
+            yield _downcast_chunk(chunk)
+
+
+def reservoir_sample(chunks: Iterator[pd.DataFrame], max_rows: int, seed: int = 0) -> tuple[pd.DataFrame, int]:
+    """Uniform reservoir sample of at most max_rows rows across all chunks.
+
+    Vectorized batched Algorithm R: within a chunk, later rows correctly
+    overwrite earlier rows at the same reservoir slot (pandas/numpy
+    fancy-indexing assignment resolves duplicate target positions by
+    keeping the last one in source order), matching the sequential
+    semantics of the classic single-item algorithm.
+    """
+    rng = np.random.default_rng(seed)
+    reservoir: pd.DataFrame | None = None
+    n_seen = 0
+
+    for chunk in chunks:
+        if reservoir is None:
+            reservoir = chunk.iloc[0:0].copy()
+
+        if n_seen < max_rows:
+            take = min(max_rows - n_seen, len(chunk))
+            reservoir = pd.concat([reservoir, chunk.iloc[:take]], ignore_index=True)
+            chunk = chunk.iloc[take:]
+            n_seen += take
+
+        m = len(chunk)
+        if m > 0:
+            global_idx = np.arange(n_seen, n_seen + m)
+            j = rng.integers(0, global_idx + 1)  # per-row draw in [0, i], inclusive
+            replace_mask = j < max_rows
+            if replace_mask.any():
+                reservoir.iloc[j[replace_mask]] = chunk.iloc[np.flatnonzero(replace_mask)].values
+            n_seen += m
+
+    if reservoir is None:
+        raise ValueError("No data read, chunk source was empty")
+    return reservoir, n_seen
+
+
+def load_file(
+    path: str, chunk_size: int | None, max_rows: int | None, seed: int = 0,
+    column_names: list[str] | None = None,
+) -> pd.DataFrame:
+    chunks = iter_chunks([path], chunk_size, column_names)
+    if max_rows:
+        df, n_seen = reservoir_sample(chunks, max_rows, seed)
+        logger.info("Reservoir-sampled %s: %d rows seen -> %d kept (dataset.max_rows)", path, n_seen, len(df))
+        return df
+    return pd.concat(list(chunks), ignore_index=True)
+
+
+def load_files_combined(
+    paths: list[str], chunk_size: int | None, max_rows: int | None, seed: int = 0,
+    column_names: list[str] | None = None,
+) -> pd.DataFrame:
+    chunks = iter_chunks(paths, chunk_size, column_names)
+    if max_rows:
+        df, n_seen = reservoir_sample(chunks, max_rows, seed)
+        logger.info("Reservoir-sampled %d files: %d rows seen -> %d kept (dataset.max_rows)", len(paths), n_seen, len(df))
+        return df
+    return pd.concat(list(chunks), ignore_index=True)

ids2eval/data/dataset.py

@@ -0,0 +1,208 @@
+"""Load raw or pre-split IDS data and produce a train/test split.
+
+Ported and generalized from the IDS project's data/loader.py and
+data/preprocessor.py._deduplicate, same session-grouped-split and
+feature-space-dedup logic, but driven by config instead of hardcoded
+per-dataset column names.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+from sklearn.model_selection import StratifiedGroupKFold, train_test_split
+
+from . import chunked_io
+
+logger = logging.getLogger(__name__)
+
+
+def load_raw_combined(dataset_cfg: dict, seed: int = 0) -> pd.DataFrame:
+    """Load and concatenate dataset.raw_files, before any split is applied.
+
+    Honors dataset.chunk_size/max_rows, see chunked_io module docstring
+    for what each actually bounds. seed drives reservoir_sample's
+    randomness when max_rows is set.
+    """
+    if dataset_cfg["max_rows"] and dataset_cfg["group_columns"]:
+        logger.warning(
+            "dataset.max_rows reservoir sampling is row-level, not group-aware. "
+            "session groups (dataset.group_columns) may be fragmented across the "
+            "sample boundary before any grouped split is applied."
+        )
+    return chunked_io.load_files_combined(
+        dataset_cfg["raw_files"], dataset_cfg["chunk_size"], dataset_cfg["max_rows"], seed,
+        dataset_cfg["column_names"],
+    )
+
+
+def load_split(cfg: dict) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Return (train_df, test_df), pre-split if configured, else loaded+split."""
+    dataset_cfg = cfg["dataset"]
+    seed = cfg["random_seed"]
+    if dataset_cfg["train_file"] and dataset_cfg["test_file"]:
+        chunk_size, max_rows = dataset_cfg["chunk_size"], dataset_cfg["max_rows"]
+        column_names = dataset_cfg["column_names"]
+        train_df = chunked_io.load_file(dataset_cfg["train_file"], chunk_size, max_rows, seed, column_names)
+        test_df = chunked_io.load_file(dataset_cfg["test_file"], chunk_size, max_rows, seed, column_names)
+        return train_df, test_df
+
+    combined = load_raw_combined(dataset_cfg, seed)
+    label_col = cfg["schema"]["label_column"]
+    if dataset_cfg["split_mode"] == "grouped":
+        return _grouped_split(combined, label_col, dataset_cfg, seed)
+    return _random_split(combined, label_col, dataset_cfg, seed)
+
+
+def _random_split(
+    df: pd.DataFrame, label_col: str, dataset_cfg: dict, seed: int = 0
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    train, test = train_test_split(
+        df,
+        test_size=1.0 - dataset_cfg["split_ratio"],
+        random_state=seed,
+        stratify=df[label_col],
+    )
+    return train.reset_index(drop=True), test.reset_index(drop=True)
+
+
+def _grouped_split(
+    df: pd.DataFrame, label_col: str, dataset_cfg: dict, seed: int = 0
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Session/time-grouped split via StratifiedGroupKFold.
+
+    Keeps every group_columns combination on one side of the split, so
+    session-correlated leakage across the train/test boundary is
+    structurally impossible. See the resplit-falsification audit check,
+    which uses this same split as its counterfactual.
+    """
+    group_cols = [c for c in dataset_cfg["group_columns"] if c in df.columns]
+    if not group_cols:
+        raise ValueError(
+            f"None of dataset.group_columns {dataset_cfg['group_columns']} "
+            f"are present in the loaded data"
+        )
+    # Vectorized string concatenation, not df[group_cols].astype(str).agg("|".join,
+    # axis=1): that row-wise .agg runs a Python-level join once per row, which at
+    # full scale (hundreds of millions of rows) becomes the dominant memory cost
+    # in the whole audit - a real failure this was built to avoid, not a
+    # theoretical one.
+    groups = df[group_cols[0]].astype(str)
+    for col in group_cols[1:]:
+        groups = groups + "|" + df[col].astype(str)
+
+    n_splits = round(1.0 / (1.0 - dataset_cfg["split_ratio"]))
+    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=seed)
+    train_idx, test_idx = next(sgkf.split(df, df[label_col], groups))
+    train = df.iloc[train_idx].reset_index(drop=True)
+    test = df.iloc[test_idx].reset_index(drop=True)
+
+    train_groups = set(groups.iloc[train_idx])
+    test_groups = set(groups.iloc[test_idx])
+    overlap = train_groups & test_groups
+    if overlap:
+        raise RuntimeError(
+            f"Grouped split produced {len(overlap)} groups on both sides of "
+            "train/test. StratifiedGroupKFold invariant violated, refusing "
+            "to silently continue."
+        )
+    logger.info(
+        "Grouped split: %d groups total, %d train / %d test, zero overlap confirmed.",
+        len(train_groups | test_groups), len(train_groups), len(test_groups),
+    )
+    return train, test
+
+
+def dedup(
+    train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict
+) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
+    """Drop exact feature-space duplicates within and across splits.
+
+    Compares on every column except schema.drop_columns/label_column/
+    attack_category_column. A test row whose FEATURES exactly duplicate
+    a train row measures memorization, not generalization, regardless of
+    whether its label happens to match. Zero tolerance here is deliberately
+    stricter than general ML data-quality guidance, which treats under ~10%
+    train/test overlap as acceptable "Variety" (Zhou et al. 2024,
+    arXiv:2406.19614, Table II) - any leakage at all overstates a NIDS
+    classifier's real-world generalization, so this check flags on any
+    amount rather than only past a tolerance threshold.
+    """
+    schema = cfg["schema"]
+    ignore = set(schema["drop_columns"]) | {schema["label_column"]}
+    if schema["attack_category_column"]:
+        ignore.add(schema["attack_category_column"])
+    compare_cols = [c for c in train_df.columns if c not in ignore]
+
+    n_train0, n_test0 = len(train_df), len(test_df)
+
+    n_train_dupe = int(train_df.duplicated(subset=compare_cols).sum())
+    train_df = train_df.drop_duplicates(subset=compare_cols).reset_index(drop=True)
+
+    merged = test_df.merge(
+        train_df[compare_cols].drop_duplicates(), on=compare_cols, how="left", indicator=True
+    )
+    leak_mask = (merged["_merge"] == "both").values
+    n_leak = int(leak_mask.sum())
+    test_df = test_df.loc[~leak_mask].reset_index(drop=True)
+
+    n_test_dupe = int(test_df.duplicated(subset=compare_cols).sum())
+    test_df = test_df.drop_duplicates(subset=compare_cols).reset_index(drop=True)
+
+    stats = {
+        "train_rows_raw": n_train0,
+        "train_rows_deduped": len(train_df),
+        "train_duplicates_dropped": n_train_dupe,
+        "test_rows_raw": n_test0,
+        "test_rows_deduped": len(test_df),
+        "test_leakage_dropped": n_leak,
+        "test_duplicates_dropped": n_test_dupe,
+    }
+    return train_df, test_df, stats
+
+
+def validate_loaded(train_df: pd.DataFrame, test_df: pd.DataFrame, cfg: dict) -> None:
+    """Hard-fail checks on structurally broken input, raised before anything
+    downstream (caching, audit, benchmarking) runs against it.
+
+    Distinct from audit checks: these aren't "worth a human's attention",
+    they're "nothing else in this pipeline will produce a meaningful
+    result," so they raise instead of returning a flagged finding.
+    """
+    errors: list[str] = []
+    schema = cfg["schema"]
+
+    for name, df in [("train", train_df), ("test", test_df)]:
+        if len(df) == 0:
+            errors.append(f"{name} data has zero rows after loading/splitting")
+        dupes = df.columns[df.columns.duplicated()].tolist()
+        if dupes:
+            errors.append(f"{name} data has duplicate column names: {sorted(set(dupes))}")
+
+    label_col = schema["label_column"]
+    if label_col not in train_df.columns:
+        errors.append(f"schema.label_column '{label_col}' not found in the loaded train data")
+    if label_col not in test_df.columns:
+        errors.append(f"schema.label_column '{label_col}' not found in the loaded test data")
+
+    attack_col = schema["attack_category_column"]
+    if attack_col:
+        if attack_col not in train_df.columns:
+            errors.append(f"schema.attack_category_column '{attack_col}' not found in the loaded train data")
+        if attack_col not in test_df.columns:
+            errors.append(f"schema.attack_category_column '{attack_col}' not found in the loaded test data")
+
+    if not errors:  # only meaningful to check once the columns above exist
+        common = set(train_df.columns) & set(test_df.columns)
+        ignore = set(schema["drop_columns"]) | {label_col}
+        if attack_col:
+            ignore.add(attack_col)
+        if not (common - ignore):
+            errors.append(
+                "train and test data share no feature columns after excluding "
+                "label/attack_category/drop_columns, nothing left to train or evaluate on"
+            )
+
+    if errors:
+        raise ValueError("Loaded dataset failed validation:\n" + "\n".join(f"  - {e}" for e in errors))