CPSBench 0.1.0__py3-none-any.whl

cpsbench/__init__.py

@@ -0,0 +1,76 @@
+"""CPSBench datasets, loaded like MNIST.
+
+Example::
+
+    from cpsbench import SynCAN
+    from torch.utils.data import DataLoader
+
+    train = SynCAN(root="./data", split="train", download=True)
+    window, label = train[0]
+    loader = DataLoader(train, batch_size=64, shuffle=True)
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from .datasets import MisbehaviorX, ROAD, SynCAN, VehicularDataset
+from .specs import DatasetSpec, get_spec, list_specs
+
+__all__ = [
+    "SynCAN",
+    "ROAD",
+    "MisbehaviorX",
+    "VehicularDataset",
+    "load",
+    "list_datasets",
+    "describe",
+    "get_spec",
+]
+
+__version__ = "0.1.0"
+
+_CLASSES = {
+    "syncan": SynCAN,
+    "road": ROAD,
+    "misbehaviorx": MisbehaviorX,
+    "vasp": MisbehaviorX,
+    "veremi": MisbehaviorX,
+}
+
+
+def list_datasets(status: Optional[str] = None) -> list[dict]:
+    """Return name, status, family, and shape for every registered dataset."""
+    rows = []
+    for spec in list_specs(status):
+        rows.append(
+            {
+                "name": spec.name,
+                "status": spec.status,
+                "family": spec.family,
+                "input_shape": spec.input_shape if spec.features else None,
+                "downloadable": spec.downloadable,
+                "description": spec.description,
+            }
+        )
+    return rows
+
+
+def describe(name: str) -> DatasetSpec:
+    return get_spec(name)
+
+
+def load(name: str, **kwargs) -> VehicularDataset:
+    """Load a dataset by name.
+
+    ``kwargs`` are forwarded to the dataset class (``root``, ``split``,
+    ``download``, ``window_size``, ...).
+    """
+    key = name.strip().lower()
+    if key == "x-canids":
+        spec = get_spec(key)
+        raise NotImplementedError(f"{spec.name} is not implemented yet. {spec.notes}")
+    if key not in _CLASSES:
+        known = ", ".join(sorted(set(_CLASSES) | {"x-canids"}))
+        raise KeyError(f"Unknown dataset '{name}'. Known datasets: {known}")
+    return _CLASSES[key](**kwargs)

cpsbench/__main__.py

@@ -0,0 +1,38 @@
+"""Small command-line helper: ``python -m cpsbench list``."""
+
+from __future__ import annotations
+
+import argparse
+
+from . import describe, list_datasets, load
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Download and inspect vehicular IDS datasets.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("list", help="List registered datasets")
+
+    info = sub.add_parser("info", help="Show one dataset specification")
+    info.add_argument("name")
+
+    fetch = sub.add_parser("download", help="Download and preprocess a split")
+    fetch.add_argument("name")
+    fetch.add_argument("--root", default="./data")
+    fetch.add_argument("--split", default="train", choices=["train", "test"])
+
+    args = parser.parse_args()
+    if args.command == "list":
+        for row in list_datasets():
+            shape = row["input_shape"] or "n/a"
+            print(f"{row['name']:16} {row['status']:8} {row['family']:4} {shape}")
+    elif args.command == "info":
+        spec = describe(args.name)
+        print(spec)
+    else:
+        dataset = load(args.name, root=args.root, split=args.split, download=True, verbose=True)
+        print(f"Loaded {len(dataset)} windows with shape {dataset.input_shape}")
+
+
+if __name__ == "__main__":
+    main()

cpsbench/datasets.py

@@ -0,0 +1,130 @@
+"""Public dataset classes.
+
+Usage matches torchvision:
+
+    from cpsbench import SynCAN
+
+    train = SynCAN(root="./data", split="train", download=True)
+    window, label = train[0]
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from torch.utils.data import Dataset
+
+from .download import dataset_root, ensure_downloaded, split_dir
+from .preprocess import prepare_can_split
+from .specs import DatasetSpec, get_spec
+from .v2x import V2XWindowDataset, prepare_v2x_split
+from .windows import WindowDataset
+
+
+class VehicularDataset(Dataset):
+    """Base class for datasets that download, preprocess, and yield windows."""
+
+    spec_name: str = ""
+
+    def __init__(
+        self,
+        root: str | Path = "./data",
+        split: str = "train",
+        download: bool = True,
+        transform=None,
+        target_transform=None,
+        window_size: Optional[int] = None,
+        step_size: Optional[int] = None,
+        sampling_period: Optional[int] = None,
+        return_meta: bool = False,
+        data_dir: Optional[str | Path] = None,
+        scaler_dir: Optional[str | Path] = None,
+        n_jobs: Optional[int] = None,
+        verbose: bool = False,
+    ) -> None:
+        spec = get_spec(self.spec_name).with_overrides(window_size, step_size, sampling_period)
+        if spec.status == "planned":
+            raise NotImplementedError(f"{spec.name} is not implemented yet. {spec.notes}")
+
+        self.spec = spec
+        self.root = Path(root).expanduser().resolve()
+        self.split = split
+        self.transform = transform
+        self.target_transform = target_transform
+        self.return_meta = return_meta
+
+        if data_dir is None and download:
+            ensure_downloaded(spec, self.root)
+
+        self.data_dir = Path(data_dir) if data_dir is not None else split_dir(self.root, spec.name, split)
+        if scaler_dir is not None:
+            self.scaler_path = Path(scaler_dir) / f"min_max_values_{spec.name}.csv"
+        else:
+            self.scaler_path = dataset_root(self.root, spec.name) / "scaler" / f"min_max_values_{spec.name}.csv"
+
+        fit_scaler = split.strip().lower() in {"train", "training", "ambient"}
+        if spec.family == "can":
+            prepare_can_split(spec, self.data_dir, self.scaler_path, fit_scaler, n_jobs=n_jobs)
+            self._base = WindowDataset(spec, self.data_dir, self.scaler_path, return_meta, verbose)
+        elif spec.family == "v2x":
+            prepare_v2x_split(spec, self.data_dir, self.scaler_path, fit_scaler)
+            self._base = V2XWindowDataset(spec, self.data_dir, self.scaler_path, return_meta, verbose)
+        else:
+            raise ValueError(f"Unsupported dataset family: {spec.family}")
+
+    def __len__(self) -> int:
+        return len(self._base)
+
+    def __getitem__(self, idx: int):
+        item = self._base[idx]
+        window, label = item[0], item[1]
+        if self.transform is not None:
+            window = self.transform(window)
+        if self.target_transform is not None:
+            label = self.target_transform(label)
+        if self.return_meta:
+            return window, label, item[2]
+        return window, label
+
+    @property
+    def classes(self) -> tuple[str, ...]:
+        return self.spec.classes
+
+    @property
+    def input_shape(self) -> tuple[int, int, int]:
+        return self.spec.input_shape
+
+    @property
+    def num_signals(self) -> int:
+        return self.spec.num_signals
+
+    @property
+    def window_size(self) -> int:
+        return self.spec.window_size
+
+    @property
+    def channels(self) -> int:
+        return self.spec.channels
+
+    @property
+    def features(self) -> tuple[str, ...]:
+        return self.spec.features
+
+
+class SynCAN(VehicularDataset):
+    """SynCAN intrusion dataset. ``split`` is ``'train'`` or ``'test'``."""
+
+    spec_name = "syncan"
+
+
+class ROAD(VehicularDataset):
+    """ROAD CAN intrusion dataset. ``split`` is ``'train'`` or ``'test'``."""
+
+    spec_name = "road"
+
+
+class MisbehaviorX(VehicularDataset):
+    """V2X misbehavior dataset. Requires a local copy; download is manual."""
+
+    spec_name = "misbehaviorx"

cpsbench/download.py

@@ -0,0 +1,160 @@
+"""Download raw vehicular datasets into a local root directory."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import urllib.request
+import zipfile
+from pathlib import Path
+
+from .specs import DatasetSpec
+
+
+class DownloadError(RuntimeError):
+    """Raised when a dataset cannot be fetched or unpacked."""
+
+
+def _require_tool(name: str) -> None:
+    if shutil.which(name) is None:
+        raise DownloadError(
+            f"'{name}' is required to download this dataset but was not found on PATH."
+        )
+
+
+def _download_file(url: str, destination: Path) -> None:
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    print(f"Downloading {url}")
+    try:
+        urllib.request.urlretrieve(url, destination)
+    except Exception as exc:
+        raise DownloadError(f"Failed to download {url}: {exc}") from exc
+
+
+def split_dirname(split: str) -> str:
+    key = split.strip().lower()
+    mapping = {
+        "train": "ambient",
+        "training": "ambient",
+        "ambient": "ambient",
+        "test": "attacks",
+        "testing": "attacks",
+        "attack": "attacks",
+        "attacks": "attacks",
+    }
+    if key not in mapping:
+        raise ValueError(
+            f"Unsupported split '{split}'. Use 'train'/'ambient' or 'test'/'attacks'."
+        )
+    return mapping[key]
+
+
+def dataset_root(root: str | Path, name: str) -> Path:
+    return Path(root).expanduser().resolve() / name
+
+
+def split_dir(root: str | Path, name: str, split: str) -> Path:
+    return dataset_root(root, name) / split_dirname(split)
+
+
+def raw_csvs_present(path: Path) -> bool:
+    return path.is_dir() and any(path.glob("*.csv"))
+
+
+def download_syncan(destination: Path) -> None:
+    """Clone SynCAN and unpack ambient/attacks next to each other."""
+    _require_tool("git")
+    destination.mkdir(parents=True, exist_ok=True)
+    if raw_csvs_present(destination / "ambient") and raw_csvs_present(destination / "attacks"):
+        print(f"SynCAN already present at {destination}")
+        return
+
+    clone_dir = destination / "_raw"
+    if clone_dir.exists():
+        shutil.rmtree(clone_dir)
+    print(f"Cloning SynCAN into {clone_dir}")
+    subprocess.run(
+        ["git", "clone", "--depth", "1", "https://github.com/etas/SynCAN.git", str(clone_dir)],
+        check=True,
+    )
+
+    for pattern, folder in (("train_*.zip", "ambient"), ("test_*.zip", "attacks")):
+        zips = sorted(clone_dir.glob(pattern))
+        if not zips:
+            raise DownloadError(f"SynCAN clone is missing {pattern} archives.")
+        out = destination / folder
+        out.mkdir(parents=True, exist_ok=True)
+        for archive in zips:
+            print(f"Extracting {archive.name} -> {out}")
+            with zipfile.ZipFile(archive) as zf:
+                zf.extractall(out)
+
+    for normal in (destination / "attacks").glob("test_normal*"):
+        if normal.is_file():
+            normal.unlink()
+    shutil.rmtree(clone_dir, ignore_errors=True)
+    print(f"SynCAN downloaded to {destination}")
+
+
+def download_road(destination: Path) -> None:
+    """Download the ROAD signal-extraction release from Zenodo."""
+    destination.mkdir(parents=True, exist_ok=True)
+    if raw_csvs_present(destination / "ambient") and raw_csvs_present(destination / "attacks"):
+        print(f"ROAD already present at {destination}")
+        return
+
+    archive = destination / "road.zip"
+    _download_file("https://zenodo.org/records/10462796/files/road.zip", archive)
+    extract_root = destination / "_raw"
+    if extract_root.exists():
+        shutil.rmtree(extract_root)
+    extract_root.mkdir(parents=True, exist_ok=True)
+    print(f"Extracting {archive.name}")
+    with zipfile.ZipFile(archive) as zf:
+        zf.extractall(extract_root)
+
+    signal_dir = next(extract_root.rglob("signal_extractions"), None)
+    source = signal_dir if signal_dir is not None else extract_root
+    for folder in ("ambient", "attacks"):
+        matches = [path for path in source.rglob(folder) if path.is_dir()]
+        if not matches:
+            raise DownloadError(
+                f"ROAD archive does not contain an '{folder}' directory. "
+                f"Inspect {extract_root} and place the CSVs manually."
+            )
+        target = destination / folder
+        if target.exists():
+            shutil.rmtree(target)
+        shutil.move(str(matches[0]), str(target))
+
+    # Original release includes non-ambient files inside ambient/.
+    for extra in (destination / "ambient").iterdir():
+        if extra.is_file() and not extra.name.startswith("ambient_"):
+            extra.unlink()
+
+    archive.unlink(missing_ok=True)
+    shutil.rmtree(extract_root, ignore_errors=True)
+    print(f"ROAD downloaded to {destination}")
+
+
+def ensure_downloaded(spec: DatasetSpec, root: str | Path) -> Path:
+    """Download ``spec`` under ``root/<name>`` and return that directory."""
+    destination = dataset_root(root, spec.name)
+    if spec.status == "planned":
+        raise DownloadError(
+            f"{spec.name} is registered but not implemented yet. {spec.notes}"
+        )
+    if not spec.downloadable:
+        if raw_csvs_present(destination / "ambient") or raw_csvs_present(destination / "attacks"):
+            return destination
+        raise DownloadError(
+            f"{spec.name} cannot be downloaded automatically. {spec.notes} "
+            f"Expected files under {destination}."
+        )
+    if spec.name == "syncan":
+        download_syncan(destination)
+    elif spec.name == "road":
+        download_road(destination)
+    else:
+        raise DownloadError(f"No downloader registered for {spec.name}.")
+    return destination

cpsbench/preprocess.py

@@ -0,0 +1,158 @@
+"""Turn raw CAN CSVs into window-ready signal and attribute arrays."""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+from .specs import DatasetSpec
+
+
+def _process_id(target_id, frame: pd.DataFrame) -> pd.DataFrame:
+    per_id = frame[frame["ID"] == target_id].T.dropna().T
+    rename = {}
+    skip = {"ID", "Label", "Time"}
+    for column in set(per_id.columns) - skip:
+        cleaned = column.replace("Signal_", "Signal").replace("_of_ID", "")
+        rename[column] = f"ID_{target_id}_" + cleaned.replace("Signal", "Sig_")
+    renamed = per_id.rename(columns=rename)
+    keep = [name for name in rename.values() if name in renamed.columns]
+    return renamed[keep]
+
+
+def _update_minmax(spec: DatasetSpec, signals: pd.DataFrame, scaler_path: Path) -> None:
+    features = list(spec.features)
+    present = [name for name in features if name in signals.columns]
+    if not present:
+        return
+    try:
+        stored = pd.read_csv(scaler_path, index_col=0)
+        overall_min = stored["Min"].reindex(features)
+        overall_max = stored["Max"].reindex(features)
+    except (FileNotFoundError, KeyError, pd.errors.EmptyDataError):
+        scaler_path.parent.mkdir(parents=True, exist_ok=True)
+        overall_min = pd.Series(np.nan, index=features)
+        overall_max = pd.Series(np.nan, index=features)
+
+    current_min = signals[present].min(axis=0).to_numpy()
+    current_max = signals[present].max(axis=0).to_numpy()
+    stored_min = overall_min.loc[present].to_numpy(dtype=float)
+    stored_max = overall_max.loc[present].to_numpy(dtype=float)
+    overall_min.loc[present] = np.where(np.isnan(stored_min), current_min, np.minimum(stored_min, current_min))
+    overall_max.loc[present] = np.where(np.isnan(stored_max), current_max, np.maximum(stored_max, current_max))
+    pd.DataFrame({"Min": overall_min, "Max": overall_max}, index=features).to_csv(scaler_path)
+
+
+def _sparse_one_file(
+    spec: DatasetSpec,
+    file_name: str,
+    file_path: Path,
+    scaler_path: Path,
+    fit_scaler: bool,
+    n_jobs: int,
+) -> None:
+    frame = pd.read_csv(file_path, skiprows=1, names=list(spec.org_features))
+    frame["ID"] = frame["ID"].astype(str).str.replace("id", "", regex=False)
+
+    total_elements = 75_000_000
+    max_length = max(int(total_elements / max(len(spec.org_features), 1)), 1)
+    n_chunks = int(np.ceil(len(frame) / max_length))
+    chunk_length = int(np.ceil(len(frame) / max(n_chunks, 1)))
+
+    for index in range(n_chunks):
+        chunk = frame.iloc[index * chunk_length : min((index + 1) * chunk_length, len(frame))]
+        pieces = Parallel(n_jobs=n_jobs)(
+            delayed(_process_id)(target_id, chunk)
+            for target_id in tqdm(chunk["ID"].unique(), desc=file_name, leave=False)
+        )
+        extended = pd.concat(pieces, axis=1)
+        extended.insert(0, "File", file_name)
+        extended[["ID", "Label", "Time"]] = chunk[["ID", "Label", "Time"]]
+        extended = extended.sort_index()
+
+        out = file_path.parent / "generated" / f"gen_{file_name}_{index + 1}.parquet"
+        out.parent.mkdir(parents=True, exist_ok=True)
+        extended.to_parquet(out, engine="pyarrow", compression="snappy")
+        if fit_scaler:
+            _update_minmax(spec, extended, scaler_path)
+
+
+def _write_file_index(split_path: Path) -> dict[str, int]:
+    csvs = sorted(glob.glob(str(split_path / "*.csv")))
+    mapping = {Path(path).stem: index for index, path in enumerate(csvs)}
+    with open(split_path / "file_index_dict.json", "w", encoding="utf-8") as handle:
+        json.dump(mapping, handle, indent=2)
+    return mapping
+
+
+def _npy_one_file(
+    spec: DatasetSpec,
+    file_name: str,
+    parquet_path: Path,
+    file_enum: dict[str, int],
+) -> None:
+    frame = pd.read_parquet(parquet_path, engine="pyarrow")
+    frame = frame.replace(file_enum)
+    signals = frame[list(spec.features)]
+    attributes = frame[list(spec.attributes)].copy()
+    attributes["ID"] = attributes["ID"].astype(int)
+
+    total_elements = 100_000_000
+    max_length = max(int(total_elements / max(len(spec.features), 1)), 1)
+    n_chunks = int(np.ceil(len(signals) / max_length))
+    generated = parquet_path.parent
+
+    for index in range(n_chunks):
+        start = int(index * max_length)
+        stop = min(int((index + 1) * max_length), len(signals))
+        signal_chunk = signals.iloc[start:stop]
+        attr_chunk = attributes.iloc[start:stop]
+        if spec.filling == "forward":
+            signal_chunk = signal_chunk.ffill().bfill()
+        np.save(generated / f"sig_{file_name}_{index + 1}.npy", signal_chunk.to_numpy())
+        np.save(generated / f"att_{file_name}_{index + 1}.npy", attr_chunk.to_numpy())
+
+
+def prepare_can_split(
+    spec: DatasetSpec,
+    split_path: Path,
+    scaler_path: Path,
+    fit_scaler: bool,
+    n_jobs: int | None = None,
+) -> None:
+    """Download-side preprocessing for one SynCAN/ROAD split directory."""
+    split_path = Path(split_path)
+    if not any(split_path.glob("*.csv")):
+        raise FileNotFoundError(
+            f"No raw CSV files in {split_path}. Download the dataset or point root at an existing copy."
+        )
+
+    generated = split_path / "generated"
+    generated.mkdir(parents=True, exist_ok=True)
+    workers = n_jobs if n_jobs is not None else min(8, os.cpu_count() or 1)
+
+    if not list(generated.glob("gen_*.parquet")):
+        for csv_path in sorted(split_path.glob("*.csv")):
+            _sparse_one_file(spec, csv_path.stem, csv_path, scaler_path, fit_scaler, workers)
+    elif fit_scaler and not scaler_path.exists():
+        for parquet_path in sorted(generated.glob("gen_*.parquet")):
+            frame = pd.read_parquet(parquet_path, engine="pyarrow", columns=list(spec.features))
+            _update_minmax(spec, frame, scaler_path)
+
+    if not scaler_path.exists():
+        raise FileNotFoundError(
+            f"Missing scaler {scaler_path}. Prepare the train/ambient split first so min/max can be fit."
+        )
+
+    file_enum = _write_file_index(split_path)
+    for parquet_path in sorted(generated.glob("gen_*.parquet")):
+        npy_files = list(generated.glob(f"sig_{parquet_path.stem}_*.npy"))
+        if not npy_files:
+            _npy_one_file(spec, parquet_path.stem, parquet_path, file_enum)

cpsbench/py.typed

@@ -0,0 +1 @@
+

cpsbench/specs.py

@@ -0,0 +1,179 @@
+"""Built-in dataset specifications.
+
+These replace per-experiment YAML so a caller can load a dataset the same way
+torchvision loads MNIST: name, root, split, download.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class DatasetSpec:
+    """Static description of one vehicular cybersecurity dataset."""
+
+    name: str
+    family: str  # "can" or "v2x"
+    description: str
+    citation: str
+    source_url: str
+    downloadable: bool
+    features: tuple[str, ...]
+    attributes: tuple[str, ...]
+    org_features: tuple[str, ...] = ()
+    window_size: int = 50
+    step_size: int = 10
+    sampling_period: int = 1
+    sampling_period_factors: tuple[int, ...] = ()
+    filling: str = "forward"
+    channels: int = 1
+    label_index: int = 2
+    classes: tuple[str, ...] = ("benign", "attack")
+    status: str = "ready"  # ready | manual | planned
+    notes: str = ""
+
+    @property
+    def num_signals(self) -> int:
+        return len(self.features)
+
+    @property
+    def input_shape(self) -> tuple[int, int, int]:
+        """Channel-first window shape, matching a 1-channel image."""
+        return (self.channels, self.window_size, self.num_signals)
+
+    def with_overrides(
+        self,
+        window_size: Optional[int] = None,
+        step_size: Optional[int] = None,
+        sampling_period: Optional[int] = None,
+    ) -> "DatasetSpec":
+        updates = {}
+        if window_size is not None:
+            updates["window_size"] = int(window_size)
+        if step_size is not None:
+            updates["step_size"] = int(step_size)
+        if sampling_period is not None:
+            updates["sampling_period"] = int(sampling_period)
+        return replace(self, **updates) if updates else self
+
+
+SYNCAN_FEATURES = (
+    "ID_2_Sig_1", "ID_7_Sig_1", "ID_3_Sig_2", "ID_10_Sig_1", "ID_9_Sig_1",
+    "ID_1_Sig_1", "ID_10_Sig_4", "ID_2_Sig_2", "ID_10_Sig_3", "ID_6_Sig_1",
+    "ID_5_Sig_2", "ID_4_Sig_1", "ID_5_Sig_1", "ID_2_Sig_3", "ID_8_Sig_1",
+    "ID_6_Sig_2", "ID_10_Sig_2", "ID_7_Sig_2", "ID_1_Sig_2", "ID_3_Sig_1",
+)
+
+# ID-2,6,9 -> 15/5, ID-4,10 -> 25/10, ID-1,3,5,7,8 -> 5/1
+SYNCAN_PERIOD_FACTORS = (5, 1, 1, 10, 5, 1, 10, 5, 10, 5, 1, 10, 1, 5, 1, 5, 10, 1, 1, 1)
+
+ROAD_FEATURES = (
+    "ID_1413_Sig_7", "ID_930_Sig_5", "ID_1621_Sig_6", "ID_186_Sig_7", "ID_692_Sig_2",
+    "ID_1628_Sig_4", "ID_1255_Sig_2", "ID_1668_Sig_5", "ID_1760_Sig_4", "ID_1760_Sig_3",
+    "ID_208_Sig_6", "ID_1760_Sig_2", "ID_1760_Sig_1", "ID_526_Sig_2", "ID_1176_Sig_4",
+    "ID_167_Sig_6", "ID_208_Sig_3", "ID_1455_Sig_14", "ID_661_Sig_1", "ID_192_Sig_1",
+)
+
+CAN_ORG_FEATURES = (
+    "Label", "Time", "ID",
+    "Signal1_of_ID", "Signal2_of_ID", "Signal3_of_ID", "Signal4_of_ID",
+)
+
+ROAD_ORG_FEATURES = CAN_ORG_FEATURES + tuple(
+    f"Signal{i}_of_ID" for i in range(5, 23)
+)
+
+CAN_ATTRIBUTES = ("Time", "ID", "Label", "File")
+
+MISBEHAVIORX_FEATURES = (
+    "speed_x", "del_pos_x", "speed_y", "del_pos_y",
+    "accel_x", "del_speed_x", "accel_y", "del_speed_y",
+    "del_heading_x", "yaw_rate_x", "del_heading_y", "yaw_rate_y",
+)
+
+MISBEHAVIORX_ATTRIBUTES = ("id", "time_chunk", "attack_gt", "attack_name")
+
+
+SPECS: dict[str, DatasetSpec] = {
+    "syncan": DatasetSpec(
+        name="syncan",
+        family="can",
+        description="Synthetic CAN dataset with signal-level intrusion traces.",
+        citation="Hanselmann et al., SynCAN, ETAS.",
+        source_url="https://github.com/etas/SynCAN",
+        downloadable=True,
+        features=SYNCAN_FEATURES,
+        attributes=CAN_ATTRIBUTES,
+        org_features=CAN_ORG_FEATURES,
+        window_size=50,
+        step_size=10,
+        sampling_period_factors=SYNCAN_PERIOD_FACTORS,
+    ),
+    "road": DatasetSpec(
+        name="road",
+        family="can",
+        description="Real ORNL Automotive Dynamometer (ROAD) CAN intrusion dataset.",
+        citation="Verma et al., ROAD, ORNL.",
+        source_url="https://zenodo.org/records/10462796",
+        downloadable=True,
+        features=ROAD_FEATURES,
+        attributes=CAN_ATTRIBUTES,
+        org_features=ROAD_ORG_FEATURES,
+        window_size=50,
+        step_size=5,
+        sampling_period_factors=tuple(1 for _ in ROAD_FEATURES),
+    ),
+    "misbehaviorx": DatasetSpec(
+        name="misbehaviorx",
+        family="v2x",
+        description="VeReMi-extension / MisbehaviorX V2X misbehavior traces.",
+        citation="Kamel et al., VeReMi Extension.",
+        source_url="https://github.com/josephkamel/VeReMi-Dataset",
+        downloadable=False,
+        features=MISBEHAVIORX_FEATURES,
+        attributes=MISBEHAVIORX_ATTRIBUTES,
+        window_size=10,
+        step_size=10,
+        status="manual",
+        notes=(
+            "Place curated ambient/ and attacks/ folders under the dataset root. "
+            "Automatic download is not wired yet."
+        ),
+    ),
+    "x-canids": DatasetSpec(
+        name="x-canids",
+        family="can",
+        description="X-CANIDS in-vehicle intrusion dataset.",
+        citation="Jeong et al., X-CANIDS.",
+        source_url="https://ieee-dataport.org/open-access/x-canids-dataset",
+        downloadable=False,
+        features=(),
+        attributes=CAN_ATTRIBUTES,
+        status="planned",
+        notes="Registered so experiments can target it, but the loader is not implemented yet.",
+    ),
+}
+
+ALIASES = {
+    "vasp": "misbehaviorx",
+    "veremi": "misbehaviorx",
+    "syncan_robids": "syncan",
+}
+
+
+def get_spec(name: str) -> DatasetSpec:
+    key = name.strip().lower()
+    key = ALIASES.get(key, key)
+    if key not in SPECS:
+        known = ", ".join(sorted(SPECS))
+        raise KeyError(f"Unknown dataset '{name}'. Known datasets: {known}")
+    return SPECS[key]
+
+
+def list_specs(status: Optional[str] = None) -> list[DatasetSpec]:
+    specs = list(SPECS.values())
+    if status is not None:
+        specs = [spec for spec in specs if spec.status == status]
+    return specs

cpsbench/v2x.py

@@ -0,0 +1,142 @@
+"""V2X / MisbehaviorX window dataset.
+
+Automatic download is not available. If curated CSVs already exist under
+``<root>/misbehaviorx/{ambient,attacks}/generated``, they are converted to the
+same window format as the CAN datasets.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+
+from .specs import DatasetSpec
+
+
+def prepare_v2x_split(
+    spec: DatasetSpec,
+    split_path: Path,
+    scaler_path: Path,
+    fit_scaler: bool,
+) -> None:
+    split_path = Path(split_path)
+    generated = split_path / "generated"
+    csvs = sorted(generated.glob("*.csv"))
+    if not csvs and not list(generated.glob("sig_*.npy")):
+        raise FileNotFoundError(
+            f"MisbehaviorX files were not found in {generated}. "
+            "Automatic download is not implemented; place curated CSVs there first."
+        )
+
+    if fit_scaler and csvs and not scaler_path.exists():
+        scaler_path.parent.mkdir(parents=True, exist_ok=True)
+        overall_min = None
+        overall_max = None
+        for csv_path in csvs:
+            frame = pd.read_csv(csv_path, index_col=0)
+            current_min = frame[list(spec.features)].min(axis=0)
+            current_max = frame[list(spec.features)].max(axis=0)
+            overall_min = current_min if overall_min is None else np.minimum(overall_min, current_min)
+            overall_max = current_max if overall_max is None else np.maximum(overall_max, current_max)
+        pd.DataFrame({"Min": overall_min, "Max": overall_max}).to_csv(scaler_path)
+
+    if list(generated.glob("sig_*.npy")):
+        return
+    if not scaler_path.exists():
+        raise FileNotFoundError(
+            f"Missing scaler {scaler_path}. Prepare the train/ambient split first."
+        )
+
+    for csv_path in csvs:
+        frame = pd.read_csv(csv_path, index_col=0)
+        if not fit_scaler and "attack_name" in frame.columns:
+            frame = frame[frame["attack_name"] != "No Attack"]
+        signals = frame[list(spec.features)]
+        attributes = frame[list(spec.attributes)].copy()
+        if spec.filling == "forward":
+            signals = signals.ffill().bfill()
+        if "attack_name" in attributes.columns:
+            attributes["attack_name"], categories = pd.factorize(attributes["attack_name"])
+            mapping = {str(category): int(code) for code, category in enumerate(categories)}
+            with open(split_path / "file_index_dict.json", "w", encoding="utf-8") as handle:
+                json.dump(mapping, handle, indent=2)
+        np.save(generated / f"sig_{csv_path.stem}_1.npy", signals.to_numpy())
+        np.save(generated / f"att_{csv_path.stem}_1.npy", attributes.to_numpy())
+
+
+class V2XWindowDataset(Dataset):
+    """Windowed V2X dataset. Returns ``(window, label)`` like MNIST."""
+
+    def __init__(
+        self,
+        spec: DatasetSpec,
+        data_dir: str | Path,
+        scaler_path: str | Path,
+        return_meta: bool = False,
+        verbose: bool = False,
+    ) -> None:
+        self.spec = spec
+        self.data_dir = Path(data_dir)
+        self.return_meta = return_meta
+        self.features = list(spec.features)
+        generated = self.data_dir / "generated"
+        self.sig_files = sorted(generated.glob("sig_*.npy"))
+        self.att_files = sorted(generated.glob("att_*.npy"))
+        if not self.sig_files or len(self.sig_files) != len(self.att_files):
+            raise FileNotFoundError(f"Expected paired npy files in {generated}.")
+
+        table = pd.read_csv(scaler_path, index_col=0)
+        self.min_vals = table["Min"].loc[self.features].to_numpy(dtype=np.float32)
+        maxs = table["Max"].loc[self.features].to_numpy(dtype=np.float32)
+        span = np.where(maxs - self.min_vals == 0, 1.0, maxs - self.min_vals)
+        self.max_vals = self.min_vals + span
+
+        with open(self.data_dir / "file_index_dict.json", encoding="utf-8") as handle:
+            mapping = json.load(handle)
+        self.file_names = {int(value): key for key, value in mapping.items()}
+        self.index_map = self._index_windows()
+        if verbose:
+            print(f"{spec.name}: {len(self.index_map)} windows from {self.data_dir}")
+
+    def _index_windows(self) -> list[tuple[int, int]]:
+        index_map: list[tuple[int, int]] = []
+        for file_idx, (sig_file, att_file) in enumerate(zip(self.sig_files, self.att_files)):
+            sig = np.load(sig_file, mmap_mode="r")
+            att = pd.DataFrame(np.load(att_file, mmap_mode="r"), columns=list(self.spec.attributes))
+            groups = att[["id", "time_chunk", "attack_name"]].drop_duplicates()
+            for _, row in groups.iterrows():
+                mask = (
+                    (att["id"] == row["id"])
+                    & (att["time_chunk"] == row["time_chunk"])
+                    & (att["attack_name"] == row["attack_name"])
+                )
+                indices = att.index[mask]
+                if len(indices) == 0 or indices[-1] - indices[0] + 1 != len(indices):
+                    continue
+                n_windows = (len(indices) - self.spec.window_size) // self.spec.step_size + 1
+                start = int(indices[0])
+                for offset in range(max(n_windows, 0)):
+                    index_map.append((file_idx, start + offset * self.spec.step_size))
+        return index_map
+
+    def __len__(self) -> int:
+        return len(self.index_map)
+
+    def __getitem__(self, idx: int):
+        file_idx, start = self.index_map[idx]
+        stop = start + self.spec.window_size
+        sig = np.load(self.sig_files[file_idx], mmap_mode="r")
+        att = np.load(self.att_files[file_idx], mmap_mode="r")
+        window = (sig[start:stop] - self.min_vals) / (self.max_vals - self.min_vals)
+        tensor = torch.tensor(window, dtype=torch.float32).unsqueeze(0)
+        label = int(np.sum(att[start:stop, self.spec.label_index]) > 0.0)
+        if not self.return_meta:
+            return tensor, label
+        file_id = int(att[start, 3])
+        meta = {"idx": int(start), "file": self.file_names.get(file_id, str(file_id))}
+        return tensor, label, meta

cpsbench/windows.py

@@ -0,0 +1,111 @@
+"""Windowed PyTorch datasets over preprocessed signal arrays."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+
+from .specs import DatasetSpec
+
+
+class WindowDataset(Dataset):
+    """Sliding-window CAN dataset.
+
+    ``__getitem__`` returns ``(window, label)`` like MNIST. Pass
+    ``return_meta=True`` to also get ``{"file", "idx"}``.
+    """
+
+    def __init__(
+        self,
+        spec: DatasetSpec,
+        data_dir: str | Path,
+        scaler_path: str | Path,
+        return_meta: bool = False,
+        verbose: bool = False,
+    ) -> None:
+        self.spec = spec
+        self.data_dir = Path(data_dir)
+        self.scaler_path = Path(scaler_path)
+        self.return_meta = return_meta
+        self.features = list(spec.features)
+        self.sampling_periods = np.asarray(spec.sampling_period_factors, dtype=int) * int(spec.sampling_period)
+        if len(self.sampling_periods) != len(self.features):
+            raise ValueError(
+                f"{spec.name} has {len(self.features)} features but "
+                f"{len(self.sampling_periods)} sampling-period factors."
+            )
+        self.max_sampling_period = int(self.sampling_periods.max())
+
+        generated = self.data_dir / "generated"
+        self.sig_files = sorted(generated.glob("sig_*.npy"))
+        self.att_files = sorted(generated.glob("att_*.npy"))
+        if not self.sig_files or len(self.sig_files) != len(self.att_files):
+            raise FileNotFoundError(
+                f"Expected paired sig_*.npy and att_*.npy files in {generated}."
+            )
+
+        self.min_vals, self.max_vals = self._load_scaler()
+        self.file_names = self._load_file_names()
+        self.index_map = self._index_windows()
+        if verbose:
+            print(f"{spec.name}: {len(self.index_map)} windows from {self.data_dir}")
+
+    def _load_file_names(self) -> dict[int, str]:
+        with open(self.data_dir / "file_index_dict.json", encoding="utf-8") as handle:
+            mapping = json.load(handle)
+        return {int(value): key for key, value in mapping.items()}
+
+    def _load_scaler(self) -> tuple[np.ndarray, np.ndarray]:
+        table = pd.read_csv(self.scaler_path, index_col=0)
+        mins = table["Min"].loc[self.features].to_numpy(dtype=np.float32)
+        maxs = table["Max"].loc[self.features].to_numpy(dtype=np.float32)
+        span = np.where(maxs - mins == 0, 1.0, maxs - mins)
+        return mins, mins + span
+
+    def _index_windows(self) -> list[tuple[int, int]]:
+        index_map: list[tuple[int, int]] = []
+        span = self.spec.window_size * self.max_sampling_period
+        for file_idx, (sig_file, att_file) in enumerate(zip(self.sig_files, self.att_files)):
+            sig = np.load(sig_file, mmap_mode="r")
+            att = np.load(att_file, mmap_mode="r")
+            if sig.shape[1] != len(self.features) or att.shape[1] != len(self.spec.attributes):
+                raise ValueError(
+                    f"Unexpected array shape in {sig_file.name}: signals {sig.shape}, attributes {att.shape}."
+                )
+            if sig.shape[0] != att.shape[0]:
+                raise ValueError(f"Signal/attribute length mismatch in {sig_file.name}.")
+            n_windows = (sig.shape[0] - span) // self.spec.step_size + 1
+            for offset in range(max(n_windows, 0)):
+                index_map.append((file_idx, offset * self.spec.step_size))
+        return index_map
+
+    def __len__(self) -> int:
+        return len(self.index_map)
+
+    def __getitem__(self, idx: int):
+        file_idx, start = self.index_map[idx]
+        span = self.spec.window_size * self.max_sampling_period
+        sig = np.load(self.sig_files[file_idx], mmap_mode="r")
+        att = np.load(self.att_files[file_idx], mmap_mode="r")
+        stop = start + span
+        if stop > sig.shape[0]:
+            raise IndexError(f"Window {idx} exceeds {self.sig_files[file_idx].name}.")
+
+        rows = np.arange(self.spec.window_size)[:, None] * self.sampling_periods
+        window = sig[start:stop][rows, np.arange(len(self.features))]
+        window = (window - self.min_vals) / (self.max_vals - self.min_vals)
+        tensor = torch.tensor(window, dtype=torch.float32).unsqueeze(0)
+
+        att_window = att[start:stop]
+        label = int(np.sum(att_window[:, self.spec.label_index]) > 0.0)
+        if not self.return_meta:
+            return tensor, label
+
+        file_id = int(att_window[0, 3])
+        meta = {"idx": int(start), "file": self.file_names.get(file_id, str(file_id))}
+        return tensor, label, meta

cpsbench-0.1.0.dist-info/METADATA

@@ -0,0 +1,110 @@
+Metadata-Version: 2.4
+Name: CPSBench
+Version: 0.1.0
+Summary: Cyber-physical security datasets that download, preprocess, and load like MNIST.
+Author: Md Hasan Shahriar
+License: MIT
+Project-URL: Homepage, https://github.com/shahriar0651/CPSBench
+Keywords: dataset,cps,can,ids,vehicular,cybersecurity,pytorch
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=2.0
+Requires-Dist: pyarrow>=14
+Requires-Dist: joblib>=1.3
+Requires-Dist: torch>=2.0
+Requires-Dist: tqdm>=4.60
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# CPSBench
+
+Installable loader for cyber-physical security datasets. It downloads the raw traces, builds the windowed tensors, and returns a PyTorch dataset with the same contract as MNIST: `(window, label)`.
+
+```python
+from cpsbench import SynCAN
+from torch.utils.data import DataLoader
+
+train = SynCAN(root="./data", split="train", download=True)
+test = SynCAN(root="./data", split="test", download=True)
+
+window, label = train[0]          # window: (1, time, signals), label: 0 benign / 1 attack
+loader = DataLoader(train, batch_size=64, shuffle=True)
+```
+
+Or load by name:
+
+```python
+import cpsbench
+
+dataset = cpsbench.load("road", root="./data", split="test", download=True)
+print(dataset.input_shape)   # (channels, window, signals)
+```
+
+## Install
+
+```bash
+pip install cpsbench
+```
+
+That works after the package is published on PyPI. Until then, install this repository directly:
+
+```bash
+pip install "git+https://github.com/shahriar0651/CPSBench.git"
+```
+
+From a local clone, for development:
+
+```bash
+pip install -e .
+```
+
+Python 3.10+. SynCAN also needs `git` on `PATH`. ROAD is fetched from Zenodo with the standard library, so `wget` is not required.
+
+## Datasets
+
+| Name | Status | What you get |
+| --- | --- | --- |
+| `syncan` | ready, auto-download | Synthetic CAN intrusion traces |
+| `road` | ready, auto-download | ROAD dynamometer CAN traces |
+| `misbehaviorx` | loader ready, manual files | V2X misbehavior (also accepted as `vasp`) |
+| `x-canids` | registered, not implemented | Raises a clear error until a loader is added |
+
+```bash
+python -m cpsbench list
+python -m cpsbench info syncan
+python -m cpsbench download syncan --root ./data --split train
+```
+
+Downloaded files land in `<root>/<name>/{ambient,attacks}` plus a fitted min/max scaler under `<root>/<name>/scaler`. Later calls reuse those files.
+
+## Overrides
+
+Windowing defaults live in the library so a new project does not need the old Hydra YAML. Override them per call:
+
+```python
+from cpsbench import ROAD
+
+dataset = ROAD(root="./data", split="train", download=True, window_size=50, step_size=5)
+```
+
+Pass `return_meta=True` if you also need the source file and row index: `(window, label, {"file", "idx"})`.
+
+## Layout
+
+Each sample is a min-max scaled window with a channel axis, so the same convolutional IDS can run on every dataset. Shape is always `(channels, window_size, num_signals)`. Label `0` is benign and `1` is attack (any attack flag inside the window).
+
+## Adding a dataset
+
+1. Add a `DatasetSpec` in `src/cpsbench/specs.py`.
+2. Add a downloader in `src/cpsbench/download.py` if the files can be fetched automatically.
+3. Register the class in `src/cpsbench/datasets.py` and `_CLASSES` in `__init__.py`.
+
+The IDS experiments that consume this package live in the sibling [RobIDS](https://github.com/shahriar0651/robids) repo.

cpsbench-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+cpsbench/__init__.py,sha256=Q_CO_KyeJV0M-19XUfhwiBgwsRO9eES5WviSzpX-I_Y,1999
+cpsbench/__main__.py,sha256=MEWP6zPkTYeV-kAXOG10J3_vv0ANIBHTav_3MG0Cne8,1293
+cpsbench/datasets.py,sha256=OiZUQkueOvNgRwYfLLObaHZd1DJYqIruXzwbTwOpUkc,4145
+cpsbench/download.py,sha256=ZTUI2sCJuRP-9tPegw2b0QlXIerl4B5NZGHKP3wOHLA,5627
+cpsbench/preprocess.py,sha256=FdNI5RuP0e2zDX41mHvNyYiSUI79PB4kRNbqJxSd3rU,6372
+cpsbench/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+cpsbench/specs.py,sha256=3yqpMLhFKBnXSDtTR5ICZ39y__cmnr4ahG39gCkBLnM,5926
+cpsbench/v2x.py,sha256=hso30XucXdFDMvYoIAfB73E1tKr1r33bEqHvg6Rjw3M,6157
+cpsbench/windows.py,sha256=Xd3nbtcLP5lQBFVfStwpZTZ-OuyqpWDkH8bWe2Ci4d0,4641
+cpsbench-0.1.0.dist-info/licenses/LICENSE,sha256=puRg9gSMJy3domQ53eusoFeQHNemw5TNmcSDIwoSMUY,1074
+cpsbench-0.1.0.dist-info/METADATA,sha256=B3--wcOhwMwXUOZ_-Y28SMX0K5gsTq7hDx8oRs-5AxI,3731
+cpsbench-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+cpsbench-0.1.0.dist-info/top_level.txt,sha256=22JwhlbHTxWz-RtuXKzL9VTGRMl6jayXan1wH9LkE4c,9
+cpsbench-0.1.0.dist-info/RECORD,,

cpsbench-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cpsbench-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Md Hasan Shahriar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cpsbench-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cpsbench