idslab 0.3.0__py3-none-any.whl

idslab/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""IDSLab: evaluation framework for staged intrusion detection systems."""
+
+__version__ = "0.3.0"

idslab/adapters.py

@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""Model adapters: build estimators and read their output as a score contract.
+
+Contracts (DESIGN.md section 0.1):
+
+* AttackScore: one value per row, P(attack), read from ``predict_proba`` at the
+  column of class 1.
+* ClassProba: per row, probabilities over named types, read from ``predict_proba``
+  with the columns named by ``classes_``.
+* AnomalyScore: one value per row, higher means more anomalous. scikit-learn
+  detectors use ``score_samples`` (higher means more normal, so it is negated);
+  PyOD detectors use ``decision_function`` (already higher means more anomalous);
+  a candidate may override both with ``score: {method, higher_is_anomalous}``.
+* BatchAnomaly: the detector is fitted on one batch of flows and scores that same
+  batch, higher = more anomalous (LOF without novelty, PyOD ``decision_scores_``).
+
+A class that is not a scikit-learn estimator can still be used: give it a class
+attribute ``idslab_contract`` naming its contract, with ``fit(X, y)`` and
+``attack_score(X)`` (AttackScore), ``fit(X, y)`` and ``class_proba(X)`` plus
+``classes_`` (ClassProba), ``fit(X)`` and ``anomaly_score(X)`` (AnomalyScore), or
+``fit_score(X)`` (BatchAnomaly).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+
+from idslab.config import Candidate, import_class
+
+ATTACK = 1
+
+
+class ContractError(RuntimeError):
+    """A model's output does not meet its score contract."""
+
+
+def is_custom(model: Any) -> bool:
+    return getattr(model, "idslab_contract", None) is not None
+
+
+@dataclass(frozen=True)
+class ClassScores:
+    """Probabilities over named types (ClassProba contract); rows follow the data, columns ``classes``."""
+
+    proba: np.ndarray
+    classes: tuple[str, ...]
+
+    def __len__(self) -> int:
+        return len(self.proba)
+
+
+def build_estimator(cand: Candidate, seed: int, n_jobs: int, *, novelty: bool = True) -> Any:
+    cls = import_class(cand.cls)
+    params = dict(cand.params)
+    if getattr(cls, "idslab_contract", None) is not None:
+        return cls(**params)
+    est = cls(**params)
+    known = est.get_params()
+    extra = {}
+    if "random_state" in known and "random_state" not in params:
+        extra["random_state"] = seed
+    if "n_jobs" in known and "n_jobs" not in params and n_jobs != 1:
+        extra["n_jobs"] = n_jobs
+    if "novelty" in known:
+        extra["novelty"] = novelty
+    if extra:
+        est.set_params(**extra)
+    return est
+
+
+def build_pipeline(steps: tuple[dict, ...], estimator: Any, seed: int) -> Any:
+    """Pipeline of the stage's steps with the model last, named ``model``.
+
+    An imbalanced-learn sampler in the steps switches to its Pipeline, which
+    resamples during fit only.
+    """
+    built = []
+    for i, step in enumerate(steps):
+        obj = import_class(step["class"])(**(step.get("params") or {}))
+        if "random_state" in obj.get_params() and "random_state" not in (step.get("params") or {}):
+            obj.set_params(random_state=seed)
+        built.append((f"step{i}", obj))
+    if is_custom(estimator):
+        if built:
+            raise ContractError("a custom adapter class cannot be combined with pipeline steps")
+        return estimator
+    if any(hasattr(obj, "fit_resample") for _, obj in built):
+        from imblearn.pipeline import Pipeline as ImbPipeline  # noqa: PLC0415  (optional extra)
+
+        return ImbPipeline([*built, ("model", estimator)])
+    from sklearn.pipeline import Pipeline  # noqa: PLC0415
+
+    return Pipeline([*built, ("model", estimator)])
+
+
+def attack_score(model: Any, X: Any) -> np.ndarray:
+    """P(attack) per row (AttackScore contract)."""
+    if is_custom(model):
+        out = np.asarray(model.attack_score(X), dtype=np.float64)
+    else:
+        classes = list(np.asarray(model.classes_).tolist())
+        if ATTACK not in classes:
+            raise ContractError(f"classifier classes {classes} do not include the attack class 1")
+        out = np.asarray(model.predict_proba(X), dtype=np.float64)[:, classes.index(ATTACK)]
+    _check(out, X, "AttackScore")
+    if out.min() < 0 or out.max() > 1:
+        raise ContractError("AttackScore must be a probability in [0, 1]")
+    return out
+
+
+def class_proba(model: Any, X: Any) -> ClassScores:
+    """Probabilities over the model's types (ClassProba contract)."""
+    raw = model.class_proba(X) if is_custom(model) else model.predict_proba(X)
+    proba = np.asarray(raw, dtype=np.float64)
+    classes = tuple(str(c) for c in np.asarray(model.classes_).tolist())
+    if proba.ndim != 2 or proba.shape != (len(X), len(classes)):
+        raise ContractError(f"ClassProba must be rows x {len(classes)} classes; got shape {proba.shape}")
+    if not np.isfinite(proba).all() or proba.min() < 0 or proba.max() > 1:
+        raise ContractError("ClassProba must hold finite probabilities in [0, 1]")
+    return ClassScores(proba=proba, classes=classes)
+
+
+class PoolDetector:
+    """A detector refitted on each batch of flows reaching its stage (BatchAnomaly contract).
+
+    The preprocessing steps are fitted once on FIT rows; the detector itself is built
+    afresh for every batch, fitted on that batch's features only and asked to score it.
+    """
+
+    def __init__(self, cand: Candidate, steps: tuple[dict, ...], seed: int, n_jobs: int):
+        self.cand, self.steps, self.seed, self.n_jobs = cand, steps, seed, n_jobs
+        self.prep: Any = None
+
+    def fit(self, X: Any) -> PoolDetector:
+        if self.steps:
+            from sklearn.pipeline import Pipeline  # noqa: PLC0415
+
+            built = []
+            for i, step in enumerate(self.steps):
+                obj = import_class(step["class"])(**(step.get("params") or {}))
+                if "random_state" in obj.get_params() and "random_state" not in (step.get("params") or {}):
+                    obj.set_params(random_state=self.seed)
+                built.append((f"step{i}", obj))
+            self.prep = Pipeline(built).fit(X)
+        return self
+
+    def score_pool(self, X: Any) -> np.ndarray:
+        data = self.prep.transform(X) if self.prep is not None else X
+        det = build_estimator(self.cand, self.seed, self.n_jobs, novelty=False)
+        if is_custom(det):
+            out = np.asarray(det.fit_score(data), dtype=np.float64)
+        else:
+            det.fit(data)
+            if hasattr(det, "negative_outlier_factor_"):
+                out = -np.asarray(det.negative_outlier_factor_, dtype=np.float64)
+            elif hasattr(det, "decision_scores_"):  # PyOD: training scores, higher = more anomalous
+                out = np.asarray(det.decision_scores_, dtype=np.float64)
+            elif hasattr(det, "score_samples"):
+                out = -np.asarray(det.score_samples(data), dtype=np.float64)
+            else:
+                raise ContractError(f"{self.cand.cls} gives no batch scores (BatchAnomaly contract)")
+        _check(out, X, "BatchAnomaly")
+        return out
+
+
+def anomaly_score(model: Any, X: Any, cls_path: str, override: dict | None) -> np.ndarray:
+    """Anomaly score per row, higher = more anomalous (AnomalyScore contract)."""
+    if is_custom(model):
+        out = np.asarray(model.anomaly_score(X), dtype=np.float64)
+    else:
+        detector, data = final_step(model, X)
+        if override is not None:
+            method, higher_is_anomalous = override["method"], bool(override["higher_is_anomalous"])
+        elif cls_path.startswith("pyod."):
+            method, higher_is_anomalous = "decision_function", True
+        elif hasattr(detector, "score_samples"):
+            method, higher_is_anomalous = "score_samples", False
+        else:
+            method, higher_is_anomalous = "decision_function", False
+        if not hasattr(detector, method):
+            raise ContractError(f"{cls_path} has no method {method!r}")
+        raw = np.asarray(getattr(detector, method)(data), dtype=np.float64)
+        out = raw if higher_is_anomalous else -raw
+    _check(out, X, "AnomalyScore")
+    return out
+
+
+def final_step(model: Any, X: Any) -> tuple[Any, Any]:
+    """The last estimator of a pipeline and ``X`` passed through the steps before it.
+
+    Scoring methods are called on the detector itself, so any method it has can be
+    named in a ``score`` override (a Pipeline passes only a fixed set through).
+    """
+    steps = getattr(model, "steps", None)
+    if not steps:
+        return model, X
+    data = X
+    for _, step in steps[:-1]:
+        if hasattr(step, "transform"):
+            data = step.transform(data)
+    return steps[-1][1], data
+
+
+def _check(out: np.ndarray, X: Any, contract: str) -> None:
+    if out.ndim != 1 or len(out) != len(X):
+        raise ContractError(
+            f"{contract} must give one value per row; got shape {out.shape} for {len(X)} rows"
+        )
+    if not np.isfinite(out).all():
+        raise ContractError(f"{contract} has {int((~np.isfinite(out)).sum())} non-finite values")

idslab/cli.py

@@ -0,0 +1,301 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""Command line: idslab validate | run | report | export-splits | example | compare | sweep | dashboard."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+from importlib import resources
+
+from idslab import __version__
+from idslab.adapters import ContractError
+from idslab.compare import CompareError
+from idslab.config import Config, ConfigError, load
+from idslab.data import DataError
+from idslab.fitting import FitError
+from idslab.report import write_report
+from idslab.runner import data_fingerprints, export_splits, run
+from idslab.sweep import SweepError
+from idslab.synthetic import write_example
+
+EXAMPLES = ("r1_router_gate.yaml", "r2_anomaly_first.yaml", "r1_four_layer.yaml")
+
+
+def validation_text(path: str, cfg: Config) -> str:
+    """What ``idslab validate`` prints for a valid config (the dashboard shows the same lines)."""
+    lines = [f"{path}: valid", f"  stages in selection order: {' > '.join(cfg.order())}"]
+    for name in cfg.order():
+        st = cfg.stages[name]
+        tune = "chosen on VAL" if st.decision.tunable else "fixed"
+        cands = ", ".join(c.label for c in st.candidates)
+        lines.append(
+            f"  {name}: {st.kind}, {st.decision.mode} {list(st.decision.values)} ({tune}), candidates {cands}"
+        )
+    lines.append(f"  controls compare against: {cfg.controls.first_stage}; seeds {list(cfg.protocol.seeds)}")
+    return "\n".join(lines)
+
+
+def _validate(args: argparse.Namespace) -> int:
+    print(validation_text(args.config, load(args.config)))
+    return 0
+
+
+def _run(args: argparse.Namespace) -> int:
+    cfg = load(args.config)
+    data_fps = data_fingerprints(cfg)  # hash the data once for the run and the report
+    results = run(
+        cfg, out_root=args.out, seeds=args.seeds, jobs=args.jobs, force=args.force, data_fps=data_fps
+    )
+    goal = cfg.protocol.objective
+    for r in results:
+        s = r["summary"]
+        sys_ = s["controls"]["systems_test"]
+        tag = "skipped (already done)" if r["skipped"] else f"{s['runtime_seconds']}s"
+        print(
+            f"seed {r['seed']}{' family ' + r['family'] if r.get('family') else ''}: {tag}; "
+            f"TEST {goal} cascade {sys_['cascade'][goal]:.4f}, "
+            f"first stage {sys_['B1_first_stage'][goal]:.4f}"
+        )
+    if not args.no_report:
+        md, _ = write_report(args.out, cfg, args.seeds or (), data_fps)
+        print(f"report: {md} (and report.html next to it)")
+    return 0
+
+
+def _report(args: argparse.Namespace) -> int:
+    cfg = load(args.config)
+    md, js = write_report(args.out, cfg, args.seeds or ())
+    print(f"report: {md}\nreport page: {os.path.splitext(md)[0]}.html\nreport data: {js}")
+    return 0
+
+
+def _export(args: argparse.Namespace) -> int:
+    cfg = load(args.config)
+    paths = export_splits(cfg, args.out, args.seeds)
+    print(
+        f"wrote {len(paths)} split files (each with a .json sidecar) under {os.path.join(args.out, cfg.name)}"
+    )
+    return 0
+
+
+def _example(args: argparse.Namespace) -> int:
+    target = args.directory
+    data_dir = os.path.join(target, "data")
+    targets = [os.path.join(target, n) for n in EXAMPLES]
+    targets += [os.path.join(data_dir, n) for n in ("train.parquet", "test.parquet")]
+    existing = [t for t in targets if os.path.exists(t)]
+    if existing and not args.overwrite:
+        print(f"already present: {', '.join(existing)}; use --overwrite to replace them", file=sys.stderr)
+        return 2
+    os.makedirs(target, exist_ok=True)
+    for name in EXAMPLES:
+        with resources.as_file(resources.files("idslab.examples") / name) as src:
+            shutil.copyfile(src, os.path.join(target, name))
+    train, test = write_example(data_dir)
+    print(f"wrote {train}, {test} and {', '.join(EXAMPLES)} in {target}")
+    print(f"try: idslab run {os.path.join(target, EXAMPLES[0])} --out {os.path.join(target, 'runs')}")
+    return 0
+
+
+def _compare(args: argparse.Namespace) -> int:
+    from idslab.compare import compare, write_comparison  # noqa: PLC0415
+
+    c = compare(args.experiments, bootstrap=args.bootstrap)
+    out = args.out or os.path.join(
+        "comparisons", "-vs-".join(os.path.basename(os.path.normpath(p)) for p in args.experiments)[:120]
+    )
+    md, _, html = write_comparison(c, out)
+    goal = c["objective"]
+    for h in c["experiments"]:
+        cas, first = (h["cascade"] or {}).get("mean"), (h["first_stage"] or {}).get("mean")
+        print(f"{h['name']}: TEST {goal} cascade {_num(cas)}, first stage alone {_num(first)}")
+    for p in c["paired"]:
+        n = len(p["per_unit"])
+        print(
+            f"{p['other']} minus {p['base']}: {_num(p['mean_diff'], signed=True)} over {n} paired units "
+            f"(CI above 0 in {p['units_ci_above_zero']}, below 0 in {p['units_ci_below_zero']})"
+        )
+    for w in c["warnings"]:
+        print(f"warning: {w}")
+    print(f"comparison: {md} (and {html})")
+    return 0
+
+
+def _num(v: float | None, signed: bool = False) -> str:
+    return "n/a" if v is None else (f"{v:+.4f}" if signed else f"{v:.4f}")
+
+
+def _sweep(args: argparse.Namespace) -> int:
+    from idslab.sweep import load_sweep, run_sweep  # noqa: PLC0415
+
+    if args.dry_run:
+        sw = load_sweep(args.sweep)
+        print(f"{args.sweep}: valid; {len(sw.variants)} variants of {sw.base_path}")
+        for v in sw.variants:
+            print(f"  {v.name}: {v.label}")
+        return 0
+    md, _, html = run_sweep(
+        args.sweep,
+        out_root=args.out,
+        seeds=args.seeds,
+        jobs=args.jobs,
+        force=args.force,
+        bootstrap=args.bootstrap,
+    )
+    print(f"comparison: {md} (and {html})")
+    return 0
+
+
+def _name(folder: str) -> str:
+    """A folder as typed, when it is a plain name every shell passes unchanged; otherwise a
+    placeholder rather than a command that would paste wrongly."""
+    plain = folder and folder[0] != "-" and all(c.isalnum() or c in "_.-" for c in folder)
+    return folder if plain else "<this folder>"
+
+
+def _dashboard(args: argparse.Namespace) -> int:
+    from idslab.dashboard.server import App, make_server  # noqa: PLC0415  (only this command needs it)
+    from idslab.dashboard.workspace import PathError, WorkspaceMissingError  # noqa: PLC0415
+
+    try:
+        app = App(args.workspace)
+    except PathError as e:  # the workspace folder: told plainly, it is not an unexpected error
+        if args.debug:
+            raise
+        print(f"idslab: {e}", file=sys.stderr)
+        if isinstance(e, WorkspaceMissingError):
+            print(
+                "If it should hold your own data, check the path for a typo. To try the example data, "
+                "create the folder first with the example command, started the same way as this one: "
+                f"idslab example {_name(args.workspace)}",
+                file=sys.stderr,
+            )
+        return 2
+    server = make_server(args.host, args.port, app)
+    host, port = server.server_address[:2]
+    url = f"http://{'127.0.0.1' if host in ('0.0.0.0', '::') else host}:{port}/"  # noqa: S104
+    print(f"IDSLab dashboard for {app.ws.root} at {url} (Ctrl+C to stop)", flush=True)
+    if args.host not in ("127.0.0.1", "localhost", "::1"):
+        print("warning: the dashboard runs commands and has no login; keep it on a trusted network")
+    if args.open:
+        import webbrowser  # noqa: PLC0415
+
+        webbrowser.open(url)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        app.job.stop()
+        server.server_close()
+    return 0
+
+
+def _jobs(text: str) -> int:
+    v = int(text)
+    if v == 0 or v < -1:
+        raise argparse.ArgumentTypeError("--jobs must be a positive integer or -1")
+    return v
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="idslab", description="Evaluate staged intrusion detection designs.")
+    p.add_argument("--version", action="version", version=f"idslab {__version__}")
+    p.add_argument("--debug", action="store_true", help="show the full traceback on errors")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    v = sub.add_parser("validate", help="check a config without reading data")
+    v.add_argument("config")
+    v.set_defaults(func=_validate)
+
+    r = sub.add_parser("run", help="fit, select on VAL, evaluate on TEST, write results and the report")
+    r.add_argument("config")
+    r.add_argument("--out", default="runs", help="output root (default: runs)")
+    r.add_argument("--seeds", type=int, nargs="+", help="run only these seeds")
+    r.add_argument("--jobs", type=_jobs, default=1, help="seeds run in parallel (default 1, -1 = all cores)")
+    r.add_argument("--force", action="store_true", help="rerun seeds that are already done")
+    r.add_argument("--no-report", action="store_true", help="skip writing report.md")
+    r.set_defaults(func=_run)
+
+    rep = sub.add_parser("report", help="aggregate finished seeds into report.md and report.json")
+    rep.add_argument("config")
+    rep.add_argument("--out", default="runs")
+    rep.add_argument("--seeds", type=int, nargs="+", help="also include these seeds")
+    rep.set_defaults(func=_report)
+
+    x = sub.add_parser("export-splits", help="write row ids, folds and fingerprints for external score jobs")
+    x.add_argument("config")
+    x.add_argument("--out", default="splits", help="output root (default: splits)")
+    x.add_argument("--seeds", type=int, nargs="+", help="only these seeds")
+    x.set_defaults(func=_export)
+
+    e = sub.add_parser("example", help="write the synthetic dataset and the two reference configs")
+    e.add_argument("directory")
+    e.add_argument("--overwrite", action="store_true")
+    e.set_defaults(func=_example)
+
+    cp = sub.add_parser("compare", help="compare finished experiments side by side, paired where possible")
+    cp.add_argument(
+        "experiments", nargs="+", help="experiment folders (<out>/<name>); the first is the baseline"
+    )
+    cp.add_argument("--out", help="folder for compare.md/.json/.html (default: comparisons/<names>)")
+    cp.add_argument("--bootstrap", type=int, default=1000, help="paired bootstrap replicates (default 1000)")
+    cp.set_defaults(func=_compare)
+
+    sw = sub.add_parser("sweep", help="run every variant of a sweep file and compare them")
+    sw.add_argument("sweep", help="sweep file (YAML: name, base, and variants or grid)")
+    sw.add_argument("--out", default="runs", help="output root (default: runs)")
+    sw.add_argument("--seeds", type=int, nargs="+", help="run only these seeds")
+    sw.add_argument("--jobs", type=_jobs, default=1, help="seeds run in parallel (default 1, -1 = all cores)")
+    sw.add_argument("--force", action="store_true", help="rerun seeds that are already done")
+    sw.add_argument("--bootstrap", type=int, default=1000, help="paired bootstrap replicates (default 1000)")
+    sw.add_argument(
+        "--dry-run", action="store_true", help="check the sweep and list its variants, run nothing"
+    )
+    sw.set_defaults(func=_sweep)
+
+    d = sub.add_parser("dashboard", help="open the web dashboard over a workspace folder")
+    d.add_argument("--workspace", default=".", help="folder with configs, data and runs (default: .)")
+    d.add_argument("--host", default="127.0.0.1", help="address to bind (default: 127.0.0.1)")
+    d.add_argument("--port", type=int, default=8765, help="port (default 8765, 0 = any free port)")
+    d.add_argument("--open", action="store_true", help="open the page in the default browser")
+    d.set_defaults(func=_dashboard)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command in ("run", "export-splits", "sweep"):  # progress lines on stderr; results stay on stdout
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter("%(asctime)s %(message)s", "%H:%M:%S"))
+        log = logging.getLogger("idslab")
+        if not log.handlers:  # main() can run more than once in a process (tests)
+            log.addHandler(handler)
+        log.setLevel(logging.INFO)
+    try:
+        return int(args.func(args))
+    except (
+        ConfigError,
+        DataError,
+        FitError,
+        ContractError,
+        CompareError,
+        SweepError,
+        ImportError,
+        OSError,
+    ) as e:
+        print(f"idslab: {e}", file=sys.stderr)
+        return 2
+    except (ValueError, RuntimeError, IndexError) as e:
+        if args.debug:
+            raise
+        print(f"idslab: {type(e).__name__}: {e} (rerun with --debug for the traceback)", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())