idslab 0.3.0__py3-none-any.whl

idslab/controls.py

@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""Controls computed after the configuration is frozen (DESIGN.md section 6).
+
+Everything here takes the frozen selection as given; nothing feeds back into it.
+All controls use the same fitted first-stage model as the cascade.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from dataclasses import replace
+
+import numpy as np
+
+from idslab.config import Config
+from idslab.graph import Routing, StageState, recorded_types, route, with_off
+from idslab.metrics import (
+    binary_metrics,
+    metric_names,
+    em_prior,
+    frontier_ci,
+    frontier_gap,
+    objective,
+    auc,
+    paired_bootstrap,
+    type_metrics,
+    unseen_metrics,
+)
+from idslab.selection import Scores, Selection, active_scores
+
+
+def budget_matched_threshold(p_val: np.ndarray, y_val: np.ndarray, target_fpr: float) -> tuple[float, float]:
+    """Highest threshold whose VAL FPR is at least ``target_fpr``; returns (threshold, achieved FPR)."""
+    benign = np.sort(p_val[y_val == 0])[::-1]
+    if len(benign) == 0:
+        raise ValueError("no benign VAL rows to match a false-alarm budget")
+    k = math.ceil(target_fpr * len(benign) - 1e-12)
+    if k <= 0:
+        return math.inf, 0.0
+    t = float(benign[min(k, len(benign)) - 1])
+    return t, float((benign >= t).mean())
+
+
+def best_cutoff(p_val: np.ndarray, y_val: np.ndarray, grid: tuple[float, ...], goal: str) -> float:
+    values = [objective(y_val, (p_val >= c).astype(int), goal) for c in grid]
+    return float(grid[int(np.argmax(values))])
+
+
+def evaluate(
+    cfg: Config,
+    sel: Selection,
+    *,
+    val_scores: Scores,
+    test_scores: Scores,
+    y_val: np.ndarray,
+    y_test: np.ndarray,
+    shares: dict[str, float],
+    first_train_share: float,
+    seed: int,
+    types_test: np.ndarray | None = None,
+    unseen: np.ndarray | None = None,
+    extra_states: Mapping[str, Mapping[str, StageState]] | None = None,
+) -> tuple[dict, dict[str, np.ndarray], Routing]:
+    """TEST evaluation of the frozen cascade and every control.
+
+    Returns the summary block, the per-flow predictions of each system and the TEST routing.
+    """
+    goal = cfg.protocol.objective
+    order, state = sel.order, sel.state
+    n_val, n_test = len(y_val), len(y_test)
+    first = cfg.controls.first_stage
+    fs = state[first]
+    if fs.off or fs.candidate is None or fs.threshold is None:
+        raise RuntimeError("the control first stage must be active in the frozen configuration")
+    p1_val, p1_test = val_scores[first][fs.candidate], test_scores[first][fs.candidate]
+
+    def run(st, scores, n):
+        return route(order, cfg.stages, active_scores(scores, st), st, n)
+
+    cascade_val = run(state, val_scores, n_val).verdict
+    routing = run(state, test_scores, n_test)
+    cascade = routing.verdict
+
+    preds = {"cascade": cascade, "B1_first_stage": (p1_test >= fs.threshold).astype(np.int8)}
+    b2_cut = best_cutoff(p1_val, y_val, cfg.controls.cutoff_grid, goal)
+    preds["B2_first_stage_val_cutoff"] = (p1_test >= b2_cut).astype(np.int8)
+    target = binary_metrics(y_val, cascade_val)["fpr"]
+    t_budget, achieved = budget_matched_threshold(p1_val, y_val, target)
+    preds["budget_matched_first_stage"] = (p1_test >= t_budget).astype(np.int8)
+    for name, st_map in (extra_states or {}).items():
+        preds[name] = run(dict(st_map), test_scores, n_test).verdict
+
+    systems = {name: binary_metrics(y_test, p, goal) for name, p in preds.items()}
+    frontier = frontier_gap(y_test, p1_test, cascade)
+    frontier["ci95_interp"] = (
+        frontier_ci(y_test, p1_test, cascade, cfg.controls.frontier_bootstrap, seed)
+        if frontier["defined"]
+        else None
+    )
+    boot = {
+        name: paired_bootstrap(
+            y_test, cascade, preds[name], n=cfg.controls.bootstrap, seed=seed + i, names=metric_names(goal)
+        )
+        for i, name in enumerate(preds)
+        if name != "cascade"
+    }
+
+    removal = {}
+    for s in order:
+        if state[s].off:
+            continue
+        p = run(with_off(state, [s]), test_scores, n_test).verdict
+        m = binary_metrics(y_test, p, goal)
+        removal[s] = {"metrics": m, f"cascade_minus_removed_{goal}": systems["cascade"][goal] - m[goal]}
+
+    prefix = []
+    for k in range(len(order) + 1):
+        st_k = {s: (state[s] if i < k else replace(state[s], off=True)) for i, s in enumerate(order)}
+        m = binary_metrics(y_test, run(st_k, test_scores, n_test).verdict, goal)
+        prefix.append({"stages_on": [s for s in order[:k] if not state[s].off], goal: m[goal], "metrics": m})
+    for k in range(1, len(prefix)):
+        prefix[k][f"step_{goal}"] = prefix[k][goal] - prefix[k - 1][goal]
+
+    calibrated = cfg.stages[first].calibrate != "none"
+    prevalence = {
+        "attack_share": {**shares, "TEST": float(np.mean(y_test))},
+        "em_estimate_TEST": em_prior(p1_test, first_train_share) if calibrated else None,
+        "note": "EM assumes label shift only"
+        if calibrated
+        else "EM estimate needs a calibrated first stage; not computed",
+    }
+    summary = {
+        "objective": goal,
+        "first_stage": first,
+        "systems_test": systems,
+        "cascade_val": binary_metrics(y_val, cascade_val, goal),
+        "b1_threshold": fs.threshold,
+        "b2_val_cutoff": b2_cut,
+        "budget_matched": {
+            "target_val_fpr": target,
+            "achieved_val_fpr": achieved,
+            "threshold": None if math.isinf(t_budget) else t_budget,
+        },
+        "frontier_gap_test": frontier,
+        "bootstrap_cascade_minus": boot,
+        "stage_removal_test": removal,
+        "prefix_test": prefix,
+        "prevalence": prevalence,
+    }
+    typed = [s for s in order if cfg.stages[s].kind == "multiclass" and not state[s].off]
+    if typed and types_test is not None:
+        recorded = recorded_types(routing, cfg.dataset.normal_type)
+        summary["types_test"] = {
+            "stages": typed,
+            "cascade": type_metrics(types_test, recorded, y_test == 1),
+        }
+        preds["type_recorded"] = recorded
+    if routing.pool_off:
+        summary["pool_stages_off_on_test"] = routing.pool_off
+    if unseen is not None:
+        summary["unseen_test"] = unseen_block(
+            y_test, unseen, p1_test, preds, n_boot=cfg.controls.frontier_bootstrap, seed=seed
+        )
+    return summary, preds, routing
+
+
+def unseen_block(
+    y: np.ndarray, unseen: np.ndarray, p1: np.ndarray, preds: dict[str, np.ndarray], *, n_boot: int, seed: int
+) -> dict:
+    """Held-out or novel attacks on TEST: per-system recall and FPR, plus the frontier gap and the
+    first-stage AUC on the {benign, unseen} subset (DESIGN.md section 7)."""
+    u = np.asarray(unseen).astype(bool) & (y == 1)
+    systems = ("cascade", "B1_first_stage", "B2_first_stage_val_cutoff", "budget_matched_first_stage")
+    out: dict = {"n_unseen": int(u.sum()), "systems": {k: unseen_metrics(y, u, preds[k]) for k in systems}}
+    sub = u | (y == 0)
+    if u.any() and (y == 0).any():
+        gap = frontier_gap(y[sub], p1[sub], preds["cascade"][sub])
+        gap["ci95_interp"] = (
+            frontier_ci(y[sub], p1[sub], preds["cascade"][sub], n_boot, seed) if gap["defined"] else None
+        )
+        out["frontier_gap_unseen"] = gap
+        out["first_stage_auc_unseen_vs_benign"] = auc(y[sub], p1[sub])
+    return out

idslab/dashboard/README.md

@@ -0,0 +1,113 @@
+# IDSLab dashboard: source reference
+
+A local web UI over one workspace folder. Standard library only (`http.server`), so it adds no
+dependencies.
+
+**Using it?** See [docs/dashboard.md](../../../docs/dashboard.md) for how to start it and what each
+page does. This page is the source-level reference: file layout, the HTTP API and the security model,
+for anyone reading or extending the implementation.
+
+```bash
+idslab dashboard --workspace demo            # http://127.0.0.1:8765/
+```
+
+![The YAML view of an experiment](../../../docs/img/dashboard-editor.png)
+
+## Layout
+
+| File | What it is |
+|---|---|
+| `server.py` | the HTTP server: `App` (workspace, job, validation, the experiment list with the running job marked), the request handler with its guards, and `make_server` |
+| `workspace.py` | `Workspace`: path checks (relative, inside the root, no hidden parts), config and run discovery by layout (`<name>/<mode>/seed_<s>`), `experiment_index` (configs grouped with their outputs, see below), `history` and `datasets` read from files, file listing; `describe(cfg)` for the cascade preview |
+| `catalog.py` | the models and pipeline steps the builder offers, with parameter forms read from each class's constructor, plus plugin entries |
+| `jobs.py` | `build_command` turns a request into the argv of `python -m idslab.cli`, refusing what the CLI parser would; `Job` runs one subprocess at a time in its own process group, keeps the last 5,000 log lines and stops the whole group; `progress` reads phase and finished units from the log |
+| `static/index.html` | the page shell: sidebar navigation, top bar with the running-job indicator, the `<main>` the pages render into, a live region for screen readers |
+| `static/core.js` | helpers: `el()` (DOM from `textContent`, never `innerHTML`), `api()`, formatting, objective-aware numbers, plain-language errors, dialogs, a line diff, the shared job poller |
+| `static/app.js` | the hash router, the redirects from earlier addresses, theme, job indicator |
+| `static/draft.js` | `Draft`: one config being edited, shared by the setup steps, the builder and the YAML view; validation, rewrite consent, save, revert; `bound()` form fields |
+| `static/experiments.js` | the home list with next actions, New experiment, and one experiment's shell (header, save controls, pages, leave guard) |
+| `static/setup.js` | the Data, Cascade, Evaluation and Review & run steps and the YAML view |
+| `static/builder.js` | the visual builder component: canvas, ports, click and keyboard routing, drawing problems, the stage inspector |
+| `static/results.js` | an experiment's Runs, Results, Report and Files pages |
+| `static/library.js` | Templates, Datasets, Reports and Help |
+| `static/jobs.js` | the Jobs page: progress, stop, failures, past activity |
+| `static/compare.js` | the Compare page (pick experiments, paired comparison) and the Sweeps pages (list, new sweep, editor with live check and run panel) |
+| `static/views.js` | one unit in detail, and the file browser |
+| `static/charts.js` | SVG charts: grouped bars with error bars, and the cascade map drawn from parsed routes |
+| `static/styles.css` | all styling; colours are tokens on `:root` with a dark set |
+
+Routes: `#/` experiments, `#/new[/blank|/template/<file>]`, `#/e/<key>/<page>[/<step>]` one experiment
+(`key` is `c:<config path>`, `o:<output folder>` or `n:<unsaved draft>`; pages `setup`, `runs`,
+`results`, `report`, `files`; steps `data`, `cascade`, `evaluation`, `review`, `yaml`), `#/templates`,
+`#/datasets[/<file>]`, `#/jobs`, `#/reports`, `#/help`, `#/compare`, `#/sweeps`, `#/sweep/<file>`,
+`#/unit/<folder>`, `#/files/<folder>`. Earlier addresses (`#/config`, `#/builder`, `#/run`, `#/job`,
+`#/command/...`, `#/example`, `#/configs`, `#/runs`) redirect to their new pages.
+
+## Experiments
+
+An experiment is not stored anywhere: `experiment_index` builds it from the files each time. Each
+config is one experiment. A run output folder belongs to the config whose `name` it carries. When
+several configs share that name, the config fingerprints in the folder's `DONE` files must match
+exactly one of them. Otherwise the folder is listed on its own (`o:` key) with the reason, and the
+page offers no edit, run or rebuild for it.
+
+The state (`draft`, `needs_data`, `invalid`, `running`, `partial`, `completed`) and
+`changed_since_run` come from the config's validation, its data paths, the outputs and the job.
+
+Static files are served as they are, so a change shows up on a browser reload; a change to the Python
+files needs a restart.
+
+## HTTP API
+
+| Method | Path | Returns or does |
+|---|---|---|
+| GET | `/`, `/static/<file>` | the page and its assets (only the files listed in `server.STATIC`) |
+| GET | `/api/meta` | version, workspace, commands, default output folders, modes, objectives, installed extras, CPU count |
+| GET | `/api/configs` | YAML files in the workspace that look like configs: path, name, stage count, mode |
+| GET | `/api/config?path=` | the text of one config |
+| GET | `/api/templates` | the shipped reference configs |
+| GET | `/api/experiments` | every experiment: key, state, config record, output folders, next-action inputs |
+| GET | `/api/experiment?key=` (or `?output=<folder>`) | one experiment with its chosen output in full (`&out=` picks the folder) and the config's validation |
+| GET | `/api/runs` | every run output found: units done, report headline and report state |
+| GET | `/api/run?path=` | one run output: units, `report.json`, matching configs |
+| GET | `/api/datasets` | data files with size, time and the configs that read them |
+| GET | `/api/history` | finished units, reports and comparisons read from files, newest first |
+| GET | `/api/unit?path=` | one unit's `summary.json` and `DONE` |
+| GET | `/api/files?path=` | a folder listing |
+| GET | `/api/catalog` | models per stage kind and pipeline steps, with parameter forms |
+| GET | `/api/datafiles` | Parquet and CSV files in the workspace, outside run outputs |
+| GET | `/api/dataset?path=` | columns of a data file from a sample, with suggested label, type, normal value and categorical columns |
+| GET | `/api/sweeps` | sweep files in the workspace |
+| GET | `/api/layout?path=` | saved node positions of a config |
+| GET | `/api/job[?tail=N]` | the job: state, command and argv, target, timings, log, progress read from the log, units finished on disk |
+| GET | `/files/<path>[?download=1]` | a workspace file: report pages as HTML under a no-script policy, text as plain text, anything else as a download |
+| POST | `/api/validate` | `{yaml, path}`: the CLI's loader; returns the parsed design or the error |
+| POST | `/api/config/save` | `{path, yaml, overwrite}`: writes a `.yaml` or `.yml` file, then validates it |
+| POST | `/api/parse` | `{yaml}`: the config as data, for the builder |
+| POST | `/api/dump` | `{config, path}`: the config written as YAML (the CLI's rules), then validated |
+| POST | `/api/compare` | `{experiments, bootstrap}`: the comparison of finished experiments, computed in process |
+| POST | `/api/sweep/check` | `{yaml, path}`: checks a sweep; returns its variants and the base config's dotted keys |
+| POST | `/api/layout` | `{path, positions}`: writes `<config>.layout.json` |
+| POST | `/api/job` | `{command, config, out, seeds, jobs, force, no_report, debug}` or `{command: example, directory, overwrite}`: starts a CLI job |
+| POST | `/api/job/stop` | stops the running job |
+
+## Security model
+
+The dashboard runs commands and has no login, so it is built for one user on one machine:
+
+- it binds to 127.0.0.1 unless told otherwise, and prints a warning for any other address;
+- every request must carry the server's own `Host` (blocks DNS rebinding);
+- POST takes JSON only, with a same-origin `Origin` when one is sent, so another site cannot send a
+  request without a CORS preflight, which the server never approves;
+- paths are relative to the workspace, resolved with symlinks, and refused if they leave it or contain
+  a hidden part; configs are written only as `.yaml` or `.yml`;
+- the page has a strict Content Security Policy (scripts and styles from the server only, no inline
+  code); report pages get a policy that allows no scripts at all, and other text is served as
+  `text/plain`;
+- the browser receives data only; every string from a file reaches the DOM through `textContent`.
+
+`tests/test_dashboard.py` covers the path rules, discovery, experiment grouping, progress, command
+building and these guards through a live server. `tests/test_dashboard_browser.py` drives the pages in
+Chromium (every page and redirect, narrow screens in both themes, the template to report journey, the
+shared YAML and visual draft); it is skipped when Playwright is not installed
+(`pip install -e .[browser-tests]` and `playwright install chromium`).

idslab/dashboard/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""Local web dashboard: every CLI command from the browser, plus run and report browsing."""

idslab/dashboard/catalog.py

@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 hashtagRR
+"""Models and pipeline steps the builder offers, with parameter forms read from each class.
+
+The lists are a curated starting point, not a limit: the builder also takes any class path, and
+plugins (``idslab.plugins`` entry points) add their own entries. A class from a missing optional
+library is listed as not installed, with the extra that provides it.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import inspect
+from typing import Any
+
+from idslab.config import EXTRAS, import_class
+from idslab.plugins import load as load_plugins
+
+# (class path, short label, note)
+CLASSIFIERS = [
+    ("sklearn.ensemble.RandomForestClassifier", "Random forest", "strong default for flow features"),
+    ("sklearn.ensemble.ExtraTreesClassifier", "Extra trees", "faster, more random random forest"),
+    ("sklearn.ensemble.HistGradientBoostingClassifier", "Gradient boosting", "handles large tables fast"),
+    ("sklearn.linear_model.LogisticRegression", "Logistic regression", "linear; scale the features"),
+    ("sklearn.neighbors.KNeighborsClassifier", "k nearest neighbours", "scale the features"),
+    ("sklearn.tree.DecisionTreeClassifier", "Decision tree", "one readable tree"),
+    ("sklearn.neural_network.MLPClassifier", "MLP (scikit-learn)", "small neural network"),
+    ("sklearn.naive_bayes.GaussianNB", "Gaussian naive Bayes", "very fast baseline"),
+    ("xgboost.XGBClassifier", "XGBoost", "extra: xgboost"),
+    ("lightgbm.LGBMClassifier", "LightGBM", "extra: lightgbm"),
+    ("idslab.torch_models.TorchMLPClassifier", "PyTorch MLP", "extra: torch; scale the features"),
+]
+DETECTORS = [
+    ("sklearn.ensemble.IsolationForest", "Isolation forest", "fit once; no scaling needed"),
+    ("sklearn.neighbors.LocalOutlierFactor", "Local outlier factor", "fit once or per batch; scale"),
+    ("sklearn.svm.OneClassSVM", "One-class SVM", "scale the features; slow on large data"),
+    ("sklearn.covariance.EllipticEnvelope", "Elliptic envelope", "assumes roughly Gaussian features"),
+    ("pyod.models.ecod.ECOD", "ECOD (PyOD)", "extra: pyod; no parameters to tune"),
+    ("pyod.models.copod.COPOD", "COPOD (PyOD)", "extra: pyod"),
+    ("pyod.models.hbos.HBOS", "HBOS (PyOD)", "extra: pyod; very fast"),
+    ("idslab.torch_models.TorchAutoencoder", "PyTorch autoencoder", "extra: torch; scale the features"),
+]
+STEPS = [
+    ("idslab.prep.Encode", "Encode categories", "required for text columns; list them in categorical"),
+    ("sklearn.preprocessing.StandardScaler", "Standard scaler", "zero mean, unit variance"),
+    ("sklearn.preprocessing.MinMaxScaler", "Min-max scaler", "scale to [0, 1]"),
+    ("sklearn.preprocessing.RobustScaler", "Median and quartile scaler", "less sensitive to outliers"),
+    ("sklearn.preprocessing.QuantileTransformer", "Quantile transformer", "map to a uniform or normal shape"),
+    ("sklearn.decomposition.PCA", "PCA", "n_components below 1 keeps that share of variance"),
+    ("sklearn.feature_selection.VarianceThreshold", "Drop constant features", ""),
+    ("sklearn.feature_selection.SelectKBest", "Select k best", "univariate feature selection"),
+    ("imblearn.over_sampling.SMOTE", "SMOTE", "extra: imbalance; oversample during fit only"),
+    ("imblearn.under_sampling.RandomUnderSampler", "Random undersampler", "extra: imbalance"),
+]
+SKIP_PARAMS = {"random_state", "n_jobs", "verbose", "warm_start", "copy", "novelty", "device"}
+
+
+def _installed(path: str) -> bool:
+    return importlib.util.find_spec(path.split(".", 1)[0]) is not None
+
+
+def params_of(path: str) -> list[dict]:
+    """Constructor parameters with simple defaults: name, default and a form type."""
+    try:
+        cls = import_class(path)
+        sig = inspect.signature(cls.__init__)
+    except (ImportError, TypeError, ValueError):
+        return []
+    out = []
+    for name, p in sig.parameters.items():
+        if name == "self" or name in SKIP_PARAMS or p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD):
+            continue
+        d = None if p.default is inspect.Parameter.empty else p.default
+        if isinstance(d, bool):
+            kind = "bool"
+        elif isinstance(d, int):
+            kind = "int"
+        elif isinstance(d, float):
+            kind = "float"
+        elif isinstance(d, str) or d is None:
+            kind = "text"
+        elif isinstance(d, tuple) and all(isinstance(x, (int, float)) for x in d):
+            kind, d = "list", list(d)
+        else:
+            continue  # objects and callables: set them in YAML
+        out.append({"name": name, "default": d, "type": kind})
+    return out
+
+
+def _entry(path: str, label: str, note: str) -> dict:
+    installed = _installed(path)
+    top = path.split(".", 1)[0]
+    return {
+        "class": path,
+        "label": label,
+        "note": note,
+        "installed": installed,
+        "extra": EXTRAS.get(top),
+        "params": params_of(path) if installed else [],
+    }
+
+
+def plugin_entries() -> dict[str, list[tuple[str, str, str]]]:
+    """Catalog entries from installed plugins (``idslab.plugins`` entry points), by section."""
+    loaded = load_plugins()
+    return {
+        section: [
+            (i["class"], str(i.get("label", i["class"])), str(i.get("note", ""))) for i in loaded[section]
+        ]
+        for section in ("classifier", "detector", "step")
+    }
+
+
+def catalog() -> dict[str, Any]:
+    extra = plugin_entries()
+    classifiers = [_entry(*c) for c in CLASSIFIERS + extra["classifier"]]
+    return {
+        "classifier": classifiers,
+        "multiclass": classifiers,
+        "detector": [_entry(*c) for c in DETECTORS + extra["detector"]],
+        "step": [_entry(*c) for c in STEPS + extra["step"]],
+    }