telemetry-anomdet 0.1.0__py3-none-any.whl

telemetry_anomdet/__init__.py

@@ -0,0 +1,20 @@
+# telemetry_anomdet/__init__.py
+
+"""
+Telemetry Anomaly Detection Toolkit
+
+This package provides tools for:
+- Preprocessing telemetry data
+- Extracting features
+- Running unsupervised anomaly detection models
+
+Usage:
+    import telemetry_anomdet
+    # Access modules like telemetry_anomdet.preprocessing
+"""
+
+from .version import __version__
+
+__all__ = [
+    "__version__",
+]

telemetry_anomdet/evaluation.py

@@ -0,0 +1,167 @@
+# src/telemetry_anomdet/evaluation.py
+
+"""
+Point-adjusted evaluation metrics for time-series anomaly detection.
+
+Point adjustment is the standard SMAP / MSL protocol (Xu et al., 2018): if a
+detector flags any point inside a labeled anomaly segment, the whole segment
+counts as detected. It rewards catching an event even when not every point of
+it is flagged, and makes results comparable to the telemanom and MEMTO
+baselines that report point-adjusted F1.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+
+
+def point_adjust(pred: Sequence[bool], truth: Sequence[bool]) -> np.ndarray:
+    """
+    Apply point adjustment to point-level predictions.
+
+    For each contiguous anomaly segment in ``truth``, if ``pred`` flags any point
+    inside it, the entire segment is marked as predicted.
+
+    Arguments:
+        pred: Point-level boolean predictions.
+        truth: Point-level boolean ground truth, same length as ``pred``.
+    Returns:
+        np.ndarray: The adjusted boolean prediction array.
+    """
+
+    pred = np.asarray(pred, dtype=bool).copy()
+    truth = np.asarray(truth, dtype=bool)
+    if pred.shape != truth.shape:
+        raise ValueError(f"pred and truth must match: {pred.shape} vs {truth.shape}")
+
+    n = len(truth)
+    i = 0
+    while i < n:
+        if not truth[i]:
+            i += 1
+            continue
+        j = i
+        while j < n and truth[j]:
+            j += 1
+        if pred[i:j].any():
+            pred[i:j] = True
+        i = j
+    return pred
+
+
+def prf(pred: Sequence[bool], truth: Sequence[bool]) -> dict:
+    """
+    Precision, recall, and F1 for point-level boolean arrays.
+
+    Returns:
+        dict: {'precision', 'recall', 'f1', 'tp', 'fp', 'fn'}.
+    """
+
+    pred = np.asarray(pred, dtype=bool)
+    truth = np.asarray(truth, dtype=bool)
+    tp = int(np.sum(pred & truth))
+    fp = int(np.sum(pred & ~truth))
+    fn = int(np.sum(~pred & truth))
+    precision = tp / (tp + fp) if (tp + fp) else 0.0
+    recall = tp / (tp + fn) if (tp + fn) else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
+    return {"precision": precision, "recall": recall, "f1": f1, "tp": tp, "fp": fp, "fn": fn}
+
+
+def point_adjusted_f1(pred: Sequence[bool], truth: Sequence[bool]) -> dict:
+    """
+    Point-adjusted precision, recall, and F1 (the standard SMAP metric).
+
+    Equivalent to :func:`prf` computed on :func:`point_adjust` output.
+    """
+
+    return prf(point_adjust(pred, truth), truth)
+
+
+def windows_to_points(
+    window_flags: Sequence[bool], n_points: int, *, window_size: int, step: int
+) -> np.ndarray:
+    """
+    Expand window-level flags to a point-level mask.
+
+    Point ``t`` is flagged if it falls inside any flagged window. Window ``i``
+    spans ``[i*step, i*step + window_size)``, matching ``windowify``.
+
+    Arguments:
+        window_flags: Boolean flag per window.
+        n_points: Length of the point-level series the windows came from.
+        window_size: Samples per window.
+        step: Stride between windows.
+    Returns:
+        np.ndarray: Boolean mask of length ``n_points``.
+    """
+
+    window_flags = np.asarray(window_flags, dtype=bool)
+    points = np.zeros(int(n_points), dtype=bool)
+    for i, flag in enumerate(window_flags):
+        if flag:
+            s = i * step
+            points[s : s + window_size] = True
+    return points
+
+
+def windows_to_point_scores(
+    window_scores: Sequence[float], n_points: int, *, window_size: int, step: int
+) -> np.ndarray:
+    """
+    Expand window-level scores to a point-level score array.
+
+    Each point takes the maximum score among the windows covering it (window
+    ``i`` spans ``[i*step, i*step + window_size)``). Points not covered by any
+    window fall back to the minimum window score.
+
+    Arguments:
+        window_scores: Anomaly score per window (higher = more anomalous).
+        n_points: Length of the point-level series.
+        window_size: Samples per window.
+        step: Stride between windows.
+    Returns:
+        np.ndarray: Float score array of length ``n_points``.
+    """
+    window_scores = np.asarray(window_scores, dtype=float)
+    points = np.full(int(n_points), -np.inf)
+    for i, s in enumerate(window_scores):
+        a = i * step
+        points[a : a + window_size] = np.maximum(points[a : a + window_size], s)
+    fill = window_scores.min() if window_scores.size else 0.0
+    points[~np.isfinite(points)] = fill
+    return points
+
+
+def best_point_adjusted_f1(
+    scores: Sequence[float], truth: Sequence[bool], *, n_thresholds: int = 200
+) -> dict:
+    """
+    Best point-adjusted F1 over a threshold sweep on point-level scores.
+
+    Selects the threshold that maximizes point-adjusted F1. This is the standard
+    SMAP / MSL "best F1" protocol used by telemanom and MEMTO, which makes those
+    baselines comparable. Note that it selects the threshold using the labels, so
+    it should be reported as an oracle-threshold upper bound, not a deployable
+    operating point.
+
+    Arguments:
+        scores: Point-level anomaly scores (higher = more anomalous).
+        truth: Point-level boolean ground truth.
+        n_thresholds: Number of candidate thresholds sampled across the score range.
+    Returns:
+        dict: The best {'precision', 'recall', 'f1', 'tp', 'fp', 'fn', 'threshold'}.
+    """
+    scores = np.asarray(scores, dtype=float)
+    truth = np.asarray(truth, dtype=bool)
+    candidates = np.unique(np.quantile(scores, np.linspace(0.0, 1.0, n_thresholds)))
+    best = {"precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 0, "fn": 0}
+    best["threshold"] = float("inf")
+    for t in candidates:
+        m = prf(point_adjust(scores >= t, truth), truth)
+        if m["f1"] > best["f1"]:
+            m["threshold"] = float(t)
+            best = m
+    return best

telemetry_anomdet/feature_extraction/features.py

@@ -0,0 +1,200 @@
+# telemetry_anomdet/features.py
+
+"""
+Feature extraction methods for telemetry analysis.
+
+This module defines functions to compute relevant
+features from preprocessed telemetry data.
+"""
+
+# Long form telemetry means one observation per row, wide means all variables get their own column, with timestamps as the index
+
+import numpy as np
+import pandas as pd
+
+# Canonical column names
+_TS, _VAR, _VAL = "timestamp", "variable", "value"
+
+_REQURED_COLS = {_TS, _VAR, _VAL}
+
+
+def pivot_wide(df: pd.DataFrame, *, variables=None) -> pd.DataFrame:
+    """
+    Convert long form telemetry data to wide format.
+
+    Arguments:
+        df (pd.DataFrame): Long-form data with ['timestamp','variable','value'].
+        variables (list[str], optional): Specific variables to include; all by default.
+    Returns:
+        pd.DataFrame: Wide table (timestamp index, variables as columns).
+    """
+
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("pivot_table() expects a pandas DataFrame")
+
+    missing = _REQURED_COLS - set(df.columns)
+
+    if missing:
+        raise ValueError(f"pivot_wide() missing required columns: {missing}")
+
+    data = df
+
+    # (Optional): filter to a subset of variables/sensors
+    if variables is not None:
+        data = data[data[_VAR].isin(variables)]
+
+    if data.empty:
+        return pd.DataFrame(index=pd.Index([], name=_TS))
+
+    # Pivot from long to wide
+    wide = data.pivot_table(
+        index=_TS,
+        columns=_VAR,
+        values=_VAL,
+        aggfunc="last",
+    ).sort_index()
+
+    # Drop channels that are entirely NaN
+    wide = wide.dropna(axis=1, how="all")
+
+    # Deterministic column order
+    if wide.shape[1] > 0:
+        wide = wide.loc[:, sorted(wide.columns)]
+
+    # Tidy up the labels
+    wide.index.name = _TS
+    wide.columns.name = None
+
+    return wide
+
+
+def windowify(wide_df: pd.DataFrame, *, window_size: int = 50, step: int = 10) -> np.ndarray:
+    """
+    Slice wide form telemetry data into overlapping windows.
+
+    Arguments:
+        wide_df (pd.DataFrame): Wide telemetry table.
+        window_size (int): Number of samples per window.
+        step (int): Step size between consecutive windows.
+    Returns:
+        np.ndarray: Array with shape (n_windows, window_size, n_features).
+    """
+
+    if not isinstance(wide_df, pd.DataFrame):
+        raise TypeError("windoify() expects a pandas DataFrame")
+
+    # Convert to numeric matrix: shape (n_samples, n_features)
+    values = wide_df.to_numpy(dtype=float)
+    n_samples, n_features = values.shape
+
+    windows = []
+
+    # Slide a fixed size window across time axis
+    for start in range(0, n_samples - window_size + 1, step):
+        end = start + window_size
+        windows.append(values[start:end, :])
+
+    # If not enough data for a single window, return an empty array
+    if not windows:
+        return np.empty((0, window_size, n_features), dtype=float)
+
+    # Stack list of (window_size, n_features) into (n_windows, window_size, n_features)
+    return np.stack(windows, axis=0)
+
+
+def features_stat(X3d: np.ndarray) -> np.ndarray:
+    """
+    Generate a 2D feature table from windowed telemetry.
+
+    Arguments:
+        X3d (np.ndarray): Windowed telemetry tensor with shape
+            (n_windows, window_size, n_features), typically produced by
+            `make_feature_table()`.
+
+    Returns:
+        np.ndarray:
+            Shape (n_windows, n_features * 4).
+            For each window and each variable, this function computes
+            simple statistical features (mean, standard deviation, min,
+            and max) along the time axis and concatenates them into a
+            flat feature vector ready for downstream modeling.
+    """
+
+    X3d = np.asarray(X3d)
+
+    if X3d.ndim != 3:
+        raise ValueError(
+            f"features_stat() expects a 3D array "
+            f"(n_windows, window_size, n_features), got {X3d.shape}"
+        )
+
+    n_windows, window_size, n_features = X3d.shape
+
+    if n_windows == 0 or n_features == 0:
+        # If no data return an empty 2D array with 0 rows
+        return np.empty((0, 0), dtype=float)
+
+    # Compute stats along the time axis (axis = 1)
+    means = np.nanmean(X3d, axis=1)  # (n_windows, n_features)
+    stds = np.nanstd(X3d, axis=1)  # (n_windows, n_features)
+    mins = np.nanmin(X3d, axis=1)  # (n_windows, n_features)
+    maxs = np.nanmax(X3d, axis=1)  # (n_windows, n_features)
+    medians = np.nanmedian(X3d, axis=1)  # (n_windows, n_features)
+
+    # Linear slope per variable for each window
+    times = np.arange(window_size)
+    slopes = np.zeros((n_windows, n_features), dtype=float)
+
+    for i in range(n_windows):
+        for j in range(n_features):
+            y = X3d[i, :, j]
+            mask = np.isfinite(y)
+
+            # If all NaN or fewer than 2 finite points, slope stays 0
+            if mask.sum() < 2:
+                slopes[i, j] = 0.0
+            else:
+                a, b = np.polyfit(times[mask], y[mask], 1)
+                slopes[i, j] = a
+
+    # Concatenate into a single feature vector per window: [mean | std | min | max]
+    X_feat = np.concatenate([means, stds, mins, maxs, medians, slopes], axis=1)
+
+    # Replace any remaining NaN/inf with column means
+    col_means = np.nanmean(X_feat, axis=0)
+    col_means = np.where(np.isfinite(col_means), col_means, 0.0)
+    bad = ~np.isfinite(X_feat)
+    if bad.any():
+        X_feat[bad] = col_means[np.where(bad)[1]]
+
+    return X_feat
+
+
+def make_feature_table(df: pd.DataFrame, *, variables=None, window_size=50, step=10) -> np.ndarray:
+    """
+    Generate a complete feature table from preprocessed telemetry.
+
+    Arguments:
+        df (pd.DataFrame): Preprocessed long-form telemetry data.
+        variables (list[str], optional): Subset of sensors to include.
+        window_size (int): Window length (default 50 samples).
+        step (int): Step size between windows.
+    Returns:
+         np.ndarray:
+            Shape (n_windows, window_size, n_features).
+            Ready for downstream modeling or for passing into a
+            future features_stat() to get a 2D feature table.
+    """
+
+    # Convert long form to wide form
+    wide = pivot_wide(df, variables=variables)
+
+    # If no usable data remains, stop early
+    if wide.empty:
+        return np.empty((0, window_size, 0))
+
+    # Slice the wide data into fixed length windows
+    X3d = windowify(wide, window_size=window_size, step=step)
+
+    # Return the 3D telemetry tensor ready for feature extraction
+    return X3d

telemetry_anomdet/ingest/__init__.py

@@ -0,0 +1,21 @@
+# src/telemetry_anomdet/ingest/__init__.py
+
+"""
+Ingest subpackage: dataset loaders and parsers.
+"""
+
+from .dataset import TelemetryDataset
+from .smap import (
+    anomaly_point_mask,
+    load_smap,
+    load_smap_channel,
+    load_smap_labels,
+)
+
+__all__ = [
+    "TelemetryDataset",
+    "load_smap",
+    "load_smap_channel",
+    "load_smap_labels",
+    "anomaly_point_mask",
+]

telemetry_anomdet/ingest/ccsds_loader.py

@@ -0,0 +1,18 @@
+# src/telemetry_anomdet/ingest/ccsds_loader.pyv
+
+from __future__ import annotations
+
+import pandas as pd
+
+from telemetry_anomdet.ingest.dataset import TelemetryDataset
+
+
+def load_from_ccsds(file_or_stream) -> TelemetryDataset:
+    """
+    Load telemetry from a CCSDS file or stream.
+    """
+
+    df = pd.DataFrame()  # Placeholder
+    # TODO: Implement CCSDS packet parsing logic
+
+    return TelemetryDataset(df)

telemetry_anomdet/ingest/csv_loader.py

@@ -0,0 +1,133 @@
+# src/telemetry_anomdet/ingest/csv_loader.py
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+
+import pandas as pd
+
+from telemetry_anomdet.ingest.dataset import TelemetryDataset
+
+# Canonical columns
+_TS, _VAR, _VAL = "timestamp", "variable", "value"
+
+# Common aliases
+DEFAULT_ALIASES: Mapping[str, Iterable[str]] = {
+    _TS: {"timestamp", "time", "datetime", "date", "ts"},
+    _VAR: {"name", "key", "channel", "sensor", "variable"},
+    _VAL: {"value", "reading", "val", "y"},
+}
+
+
+def load_from_csv(
+    path: str, *, time_col: str | None = None, value_cols: Sequence[str] | None = None
+) -> TelemetryDataset:
+    """
+    Load telemetry from a CSV file.
+
+    Notes:
+    CSV must contain at least colums: timestamp, variable, value. timestamp will be converted to pandas datetime.
+
+    Arguments:
+    path - Path to CSV file.
+    time_col - Explicit time column, if ommitted, guessed from aliases
+    value_cols - Explicit value columns
+
+    Returns:
+    TelemetryDataset
+    """
+
+    df = pd.read_csv(path)
+    aliases_local = DEFAULT_ALIASES
+
+    # If in long form, normalize column names to canonical
+    if is_long_form(df.columns.tolist(), aliases_local):
+        ren = {}
+        lowmap = {c.lower(): c for c in df.columns}
+        ren[lowmap[next(a for a in aliases_local[_TS] if a in lowmap)]] = _TS
+        ren[lowmap[next(a for a in aliases_local[_VAR] if a in lowmap)]] = _VAR
+        ren[lowmap[next(a for a in aliases_local[_VAL] if a in lowmap)]] = _VAL
+
+        out = df.rename(columns=ren)
+        out = coerce_long(out)
+
+        return TelemetryDataset(out)
+
+    # If not in long form, convert from Wide to melt
+    tcol = pick_time_column(df.columns.tolist(), time_col=time_col, aliases=aliases_local)
+    wide = df.copy()
+    wide[tcol] = pd.to_datetime(wide[tcol], errors="coerce", utc=True)
+    wide = wide.dropna(subset=[tcol])
+
+    # Choose measurement columns
+    if value_cols:
+        missing = [c for c in value_cols if c not in wide.columns]
+        if missing:
+            raise KeyError(f"value_cols not found: {missing}. Columns: {list(wide.columns)}")
+        meas = list(value_cols)
+    else:
+        meas = [c for c in wide.columns if c != tcol]
+        if not meas:
+            raise ValueError("No measurement columns found. Provide value_cols=[].")
+
+    long = wide.melt(id_vars=tcol, value_vars=meas, var_name=_VAR, value_name=_VAL)
+    long = long.rename(columns={tcol: _TS})
+    long = coerce_long(long)
+
+    return TelemetryDataset(long)
+
+
+# Helper for finding datasets with timestamps/variables/values - returns t/f
+def is_long_form(cols: Sequence[str], aliases: Mapping[str, Iterable[str]]) -> bool:
+    """
+    Determine whether a CSV is already in "long" form by checking that all three semantic roles appear under some alias.
+    """
+
+    lc = set(c.lower() for c in cols)
+    has_ts = any(a in lc for a in aliases[_TS])
+    has_var = any(a in lc for a in aliases[_VAR])
+    has_val = any(a in lc for a in aliases[_VAL])
+
+    return has_ts and has_var and has_val
+
+
+def coerce_long(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Final cleanup for a long form DataFrame
+    - sort by time, then variable
+    - ensure variable is string
+    - parse timestamp to utc
+    """
+
+    df[_TS] = pd.to_datetime(df[_TS], errors="coerce", utc=True)
+    df[_VAL] = pd.to_numeric(df[_VAL], errors="coerce")
+    df[_VAR] = df[_VAR].astype(str)
+    df = df.dropna(subset=[_TS, _VAR, _VAL]).sort_values([_TS, _VAR]).reset_index(drop=True)
+
+    return df[[_TS, _VAR, _VAL]]
+
+
+def pick_time_column(
+    cols: Sequence[str], *, time_col: str | None, aliases: Mapping[str, Iterable[str]]
+) -> str:
+    """
+    Choose which colun is the time axis for *wide* CSV.
+    - if time_col is explicitly provided, verify it exists and return it
+    - else, try and match from alias candidates (like: "timestamp", "time")
+    - if nothing can be found, raise error listing options and actual columns
+    """
+
+    if time_col:
+        if time_col in cols:
+            return time_col
+        raise KeyError(f"time_col='{time_col}' not found in columns: {list(cols)}")
+
+    lowered = {c.lower().strip(): c for c in cols}
+    for cand in aliases.get(_TS, ()):
+        if cand in lowered:
+            return lowered[cand]
+
+    raise KeyError(
+        "No time column found. Provide time_col= or include one of the aliases: "
+        f"{aliases.get(_TS)}. Columns seen: {list(cols)}"
+    )

telemetry_anomdet/ingest/dataset.py

@@ -0,0 +1,80 @@
+# src/telemetry_anomdet/ingest/dataset.py
+
+"""
+Telemetry dataset loader.
+
+This module a lightweight class called TelemetryDataset that wraps telemetry data in a consistent form factor for downstream processing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+try:
+    from typing import Self
+except ImportError:
+    from typing_extensions import Self
+
+
+@dataclass
+class TelemetryDataset:
+    """
+    Container for telemetry data.
+    """
+
+    # Pandas DataFrame containing telemetry rows.
+    _df: pd.DataFrame
+
+    @classmethod
+    def synthetic(cls) -> Self:
+        """
+        Generate a small synthetic telemetry dataset for development / testing.
+
+        Parameters:
+        N/A for now.
+
+        Returns:
+        TelemetryDataset
+        """
+
+        df = pd.DataFrame(
+            {
+                "timestamp": pd.to_datetime(
+                    ["2025-01-01T00:00:00Z", "2025-01-01T00:01:00Z"], utc=True
+                ),
+                "variable": ["temp", "pressure"],
+                "value": [25.5, 1013.25],
+            }
+        ).sort_values(["timestamp", "variable"], ignore_index=True)
+
+        return cls(df)
+
+    def to_pandas(self) -> pd.DataFrame:
+        """
+        Return a copy of the Dataframe to prevent accidental mutation.
+        """
+
+        return self._df.copy()
+
+    @property
+    def data(self) -> pd.DataFrame:
+        """
+        Direct access to the underlying DataFrame.
+        """
+        return self._df
+
+    def head(self, n: int = 5) -> pd.DataFrame:
+        """
+        Head of the underlying DataFrame (copy).
+        """
+
+        return self._df.head(n).copy()
+
+    def __len__(self) -> int:
+        return len(self._df)
+
+    def __repr__(self) -> str:
+        cols = list(self._df.columns)
+        return f"TelemetryDataset(n={len(self)}, cols={cols})"