sensor-modeling 0.2.0__py3-none-any.whl

sensor_modeling/analysis/granger_causality.py

@@ -0,0 +1,314 @@
+"""
+Granger Causality Testing for Binary Time Series
+
+This module implements Granger causality tests specifically adapted
+for binary sensor time series data.
+"""
+
+import logging
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2
+from sklearn.linear_model import LogisticRegression
+
+logger = logging.getLogger(__name__)
+
+CAUSALITY_RESULT_COLUMNS = [
+    "cause",
+    "effect",
+    "test_statistic",
+    "p_value",
+    "causality_detected",
+    "lags_used",
+]
+
+
+class GrangerCausalityTest:
+    """
+    Granger causality test for binary time series sensor data.
+    Tests whether past values of sensor X help predict sensor Y beyond what
+    past values of Y already explain.
+    """
+
+    def __init__(self, max_lags: int = 5):
+        """
+        Initialize Granger causality test.
+
+        Args:
+            max_lags: Maximum number of lags to consider
+        """
+        if max_lags < 1:
+            raise ValueError("max_lags must be at least 1")
+        self.max_lags = max_lags
+
+    def test(self, x: np.ndarray, y: np.ndarray, lags: int | None = None) -> dict:
+        """
+        Perform Granger causality test.
+
+        Args:
+            x: Time series of sensor X (potential cause)
+            y: Time series of sensor Y (potential effect)
+            lags: Number of lags to use (if None, automatically selected)
+
+        Returns:
+            Dictionary with test results
+        """
+        x, y = self._validate_series(x, y)
+        if lags is None:
+            lags = self._select_optimal_lags(x, y)
+        self._validate_lags(lags, len(y))
+
+        # Restricted model: Y ~ lags of Y
+        y_lagged = self._create_lag_matrix(y, lags, include_current=False)
+        y_current = y[lags:]
+
+        # Unrestricted model: Y ~ lags of Y + lags of X
+        x_lagged = self._create_lag_matrix(x, lags, include_current=False)
+
+        # Fit restricted model (Y predicted by its own lags only)
+        restricted_ll = self._fit_binary_regression(y_lagged, y_current)
+
+        # Fit unrestricted model (Y predicted by its own lags + X lags)
+        combined_features = np.column_stack([y_lagged, x_lagged])
+        unrestricted_ll = self._fit_binary_regression(combined_features, y_current)
+
+        # Likelihood ratio test
+        lr_stat = max(0.0, float(2 * (unrestricted_ll - restricted_ll)))
+        p_value = float(1 - chi2.cdf(lr_stat, df=lags))
+
+        return {
+            "test_statistic": lr_stat,
+            "p_value": p_value,
+            "lags_used": lags,
+            "causality_detected": p_value < 0.05,
+            "restricted_ll": restricted_ll,
+            "unrestricted_ll": unrestricted_ll,
+        }
+
+    def _validate_series(
+        self, x: np.ndarray, y: np.ndarray
+    ) -> tuple[np.ndarray, np.ndarray]:
+        """Validate and normalize input series."""
+        x_array = np.asarray(x, dtype=float).ravel()
+        y_array = np.asarray(y, dtype=float).ravel()
+        if len(x_array) != len(y_array):
+            raise ValueError("x and y must have the same length")
+        if len(x_array) < 3:
+            raise ValueError("x and y must contain at least 3 observations")
+        if np.isnan(x_array).any() or np.isnan(y_array).any():
+            raise ValueError("x and y must not contain NaN values")
+        for name, values in {"x": x_array, "y": y_array}.items():
+            unique = set(np.unique(values))
+            if not unique <= {0.0, 1.0}:
+                raise ValueError(f"{name} must be binary with values 0 and 1")
+        return x_array, y_array
+
+    def _validate_lags(self, lags: int, n_obs: int) -> None:
+        """Validate requested lag count."""
+        if lags < 1:
+            raise ValueError("lags must be at least 1")
+        if lags >= n_obs:
+            raise ValueError("lags must be smaller than the number of observations")
+
+    def _create_lag_matrix(
+        self, series: np.ndarray, lags: int, include_current: bool = False
+    ) -> np.ndarray:
+        """Create matrix of lagged variables."""
+        n = len(series)
+        lag_matrix = np.zeros((n - lags, lags + (1 if include_current else 0)))
+
+        for row, t in enumerate(range(lags, n)):
+            col = 0
+            if include_current:
+                lag_matrix[row, col] = series[t]
+                col += 1
+            for lag in range(1, lags + 1):
+                lag_matrix[row, col] = series[t - lag]
+                col += 1
+
+        return lag_matrix
+
+    def _fit_binary_regression(self, X: np.ndarray, y: np.ndarray) -> float:
+        """Fit logistic regression and return log-likelihood."""
+        try:
+            if len(np.unique(y)) < 2:
+                return self._constant_log_likelihood(y)
+
+            # Add intercept
+            X_with_intercept = np.column_stack([np.ones(X.shape[0]), X])
+
+            # Fit using sklearn for numerical stability
+            model = LogisticRegression(fit_intercept=False, max_iter=1000)
+            model.fit(X_with_intercept, y)
+
+            # Calculate log-likelihood manually
+            linear_pred = X_with_intercept @ model.coef_.T
+            probs = 1 / (1 + np.exp(-linear_pred.flatten()))
+
+            # Avoid log(0) issues
+            probs = np.clip(probs, 1e-15, 1 - 1e-15)
+
+            log_likelihood = np.sum(y * np.log(probs) + (1 - y) * np.log(1 - probs))
+            return log_likelihood
+
+        except (FloatingPointError, ValueError) as e:
+            logger.warning("Error in binary regression: %s", e)
+            return -np.inf
+
+    def _constant_log_likelihood(self, y: np.ndarray) -> float:
+        """Return Bernoulli log-likelihood for a constant target."""
+        p = float(np.clip(y.mean(), 1e-15, 1 - 1e-15))
+        return float(np.sum(y * np.log(p) + (1 - y) * np.log(1 - p)))
+
+    def _select_optimal_lags(self, x: np.ndarray, y: np.ndarray) -> int:
+        """Select optimal number of lags using BIC."""
+        best_lags = 1
+        best_bic = np.inf
+
+        for lags in range(1, min(self.max_lags + 1, len(y) // 4)):
+            try:
+                # Fit model with this number of lags
+                y_lagged = self._create_lag_matrix(y, lags, include_current=False)
+                x_lagged = self._create_lag_matrix(x, lags, include_current=False)
+                y_current = y[lags:]
+
+                combined_features = np.column_stack([y_lagged, x_lagged])
+                log_likelihood = self._fit_binary_regression(
+                    combined_features, y_current
+                )
+
+                n_params = combined_features.shape[1] + 1  # +1 for intercept
+                n_obs = len(y_current)
+                bic = -2 * log_likelihood + n_params * np.log(n_obs)
+
+                if bic < best_bic:
+                    best_bic = bic
+                    best_lags = lags
+
+            except (FloatingPointError, ValueError):
+                continue
+
+        return best_lags
+
+    def test_all_pairs(self, data: pd.DataFrame) -> pd.DataFrame:
+        """
+        Test Granger causality for all sensor pairs.
+
+        Args:
+            data: DataFrame with sensor data
+
+        Returns:
+            DataFrame with causality test results for all pairs
+        """
+        sensors = data.columns.tolist()
+        results = []
+
+        for cause in sensors:
+            for effect in sensors:
+                if cause != effect:
+                    try:
+                        result = self.test(data[cause].values, data[effect].values)
+                        results.append(
+                            {
+                                "cause": cause,
+                                "effect": effect,
+                                "test_statistic": result["test_statistic"],
+                                "p_value": result["p_value"],
+                                "causality_detected": result["causality_detected"],
+                                "lags_used": result["lags_used"],
+                            }
+                        )
+                    except (FloatingPointError, ValueError) as e:
+                        logger.warning("Error testing %s -> %s: %s", cause, effect, e)
+                        results.append(
+                            {
+                                "cause": cause,
+                                "effect": effect,
+                                "test_statistic": np.nan,
+                                "p_value": np.nan,
+                                "causality_detected": False,
+                                "lags_used": np.nan,
+                            }
+                        )
+
+        return pd.DataFrame(results, columns=CAUSALITY_RESULT_COLUMNS)
+
+    def create_causality_summary(self, causality_results: pd.DataFrame) -> dict:
+        """
+        Create summary statistics from causality test results.
+
+        Args:
+            causality_results: DataFrame from test_all_pairs()
+
+        Returns:
+            Dictionary with summary statistics
+        """
+        significant = causality_results[
+            causality_results["causality_detected"].eq(True)
+        ]
+
+        # Count relationships
+        total_pairs = len(causality_results)
+        significant_pairs = len(significant)
+        causality_rate = significant_pairs / total_pairs if total_pairs > 0 else 0
+
+        # Identify most influential sensors
+        if len(significant) > 0:
+            cause_counts = (
+                significant.groupby("cause").size().sort_values(ascending=False)
+            )
+            effect_counts = (
+                significant.groupby("effect").size().sort_values(ascending=False)
+            )
+
+            top_causes = cause_counts.head(3).to_dict()
+            top_effects = effect_counts.head(3).to_dict()
+        else:
+            top_causes = {}
+            top_effects = {}
+
+        # Average test statistics
+        avg_test_stat = self._finite_mean_or_zero(causality_results["test_statistic"])
+        avg_significant_stat = self._finite_mean_or_zero(significant["test_statistic"])
+
+        return {
+            "total_pairs_tested": total_pairs,
+            "significant_relationships": significant_pairs,
+            "causality_rate": causality_rate,
+            "average_test_statistic": avg_test_stat,
+            "average_significant_statistic": avg_significant_stat,
+            "top_causes": top_causes,
+            "top_effects": top_effects,
+            "bidirectional_relationships": self._find_bidirectional(significant),
+        }
+
+    def _finite_mean_or_zero(self, values: pd.Series) -> float:
+        """Return the finite mean of values, or zero when none are finite."""
+        finite_values = pd.to_numeric(values, errors="coerce").replace(
+            [np.inf, -np.inf], np.nan
+        )
+        mean = finite_values.mean()
+        if pd.isna(mean):
+            return 0.0
+        return float(mean)
+
+    def _find_bidirectional(
+        self, significant_results: pd.DataFrame
+    ) -> list[tuple[str, str]]:
+        """Find bidirectional causal relationships."""
+        bidirectional = []
+
+        for _, row in significant_results.iterrows():
+            # Check if reverse relationship also exists
+            reverse = significant_results[
+                (significant_results["cause"] == row["effect"])
+                & (significant_results["effect"] == row["cause"])
+            ]
+
+            if len(reverse) > 0:
+                pair = tuple(sorted([row["cause"], row["effect"]]))
+                if pair not in bidirectional:
+                    bidirectional.append(pair)
+
+        return bidirectional

sensor_modeling/analysis/pipeline.py

@@ -0,0 +1,168 @@
+"""Unified analysis pipeline for sensor modeling.
+
+This module exposes :class:`AnalysisPipeline` which orchestrates the
+available modeling approaches (autoregressive, HMM, change-point and NHPP)
+on a common :class:`~sensor_modeling.utils.data_io.SensorDataset` input.
+
+The pipeline hides the complexity of individual models while presenting a
+simple interface for end users::
+
+    from sensor_modeling.analysis import AnalysisPipeline
+    from sensor_modeling.utils.data_io import SensorDataset
+
+    pipeline = AnalysisPipeline()
+    results = pipeline.run(dataset)
+    pipeline.generate_report(results, output_dir="out")
+
+Each model is executed in parallel and their results are collected in a
+single dictionary that can be further analysed or exported.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Dict
+
+import numpy as np
+import pandas as pd
+
+from ..change_point.embedding_cpd import EmbeddingCPD
+from ..hmm.base import BaseHMM
+from ..models.bernoulli_ar.base_model import BernoulliAutoregressiveModel
+from ..models.nhpp_pelt.model import NHPPPELT, NHPPConfig
+from ..utils.data_io import SensorDataset
+
+logger = logging.getLogger(__name__)
+
+ModelRunError = (
+    AttributeError,
+    FloatingPointError,
+    RuntimeError,
+    TypeError,
+    ValueError,
+)
+
+
+class AnalysisPipeline:
+    """Master analysis class coordinating all modeling approaches."""
+
+    def __init__(self, models: Dict[str, Any] | None = None):
+        self.models = models or {}
+        self.results: Dict[str, Any] = {}
+        self.dataset: SensorDataset | None = None
+
+    # ------------------------------------------------------------------
+    def _default_models(self, sensor_names: list[str]) -> Dict[str, Any]:
+        """Create default model instances for the given sensors."""
+        ar_model = BernoulliAutoregressiveModel(sensor_names, sensor_names[0])
+        hmm_model = BaseHMM()
+        cpd_model = EmbeddingCPD()
+        # n_basis must be at least degree+1; the cubic default needs four
+        # basis functions. Below that the model raises on every fit and the
+        # pipeline silently records an error instead of a result.
+        nhpp_model = NHPPPELT(NHPPConfig(n_basis=4, min_seg_len=1))
+        return {
+            "ar": ar_model,
+            "hmm": hmm_model,
+            "cpd": cpd_model,
+            "nhpp": nhpp_model,
+        }
+
+    # ------------------------------------------------------------------
+    def _prepare_dataset(self, data: pd.DataFrame | SensorDataset) -> pd.DataFrame:
+        """Store and validate the input dataset for a pipeline run."""
+        self.dataset = data if isinstance(data, SensorDataset) else SensorDataset(data)
+        df = self.dataset.to_dataframe()
+        if len(df.columns) == 0:
+            raise ValueError("analysis pipeline requires at least one sensor column")
+        if len(df) == 0:
+            raise ValueError("analysis pipeline requires at least one row")
+        return df
+
+    # ------------------------------------------------------------------
+    def _run_model(
+        self,
+        name: str,
+        model: Any,
+        df: pd.DataFrame,
+        sensor_names: list[str],
+    ) -> Dict[str, Any]:
+        """Run one configured model and return serializable summary output."""
+        target_sensor = sensor_names[0]
+        if name == "nhpp":
+            model.fit(self.dataset, sensor=target_sensor)
+            return {"changepoints": getattr(model, "changepoints_", [])}
+        if name == "cpd":
+            model.fit(df[target_sensor].values)
+            return {"changepoints": model.predict()}
+        if name == "hmm":
+            x = df.values
+            model.fit(x)
+            return {"states": model.predict(x)}
+        if name == "ar":
+            model.fit(df)
+            probabilities = np.asarray(model.predict_probabilities(df))
+            return {"probabilities": probabilities[:5].tolist()}
+        return {"status": "skipped"}
+
+    # ------------------------------------------------------------------
+    def run(self, data: pd.DataFrame | SensorDataset) -> Dict[str, Any]:
+        """Run all configured models on *data* in parallel."""
+        df = self._prepare_dataset(data)
+        sensor_names = list(df.columns)
+        if not self.models:
+            self.models = self._default_models(sensor_names)
+
+        def _run(name: str, model: Any) -> Dict[str, Any]:
+            try:
+                return self._run_model(name, model, df, sensor_names)
+            except ModelRunError as exc:
+                logger.error("Model %s failed: %s", name, exc)
+                return {"error": str(exc)}
+
+        with ThreadPoolExecutor() as ex:
+            futures = {n: ex.submit(_run, n, m) for n, m in self.models.items()}
+            self.results = {n: f.result() for n, f in futures.items()}
+        return self.results
+
+    # ------------------------------------------------------------------
+    def generate_report(
+        self, results: Dict[str, Any], output_dir: str
+    ) -> Dict[str, Path]:
+        """Generate LaTeX, HTML and FHIR reports for *results*."""
+        from .reporting import (
+            create_html_dashboard,
+            export_to_fhir,
+            generate_latex_report,
+        )
+
+        out_dir = Path(output_dir)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        return {
+            "latex": generate_latex_report(results, out_dir / "analysis.tex"),
+            "html": create_html_dashboard(results, out_dir / "dashboard.html"),
+            "fhir": export_to_fhir(results, out_dir / "analysis_fhir.json"),
+        }
+
+
+def main() -> None:
+    """Run the analysis pipeline from the command line."""
+    parser = argparse.ArgumentParser(description="Run the sensor analysis pipeline")
+    parser.add_argument("data", help="Path to CSV sensor data")
+    parser.add_argument(
+        "output_dir",
+        help="Directory where the generated reports should be written",
+    )
+    args = parser.parse_args()
+
+    dataset = SensorDataset.from_csv(args.data)
+    pipeline = AnalysisPipeline()
+    results = pipeline.run(dataset)
+    pipeline.generate_report(results, args.output_dir)
+
+
+if __name__ == "__main__":
+    main()

sensor_modeling/analysis/reporting.py

@@ -0,0 +1,109 @@
+"""Reporting utilities for analysis results."""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from os import PathLike
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+ReportPath = str | PathLike[str]
+
+
+def _prepare_output_path(path: ReportPath) -> Path:
+    """Return an output path with its parent directory created."""
+    output_path = Path(path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    return output_path
+
+
+# ---------------------------------------------------------------------------
+def generate_latex_report(results: dict[str, Any], path: ReportPath) -> Path:
+    """Generate a minimal LaTeX report summarizing *results*."""
+    output_path = _prepare_output_path(path)
+    content = [
+        r"\documentclass{article}",
+        r"\begin{document}",
+        r"\section*{Sensor Modeling Report}",
+        r"\begin{verbatim}",
+        json.dumps(results, indent=2, default=str),
+        r"\end{verbatim}",
+        r"\end{document}",
+    ]
+    output_path.write_text("\n".join(content), encoding="utf-8")
+    logger.info("LaTeX report written to %s", output_path)
+    return output_path
+
+
+# ---------------------------------------------------------------------------
+def create_html_dashboard(results: dict[str, Any], path: ReportPath) -> Path:
+    """Generate a simple HTML dashboard for *results*."""
+    output_path = _prepare_output_path(path)
+    html = [
+        "<html><body><h1>Sensor Modeling Dashboard</h1><pre>",
+        json.dumps(results, indent=2, default=str),
+        "</pre></body></html>",
+    ]
+    output_path.write_text("\n".join(html), encoding="utf-8")
+    logger.info("HTML dashboard written to %s", output_path)
+    return output_path
+
+
+# ---------------------------------------------------------------------------
+def export_to_fhir(results: dict[str, Any], path: ReportPath) -> Path:
+    """Export *results* as a minimal FHIR-like Observation resource.
+
+    The export preserves each top-level analysis result as an Observation
+    component. It is intended as an interoperable starting point, not a full
+    clinical profile implementation.
+    """
+    fhir = {
+        "resourceType": "Observation",
+        "id": "sensor-analysis",
+        "status": "final",
+        "category": [
+            {
+                "coding": [
+                    {
+                        "system": (
+                            "http://terminology.hl7.org/CodeSystem/"
+                            "observation-category"
+                        ),
+                        "code": "activity",
+                        "display": "Activity",
+                    }
+                ]
+            }
+        ],
+        "code": {
+            "text": "Sensor modeling analysis summary",
+        },
+        "effectiveDateTime": datetime.now(timezone.utc)
+        .isoformat()
+        .replace("+00:00", "Z"),
+        "component": [
+            {
+                "code": {"text": name},
+                "valueString": json.dumps(value, default=str),
+            }
+            for name, value in results.items()
+        ],
+    }
+    output_path = _prepare_output_path(path)
+    output_path.write_text(json.dumps(fhir, indent=2), encoding="utf-8")
+    logger.info("FHIR export written to %s", output_path)
+    return output_path
+
+
+# ---------------------------------------------------------------------------
+def render_template(template: str, context: dict[str, Any]) -> str:
+    """Render *template* using ``str.format`` with the provided *context*."""
+    try:
+        return template.format(**context)
+    except (KeyError, IndexError, ValueError, AttributeError) as exc:
+        logger.error("Template rendering failed: %s", exc)
+        return template

sensor_modeling/baseline/__init__.py

@@ -0,0 +1,30 @@
+"""Adaptive personal baselines over non-stationary behaviour.
+
+Normal behaviour is modelled as a slowly moving distribution rather than a
+fixed calibration window, and the reasons a day can look unusual -- ordinary
+variability, weekly rhythm, a temporary disturbance, a persistent change, a
+gradual drift, or simply a day the sensors did not watch -- are distinguished
+rather than collapsed into a single anomaly score.
+"""
+
+from .adaptive import (
+    MAD_TO_SIGMA,
+    AdaptiveBaseline,
+    BaselineConfig,
+    BaselineReference,
+    BehaviouralChange,
+    ChangeKind,
+)
+from .features import DailySummary, feature_series, summarise_days
+
+__all__ = [
+    "MAD_TO_SIGMA",
+    "AdaptiveBaseline",
+    "BaselineConfig",
+    "BaselineReference",
+    "BehaviouralChange",
+    "ChangeKind",
+    "DailySummary",
+    "feature_series",
+    "summarise_days",
+]