sensor-modeling 0.2.0__py3-none-any.whl

sensor_modeling/analysis/behavioral_analysis.py

@@ -0,0 +1,57 @@
+"""High-level behavioral analysis utilities."""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+import pandas as pd
+
+from ._frame import prepare_sensor_frame
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+def recognize_activity_patterns(data: pd.DataFrame) -> dict[str, int]:
+    """Identify peak and quiet hours of activity."""
+    sensor_data = prepare_sensor_frame(data, context="recognize_activity_patterns")
+    hourly = sensor_data.groupby(sensor_data.index.hour).sum()
+    totals = hourly.sum(axis=1)
+    return {
+        "peak_hours": int(totals.idxmax()),
+        "quiet_hours": int(totals.idxmin()),
+    }
+
+
+# ---------------------------------------------------------------------------
+def score_anomalies(data: pd.DataFrame) -> pd.Series:
+    """Simple z-score based anomaly metric for each timestamp."""
+    sensor_data = prepare_sensor_frame(data, context="score_anomalies")
+    std = sensor_data.std(ddof=0).replace(0, np.nan)
+    zscores = (sensor_data - sensor_data.mean()) / std
+    return zscores.fillna(0.0).abs().sum(axis=1)
+
+
+# ---------------------------------------------------------------------------
+def detect_trends(data: pd.DataFrame, window: int = 24) -> pd.DataFrame:
+    """Rolling mean trend indicator."""
+    if window < 1:
+        raise ValueError("window must be at least 1")
+    sensor_data = prepare_sensor_frame(data, context="detect_trends")
+    return sensor_data.rolling(window=window, min_periods=1).mean()
+
+
+# ---------------------------------------------------------------------------
+def health_indicators(data: pd.DataFrame) -> dict[str, float]:
+    """Basic health status indicators derived from activity levels."""
+    sensor_data = prepare_sensor_frame(data, context="health_indicators")
+    activity = sensor_data.sum(axis=1)
+    overall = float(activity.mean())
+    variability = float(activity.std(ddof=0))
+    sedentary_ratio = float((activity == 0).mean())
+    return {
+        "overall_activity": overall,
+        "activity_variability": variability,
+        "sedentary_ratio": sedentary_ratio,
+    }

sensor_modeling/analysis/behavioral_metrics.py

@@ -0,0 +1,66 @@
+"""Behavioral metrics calculated from sensor datasets."""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+
+from ._frame import prepare_sensor_frame
+
+logger = logging.getLogger(__name__)
+
+
+def calculate_behavioral_metrics(data: pd.DataFrame) -> dict[str, object]:
+    """Calculate basic behavioral pattern metrics from sensor data."""
+    sensor_data = prepare_sensor_frame(data, context="calculate_behavioral_metrics")
+    metrics: dict[str, object] = {}
+
+    total_activations = sensor_data.sum().sum()
+    total_possible = len(sensor_data) * len(sensor_data.columns)
+    metrics["overall_activity_rate"] = float(total_activations / total_possible)
+    metrics["total_activations"] = int(total_activations)
+
+    hourly_activity = sensor_data.groupby(sensor_data.index.hour).sum()
+    metrics["peak_activity_hour"] = int(hourly_activity.sum(axis=1).idxmax())
+    metrics["quietest_hour"] = int(hourly_activity.sum(axis=1).idxmin())
+
+    sensor_metrics: dict[str, dict[str, float | int]] = {}
+    for sensor in sensor_data.columns:
+        series = sensor_data[sensor]
+        daily_variance = series.groupby(series.index.date).sum().var()
+        sensor_metrics[sensor] = {
+            "activation_rate": float(series.mean()),
+            "total_activations": int(series.sum()),
+            "longest_inactive_period": _find_longest_streak(series, 0),
+            "longest_active_period": _find_longest_streak(series, 1),
+            "daily_variance": 0.0 if pd.isna(daily_variance) else float(daily_variance),
+        }
+    metrics["sensor_metrics"] = sensor_metrics
+
+    dow_activity = sensor_data.groupby(sensor_data.index.dayofweek).sum()
+    weekday_names = [
+        "Monday",
+        "Tuesday",
+        "Wednesday",
+        "Thursday",
+        "Friday",
+        "Saturday",
+        "Sunday",
+    ]
+    metrics["most_active_day"] = weekday_names[dow_activity.sum(axis=1).idxmax()]
+    metrics["least_active_day"] = weekday_names[dow_activity.sum(axis=1).idxmin()]
+
+    return metrics
+
+
+def _find_longest_streak(series: pd.Series, value: int) -> int:
+    max_streak = 0
+    current_streak = 0
+    for val in series:
+        if val == value:
+            current_streak += 1
+            max_streak = max(max_streak, current_streak)
+        else:
+            current_streak = 0
+    return max_streak

sensor_modeling/analysis/comparison.py

@@ -0,0 +1,164 @@
+"""Utilities for comparing different sensor models."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from scipy.stats import ttest_rel
+from sklearn.model_selection import TimeSeriesSplit
+
+from ..utils.data_io import SensorDataset
+
+logger = logging.getLogger(__name__)
+
+ModelScorer = Callable[[Any, pd.DataFrame, pd.DataFrame], float]
+FoldError = (AttributeError, FloatingPointError, RuntimeError, TypeError, ValueError)
+
+
+def time_series_splits(
+    data: SensorDataset | pd.DataFrame, n_splits: int = 3
+) -> list[tuple[np.ndarray, np.ndarray]]:
+    """Return chronological train/test splits for sensor time series."""
+    if n_splits < 1:
+        raise ValueError("n_splits must be at least 1")
+    dataset = data if isinstance(data, SensorDataset) else SensorDataset(data)
+    df = dataset.to_dataframe()
+    if len(df) < 2:
+        raise ValueError("At least two observations are required for cross-validation")
+    split_count = min(n_splits, len(df) - 1)
+    splitter = TimeSeriesSplit(n_splits=split_count)
+    return list(splitter.split(df))
+
+
+def _fit_and_score_model(
+    name: str,
+    model: Any,
+    train_df: pd.DataFrame,
+    test_df: pd.DataFrame,
+    scorers: dict[str, ModelScorer],
+) -> float:
+    """Fit one model on a fold and return its score."""
+    model.fit(train_df.values if name == "hmm" else train_df)
+    if name in scorers:
+        return float(scorers[name](model, train_df, test_df))
+    return float(model.score(test_df.values))
+
+
+def _mean_score(values: list[float]) -> float:
+    """Return the mean of valid fold scores, or NaN when every fold failed."""
+    finite_scores = [score for score in values if not np.isnan(score)]
+    if not finite_scores:
+        return float("nan")
+    return float(np.mean(finite_scores))
+
+
+# ---------------------------------------------------------------------------
+def cross_validate(
+    models: dict[str, Any],
+    data: SensorDataset | pd.DataFrame,
+    n_splits: int = 3,
+    scorers: dict[str, ModelScorer] | None = None,
+) -> dict[str, float]:
+    """Perform chronological cross-validation across *models*.
+
+    Parameters
+    ----------
+    models : dict[str, Any]
+        Mapping from model name to model instance. Each model must implement a
+        :py:meth:`fit` method and either a :py:meth:`score` method or a scorer
+        supplied via ``scorers``.
+    data : SensorDataset | pd.DataFrame
+        Input dataset.
+    n_splits : int
+        Number of chronological folds.
+    scorers : dict[str, ModelScorer] | None
+        Optional per-model scoring functions accepting
+        ``(model, train_df, test_df)``.
+    """
+    dataset = data if isinstance(data, SensorDataset) else SensorDataset(data)
+    df = dataset.to_dataframe()
+    scorers = scorers or {}
+    unsupported = [
+        name
+        for name, model in models.items()
+        if name not in scorers and not hasattr(model, "score")
+    ]
+    if unsupported:
+        names = ", ".join(sorted(unsupported))
+        raise TypeError(
+            "Cross-validation requires each model to define score() or an "
+            f"explicit scorer. Missing scorer for: {names}"
+        )
+
+    scores: dict[str, list[float]] = {name: [] for name in models}
+    for train_idx, test_idx in time_series_splits(dataset, n_splits=n_splits):
+        train_df = df.iloc[train_idx]
+        test_df = df.iloc[test_idx]
+        for name, model in models.items():
+            try:
+                score = _fit_and_score_model(name, model, train_df, test_df, scorers)
+                scores[name].append(score)
+            except FoldError as exc:
+                logger.error("Failed fold for %s: %s", name, exc)
+                scores[name].append(float("nan"))
+    return {name: _mean_score(vals) for name, vals in scores.items()}
+
+
+# ---------------------------------------------------------------------------
+def significance_test(scores_a: list[float], scores_b: list[float]) -> float:
+    """Paired t-test returning the p-value."""
+    values_a = np.asarray(scores_a, dtype=float)
+    values_b = np.asarray(scores_b, dtype=float)
+    if values_a.shape != values_b.shape:
+        raise ValueError("scores_a and scores_b must have the same length")
+
+    valid_pairs = np.isfinite(values_a) & np.isfinite(values_b)
+    if np.count_nonzero(valid_pairs) < 2:
+        raise ValueError("at least two paired finite scores are required")
+
+    differences = values_a[valid_pairs] - values_b[valid_pairs]
+    if np.allclose(differences, 0.0):
+        return 1.0
+
+    _, pvalue = ttest_rel(values_a[valid_pairs], values_b[valid_pairs])
+    if not np.isfinite(pvalue):
+        return 1.0
+    return float(pvalue)
+
+
+# ---------------------------------------------------------------------------
+def standardize_metrics(metrics: dict[str, float]) -> dict[str, float]:
+    """Scale metric values to the [0,1] range."""
+    if not metrics:
+        return {}
+
+    vals = np.array(list(metrics.values()), dtype=float)
+    finite_vals = vals[np.isfinite(vals)]
+    if finite_vals.size == 0:
+        return {name: float("nan") for name in metrics}
+
+    vmin = np.min(finite_vals)
+    vmax = np.max(finite_vals)
+    rng = vmax - vmin if vmax != vmin else 1.0
+    return {
+        name: float("nan") if not np.isfinite(value) else float((value - vmin) / rng)
+        for name, value in zip(metrics, vals)
+    }
+
+
+# ---------------------------------------------------------------------------
+def visualize_comparison(metrics: dict[str, float], ax=None):
+    """Visualize model comparison scores as a bar chart."""
+    if ax is None:
+        _, ax = plt.subplots()
+    names = list(metrics.keys())
+    vals = [metrics[n] for n in names]
+    ax.bar(names, vals)
+    ax.set_ylabel("Score")
+    ax.set_title("Model Comparison")
+    return ax

sensor_modeling/analysis/dependency_network.py

@@ -0,0 +1,408 @@
+"""
+Sensor Dependency Network Analysis
+
+This module builds and analyzes cross-sensor dependency networks
+using Granger causality and network analysis techniques.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+import matplotlib.pyplot as plt
+import networkx as nx
+import numpy as np
+import pandas as pd
+import seaborn as sns
+from sklearn.metrics import mutual_info_score
+
+from .granger_causality import GrangerCausalityTest
+
+logger = logging.getLogger(__name__)
+
+
+class SensorDependencyNetwork:
+    """
+    Build and analyze cross-sensor dependency networks.
+    Creates directed graphs showing causal relationships between sensors.
+    """
+
+    def __init__(self, significance_level: float = 0.05):
+        """
+        Initialize dependency network builder.
+
+        Args:
+            significance_level: P-value threshold for including edges
+        """
+        if not 0 < significance_level < 1:
+            raise ValueError("significance_level must be between 0 and 1")
+        self.significance_level = significance_level
+        self.granger_test = GrangerCausalityTest()
+        self.network: nx.DiGraph | None = None
+        self.causality_results: pd.DataFrame | None = None
+
+    def build_network(self, data: pd.DataFrame) -> nx.DiGraph:
+        """
+        Build sensor dependency network using Granger causality.
+
+        Args:
+            data: DataFrame with sensor data
+
+        Returns:
+            NetworkX directed graph
+        """
+        logger.info("Building sensor dependency network...")
+        sensor_data = self._prepare_sensor_data(data)
+
+        # Test all pairs for Granger causality
+        self.causality_results = self.granger_test.test_all_pairs(sensor_data)
+
+        # Create directed graph
+        self.network = nx.DiGraph()
+
+        # Add all sensors as nodes
+        sensors = sensor_data.columns.tolist()
+        self.network.add_nodes_from(sensors)
+
+        # Add edges for significant causal relationships
+        significant_results = self.causality_results[
+            self.causality_results["causality_detected"]
+        ]
+
+        for _, row in significant_results.iterrows():
+            self.network.add_edge(
+                row["cause"],
+                row["effect"],
+                weight=row["test_statistic"],
+                p_value=row["p_value"],
+                lags=row["lags_used"],
+            )
+
+        logger.info(
+            "Network created with %d nodes and %d edges",
+            len(self.network.nodes),
+            len(self.network.edges),
+        )
+        return self.network
+
+    def _prepare_sensor_data(self, data: pd.DataFrame) -> pd.DataFrame:
+        """Return a numeric binary sensor frame for network analysis."""
+        if data.empty:
+            raise ValueError("dependency network analysis requires at least one row")
+
+        sensor_data = data.select_dtypes(include="number")
+        if sensor_data.empty:
+            raise ValueError(
+                "dependency network analysis requires at least one numeric sensor column"
+            )
+
+        if sensor_data.isna().any().any():
+            raise ValueError("dependency network analysis does not accept NaN values")
+
+        non_binary = [
+            column
+            for column in sensor_data.columns
+            if not set(sensor_data[column].unique()) <= {0, 1}
+        ]
+        if non_binary:
+            raise ValueError(
+                "dependency network analysis requires binary sensor columns: "
+                + ", ".join(map(str, non_binary))
+            )
+
+        return sensor_data.copy()
+
+    def get_network_statistics(self) -> dict[str, Any]:
+        """Get network topology statistics."""
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        is_connected = (
+            nx.is_weakly_connected(self.network)
+            if self.network.number_of_nodes()
+            else False
+        )
+
+        return {
+            "num_nodes": len(self.network.nodes),
+            "num_edges": len(self.network.edges),
+            "density": nx.density(self.network),
+            "is_connected": is_connected,
+            "num_components": nx.number_weakly_connected_components(self.network),
+            "avg_clustering": nx.average_clustering(self.network.to_undirected()),
+            "in_degree_centrality": nx.in_degree_centrality(self.network),
+            "out_degree_centrality": nx.out_degree_centrality(self.network),
+        }
+
+    def identify_sensor_roles(self) -> dict[str, list[str]]:
+        """
+        Identify different roles of sensors in the network.
+
+        Returns:
+            Dictionary categorizing sensors by their network roles
+        """
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        # Calculate centrality measures
+        in_degree = dict(self.network.in_degree())
+        out_degree = dict(self.network.out_degree())
+
+        roles: dict[str, list[str]] = {
+            "triggers": [],  # High out-degree, low in-degree
+            "responders": [],  # High in-degree, low out-degree
+            "hubs": [],  # High both in and out degree
+            "isolated": [],  # Low both in and out degree
+        }
+
+        if self.network.number_of_nodes() == 0:
+            return roles
+        if all(
+            degree == 0
+            for degree in list(in_degree.values()) + list(out_degree.values())
+        ):
+            roles["isolated"] = list(self.network.nodes())
+            return roles
+
+        # Define thresholds (can be adjusted)
+        high_threshold = np.percentile(
+            list(in_degree.values()) + list(out_degree.values()), 75
+        )
+        low_threshold = np.percentile(
+            list(in_degree.values()) + list(out_degree.values()), 25
+        )
+
+        for node in self.network.nodes():
+            in_deg = in_degree[node]
+            out_deg = out_degree[node]
+
+            if out_deg >= high_threshold and in_deg <= low_threshold:
+                roles["triggers"].append(node)
+            elif in_deg >= high_threshold and out_deg <= low_threshold:
+                roles["responders"].append(node)
+            elif in_deg >= high_threshold and out_deg >= high_threshold:
+                roles["hubs"].append(node)
+            else:
+                roles["isolated"].append(node)
+
+        return roles
+
+    def detect_communities(self) -> list[list[str]]:
+        """Detect communities/clusters in the sensor network."""
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        if self.network.number_of_nodes() == 0:
+            return []
+        if self.network.number_of_edges() == 0:
+            return [[node] for node in self.network.nodes()]
+
+        try:
+            # Convert to undirected for community detection
+            undirected_network = self.network.to_undirected()
+
+            # Use greedy modularity optimization
+            communities = nx.community.greedy_modularity_communities(undirected_network)
+            return [list(community) for community in communities]
+
+        except (nx.NetworkXException, ValueError) as exc:
+            logger.warning("Community detection failed: %s", exc)
+            return []
+
+    def calculate_mutual_information(self, data: pd.DataFrame) -> pd.DataFrame:
+        """Calculate mutual information between all sensor pairs."""
+        sensor_data = self._prepare_sensor_data(data)
+        sensors = sensor_data.columns.tolist()
+        n_sensors = len(sensors)
+        mi_matrix = np.zeros((n_sensors, n_sensors))
+
+        for i, sensor1 in enumerate(sensors):
+            for j, sensor2 in enumerate(sensors):
+                if i != j:
+                    mi_matrix[i, j] = mutual_info_score(
+                        sensor_data[sensor1].values, sensor_data[sensor2].values
+                    )
+
+        return pd.DataFrame(mi_matrix, index=sensors, columns=sensors)
+
+    def find_critical_sensors(self) -> dict[str, Any]:
+        """
+        Identify critical sensors whose removal would significantly affect network connectivity.
+
+        Returns:
+            Dictionary with criticality analysis
+        """
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        original_components = nx.number_weakly_connected_components(self.network)
+
+        criticality_scores: dict[str, dict[str, float]] = {}
+
+        for node in self.network.nodes():
+            # Create network without this node
+            temp_network = self.network.copy()
+            temp_network.remove_node(node)
+
+            # Calculate impact on connectivity
+            new_components = nx.number_weakly_connected_components(temp_network)
+            component_change = new_components - original_components
+
+            # Calculate impact on edges
+            edges_lost = len(self.network.edges()) - len(temp_network.edges())
+
+            # Criticality score combines connectivity and edge impact
+            criticality_scores[node] = {
+                "component_change": component_change,
+                "edges_lost": edges_lost,
+                "criticality_score": component_change + 0.1 * edges_lost,
+            }
+
+        # Sort by criticality
+        sorted_sensors = sorted(
+            criticality_scores.items(),
+            key=lambda x: x[1]["criticality_score"],
+            reverse=True,
+        )
+
+        return {
+            "criticality_rankings": sorted_sensors,
+            "most_critical": sorted_sensors[0][0] if sorted_sensors else None,
+            "original_components": original_components,
+        }
+
+    def plot_network(
+        self,
+        figsize: tuple[int, int] = (12, 8),
+        node_size_factor: int = 500,
+        *,
+        show: bool = True,
+    ):
+        """
+        Plot the sensor dependency network.
+
+        Args:
+            figsize: Figure size
+            node_size_factor: Factor to scale node sizes
+            show: Whether to display the figure immediately
+        """
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        fig, ax = plt.subplots(figsize=figsize)
+
+        # Calculate node sizes based on degree centrality
+        centrality = nx.degree_centrality(self.network.to_undirected())
+        node_sizes = [
+            centrality[node] * node_size_factor + 100 for node in self.network.nodes()
+        ]
+
+        # Calculate edge weights for visualization
+        edge_weights = [
+            self.network[u][v]["weight"] / 10 for u, v in self.network.edges()
+        ]
+
+        # Use spring layout for positioning
+        pos = nx.spring_layout(self.network, k=1, iterations=50)
+
+        # Draw network
+        nx.draw_networkx_nodes(
+            self.network,
+            pos,
+            node_size=node_sizes,
+            node_color="lightblue",
+            alpha=0.7,
+            ax=ax,
+        )
+
+        nx.draw_networkx_edges(
+            self.network,
+            pos,
+            width=edge_weights,
+            edge_color="gray",
+            arrows=True,
+            arrowsize=20,
+            alpha=0.6,
+            ax=ax,
+        )
+
+        nx.draw_networkx_labels(self.network, pos, font_size=10, ax=ax)
+
+        ax.set_title("Sensor Dependency Network\n(Arrows show causal direction)")
+        ax.axis("off")
+        fig.tight_layout()
+        if show:
+            plt.show()
+        return fig
+
+    def plot_causality_matrix(
+        self, figsize: tuple[int, int] = (10, 8), *, show: bool = True
+    ):
+        """
+        Plot heatmap of causality test results.
+
+        Args:
+            figsize: Figure size
+            show: Whether to display the figure immediately
+        """
+        if self.causality_results is None:
+            raise ValueError("Network must be built first")
+
+        # Create pivot table for heatmap
+        pivot_data = self.causality_results.pivot(
+            index="effect", columns="cause", values="test_statistic"
+        )
+
+        fig, ax = plt.subplots(figsize=figsize)
+
+        # Create heatmap
+        mask = pivot_data.isna()
+        sns.heatmap(
+            pivot_data,
+            annot=True,
+            fmt=".2f",
+            mask=mask,
+            cmap="Reds",
+            cbar_kws={"label": "Granger Causality Test Statistic"},
+            ax=ax,
+        )
+
+        ax.set_title("Sensor Causality Matrix\n(Rows: Effects, Columns: Causes)")
+        ax.set_xlabel("Potential Causes")
+        ax.set_ylabel("Effects")
+        fig.tight_layout()
+        if show:
+            plt.show()
+        return fig
+
+    def export_network_data(self, filename: str | Path | None = None) -> dict[str, Any]:
+        """
+        Export network data for external analysis.
+
+        Args:
+            filename: Optional filename to save data
+
+        Returns:
+            Dictionary with all network data
+        """
+        if self.network is None:
+            raise ValueError("Network must be built first")
+
+        export_data = {
+            "nodes": list(self.network.nodes()),
+            "edges": [(u, v, self.network[u][v]) for u, v in self.network.edges()],
+            "network_statistics": self.get_network_statistics(),
+            "sensor_roles": self.identify_sensor_roles(),
+            "communities": self.detect_communities(),
+            "causality_results": self.causality_results.to_dict("records"),
+            "critical_sensors": self.find_critical_sensors(),
+        }
+
+        if filename:
+            import json
+
+            path = Path(filename)
+            path.write_text(json.dumps(export_data, indent=2, default=str))
+            logger.info("Network data exported to %s", path)
+
+        return export_data