tsagentkit 1.0.2__py3-none-any.whl

tsagentkit/features/extra/native.py

@@ -0,0 +1,179 @@
+"""Native (hand-rolled) feature engineering backend.
+
+This module preserves the original feature-engineering logic but is treated
+as a non-default backend for Phase 2+.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+import pandas as pd
+
+from tsagentkit.features.matrix import FeatureMatrix
+from tsagentkit.features.versioning import FeatureConfig, compute_feature_hash
+
+
+def extract_panel(dataset: Any) -> pd.DataFrame:
+    if hasattr(dataset, "df"):
+        return dataset.df.copy()
+    if hasattr(dataset, "data"):
+        return dataset.data.copy()
+    return dataset.copy()
+
+
+def prepare_panel(df: pd.DataFrame, reference_time: datetime | None) -> pd.DataFrame:
+    required = ["unique_id", "ds", "y"]
+    missing = [col for col in required if col not in df.columns]
+    if missing:
+        raise ValueError(f"Missing required columns: {missing}")
+
+    if not pd.api.types.is_datetime64_any_dtype(df["ds"]):
+        df["ds"] = pd.to_datetime(df["ds"])
+
+    if reference_time is None:
+        reference_time = df["ds"].max()
+
+    df = df[df["ds"] <= reference_time].copy()
+    return df.sort_values(["unique_id", "ds"]).reset_index(drop=True)
+
+
+def create_lag_features(df: pd.DataFrame, lags: list[int]) -> pd.DataFrame:
+    for lag in lags:
+        df[f"y_lag_{lag}"] = df.groupby("unique_id")["y"].shift(lag)
+    return df
+
+
+def create_calendar_features(df: pd.DataFrame, features: list[str]) -> pd.DataFrame:
+    ds = pd.to_datetime(df["ds"])
+    feature_map = {
+        "dayofweek": lambda d: d.dt.dayofweek,
+        "month": lambda d: d.dt.month,
+        "quarter": lambda d: d.dt.quarter,
+        "year": lambda d: d.dt.year,
+        "dayofmonth": lambda d: d.dt.day,
+        "dayofyear": lambda d: d.dt.dayofyear,
+        "weekofyear": lambda d: d.dt.isocalendar().week,
+        "hour": lambda d: d.dt.hour,
+        "minute": lambda d: d.dt.minute,
+        "is_month_start": lambda d: d.dt.is_month_start.astype(int),
+        "is_month_end": lambda d: d.dt.is_month_end.astype(int),
+        "is_quarter_start": lambda d: d.dt.is_quarter_start.astype(int),
+        "is_quarter_end": lambda d: d.dt.is_quarter_end.astype(int),
+    }
+
+    for feature in features:
+        if feature in feature_map:
+            df[feature] = feature_map[feature](ds)
+
+    return df
+
+
+def create_rolling_features(df: pd.DataFrame, windows: dict[int, list[str]]) -> pd.DataFrame:
+    for window, aggs in windows.items():
+        for agg in aggs:
+            if agg == "mean":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .mean()
+                )
+            elif agg == "std":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .std()
+                )
+            elif agg == "min":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .min()
+                )
+            elif agg == "max":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .max()
+                )
+            elif agg == "sum":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .sum()
+                )
+            elif agg == "median":
+                series = df.groupby("unique_id")["y"].transform(
+                    lambda x, window=window: x.shift(1)
+                    .rolling(window=window, min_periods=1)
+                    .median()
+                )
+            else:
+                continue
+            df[f"y_rolling_{agg}_{window}"] = series
+
+    return df
+
+
+def create_observed_covariate_features(
+    df: pd.DataFrame,
+    observed_covariates: list[str],
+) -> pd.DataFrame:
+    for col in observed_covariates:
+        if col not in df.columns:
+            continue
+        lag_col = f"{col}_lag_1"
+        df[lag_col] = df.groupby("unique_id")[col].shift(1)
+    return df
+
+
+def build_native_feature_matrix(
+    dataset: Any,
+    config: FeatureConfig,
+    reference_time: datetime | None = None,
+) -> FeatureMatrix:
+    df = prepare_panel(extract_panel(dataset), reference_time)
+
+    feature_cols: list[str] = []
+
+    if config.lags:
+        df = create_lag_features(df, config.lags)
+        feature_cols.extend([f"y_lag_{lag}" for lag in config.lags])
+
+    if config.calendar_features:
+        df = create_calendar_features(df, config.calendar_features)
+        feature_cols.extend(config.calendar_features)
+
+    if config.rolling_windows:
+        df = create_rolling_features(df, config.rolling_windows)
+        for window, aggs in config.rolling_windows.items():
+            for agg in aggs:
+                feature_cols.append(f"y_rolling_{agg}_{window}")
+
+    if config.observed_covariates:
+        df = create_observed_covariate_features(df, config.observed_covariates)
+        for col in config.observed_covariates:
+            lag_col = f"{col}_lag_1"
+            if lag_col in df.columns:
+                feature_cols.append(lag_col)
+
+    if config.known_covariates:
+        for col in config.known_covariates:
+            if col in df.columns and col not in feature_cols:
+                feature_cols.append(col)
+
+    if config.include_intercept:
+        df["intercept"] = 1.0
+        feature_cols.append("intercept")
+
+    config_hash = compute_feature_hash(config)
+
+    return FeatureMatrix(
+        data=df,
+        config_hash=config_hash,
+        target_col="y",
+        feature_cols=feature_cols,
+        known_covariates=config.known_covariates,
+        observed_covariates=config.observed_covariates,
+    )

tsagentkit/features/factory.py

@@ -0,0 +1,187 @@
+"""Feature factory for point-in-time safe feature engineering."""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass, replace
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pandas as pd
+
+from tsagentkit.features.covariates import CovariateManager
+from tsagentkit.features.extra.native import (
+    build_native_feature_matrix,
+    create_calendar_features,
+    create_lag_features,
+    create_observed_covariate_features,
+    create_rolling_features,
+)
+from tsagentkit.features.matrix import FeatureMatrix
+from tsagentkit.features.tsfeatures_adapter import build_tsfeatures_matrix
+from tsagentkit.features.versioning import FeatureConfig
+
+if TYPE_CHECKING:
+    from tsagentkit.series import TSDataset
+
+
+@dataclass
+class FeatureFactory:
+    """Point-in-time safe feature engineering for time series.
+
+    This factory creates features ensuring no lookahead bias by strictly
+    enforcing that features at time t only use information available at time t.
+
+    Attributes:
+        config: Feature configuration specifying what features to create
+        covariate_manager: Manager for handling known vs observed covariates
+
+    Example:
+        >>> config = FeatureConfig(
+        ...     lags=[1, 7, 14],
+        ...     calendar_features=["dayofweek", "month"],
+        ...     rolling_windows={7: ["mean", "std"]},
+        ... )
+        >>> factory = FeatureFactory(config)
+        >>> matrix = factory.create_features(dataset)
+        >>> print(matrix.signature)
+        FeatureMatrix(c=abc123...,n=5)
+    """
+
+    config: FeatureConfig
+    covariate_manager: CovariateManager | None = None
+
+    def __post_init__(self) -> None:
+        """Initialize covariate manager if not provided."""
+        if self.covariate_manager is None:
+            self.covariate_manager = CovariateManager(
+                known_covariates=self.config.known_covariates,
+                observed_covariates=self.config.observed_covariates,
+            )
+
+    def create_features(
+        self,
+        dataset: TSDataset,
+        reference_time: datetime | None = None,
+    ) -> FeatureMatrix:
+        """Create features ensuring no lookahead bias."""
+        engine = self._resolve_engine()
+        config = self._resolved_config(engine)
+
+        if engine == "tsfeatures":
+            try:
+                return build_tsfeatures_matrix(
+                    dataset=dataset,
+                    config=config,
+                    reference_time=reference_time,
+                )
+            except ImportError as exc:
+                if not config.allow_fallback:
+                    raise
+                warnings.warn(
+                    f"tsfeatures unavailable ({exc}); falling back to native features.",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+                config = self._resolved_config("native")
+                return build_native_feature_matrix(
+                    dataset=dataset,
+                    config=config,
+                    reference_time=reference_time,
+                )
+
+        return build_native_feature_matrix(
+            dataset=dataset,
+            config=config,
+            reference_time=reference_time,
+        )
+
+    def _resolve_engine(self) -> str:
+        """Resolve feature engine based on configuration.
+
+        When engine is "auto", defaults to tsfeatures. If tsfeatures is not
+        available, raises ImportError unless allow_fallback is True.
+        """
+        if self.config.engine == "auto":
+            try:
+                import tsfeatures  # type: ignore  # noqa: F401
+
+                return "tsfeatures"
+            except Exception as exc:
+                if not self.config.allow_fallback:
+                    raise ImportError(
+                        "tsfeatures is required but not installed. "
+                        "Install with: pip install tsfeatures "
+                        "Or set allow_fallback=True to use native features."
+                    ) from exc
+                warnings.warn(
+                    "tsfeatures is not installed; falling back to native features. "
+                    "For reproducibility, install tsfeatures or explicitly set engine='native'.",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
+                return "native"
+        return self.config.engine
+
+    def _resolved_config(self, engine: str) -> FeatureConfig:
+        if engine == self.config.engine:
+            return self.config
+        return replace(self.config, engine=engine)
+
+    def _create_lag_features(
+        self,
+        df: pd.DataFrame,
+        lags: list[int],
+    ) -> pd.DataFrame:
+        return create_lag_features(df, lags)
+
+    def _create_calendar_features(
+        self,
+        df: pd.DataFrame,
+        features: list[str],
+    ) -> pd.DataFrame:
+        return create_calendar_features(df, features)
+
+    def _create_rolling_features(
+        self,
+        df: pd.DataFrame,
+        windows: dict[int, list[str]],
+    ) -> pd.DataFrame:
+        return create_rolling_features(df, windows)
+
+    def _create_observed_covariate_features(
+        self,
+        df: pd.DataFrame,
+    ) -> pd.DataFrame:
+        return create_observed_covariate_features(df, self.config.observed_covariates)
+
+    def get_feature_importance_template(
+        self,
+    ) -> dict[str, float]:
+        """Return a template for feature importance scores.
+
+        Returns:
+            Dict mapping feature names to 0.0 (template for importance scores)
+        """
+        importance: dict[str, float] = {}
+
+        for lag in self.config.lags:
+            importance[f"y_lag_{lag}"] = 0.0
+
+        for feature in self.config.calendar_features:
+            importance[feature] = 0.0
+
+        for window, aggs in self.config.rolling_windows.items():
+            for agg in aggs:
+                importance[f"y_rolling_{agg}_{window}"] = 0.0
+
+        for col in self.config.known_covariates:
+            importance[col] = 0.0
+
+        for col in self.config.observed_covariates:
+            importance[f"{col}_lag_1"] = 0.0
+
+        if self.config.include_intercept:
+            importance["intercept"] = 0.0
+
+        return importance

tsagentkit/features/matrix.py

@@ -0,0 +1,159 @@
+"""FeatureMatrix dataclass for storing engineered features with provenance."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+
+if TYPE_CHECKING:
+    pass
+
+
+@dataclass(frozen=True)
+class FeatureMatrix:
+    """Container for engineered features with provenance.
+
+    Attributes:
+        data: DataFrame with engineered features (includes unique_id, ds, target)
+        config_hash: Hash of the feature configuration used to create these features
+        target_col: Name of the target variable column
+        feature_cols: List of engineered feature column names
+        known_covariates: List of known covariate column names
+        observed_covariates: List of observed covariate column names
+        created_at: ISO 8601 timestamp of feature matrix creation
+
+    Example:
+        >>> matrix = FeatureMatrix(
+        ...     data=df_with_features,
+        ...     config_hash="abc123...",
+        ...     feature_cols=["lag_7", "rolling_mean_30", "dayofweek"],
+        ...     known_covariates=["holiday"],
+        ...     observed_covariates=["promotion"],
+        ... )
+        >>> print(matrix.signature)
+        FeatureMatrix(c=abc123...,n=3)
+    """
+
+    data: pd.DataFrame
+    config_hash: str
+    target_col: str = "y"
+    feature_cols: list[str] = field(default_factory=list)
+    known_covariates: list[str] = field(default_factory=list)
+    observed_covariates: list[str] = field(default_factory=list)
+    created_at: str = field(
+        default_factory=lambda: datetime.now(UTC).isoformat()
+    )
+
+    def __post_init__(self) -> None:
+        """Validate the feature matrix after creation."""
+        # Validate required columns exist
+        required = ["unique_id", "ds", self.target_col]
+        missing = [col for col in required if col not in self.data.columns]
+        if missing:
+            raise ValueError(f"Missing required columns: {missing}")
+
+        # Validate feature columns exist in data
+        invalid_features = [col for col in self.feature_cols if col not in self.data.columns]
+        if invalid_features:
+            raise ValueError(f"Feature columns not in data: {invalid_features}")
+
+        # Validate covariate columns exist
+        invalid_known = [col for col in self.known_covariates if col not in self.data.columns]
+        if invalid_known:
+            raise ValueError(f"Known covariates not in data: {invalid_known}")
+
+        invalid_observed = [
+            col for col in self.observed_covariates if col not in self.data.columns
+        ]
+        if invalid_observed:
+            raise ValueError(f"Observed covariates not in data: {invalid_observed}")
+
+    @property
+    def signature(self) -> str:
+        """Return feature matrix signature for provenance.
+
+        Returns:
+            String signature like "FeatureMatrix(c=abc123...,n=5)"
+        """
+        return f"FeatureMatrix(c={self.config_hash},n={len(self.feature_cols)})"
+
+    def to_pandas(self) -> pd.DataFrame:
+        """Return the feature matrix as a pandas DataFrame.
+
+        Returns:
+            Copy of the underlying DataFrame
+        """
+        return self.data.copy()
+
+    def get_feature_data(self) -> pd.DataFrame:
+        """Get only the feature columns (excluding id, timestamp, target).
+
+        Returns:
+            DataFrame with only feature columns
+        """
+        return self.data[self.feature_cols].copy()
+
+    def get_target_data(self) -> pd.Series:
+        """Get the target variable.
+
+        Returns:
+            Series with target values
+        """
+        return self.data[self.target_col].copy()
+
+    def get_covariate_data(self, covariate_type: str | None = None) -> pd.DataFrame:
+        """Get covariate columns.
+
+        Args:
+            covariate_type: "known", "observed", or None (all covariates)
+
+        Returns:
+            DataFrame with covariate columns
+        """
+        if covariate_type == "known":
+            cols = self.known_covariates
+        elif covariate_type == "observed":
+            cols = self.observed_covariates
+        else:
+            cols = self.known_covariates + self.observed_covariates
+
+        if not cols:
+            return pd.DataFrame(index=self.data.index)
+
+        return self.data[cols].copy()
+
+    def validate(self) -> list[str]:
+        """Validate the feature matrix and return any issues.
+
+        Returns:
+            List of validation error messages (empty if valid)
+        """
+        issues = []
+
+        # Check for nulls in features
+        if self.feature_cols:
+            null_counts = self.data[self.feature_cols].isnull().sum()
+            if null_counts.any():
+                cols_with_nulls = null_counts[null_counts > 0].index.tolist()
+                issues.append(f"Features contain nulls: {cols_with_nulls}")
+
+        # Check for infinite values
+        if self.feature_cols:
+            numeric_cols = self.data[self.feature_cols].select_dtypes(include=["number"])
+            if numeric_cols is not None and not numeric_cols.empty:
+                inf_counts = np.isinf(numeric_cols).sum()
+                if inf_counts.any():
+                    cols_with_inf = inf_counts[inf_counts > 0].index.tolist()
+                    issues.append(f"Features contain infinite values: {cols_with_inf}")
+
+        # Check target exists and is numeric
+        if self.target_col not in self.data.columns:
+            issues.append(f"Target column '{self.target_col}' not found")
+        elif not pd.api.types.is_numeric_dtype(self.data[self.target_col]):
+            issues.append(f"Target column '{self.target_col}' is not numeric")
+
+        return issues

tsagentkit/features/tsfeatures_adapter.py

@@ -0,0 +1,115 @@
+"""tsfeatures adapter for statistical feature extraction."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import datetime
+from typing import Any
+
+import pandas as pd
+
+from tsagentkit.features.extra.native import (
+    create_observed_covariate_features,
+    extract_panel,
+    prepare_panel,
+)
+from tsagentkit.features.matrix import FeatureMatrix
+from tsagentkit.features.versioning import FeatureConfig, compute_feature_hash
+
+
+def _import_tsfeatures():
+    import tsfeatures  # type: ignore
+
+    return tsfeatures
+
+
+def _resolve_feature_fns(tsfeatures_mod: Any, names: list[str]) -> list[Callable] | None:
+    if not names:
+        return None
+    fns: list[Callable] = []
+    for name in names:
+        fn = getattr(tsfeatures_mod, name, None)
+        if fn is None or not callable(fn):
+            raise ValueError(f"Unknown tsfeatures function: {name}")
+        fns.append(fn)
+    return fns
+
+
+def _resolve_tsfeatures_freq(dataset: Any, config: FeatureConfig) -> int | None:
+    if config.tsfeatures_freq is not None:
+        return config.tsfeatures_freq
+    task_spec = getattr(dataset, "task_spec", None)
+    if task_spec is None:
+        return None
+    return task_spec.season_length
+
+
+def _prefix_if_conflict(df: pd.DataFrame, feature_cols: list[str]) -> tuple[pd.DataFrame, list[str]]:
+    reserved = {"unique_id", "ds", "y"}
+    conflicts = [col for col in feature_cols if col in reserved]
+    if not conflicts:
+        return df, feature_cols
+
+    rename_map = {col: f"tsf_{col}" for col in conflicts}
+    df = df.rename(columns=rename_map)
+    updated = [rename_map.get(col, col) for col in feature_cols]
+    return df, updated
+
+
+def build_tsfeatures_matrix(
+    dataset: Any,
+    config: FeatureConfig,
+    reference_time: datetime | None = None,
+) -> FeatureMatrix:
+    df = prepare_panel(extract_panel(dataset), reference_time)
+
+    tsfeatures_mod = _import_tsfeatures()
+    fns = _resolve_feature_fns(tsfeatures_mod, config.tsfeatures_features)
+
+    freq = _resolve_tsfeatures_freq(dataset, config)
+    dict_freqs = config.tsfeatures_dict_freqs or None
+
+    features_df = tsfeatures_mod.tsfeatures(
+        df,
+        freq=freq,
+        features=fns,
+        dict_freqs=dict_freqs,
+    )
+
+    if "unique_id" not in features_df.columns:
+        if features_df.index.name == "unique_id":
+            features_df = features_df.reset_index()
+        else:
+            raise ValueError("tsfeatures output must include unique_id")
+
+    feature_cols = [c for c in features_df.columns if c != "unique_id"]
+    features_df, feature_cols = _prefix_if_conflict(features_df, feature_cols)
+
+    merged = df.merge(features_df, on="unique_id", how="left")
+
+    if config.observed_covariates:
+        merged = create_observed_covariate_features(merged, config.observed_covariates)
+        for col in config.observed_covariates:
+            lag_col = f"{col}_lag_1"
+            if lag_col in merged.columns:
+                feature_cols.append(lag_col)
+
+    if config.known_covariates:
+        for col in config.known_covariates:
+            if col in merged.columns and col not in feature_cols:
+                feature_cols.append(col)
+
+    if config.include_intercept:
+        merged["intercept"] = 1.0
+        feature_cols.append("intercept")
+
+    config_hash = compute_feature_hash(config)
+
+    return FeatureMatrix(
+        data=merged,
+        config_hash=config_hash,
+        target_col="y",
+        feature_cols=feature_cols,
+        known_covariates=config.known_covariates,
+        observed_covariates=config.observed_covariates,
+    )