upgini 1.2.157.dev4__tar.gz → 1.2.158a4180.dev2__tar.gz

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.157.dev4
+Version: 1.2.158a4180.dev2
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.158a4180.dev2/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.158.a4180.dev2"

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/src/upgini/autofe/date.py

@@ -1,13 +1,13 @@
 import abc
 import json
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, NamedTuple, Optional, Union
 
 import numpy as np
 import pandas as pd
 from pandas.core.arrays.timedeltas import TimedeltaArray
 from pydantic import BaseModel, __version__ as pydantic_version
 
-from upgini.autofe.operand import OperandValue
+from upgini.autofe.operand import OperandKind, OperandValue
 from upgini.autofe.operator import PandasOperator, ParametrizedOperator
 from upgini.autofe.utils import bin_index, bin_index_many, bin_index_vectorized, pydantic_validator
 
@@ -114,8 +114,57 @@ class DateDiffType2(PandasOperator, DateDiffMixin):
         return diff
 
 
-_ext_aggregations = {"nunique": (lambda x: len(np.unique(x)), 0), "count": (len, 0)}
+_ext_aggregations = {"nunique": (lambda x: float(np.unique(x).size), 0), "count": (len, 0)}
 _count_aggregations = ["nunique", "count"]
+_DATE_DIFF_LISTS_LENGTH_COL = 0
+_NS_PER_DAY = np.float64(86400 * 10**9)
+_NS_PER_YEAR = np.float64(365 * 86400 * 10**9)
+_MATRIX_AGGREGATIONS = {
+    "nunique": "_matrix_agg_nunique",
+    "count": "_matrix_agg_count",
+    "sum": "_matrix_agg_sum",
+    "mean": "_matrix_agg_mean",
+    "min": "_matrix_agg_min",
+    "max": "_matrix_agg_max",
+}
+
+
+class _MatrixAggContext(NamedTuple):
+    lengths: np.ndarray
+    masked_values: np.ndarray
+    valid_mask: np.ndarray
+    missing: np.ndarray
+    empty: np.ndarray
+    agg_source: np.ndarray
+    count_source: np.ndarray
+    has_bounds: bool
+    results: np.ndarray
+
+
+def _timedelta_ns_to_diff_unit(delta_ns: np.ndarray, diff_unit: str) -> np.ndarray:
+    if diff_unit == "D":
+        return delta_ns / _NS_PER_DAY
+    if diff_unit == "Y":
+        return (delta_ns / _NS_PER_YEAR).astype(np.int64).astype(np.float64)
+    raise ValueError(f"Unsupported difference unit: {diff_unit}")
+
+
+def _group_cumcount(group_keys: np.ndarray) -> np.ndarray:
+    n = len(group_keys)
+    if n == 0:
+        return np.zeros(0, dtype=np.intp)
+    order = np.argsort(group_keys, kind="stable")
+    sorted_keys = group_keys[order]
+    group_change = np.empty(n, dtype=bool)
+    group_change[0] = True
+    if n > 1:
+        group_change[1:] = sorted_keys[1:] != sorted_keys[:-1]
+    group_ids = np.cumsum(group_change) - 1
+    group_start_idx = np.flatnonzero(group_change)
+    sorted_cumcount = np.arange(n, dtype=np.intp) - group_start_idx[group_ids]
+    cumcount = np.empty(n, dtype=np.intp)
+    cumcount[order] = sorted_cumcount
+    return cumcount
 
 
 def _aggregate_diffs(values: np.ndarray, aggregation: str) -> float:
@@ -163,54 +212,71 @@ class DateListDiffLists(PandasOperator, DateDiffMixin, ParametrizedOperator):
                 return cls(diff_unit=diff_unit)
         return None
 
-    @staticmethod
-    def _non_empty_list_mask(right: pd.Series) -> pd.Series:
-        values = right.to_numpy()
-        mask = np.empty(len(values), dtype=bool)
-        for i, value in enumerate(values):
-            if value is None or (isinstance(value, float) and np.isnan(value)):
-                mask[i] = False
-            elif isinstance(value, (list, tuple, np.ndarray)):
-                mask[i] = len(value) > 0
-            else:
-                mask[i] = False
-        return pd.Series(mask, index=right.index)
-
-    def _convert_date_lists(self, lists: pd.Series) -> pd.Series:
-        exploded = lists.explode()
-        converted = pd.to_datetime(exploded, unit=self.right_unit, errors="coerce")
-        return pd.Series(
-            {
-                idx: pd.arrays.DatetimeArray(values.to_numpy())
-                for idx, values in converted.groupby(converted.index, sort=False)
-            }
-        )
+    def _non_empty_list_mask(self, right: pd.Series) -> pd.Series:
+        return right.map(lambda value: isinstance(value, (list, tuple, np.ndarray)) and len(value) > 0).fillna(False)
+
+    def _build_matrix(self, left: pd.Series, right: pd.Series) -> np.ndarray:
+        n = len(left)
+        if n == 0:
+            return np.empty((0, 1), dtype=np.float64)
+
+        left_dates = pd.to_datetime(left, unit=self.left_unit, errors="coerce")
+        date_unit = self.right_unit
+        right_mask = self._non_empty_list_mask(right).to_numpy()
+        right_notna = right.notna().to_numpy()
+        left_notna = left_dates.notna().to_numpy()
+
+        compute_mask = left_notna & right_notna & right_mask
+        empty_right = right_notna & ~right_mask
+
+        lengths = np.full(n, np.nan, dtype=np.float64)
+        lengths[empty_right] = 0.0
+
+        compute_idx = np.flatnonzero(compute_mask)
+        if len(compute_idx) == 0:
+            return lengths.reshape(n, 1)
+
+        exploded = right.iloc[compute_idx].explode()
+        row_indices_arr = right.index.get_indexer(exploded.index).astype(np.intp)
+        raw_dates = exploded.to_numpy()
+        pos_in_row_arr = _group_cumcount(row_indices_arr)
+        converted = pd.to_datetime(pd.Series(raw_dates), unit=date_unit, errors="coerce")
+        left_ns = left_dates.iloc[row_indices_arr].astype(np.int64).to_numpy()
+        right_ns = converted.astype(np.int64).to_numpy()
+        diffs = np.full(len(row_indices_arr), np.nan, dtype=np.float64)
+        valid_ts = converted.notna().to_numpy()
+        if valid_ts.any():
+            diffs[valid_ts] = _timedelta_ns_to_diff_unit(left_ns[valid_ts] - right_ns[valid_ts], self.diff_unit)
 
-    def _row_diffs(self, left_date, right_dates: pd.arrays.DatetimeArray) -> List[float]:
-        diffs = self._convert_diff_to_unit(left_date - right_dates)
         if self.replace_negative:
-            diffs = diffs[diffs > 0]
-        return np.atleast_1d(np.asarray(diffs, dtype=np.float64)).tolist()
-
-    def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
+            keep = diffs > 0
+            row_indices_arr = row_indices_arr[keep]
+            diffs = diffs[keep]
+            lengths[compute_mask] = 0.0
+            if len(row_indices_arr):
+                pos_in_row_arr = _group_cumcount(row_indices_arr)
+                row_lengths = np.bincount(row_indices_arr, minlength=n).astype(np.float64)
+                positive_rows = np.flatnonzero(row_lengths > 0)
+                lengths[positive_rows] = row_lengths[positive_rows]
+        else:
+            row_lengths = np.bincount(row_indices_arr, minlength=n).astype(np.float64)
+            lengths[compute_mask] = row_lengths[compute_mask]
+
+        finite_lengths = lengths[np.isfinite(lengths)]
+        k_max = int(finite_lengths.max()) if finite_lengths.size else 0
+        matrix = np.full((n, 1 + k_max), np.nan, dtype=np.float64)
+        matrix[:, _DATE_DIFF_LISTS_LENGTH_COL] = lengths
+        if k_max > 0 and len(row_indices_arr):
+            matrix[row_indices_arr, pos_in_row_arr + 1] = diffs
+        return matrix
+
+    def calculate_binary(self, left: OperandValue, right: OperandValue) -> np.ndarray:
         left = left.as_series()
         right = right.as_series()
         if left.isna().all() or right.isna().all():
-            return pd.Series([None] * len(left), index=left.index, dtype=object)
-
-        left = self._convert_to_date(left, self.left_unit)
-        right_mask = self._non_empty_list_mask(right)
-        mask = left.notna() & right.notna() & right_mask
+            return np.full((len(left), 1), np.nan, dtype=np.float64)
 
-        results = pd.Series([None] * len(left), index=left.index, dtype=object)
-        if not mask.any():
-            return results
-
-        masked_left = left[mask]
-        converted_lists = self._convert_date_lists(right[mask])
-        for idx, left_date in masked_left.items():
-            results.loc[idx] = self._row_diffs(left_date, converted_lists[idx])
-        return results
+        return self._build_matrix(left, right)
 
 
 class DateListDiffAggWithinBounds(PandasOperator, ParametrizedOperator):
@@ -263,6 +329,95 @@ class DateListDiffAggWithinBounds(PandasOperator, ParametrizedOperator):
             normalize=normalize,
         )
 
+    def _masked_values(self, matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+        lengths = matrix[:, _DATE_DIFF_LISTS_LENGTH_COL]
+        values = matrix[:, _DATE_DIFF_LISTS_LENGTH_COL + 1 :]
+        missing = np.isnan(lengths)
+        empty = (~missing) & (lengths == 0)
+        if values.shape[1] == 0:
+            valid_mask = np.zeros((len(lengths), 0), dtype=bool)
+        else:
+            valid_mask = np.arange(values.shape[1])[None, :] < lengths[:, None]
+        masked_values = np.where(valid_mask, values, np.nan)
+        return lengths, masked_values, valid_mask, missing, empty
+
+    def _matrix_agg_nunique(self, ctx: _MatrixAggContext) -> None:
+        ctx.results[ctx.empty] = 0.0
+        active = ~ctx.missing & ~ctx.empty
+        if ctx.has_bounds:
+            select_mask = ctx.count_source & active[:, None]
+        else:
+            select_mask = ctx.valid_mask & active[:, None]
+        rows, _ = np.nonzero(select_mask)
+        ctx.results[active] = 0.0
+        if rows.size:
+            vals = ctx.masked_values[select_mask]
+            counts = pd.Series(vals).groupby(rows, sort=False).nunique(dropna=False)
+            ctx.results[counts.index.to_numpy(dtype=np.intp)] = counts.to_numpy(dtype=np.float64)
+
+    def _matrix_agg_count(self, ctx: _MatrixAggContext) -> None:
+        ctx.results[~ctx.missing] = ctx.count_source[~ctx.missing].sum(axis=1).astype(np.float64)
+
+    def _matrix_agg_sum(self, ctx: _MatrixAggContext) -> None:
+        agg_rows = ~ctx.missing & ~ctx.empty
+        if agg_rows.any():
+            with np.errstate(all="ignore"):
+                ctx.results[agg_rows] = np.nansum(ctx.agg_source[agg_rows], axis=1)
+
+    def _matrix_agg_nanaxis(self, ctx: _MatrixAggContext, reducer) -> None:
+        agg_rows = ~ctx.missing & ~ctx.empty
+        if not agg_rows.any():
+            return
+        has_finite = np.any(np.isfinite(ctx.agg_source[agg_rows]), axis=1)
+        finite_rows = np.flatnonzero(agg_rows)[has_finite]
+        with np.errstate(all="ignore"):
+            ctx.results[finite_rows] = reducer(ctx.agg_source[finite_rows], axis=1)
+
+    def _matrix_agg_mean(self, ctx: _MatrixAggContext) -> None:
+        self._matrix_agg_nanaxis(ctx, np.nanmean)
+
+    def _matrix_agg_min(self, ctx: _MatrixAggContext) -> None:
+        self._matrix_agg_nanaxis(ctx, np.nanmin)
+
+    def _matrix_agg_max(self, ctx: _MatrixAggContext) -> None:
+        self._matrix_agg_nanaxis(ctx, np.nanmax)
+
+    def _calculate_unary_matrix(self, matrix: np.ndarray, index: pd.Index) -> pd.Series:
+        lengths, masked_values, valid_mask, missing, empty = self._masked_values(matrix)
+        results = np.full(len(lengths), np.nan, dtype=np.float64)
+        has_bounds = self.lower_bound is not None or self.upper_bound is not None
+
+        if has_bounds:
+            lower = self.lower_bound if self.lower_bound is not None else -np.inf
+            upper = self.upper_bound if self.upper_bound is not None else np.inf
+            in_bounds = (masked_values >= lower) & (masked_values < upper)
+            agg_source = np.where(in_bounds & valid_mask, masked_values, np.nan)
+            count_source = in_bounds & valid_mask
+        else:
+            agg_source = masked_values
+            count_source = valid_mask
+
+        ctx = _MatrixAggContext(
+            lengths=lengths,
+            masked_values=masked_values,
+            valid_mask=valid_mask,
+            missing=missing,
+            empty=empty,
+            agg_source=agg_source,
+            count_source=count_source,
+            has_bounds=has_bounds,
+            results=results,
+        )
+        method_name = _MATRIX_AGGREGATIONS.get(self.aggregation)
+        if method_name is None:
+            raise ValueError(f"Unsupported aggregation: {self.aggregation}")
+        getattr(self, method_name)(ctx)
+
+        if self.normalize:
+            normalize_mask = ~missing & ~empty & (lengths > 0)
+            results[normalize_mask] = results[normalize_mask] / lengths[normalize_mask]
+        return pd.Series(results, index=index, dtype=np.float64)
+
     def _aggregate_row(self, diffs) -> float:
         if diffs is None or (isinstance(diffs, float) and np.isnan(diffs)):
             return np.nan
@@ -279,6 +434,9 @@ class DateListDiffAggWithinBounds(PandasOperator, ParametrizedOperator):
         return agg_res
 
     def calculate_unary(self, data: OperandValue) -> pd.Series:
+        if data.kind == OperandKind.MATRIX:
+            return self._calculate_unary_matrix(data.as_matrix(), data.index)
+
         data = data.as_series()
         results = np.empty(len(data), dtype=np.float64)
         results[:] = np.nan
@@ -339,7 +497,7 @@ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
         if left.isna().all() or right.isna().all():
             return pd.Series([None] * len(left), index=left.index, dtype=np.float64)
 
-        right_mask = DateListDiffLists._non_empty_list_mask(right)
+        right_mask = self._lists_op()._non_empty_list_mask(right)
         diff_lists = self._lists_op().calculate(left=left, right=right)
         result = self._agg_op().calculate(data=diff_lists).as_series()
         if self.aggregation in _count_aggregations:

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/src/upgini/autofe/timeseries/delta.py

@@ -1,9 +1,9 @@
 import pandas as pd
-from typing import Dict, Optional, Union
+from typing import Dict, Optional
 
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
-from upgini.autofe.timeseries.lag import Lag
+from upgini.autofe.timeseries.numpy_kernels import apply_grouped_kernel, delta2_values, delta_values
 
 
 class DeltaBase(TimeSeriesBase):
@@ -20,15 +20,6 @@ class DeltaBase(TimeSeriesBase):
         )
         return res
 
-    def _calculate_delta(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
-        return_series = isinstance(x, pd.Series)
-        x = pd.DataFrame(x)
-        value_col = x.columns[-1]
-        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
-        lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
-        x.iloc[:, -1] = x.iloc[:, -1] - lag._aggregate(x.iloc[:, -1])
-        return x.iloc[:, -1] if return_series else x
-
 
 class Delta(DeltaBase, ParametrizedOperator):
     def to_formula(self) -> str:
@@ -69,7 +60,11 @@ class Delta(DeltaBase, ParametrizedOperator):
         return cls(**params)
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.apply(self._calculate_delta).iloc[:, [-1]]
+        delta_size = self.delta_size
+        delta_unit = self.delta_unit
+        return apply_grouped_kernel(
+            ts, lambda times, values: delta_values(times, values, delta_size, delta_unit)
+        )
 
 
 class Delta2(DeltaBase, ParametrizedOperator):
@@ -111,11 +106,8 @@ class Delta2(DeltaBase, ParametrizedOperator):
         return cls(**params)
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.apply(self._calculate_delta2).iloc[:, [-1]]
-
-    def _calculate_delta2(self, x):
-        # Calculate first delta
-        first_delta = self._calculate_delta(x)
-
-        # Calculate delta of delta (second derivative)
-        return self._calculate_delta(first_delta)
+        delta_size = self.delta_size
+        delta_unit = self.delta_unit
+        return apply_grouped_kernel(
+            ts, lambda times, values: delta2_values(times, values, delta_size, delta_unit)
+        )

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/src/upgini/autofe/timeseries/lag.py

@@ -1,9 +1,9 @@
-import numpy as np
 import pandas as pd
 from typing import Dict, Optional
 
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
+from upgini.autofe.timeseries.numpy_kernels import apply_grouped_kernel, lag_values
 
 
 class Lag(TimeSeriesBase, ParametrizedOperator):
@@ -58,11 +58,6 @@ class Lag(TimeSeriesBase, ParametrizedOperator):
         return res
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        lag_window = self.lag_size + 1
-        return ts.rolling(f"{lag_window}{self.lag_unit}", min_periods=1).agg(self._lag)
-
-    def _lag(self, x):
-        if x.index.min() > (x.index.max() - pd.Timedelta(self.lag_size, self.lag_unit)):
-            return np.nan
-        else:
-            return x[0]
+        lag_size = self.lag_size
+        lag_unit = self.lag_unit
+        return apply_grouped_kernel(ts, lambda times, values: lag_values(times, values, lag_size, lag_unit))

upgini-1.2.158a4180.dev2/src/upgini/autofe/timeseries/numpy_kernels.py

@@ -0,0 +1,259 @@
+"""Numpy kernels for time-series AutoFE operators.
+
+Window semantics match pandas time-based rolling with a left-open interval
+``(t - window, t]`` (see ``Lag`` / ``Roll`` operators and their tests).
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Union
+
+import numpy as np
+import pandas as pd
+
+# Aggregation applied to values in a time window (may be length 1+).
+WindowAgg = Callable[[np.ndarray], float]
+
+
+def _timedelta_ns(size: int, unit: str) -> int:
+    return int(pd.Timedelta(size, unit).to_timedelta64().astype("timedelta64[ns]").astype(np.int64))
+
+
+def window_left_indices(times_ns: np.ndarray, window_ns: int) -> np.ndarray:
+    """For each t_i, first index j with times[j] > t_i - window (left-open)."""
+    times_ns = np.asarray(times_ns, dtype=np.int64)
+    return np.searchsorted(times_ns, times_ns - window_ns, side="right")
+
+
+def lag_values(times_ns: np.ndarray, values: np.ndarray, lag_size: int, lag_unit: str) -> np.ndarray:
+    """Lag: oldest value in ``(t - (lag+1)·unit, t]`` if span covers ``lag``, else NaN."""
+    times_ns = np.asarray(times_ns, dtype=np.int64)
+    values = np.asarray(values, dtype=np.float64)
+    n = len(times_ns)
+    if n == 0:
+        return np.array([], dtype=np.float64)
+
+    lag_ns = _timedelta_ns(lag_size, lag_unit)
+    window_ns = _timedelta_ns(lag_size + 1, lag_unit)
+    lefts = window_left_indices(times_ns, window_ns)
+
+    out = np.full(n, np.nan, dtype=np.float64)
+    # Gate: oldest in window is at or before t - lag  <=>  times[left] <= t - lag
+    gate = times_ns[lefts] <= times_ns - lag_ns
+    out[gate] = values[lefts[gate]]
+    return out
+
+
+def _rolling_mean(window: np.ndarray) -> float:
+    return float(np.mean(window))
+
+
+def _rolling_min(window: np.ndarray) -> float:
+    return float(np.min(window))
+
+
+def _rolling_max(window: np.ndarray) -> float:
+    return float(np.max(window))
+
+
+def _rolling_median(window: np.ndarray) -> float:
+    return float(np.median(window))
+
+
+def _rolling_std(window: np.ndarray) -> float:
+    # pandas rolling.std default ddof=1
+    if len(window) < 2:
+        return np.nan
+    return float(np.std(window, ddof=1))
+
+
+def _rolling_norm_mean(window: np.ndarray) -> float:
+    mean = np.mean(window)
+    return float(window[-1] / mean)
+
+
+def _rolling_q25(window: np.ndarray) -> float:
+    return float(np.quantile(window, 0.25, method="linear"))
+
+
+def _rolling_q75(window: np.ndarray) -> float:
+    return float(np.quantile(window, 0.75, method="linear"))
+
+
+def _rolling_iqr(window: np.ndarray) -> float:
+    q75, q25 = np.quantile(window, [0.75, 0.25], method="linear")
+    return float(q75 - q25)
+
+
+ROLL_AGGS: dict[str, WindowAgg] = {
+    "mean": _rolling_mean,
+    "min": _rolling_min,
+    "max": _rolling_max,
+    "median": _rolling_median,
+    "std": _rolling_std,
+    "norm_mean": _rolling_norm_mean,
+    "q25": _rolling_q25,
+    "q75": _rolling_q75,
+    "iqr": _rolling_iqr,
+}
+
+
+def roll_values(
+    times_ns: np.ndarray,
+    values: np.ndarray,
+    window_size: int,
+    window_unit: str,
+    aggregation: str,
+) -> np.ndarray:
+    """Time-based rolling aggregation with ``min_periods=1`` semantics."""
+    times_ns = np.asarray(times_ns, dtype=np.int64)
+    values = np.asarray(values, dtype=np.float64)
+    n = len(times_ns)
+    if n == 0:
+        return np.array([], dtype=np.float64)
+
+    if aggregation not in ROLL_AGGS:
+        raise ValueError(f"Unsupported roll aggregation for numpy path: {aggregation}")
+
+    window_ns = _timedelta_ns(window_size, window_unit)
+    lefts = window_left_indices(times_ns, window_ns)
+    counts = np.arange(n, dtype=np.int64) - lefts + 1
+
+    if aggregation in {"mean", "norm_mean", "std"}:
+        csum = np.empty(n + 1, dtype=np.float64)
+        csum[0] = 0.0
+        np.cumsum(values, out=csum[1:])
+        window_sum = csum[1:] - csum[lefts]
+        means = window_sum / counts
+        if aggregation == "mean":
+            return means
+        if aggregation == "norm_mean":
+            return values / means
+        # std, ddof=1
+        csum2 = np.empty(n + 1, dtype=np.float64)
+        csum2[0] = 0.0
+        np.cumsum(values * values, out=csum2[1:])
+        window_sum2 = csum2[1:] - csum2[lefts]
+        out = np.full(n, np.nan, dtype=np.float64)
+        multi = counts >= 2
+        # sample variance: (sumsq - sum^2/n) / (n-1)
+        var = (window_sum2[multi] - window_sum[multi] * window_sum[multi] / counts[multi]) / (counts[multi] - 1)
+        # numerical noise can be slightly negative
+        out[multi] = np.sqrt(np.maximum(var, 0.0))
+        return out
+
+    if aggregation == "min":
+        # Falling window minima via brute force is fine for correctness; still O(n·w) worst case.
+        # Use a simple loop — windows are typically small calendar spans.
+        out = np.empty(n, dtype=np.float64)
+        for i in range(n):
+            out[i] = np.min(values[lefts[i] : i + 1])
+        return out
+
+    if aggregation == "max":
+        out = np.empty(n, dtype=np.float64)
+        for i in range(n):
+            out[i] = np.max(values[lefts[i] : i + 1])
+        return out
+
+    agg = ROLL_AGGS[aggregation]
+    out = np.empty(n, dtype=np.float64)
+    for i in range(n):
+        out[i] = agg(values[lefts[i] : i + 1])
+    return out
+
+
+def freq_pct_change(times_ns: np.ndarray, values: np.ndarray, step_size: int, step_unit: str) -> np.ndarray:
+    """Match ``Series.pct_change(freq=..., fill_method='pad').fillna(0)`` on sorted unique dates."""
+    times_ns = np.asarray(times_ns, dtype=np.int64)
+    values = np.asarray(values, dtype=np.float64)
+    n = len(times_ns)
+    if n == 0:
+        return np.array([], dtype=np.float64)
+
+    # pandas pct_change default fill_method='pad' forward-fills before differencing
+    filled = values.copy()
+    is_valid = np.isfinite(filled)
+    if not is_valid.all():
+        idx = np.where(is_valid, np.arange(n), 0)
+        np.maximum.accumulate(idx, out=idx)
+        # keep leading NaNs as NaN
+        first_valid = int(np.argmax(is_valid)) if is_valid.any() else n
+        filled = filled[idx]
+        filled[:first_valid] = np.nan
+
+    step_ns = _timedelta_ns(step_size, step_unit)
+    targets = times_ns - step_ns
+    idx = np.searchsorted(times_ns, targets, side="left")
+    out = np.zeros(n, dtype=np.float64)
+    in_range = idx < n
+    matches = np.zeros(n, dtype=bool)
+    matches[in_range] = times_ns[idx[in_range]] == targets[in_range]
+    prev = filled[idx[matches]]
+    cur = filled[matches]
+    with np.errstate(divide="ignore", invalid="ignore"):
+        pct = cur / prev - 1.0
+    out[matches] = pct
+    out[~np.isfinite(out)] = 0.0
+    return out
+
+
+def delta_values(times_ns: np.ndarray, values: np.ndarray, delta_size: int, delta_unit: str) -> np.ndarray:
+    values = np.asarray(values, dtype=np.float64)
+    return values - lag_values(times_ns, values, delta_size, delta_unit)
+
+
+def delta2_values(times_ns: np.ndarray, values: np.ndarray, delta_size: int, delta_unit: str) -> np.ndarray:
+    first = delta_values(times_ns, values, delta_size, delta_unit)
+    return delta_values(times_ns, first, delta_size, delta_unit)
+
+
+def rolling_volatility_values(
+    times_ns: np.ndarray,
+    values: np.ndarray,
+    step_size: int,
+    step_unit: str,
+    window_size: int,
+    window_unit: str,
+    abs_returns: bool = False,
+) -> np.ndarray:
+    returns = freq_pct_change(times_ns, values, step_size, step_unit)
+    if abs_returns:
+        returns = np.abs(returns)
+    return roll_values(times_ns, returns, window_size, window_unit, "std")
+
+
+def _kernel_on_frame(
+    frame: pd.DataFrame,
+    kernel: Callable[[np.ndarray, np.ndarray], np.ndarray],
+) -> pd.DataFrame:
+    if frame.empty:
+        return frame.iloc[:, [-1]].astype(np.float64) if len(frame.columns) else frame
+
+    value_col = frame.columns[-1]
+    series = pd.to_numeric(frame[value_col], errors="coerce").astype(np.float64)
+    index = frame.index
+    if isinstance(index, pd.MultiIndex):
+        times_ns = index.get_level_values(-1).asi8
+    else:
+        times_ns = index.asi8
+    out = kernel(times_ns, series.to_numpy(dtype=np.float64, copy=False))
+    return pd.DataFrame({value_col: out}, index=frame.index, dtype=np.float64)
+
+
+def apply_grouped_kernel(
+    ts: Union[pd.DataFrame, pd.core.groupby.generic.DataFrameGroupBy],
+    kernel: Callable[[np.ndarray, np.ndarray], np.ndarray],
+) -> pd.DataFrame:
+    """Apply ``kernel(times_ns, values) -> values`` on a DatetimeIndex frame or GroupBy."""
+    if isinstance(ts, pd.DataFrame):
+        return _kernel_on_frame(ts, kernel)
+
+    try:
+        applied = ts.apply(lambda g: _kernel_on_frame(g, kernel), include_groups=False)
+    except TypeError:
+        # pandas without include_groups
+        applied = ts.apply(lambda g: _kernel_on_frame(g, kernel))
+    if isinstance(applied, pd.Series):
+        return applied.to_frame()
+    return applied.iloc[:, [-1]] if getattr(applied, "shape", (0, 0))[1] > 1 else applied

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/src/upgini/autofe/timeseries/roll.py

@@ -3,14 +3,15 @@ from typing import Dict, Optional
 
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
+from upgini.autofe.timeseries.numpy_kernels import ROLL_AGGS, apply_grouped_kernel, roll_values
 from upgini.autofe.utils import pydantic_validator
 
-# Roll aggregation functions
+# Names kept for compatibility; computation uses numpy ROLL_AGGS.
 roll_aggregations = {
-    "norm_mean": lambda x: x[-1] / x.mean(),
-    "q25": lambda x: x.quantile(0.25),
-    "q75": lambda x: x.quantile(0.75),
-    "iqr": lambda x: x.quantile(0.75) - x.quantile(0.25),
+    "norm_mean": ROLL_AGGS["norm_mean"],
+    "q25": ROLL_AGGS["q25"],
+    "q75": ROLL_AGGS["q75"],
+    "iqr": ROLL_AGGS["iqr"],
 }
 
 
@@ -82,6 +83,12 @@ class Roll(TimeSeriesBase, ParametrizedOperator):
         return res
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.rolling(f"{self.window_size}{self.window_unit}", min_periods=1).agg(
-            roll_aggregations.get(self.aggregation, self.aggregation)
+        window_size = self.window_size
+        window_unit = self.window_unit
+        aggregation = self.aggregation
+        if aggregation not in ROLL_AGGS:
+            return ts.rolling(f"{window_size}{window_unit}", min_periods=1).agg(aggregation)
+        return apply_grouped_kernel(
+            ts,
+            lambda times, values: roll_values(times, values, window_size, window_unit, aggregation),
         )

{upgini-1.2.157.dev4 → upgini-1.2.158a4180.dev2}/src/upgini/autofe/timeseries/volatility.py

@@ -1,9 +1,10 @@
-from typing import Dict, Optional, Union
+from typing import Dict, Optional
 
 import numpy as np
 import pandas as pd
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
+from upgini.autofe.timeseries.numpy_kernels import apply_grouped_kernel, rolling_volatility_values
 
 
 class VolatilityBase(TimeSeriesBase):
@@ -90,18 +91,16 @@ class RollingVolBase(VolatilityBase):
         )
         return res
 
-    def _rolling_vol(
-        self, x: Union[pd.DataFrame, pd.Series], window_size: int, window_unit: str, abs_returns: bool = False
-    ) -> Union[pd.DataFrame, pd.Series]:
-        return_series = isinstance(x, pd.Series)
-        x = pd.DataFrame(x)
-        value_col = x.columns[-1]
-        x[value_col] = pd.to_numeric(x[value_col], errors="coerce").astype("float64")
-        returns = self._get_returns(x.iloc[:, -1], f"{self.step_size}{self.step_unit}")
-        if abs_returns:
-            returns = returns.abs()
-        x.iloc[:, -1] = returns.rolling(f"{window_size}{window_unit}", min_periods=1).std()
-        return x.iloc[:, -1] if return_series else x
+    def _rolling_vol_kernel(self, window_size: int, window_unit: str, abs_returns: bool = False):
+        step_size = self.step_size
+        step_unit = self.step_unit
+
+        def kernel(times_ns: np.ndarray, values: np.ndarray) -> np.ndarray:
+            return rolling_volatility_values(
+                times_ns, values, step_size, step_unit, window_size, window_unit, abs_returns=abs_returns
+            )
+
+        return kernel
 
 
 class RollingVolatility(RollingVolBase, ParametrizedOperator):
@@ -141,9 +140,9 @@ class RollingVolatility(RollingVolBase, ParametrizedOperator):
         return cls(**params)
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.apply(
-            self._rolling_vol, window_size=self.window_size, window_unit=self.window_unit, abs_returns=self.abs_returns
-        ).iloc[:, [-1]]
+        return apply_grouped_kernel(
+            ts, self._rolling_vol_kernel(self.window_size, self.window_unit, abs_returns=self.abs_returns)
+        )
 
 
 class RollingVolatility2(RollingVolBase, ParametrizedOperator):
@@ -185,12 +184,20 @@ class RollingVolatility2(RollingVolBase, ParametrizedOperator):
         return cls(**params)
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.apply(self._vol_on_vol).iloc[:, [-1]]
+        window_size = self.window_size
+        window_unit = self.window_unit
+        step_size = self.step_size
+        step_unit = self.step_unit
 
-    def _vol_on_vol(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
-        vol1 = self._rolling_vol(x, self.window_size, self.window_unit, abs_returns=True)
-        vol2 = self._rolling_vol(vol1, self.window_size, self.window_unit, abs_returns=False)
-        return vol2
+        def kernel(times_ns: np.ndarray, values: np.ndarray) -> np.ndarray:
+            vol1 = rolling_volatility_values(
+                times_ns, values, step_size, step_unit, window_size, window_unit, abs_returns=True
+            )
+            return rolling_volatility_values(
+                times_ns, vol1, step_size, step_unit, window_size, window_unit, abs_returns=False
+            )
+
+        return apply_grouped_kernel(ts, kernel)
 
 
 class VolatilityRatio(RollingVolBase, ParametrizedOperator):
@@ -252,13 +259,24 @@ class VolatilityRatio(RollingVolBase, ParametrizedOperator):
         return res
 
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        return ts.apply(self._vol_ratio).iloc[:, [-1]]
-
-    def _vol_ratio(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
-        short_vol = self._rolling_vol(x, self.short_window_size, self.short_window_unit)
-        long_vol = self._rolling_vol(x, self.window_size, self.window_unit)
-        ratio = VolatilityRatio._handle_div_errors(short_vol / long_vol)
-        return ratio
+        short_window_size = self.short_window_size
+        short_window_unit = self.short_window_unit
+        window_size = self.window_size
+        window_unit = self.window_unit
+        step_size = self.step_size
+        step_unit = self.step_unit
+
+        def kernel(times_ns: np.ndarray, values: np.ndarray) -> np.ndarray:
+            short_vol = rolling_volatility_values(
+                times_ns, values, step_size, step_unit, short_window_size, short_window_unit
+            )
+            long_vol = rolling_volatility_values(times_ns, values, step_size, step_unit, window_size, window_unit)
+            with np.errstate(divide="ignore", invalid="ignore"):
+                ratio = short_vol / long_vol
+            ratio[~np.isfinite(ratio)] = np.nan
+            return np.where(np.isnan(ratio), 1.0, ratio)
+
+        return apply_grouped_kernel(ts, kernel)
 
     @staticmethod
     def _handle_div_errors(x: pd.Series) -> pd.Series:

upgini-1.2.157.dev4/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.157.dev4"