upgini 1.2.61__tar.gz → 1.2.62a3818.dev1__tar.gz

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.61
+Version: 1.2.62a3818.dev1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.62a3818.dev1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.62a3818.dev1"

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/all_operands.py

@@ -1,4 +1,4 @@
-from upgini.autofe.operand import OperandRegistry
+from upgini.autofe.operator import OperatorRegistry
 from upgini.autofe.unary import *  # noqa
 from upgini.autofe.binary import *  # noqa
 from upgini.autofe.groupby import *  # noqa
@@ -7,4 +7,4 @@ from upgini.autofe.vector import *  # noqa
 
 
 def find_op(name):
-    return OperandRegistry.get_operand(name)
+    return OperatorRegistry.get_operand(name)

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/binary.py

@@ -5,7 +5,7 @@ import numpy as np
 import pandas as pd
 from jarowinkler import jarowinkler_similarity
 
-from upgini.autofe.operand import PandasOperand, VectorizableMixin
+from upgini.autofe.operator import PandasOperand, VectorizableMixin
 
 
 class Min(PandasOperand):

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/date.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas.core.arrays.timedeltas import TimedeltaArray
 from pydantic import BaseModel, __version__ as pydantic_version
 
-from upgini.autofe.operand import PandasOperand, ParametrizedOperand
+from upgini.autofe.operator import PandasOperand, ParametrizedOperand
 
 
 def get_pydantic_version():

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/feature.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas._typing import DtypeObj
 
 from upgini.autofe.all_operands import find_op
-from upgini.autofe.operand import Operand, PandasOperand
+from upgini.autofe.operator import Operand, PandasOperand
 
 
 class Column:

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/groupby.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 import pandas as pd
 
-from upgini.autofe.operand import PandasOperand, ParametrizedOperand, VectorizableMixin
+from upgini.autofe.operator import PandasOperand, ParametrizedOperand, VectorizableMixin
 
 
 class GroupByThenAgg(

upgini-1.2.61/src/upgini/autofe/operand.py → upgini-1.2.62a3818.dev1/src/upgini/autofe/operator.py

@@ -6,7 +6,7 @@ import pandas as pd
 from pydantic import BaseModel
 
 
-class OperandRegistry(type(BaseModel)):
+class OperatorRegistry(type(BaseModel)):
     _registry = {}
     _parametrized_registry = []
 
@@ -46,7 +46,7 @@ class OperandRegistry(type(BaseModel)):
         return None
 
 
-class Operand(BaseModel, metaclass=OperandRegistry):
+class Operand(BaseModel, metaclass=OperatorRegistry):
     name: Optional[str] = None
     alias: Optional[str] = None
     is_unary: bool = False

upgini-1.2.61/src/upgini/autofe/vector.py → upgini-1.2.62a3818.dev1/src/upgini/autofe/timeseries.py

@@ -2,33 +2,13 @@ import abc
 from typing import Dict, List, Optional
 
 import pandas as pd
+from upgini.autofe.operator import PandasOperand, ParametrizedOperand
 
 try:
     from pydantic import field_validator as validator  # V2
 except ImportError:
     from pydantic import validator  # V1
 
-from upgini.autofe.operand import PandasOperand, ParametrizedOperand, VectorizableMixin
-
-
-class Mean(PandasOperand, VectorizableMixin):
-    name: str = "mean"
-    output_type: Optional[str] = "float"
-    is_vector: bool = True
-    group_index: int = 0
-
-    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
-        return pd.DataFrame(data).T.fillna(0).mean(axis=1)
-
-
-class Sum(PandasOperand, VectorizableMixin):
-    name: str = "sum"
-    is_vector: bool = True
-    group_index: int = 0
-
-    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
-        return pd.DataFrame(data).T.fillna(0).sum(axis=1)
-
 
 class TimeSeriesBase(PandasOperand, abc.ABC):
     is_vector: bool = True
@@ -55,7 +35,7 @@ class TimeSeriesBase(PandasOperand, abc.ABC):
         ts.set_index(date.name, inplace=True)
         ts = ts[ts.index.notna()].sort_index()
         ts = (
-            ts.groupby([c.name for c in data[1:-1]])
+            ts.groupby([c.name for c in data[1:-1]], group_keys=True)
             .apply(self._shift)[data[-1].name]
             .to_frame()
             .reset_index()
@@ -84,7 +64,7 @@ class TimeSeriesBase(PandasOperand, abc.ABC):
         pass
 
 
-_roll_aggregations = {"norm_mean": lambda x: x[-1] / x.mean()}
+_roll_aggregations = {"norm_mean": lambda x: x[-1] / x.mean(), "last": lambda x: x[-1]}
 
 
 class Roll(TimeSeriesBase, ParametrizedOperand):

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/autofe/unary.py

@@ -2,7 +2,7 @@ from typing import Dict, Optional
 import numpy as np
 import pandas as pd
 
-from upgini.autofe.operand import PandasOperand, VectorizableMixin
+from upgini.autofe.operator import PandasOperand, VectorizableMixin
 
 
 class Abs(PandasOperand, VectorizableMixin):

upgini-1.2.62a3818.dev1/src/upgini/autofe/vector.py

@@ -0,0 +1,24 @@
+from typing import List, Optional
+
+import pandas as pd
+
+from upgini.autofe.operator import PandasOperand, VectorizableMixin
+
+
+class Mean(PandasOperand, VectorizableMixin):
+    name: str = "mean"
+    output_type: Optional[str] = "float"
+    is_vector: bool = True
+    group_index: int = 0
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).mean(axis=1)
+
+
+class Sum(PandasOperand, VectorizableMixin):
+    name: str = "sum"
+    is_vector: bool = True
+    group_index: int = 0
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).sum(axis=1)

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/dataset.py

@@ -40,7 +40,7 @@ from upgini.utils.email_utils import EmailSearchKeyConverter
 from upgini.utils.target_utils import (
     balance_undersample,
     balance_undersample_forced,
-    balance_undersample_time_series,
+    balance_undersample_time_series_trunc,
 )
 
 try:
@@ -58,6 +58,8 @@ class Dataset:  # (pd.DataFrame):
     FIT_SAMPLE_THRESHOLD = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_ROWS = 200_000
     FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
+    FIT_SAMPLE_THRESHOLD_TS = 54_000
+    FIT_SAMPLE_ROWS_TS = 54_000
     BINARY_MIN_SAMPLE_THRESHOLD = 5_000
     MULTICLASS_MIN_SAMPLE_THRESHOLD = 25_000
     IMBALANCE_THESHOLD = 0.6
@@ -301,7 +303,10 @@ class Dataset:  # (pd.DataFrame):
                 )
 
         # Resample over fit threshold
-        if not self.imbalanced and EVAL_SET_INDEX in self.data.columns:
+        if self.cv_type is not None and self.cv_type.is_time_series():
+            sample_threshold = self.FIT_SAMPLE_THRESHOLD_TS
+            sample_rows = self.FIT_SAMPLE_ROWS_TS
+        elif not self.imbalanced and EVAL_SET_INDEX in self.data.columns:
             sample_threshold = self.FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD
             sample_rows = self.FIT_SAMPLE_WITH_EVAL_SET_ROWS
         else:
@@ -314,7 +319,7 @@ class Dataset:  # (pd.DataFrame):
                 f"and will be downsampled to {sample_rows}"
             )
             if self.cv_type is not None and self.cv_type.is_time_series():
-                resampled_data = balance_undersample_time_series(
+                resampled_data = balance_undersample_time_series_trunc(
                     df=self.data,
                     id_columns=self.id_columns,
                     date_column=next(
@@ -584,10 +589,7 @@ class Dataset:  # (pd.DataFrame):
         return search_customization
 
     def _rename_generate_features(self, runtime_parameters: Optional[RuntimeParameters]) -> Optional[RuntimeParameters]:
-        if (
-            runtime_parameters is not None
-            and runtime_parameters.properties is not None
-        ):
+        if runtime_parameters is not None and runtime_parameters.properties is not None:
             if "generate_features" in runtime_parameters.properties:
                 generate_features = runtime_parameters.properties["generate_features"].split(",")
                 renamed_generate_features = []
@@ -607,6 +609,13 @@ class Dataset:  # (pd.DataFrame):
 
         return runtime_parameters
 
+    def _set_sample_size(self, runtime_parameters: Optional[RuntimeParameters]) -> Optional[RuntimeParameters]:
+        if runtime_parameters is not None and runtime_parameters.properties is not None:
+            if self.cv_type is not None and self.cv_type.is_time_series():
+                runtime_parameters.properties["sample_size"] = self.FIT_SAMPLE_ROWS_TS
+                runtime_parameters.properties["iter0_sample_size"] = self.FIT_SAMPLE_ROWS_TS
+        return runtime_parameters
+
     def _clean_generate_features(self, runtime_parameters: Optional[RuntimeParameters]) -> Optional[RuntimeParameters]:
         if (
             runtime_parameters is not None
@@ -638,6 +647,7 @@ class Dataset:  # (pd.DataFrame):
         file_metrics = FileMetrics()
 
         runtime_parameters = self._rename_generate_features(runtime_parameters)
+        runtime_parameters = self._set_sample_size(runtime_parameters)
 
         file_metadata = self.__construct_metadata(exclude_features_sources)
         search_customization = self.__construct_search_customization(

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/features_enricher.py

@@ -31,7 +31,7 @@ from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import BaseCrossValidator
 
 from upgini.autofe.feature import Feature
-from upgini.autofe.vector import TimeSeriesBase
+from upgini.autofe.timeseries import TimeSeriesBase
 from upgini.data_source.data_source_publisher import CommercialSchema
 from upgini.dataset import Dataset
 from upgini.errors import HttpError, ValidationError

{upgini-1.2.61 → upgini-1.2.62a3818.dev1}/src/upgini/utils/target_utils.py

@@ -9,6 +9,7 @@ from upgini.errors import ValidationError
 from upgini.metadata import SYSTEM_RECORD_ID, CVType, ModelTaskType
 from upgini.resource_bundle import ResourceBundle, bundle, get_custom_bundle
 from upgini.sampler.random_under_sampler import RandomUnderSampler
+from upgini.utils.ts_utils import get_most_frequent_time_unit, trunc_datetime
 
 TS_MIN_DIFFERENT_IDS_RATIO = 0.2
 
@@ -240,7 +241,7 @@ def balance_undersample_forced(
     df = df.copy().sort_values(by=SYSTEM_RECORD_ID)
     if cv_type is not None and cv_type.is_time_series():
         logger.warning(f"Sampling time series dataset from {len(df)} to {sample_size}")
-        resampled_data = balance_undersample_time_series(
+        resampled_data = balance_undersample_time_series_trunc(
             df,
             id_columns=id_columns,
             date_column=date_column,
@@ -279,6 +280,58 @@ def balance_undersample_forced(
     return resampled_data
 
 
+DEFAULT_HIGH_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=2, months=6), pd.DateOffset(years=2, days=7)]
+DEFAULT_LOW_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=7), pd.DateOffset(years=5)]
+DEFAULT_TIME_UNIT_THRESHOLD = pd.Timedelta(weeks=4)
+
+
+def balance_undersample_time_series_trunc(
+    df: pd.DataFrame,
+    id_columns: List[str],
+    date_column: str,
+    sample_size: int,
+    random_state: int = 42,
+    logger: Optional[logging.Logger] = None,
+    highfreq_trunc_lengths: List[pd.DateOffset] = DEFAULT_HIGH_FREQ_TRUNC_LENGTHS,
+    lowfreq_trunc_lengths: List[pd.DateOffset] = DEFAULT_LOW_FREQ_TRUNC_LENGTHS,
+    time_unit_threshold: pd.Timedelta = DEFAULT_TIME_UNIT_THRESHOLD,
+    **kwargs,
+):
+    # Convert date column to datetime
+    dates_df = df[id_columns + [date_column]].copy()
+    dates_df[date_column] = pd.to_datetime(dates_df[date_column], unit="ms")
+
+    time_unit = get_most_frequent_time_unit(dates_df, id_columns, date_column)
+    if logger is not None:
+        logger.info(f"Time unit: {time_unit}")
+
+    if time_unit is None:
+        if logger is not None:
+            logger.info("Cannot detect time unit, returning original dataset")
+        return df
+
+    if time_unit < time_unit_threshold:
+        for trunc_length in highfreq_trunc_lengths:
+            sampled_df = trunc_datetime(dates_df, id_columns, date_column, trunc_length, logger=logger)
+            if len(sampled_df) <= sample_size:
+                break
+        if len(sampled_df) > sample_size:
+            sampled_df = balance_undersample_time_series(
+                sampled_df, id_columns, date_column, sample_size, random_state, logger=logger, **kwargs
+            )
+    else:
+        for trunc_length in lowfreq_trunc_lengths:
+            sampled_df = trunc_datetime(dates_df, id_columns, date_column, trunc_length, logger=logger)
+            if len(sampled_df) <= sample_size:
+                break
+        if len(sampled_df) > sample_size:
+            sampled_df = balance_undersample_time_series(
+                sampled_df, id_columns, date_column, sample_size, random_state, logger=logger, **kwargs
+            )
+
+    return df.loc[sampled_df.index]
+
+
 def balance_undersample_time_series(
     df: pd.DataFrame,
     id_columns: List[str],

upgini-1.2.62a3818.dev1/src/upgini/utils/ts_utils.py

@@ -0,0 +1,41 @@
+import logging
+from typing import List, Optional
+import pandas as pd
+
+
+def get_most_frequent_time_unit(df: pd.DataFrame, id_columns: List[str], date_column: str) -> Optional[pd.DateOffset]:
+
+    def closest_unit(diff):
+        return pd.tseries.frequencies.to_offset(pd.Timedelta(diff, unit="s"))
+
+    all_diffs = []
+    groups = df.groupby(id_columns) if id_columns else [(None, df)]
+    for _, group in groups:
+        group_dates = group[date_column].sort_values().unique()
+        if len(group_dates) > 1:
+            diff_series = pd.Series(group_dates[1:] - group_dates[:-1])
+            diff_ns = diff_series.dt.total_seconds()
+            all_diffs.extend(diff_ns)
+
+    all_diffs = pd.Series(all_diffs)
+
+    most_frequent_unit = all_diffs.apply(closest_unit).mode().min()
+
+    return most_frequent_unit if isinstance(most_frequent_unit, pd.DateOffset) else None
+
+
+def trunc_datetime(
+    df: pd.DataFrame,
+    id_columns: List[str],
+    date_column: str,
+    length: pd.DateOffset,
+    logger: Optional[logging.Logger] = None,
+) -> pd.DataFrame:
+    if logger is not None:
+        logger.info(f"Truncating time series dataset to {length}")
+
+    if id_columns:
+        min_datetime = df.groupby(id_columns)[date_column].transform(lambda group: group.max() - length)
+    else:
+        min_datetime = df[date_column].max() - length
+    return df[df[date_column] > min_datetime]

upgini-1.2.61/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.61"