upgini 1.1.263a1__tar.gz → 1.1.264__tar.gz

{upgini-1.1.263a1/src/upgini.egg-info → upgini-1.1.264}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.263a1
+Version: 1.1.264
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.263a1 → upgini-1.1.264}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.263a1"
+version = "1.1.264"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/autofe/all_operands.py

@@ -1,4 +1,5 @@
 from typing import Dict
+from upgini.autofe.date import DateDiff, DateDiffType2
 from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
 from upgini.autofe.unary import Abs, Log, Residual, Sqrt, Square, Sigmoid, Floor, Freq
@@ -35,6 +36,8 @@ ALL_OPERANDS: Dict[str, Operand] = {
         Operand(name="GroupByThenNUnique", output_type="int", is_vectorizable=True, is_grouping=True),
         Operand(name="GroupByThenFreq", output_type="float", is_grouping=True),
         Sim(),
+        DateDiff(),
+        DateDiffType2(),
     ]
 }
 

upgini-1.1.264/src/upgini/autofe/date.py

@@ -0,0 +1,53 @@
+from typing import Optional, Union
+import numpy as np
+import pandas as pd
+
+from upgini.autofe.operand import PandasOperand
+
+
+class DateDiffMixin:
+    diff_unit: str = "D"
+    left_unit: Optional[str] = None
+    right_unit: Optional[str] = None
+
+    def _convert_to_date(
+        self, x: Union[pd.DataFrame, pd.Series], unit: Optional[str]
+    ) -> Union[pd.DataFrame, pd.Series]:
+        if isinstance(x, pd.DataFrame):
+            return x.apply(lambda y: self._convert_to_date(y, unit), axis=1)
+
+        return pd.to_datetime(x, unit=unit)
+
+
+class DateDiff(PandasOperand, DateDiffMixin):
+    name = "date_diff"
+    is_binary = True
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        left = self._convert_to_date(left, self.left_unit)
+        right = self._convert_to_date(right, self.right_unit)
+        return self.__replace_negative((left - right) / np.timedelta64(1, self.diff_unit))
+
+    def __replace_negative(self, x: Union[pd.DataFrame, pd.Series]):
+        x[x < 0] = None
+        return x
+
+
+class DateDiffType2(PandasOperand, DateDiffMixin):
+    name = "date_diff_type2"
+    is_binary = True
+    has_symmetry_importance = True
+    is_vectorizable = False
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        left = self._convert_to_date(left, self.left_unit)
+        right = self._convert_to_date(right, self.right_unit)
+        future = right + (left.dt.year - right.dt.year).apply(
+            lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
+        )
+        before = future[future < left]
+        future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
+        diff = (future - left) / np.timedelta64(1, self.diff_unit)
+
+        return diff

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/autofe/feature.py

@@ -305,7 +305,7 @@ class FeatureGroup:
         grouped_features = []
 
         def groupby_func(f: Feature) -> Tuple[Operand, Union[Column, Feature]]:
-            return (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
+            return (f.op, f.children[0 if not f.op.is_vectorizable else f.op.group_index])
 
         for op_child, features in itertools.groupby(candidates, groupby_func):
             op, main_child = op_child

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/autofe/operand.py

@@ -73,6 +73,8 @@ class PandasOperand(Operand, abc.ABC):
 
 
 class VectorizableMixin(Operand):
+    group_index: int = 1
+
     def validate_calculation(self, input_columns: List[str], **kwargs) -> Tuple[str, List[str]]:
         if not kwargs.get(MAIN_COLUMN):
             raise ValueError(f"Expected argument {MAIN_COLUMN} for grouping operator {self.name} not found")

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/autofe/unary.py

@@ -1,12 +1,13 @@
-from upgini.autofe.operand import PandasOperand
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
 import numpy as np
 import pandas as pd
 
 
-class Abs(PandasOperand):
+class Abs(PandasOperand, VectorizableMixin):
     name = "abs"
     is_unary = True
     is_vectorizable = True
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return data.abs()
@@ -15,11 +16,12 @@ class Abs(PandasOperand):
         return data.abs()
 
 
-class Log(PandasOperand):
+class Log(PandasOperand, VectorizableMixin):
     name = "log"
     is_unary = True
     is_vectorizable = True
     output_type = "float"
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return self._round_value(np.log(np.abs(data.replace(0, np.nan))), 10)
@@ -28,11 +30,12 @@ class Log(PandasOperand):
         return self._round_value(np.log(data.replace(0, np.nan).abs()), 10)
 
 
-class Sqrt(PandasOperand):
+class Sqrt(PandasOperand, VectorizableMixin):
     name = "sqrt"
     is_unary = True
     is_vectorizable = True
     output_type = "float"
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return self._round_value(np.sqrt(np.abs(data)))
@@ -41,10 +44,11 @@ class Sqrt(PandasOperand):
         return self._round_value(np.sqrt(data.abs()))
 
 
-class Square(PandasOperand):
+class Square(PandasOperand, VectorizableMixin):
     name = "square"
     is_unary = True
     is_vectorizable = True
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return np.square(data)
@@ -53,11 +57,12 @@ class Square(PandasOperand):
         return np.square(data)
 
 
-class Sigmoid(PandasOperand):
+class Sigmoid(PandasOperand, VectorizableMixin):
     name = "sigmoid"
     is_unary = True
     is_vectorizable = True
     output_type = "float"
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return self._round_value(1 / (1 + np.exp(-data)))
@@ -66,12 +71,13 @@ class Sigmoid(PandasOperand):
         return self._round_value(1 / (1 + np.exp(-data)))
 
 
-class Floor(PandasOperand):
+class Floor(PandasOperand, VectorizableMixin):
     name = "floor"
     is_unary = True
     is_vectorizable = True
     output_type = "int"
     input_type = "continuous"
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return np.floor(data)
@@ -80,11 +86,12 @@ class Floor(PandasOperand):
         return np.floor(data)
 
 
-class Residual(PandasOperand):
+class Residual(PandasOperand, VectorizableMixin):
     name = "residual"
     is_unary = True
     is_vectorizable = True
     input_type = "continuous"
+    group_index = 0
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         return data - np.floor(data)

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/autofe/vector.py

@@ -1,20 +1,22 @@
 from typing import List
 import pandas as pd
-from upgini.autofe.operand import PandasOperand
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
 
-class Mean(PandasOperand):
+class Mean(PandasOperand, VectorizableMixin):
     name = "mean"
     output_type = "float"
     is_vector = True
+    group_index = 0
 
     def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
         return pd.DataFrame(data).T.fillna(0).mean(axis=1)
 
 
-class Sum(PandasOperand):
+class Sum(PandasOperand, VectorizableMixin):
     name = "sum"
     is_vector = True
+    group_index = 0
 
     def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
         return pd.DataFrame(data).T.fillna(0).sum(axis=1)

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/features_enricher.py

@@ -70,6 +70,7 @@ from upgini.utils.datetime_utils import (
     DateTimeSearchKeyConverter,
     is_blocked_time_series,
     is_time_series,
+    validate_dates_distribution,
 )
 from upgini.utils.deduplicate_utils import (
     clean_full_duplicates,
@@ -1685,9 +1686,6 @@ class FeaturesEnricher(TransformerMixin):
             df = validated_X.copy()
 
             df[TARGET] = validated_y
-
-            df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
-
             num_samples = _num_samples(df)
             if num_samples > Dataset.FIT_SAMPLE_THRESHOLD:
                 self.logger.info(f"Downsampling from {num_samples} to {Dataset.FIT_SAMPLE_ROWS}")
@@ -1922,7 +1920,6 @@ class FeaturesEnricher(TransformerMixin):
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
             non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
-            # Don't pass 
             if email_converted_to_hem:
                 non_keys_columns.append(email_column)
 
@@ -1944,7 +1941,6 @@ class FeaturesEnricher(TransformerMixin):
             if add_fit_system_record_id:
                 df = self.__add_fit_system_record_id(df, dict(), search_keys)
                 df = df.rename(columns={SYSTEM_RECORD_ID: SORT_ID})
-                non_keys_columns.append(SORT_ID)
 
             columns_for_system_record_id = sorted(list(search_keys.keys()) + (original_features_for_transform or []))
 
@@ -2221,6 +2217,10 @@ class FeaturesEnricher(TransformerMixin):
         self.fit_search_keys = self.search_keys.copy()
         self.fit_search_keys = self.__prepare_search_keys(validated_X, self.fit_search_keys, is_demo_dataset)
 
+        validate_dates_distribution(
+            validated_X, self.fit_search_keys, self.logger, self.bundle, self.warning_counter
+        )
+
         has_date = self._get_date_column(self.fit_search_keys) is not None
         model_task_type = self.model_task_type or define_task(validated_y, has_date, self.logger)
         self._validate_binary_observations(validated_y, model_task_type)
@@ -2883,35 +2883,26 @@ class FeaturesEnricher(TransformerMixin):
 
         # order by date and idempotent order by other keys
         if self.cv not in [CVType.time_series, CVType.blocked_time_series]:
-            sort_exclude_columns = [original_order_name, ORIGINAL_INDEX, EVAL_SET_INDEX, TARGET, "__target"]
             if DateTimeSearchKeyConverter.DATETIME_COL in df.columns:
                 date_column = DateTimeSearchKeyConverter.DATETIME_COL
-                sort_exclude_columns.append(self._get_date_column(search_keys))
             else:
                 date_column = self._get_date_column(search_keys)
             sort_columns = [date_column] if date_column is not None else []
 
-            other_columns = sorted(
+            other_search_keys = sorted(
                 [
-                    c
-                    for c in df.columns
-                    if c not in sort_columns
-                    and c not in sort_exclude_columns
-                    and df[c].nunique() > 1
+                    sk
+                    for sk, key_type in search_keys.items()
+                    if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
+                    and sk in df.columns
+                    and df[sk].nunique() > 1  # don't use constant keys for hash
                 ]
-                # [
-                #     sk
-                #     for sk, key_type in search_keys.items()
-                #     if key_type not in [SearchKey.DATE, SearchKey.DATETIME]
-                #     and sk in df.columns
-                #     and df[sk].nunique() > 1  # don't use constant keys for hash
-                # ]
             )
 
             search_keys_hash = "search_keys_hash"
-            if len(other_columns) > 0:
+            if len(other_search_keys) > 0:
                 sort_columns.append(search_keys_hash)
-                df[search_keys_hash] = pd.util.hash_pandas_object(df[other_columns], index=False)
+                df[search_keys_hash] = pd.util.hash_pandas_object(df[sorted(other_search_keys)], index=False)
 
             df = df.sort_values(by=sort_columns)
 

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/resource_bundle/strings.properties

@@ -111,6 +111,7 @@ x_is_empty=X is empty
 y_is_empty=y is empty
 x_contains_reserved_column_name=Column name {} is reserved. Please rename column and try again
 missing_generate_feature=\nWARNING: Feature {} specified in `generate_features` is not present in input columns: {}
+x_unstable_by_date=\nWARNING: Your training sample is unstable in number of rows per date. It is recommended to redesign the training sample.
     # eval set validation
 unsupported_type_eval_set=Unsupported type of eval_set: {}. It should be list of tuples with two elements: X and y
 eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y
@@ -145,8 +146,7 @@ dataset_too_many_rows_registered=X rows limit for transform is {}. Please sample
 dataset_empty_column_names=Some column names are empty. Add names please
 dataset_full_duplicates=\nWARNING: {:.5f}% of the rows are fully duplicated
 dataset_diff_target_duplicates=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates with different y values. These rows will be deleted as incorrect\nIncorrect row indexes: {}
-dataset_train_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
-dataset_eval_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in eval{} X are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
+dataset_diff_target_duplicates_fintech=\nWARNING: {:.4f}% of rows ({}) in X and eval_set are duplicates, not taking into consideration dates, IP addresses and features from the training set, but have different y values. These rows have been removed to optimize search results.\nRemoved row indexes: {}
 dataset_drop_old_dates=\nWARNING: We don't have data before '2000-01-01' and removed all earlier records from the search dataset
 dataset_all_dates_old=There is empty train dataset after removing data before '2000-01-01'
 dataset_invalid_target_type=Unexpected dtype of target for binary task type: {}. Expected int or bool

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/utils/datetime_utils.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 import re
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import numpy as np
 import pandas as pd
@@ -9,7 +9,9 @@ from dateutil.relativedelta import relativedelta
 from pandas.api.types import is_numeric_dtype, is_period_dtype, is_string_dtype
 
 from upgini.errors import ValidationError
+from upgini.metadata import SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils.warning_counter import WarningCounter
 
 DATE_FORMATS = [
     "%Y-%m-%d",
@@ -225,3 +227,49 @@ def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[st
 
     is_diff_less_than_two_columns = grouped.apply(check_differences)
     return is_diff_less_than_two_columns.all()
+
+
+def validate_dates_distribution(
+    X: pd.DataFrame,
+    search_keys: Dict[str, SearchKey],
+    logger: Optional[logging.Logger] = None,
+    bundle: Optional[ResourceBundle] = None,
+    warning_counter: Optional[WarningCounter] = None,
+):
+    maybe_date_col = None
+    for key, key_type in search_keys.items():
+        if key_type in [SearchKey.DATE, SearchKey.DATETIME]:
+            maybe_date_col = key
+
+    if maybe_date_col is None:
+        for col in X.columns:
+            if col in search_keys:
+                continue
+            try:
+                pd.to_datetime(X[col])
+                maybe_date_col = col
+                break
+            except Exception:
+                pass
+
+    if maybe_date_col is None:
+        return
+
+    dates = pd.to_datetime(X[maybe_date_col]).dt.date
+
+    date_counts = dates.value_counts().sort_index()
+
+    date_counts_1 = date_counts[: round(len(date_counts) / 2)]
+    date_counts_2 = date_counts[round(len(date_counts) / 2) :]
+    ratio = date_counts_2.mean() / date_counts_1.mean()
+
+    if ratio > 1.2 or ratio < 0.8:
+        if warning_counter is not None:
+            warning_counter.increment()
+        if logger is None:
+            logger = logging.getLogger("muted_logger")
+            logger.setLevel("FATAL")
+        bundle = bundle or get_custom_bundle()
+        msg = bundle.get("x_unstable_by_date")
+        print(msg)
+        logger.warning(msg)

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/utils/deduplicate_utils.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Union
 
 import pandas as pd
 
-from upgini.metadata import EVAL_SET_INDEX, SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
+from upgini.metadata import SORT_ID, SYSTEM_RECORD_ID, TARGET, ModelTaskType, SearchKey
 from upgini.resource_bundle import ResourceBundle
 from upgini.utils.datetime_utils import DateTimeSearchKeyConverter
 from upgini.utils.target_utils import define_task
@@ -78,58 +78,20 @@ def remove_fintech_duplicates(
     rows_with_diff_target = grouped_by_personal_cols.filter(has_diff_target_within_60_days)
     if len(rows_with_diff_target) > 0:
         unique_keys_to_delete = rows_with_diff_target[personal_cols].drop_duplicates()
-        if EVAL_SET_INDEX not in df.columns:
-            rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
-            rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
-            perc = len(rows_to_remove) * 100 / len(df)
-            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
-                perc, len(rows_to_remove), rows_to_remove.index.to_list()
-            )
-            if not silent:
-                print(msg)
-            if logger:
-                logger.warning(msg)
-            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-            df = df[~df.index.isin(rows_to_remove.index)]
-            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
-        else:
-            # Indices in train and eval_set can be the same so we remove rows from them separately
-            train = df.query(f"{EVAL_SET_INDEX} == 0")
-            train_rows_to_remove = pd.merge(train.reset_index(), unique_keys_to_delete, on=personal_cols)
-            train_rows_to_remove = train_rows_to_remove.set_index(train.index.name or "index")
-            train_perc = len(train_rows_to_remove) * 100 / len(train)
-            msg = bundle.get("dataset_train_diff_target_duplicates_fintech").format(
-                train_perc, len(train_rows_to_remove), train_rows_to_remove.index.to_list()
-            )
-            if not silent:
-                print(msg)
-            if logger:
-                logger.warning(msg)
-            logger.info(f"Train dataset shape before clean fintech duplicates: {train.shape}")
-            train = train[~train.index.isin(train_rows_to_remove.index)]
-            logger.info(f"Train dataset shape after clean fintech duplicates: {train.shape}")
-
-            evals = [df.query(f"{EVAL_SET_INDEX} == {i}") for i in df[EVAL_SET_INDEX].unique() if i != 0]
-            new_evals = []
-            for i, eval in enumerate(evals):
-                eval_rows_to_remove = pd.merge(eval.reset_index(), unique_keys_to_delete, on=personal_cols)
-                eval_rows_to_remove = eval_rows_to_remove.set_index(eval.index.name or "index")
-                eval_perc = len(eval_rows_to_remove) * 100 / len(eval)
-                msg = bundle.get("dataset_eval_diff_target_duplicates_fintech").format(
-                    eval_perc, len(eval_rows_to_remove), i + 1, eval_rows_to_remove.index.to_list()
-                )
-                if not silent:
-                    print(msg)
-                if logger:
-                    logger.warning(msg)
-                logger.info(f"Eval {i + 1} dataset shape before clean fintech duplicates: {eval.shape}")
-                eval = eval[~eval.index.isin(eval_rows_to_remove.index)]
-                logger.info(f"Eval {i + 1} dataset shape after clean fintech duplicates: {eval.shape}")
-                new_evals.append(eval)
-
-            logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
-            df = pd.concat([train] + new_evals)
-            logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
+        rows_to_remove = pd.merge(df.reset_index(), unique_keys_to_delete, on=personal_cols)
+        rows_to_remove = rows_to_remove.set_index(df.index.name or "index")
+        perc = len(rows_to_remove) * 100 / len(df)
+        msg = bundle.get("dataset_diff_target_duplicates_fintech").format(
+            perc, len(rows_to_remove), rows_to_remove.index.to_list()
+        )
+        if not silent:
+            print(msg)
+        if logger:
+            logger.warning(msg)
+        logger.info(f"Dataset shape before clean fintech duplicates: {df.shape}")
+        df = df[~df.index.isin(rows_to_remove.index)]
+        logger.info(f"Dataset shape after clean fintech duplicates: {df.shape}")
+
     return df
 
 
@@ -139,18 +101,14 @@ def clean_full_duplicates(
     nrows = len(df)
     if nrows == 0:
         return df
-    # Remove full duplicates (exclude system_record_id, sort_id and eval_set_index)
+    # Remove absolute duplicates (exclude system_record_id)
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
     if SORT_ID in unique_columns:
         unique_columns.remove(SORT_ID)
-    if EVAL_SET_INDEX in unique_columns:
-        unique_columns.remove(EVAL_SET_INDEX)
     logger.info(f"Dataset shape before clean duplicates: {df.shape}")
-    # Train segment goes first so if duplicates are found in train and eval set
-    # then we keep unique rows in train segment
-    df = df.drop_duplicates(subset=unique_columns, keep="first")
+    df = df.drop_duplicates(subset=unique_columns)
     logger.info(f"Dataset shape after clean duplicates: {df.shape}")
     nrows_after_full_dedup = len(df)
     share_full_dedup = 100 * (1 - nrows_after_full_dedup / nrows)
@@ -165,7 +123,7 @@ def clean_full_duplicates(
         marked_duplicates = df.duplicated(subset=unique_columns, keep=False)
         if marked_duplicates.sum() > 0:
             dups_indices = df[marked_duplicates].index.to_list()
-            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns, keep=False))
+            nrows_after_tgt_dedup = len(df.drop_duplicates(subset=unique_columns))
             num_dup_rows = nrows_after_full_dedup - nrows_after_tgt_dedup
             share_tgt_dedup = 100 * num_dup_rows / nrows_after_full_dedup
 
@@ -175,7 +133,6 @@ def clean_full_duplicates(
                 print(msg)
             df = df.drop_duplicates(subset=unique_columns, keep=False)
             logger.info(f"Dataset shape after clean invalid target duplicates: {df.shape}")
-
     return df
 
 

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini/utils/target_utils.py

@@ -132,9 +132,7 @@ def balance_undersample(
                 class_value = classes[class_idx]
                 class_count = vc[class_value]
                 sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
-            sampler = RandomUnderSampler(
-                sampling_strategy=sample_strategy, random_state=random_state
-            )
+            sampler = RandomUnderSampler(sampling_strategy=sample_strategy, random_state=random_state)
             X = df[SYSTEM_RECORD_ID]
             X = X.to_frame(SYSTEM_RECORD_ID)
             new_x, _ = sampler.fit_resample(X, target)  # type: ignore
@@ -153,9 +151,7 @@ def balance_undersample(
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
         sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
-        sampled_majority_class = majority_class.sample(
-            n=sample_size, random_state=random_state
-        )
+        sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))

{upgini-1.1.263a1 → upgini-1.1.264/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.263a1
+Version: 1.1.264
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.263a1 → upgini-1.1.264}/src/upgini.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ src/upgini/ads_management/ads_manager.py
 src/upgini/autofe/__init__.py
 src/upgini/autofe/all_operands.py
 src/upgini/autofe/binary.py
+src/upgini/autofe/date.py
 src/upgini/autofe/feature.py
 src/upgini/autofe/groupby.py
 src/upgini/autofe/operand.py
@@ -64,6 +65,7 @@ src/upgini/utils/sklearn_ext.py
 src/upgini/utils/target_utils.py
 src/upgini/utils/track_info.py
 src/upgini/utils/warning_counter.py
+tests/test_autofe_operands.py
 tests/test_binary_dataset.py
 tests/test_blocked_time_series.py
 tests/test_categorical_dataset.py

upgini-1.1.264/tests/test_autofe_operands.py

@@ -0,0 +1,27 @@
+import pandas as pd
+from upgini.autofe.date import DateDiff, DateDiffType2
+
+from datetime import datetime
+from pandas.testing import assert_series_equal
+
+
+def test_date_diff():
+    df = pd.DataFrame(
+        [[datetime(1993, 12, 10), datetime(2022, 10, 10)], [datetime(2023, 10, 10), datetime(2022, 10, 10)]],
+        columns=["date1", "date2"],
+    )
+
+    operand = DateDiff()
+    expected_result = pd.Series([10531, None])
+    assert_series_equal(operand.calculate_binary(df.date2, df.date1), expected_result)
+
+
+def test_date_diff_future():
+    df = pd.DataFrame(
+        [[datetime(1993, 12, 10), datetime(2022, 10, 10)], [datetime(1993, 4, 10), datetime(2022, 10, 10)]],
+        columns=["date1", "date2"],
+    )
+
+    operand = DateDiffType2()
+    expected_result = pd.Series([61.0, 182.0])
+    assert_series_equal(operand.calculate_binary(df.date2, df.date1), expected_result)

{upgini-1.1.263a1 → upgini-1.1.264}/tests/test_datetime_utils.py

@@ -1,7 +1,13 @@
-import pandas as pd
 import numpy as np
+import pandas as pd
 
-from upgini.utils.datetime_utils import is_blocked_time_series, is_time_series
+from upgini.metadata import SearchKey
+from upgini.utils.datetime_utils import (
+    is_blocked_time_series,
+    is_time_series,
+    validate_dates_distribution,
+)
+from upgini.utils.warning_counter import WarningCounter
 
 pd.set_option("mode.chained_assignment", "raise")
 
@@ -183,3 +189,25 @@ def test_multivariate_time_series():
     assert not is_blocked_time_series(df, "date", ["date"])
 
     assert is_blocked_time_series(df, "date", ["date", "feature3"])
+
+
+def test_validate_dates_distribution():
+    df = pd.DataFrame({"date": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {}, warning_counter=warning_counter)
+    assert warning_counter.has_warnings()
+
+    df = pd.DataFrame({"date": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {}, warning_counter=warning_counter)
+    assert not warning_counter.has_warnings()
+
+    df = pd.DataFrame(
+        {
+            "date2": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
+            "date1": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
+        }
+    )
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {"date1": SearchKey.DATE}, warning_counter=warning_counter)
+    assert warning_counter.has_warnings()

{upgini-1.1.263a1 → upgini-1.1.264}/tests/test_features_enricher.py

@@ -2164,6 +2164,8 @@ def test_idempotent_order_with_imbalanced_dataset(requests_mock: Mocker):
 
             actual_result_df = result_wrapper.df.sort_values(by="system_record_id").reset_index(drop=True)
             # actual_result_df.to_parquet(expected_result_path)
+            actual_result_df["phone_num_a54a33"] = actual_result_df["phone_num_a54a33"].astype("Int64")
+            actual_result_df["rep_date_f5d6bb"] = actual_result_df["rep_date_f5d6bb"].astype("Int64")
             assert_frame_equal(actual_result_df, expected_result_df)
 
         for i in range(5):