upgini 1.1.264a1__tar.gz → 1.1.265__tar.gz

{upgini-1.1.264a1/src/upgini.egg-info → upgini-1.1.265}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.264a1
+Version: 1.1.265
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.264a1 → upgini-1.1.265}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.264a1"
+version = "1.1.265"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.264a1 → upgini-1.1.265}/src/upgini/features_enricher.py

@@ -70,6 +70,7 @@ from upgini.utils.datetime_utils import (
     DateTimeSearchKeyConverter,
     is_blocked_time_series,
     is_time_series,
+    validate_dates_distribution,
 )
 from upgini.utils.deduplicate_utils import (
     clean_full_duplicates,
@@ -1922,7 +1923,7 @@ class FeaturesEnricher(TransformerMixin):
 
             meaning_types = {col: key.value for col, key in search_keys.items()}
             non_keys_columns = [column for column in df.columns if column not in search_keys.keys()]
-            # Don't pass 
+
             if email_converted_to_hem:
                 non_keys_columns.append(email_column)
 
@@ -2221,6 +2222,10 @@ class FeaturesEnricher(TransformerMixin):
         self.fit_search_keys = self.search_keys.copy()
         self.fit_search_keys = self.__prepare_search_keys(validated_X, self.fit_search_keys, is_demo_dataset)
 
+        validate_dates_distribution(
+            validated_X, self.fit_search_keys, self.logger, self.bundle, self.warning_counter
+        )
+
         has_date = self._get_date_column(self.fit_search_keys) is not None
         model_task_type = self.model_task_type or define_task(validated_y, has_date, self.logger)
         self._validate_binary_observations(validated_y, model_task_type)

{upgini-1.1.264a1 → upgini-1.1.265}/src/upgini/resource_bundle/strings.properties

@@ -111,6 +111,7 @@ x_is_empty=X is empty
 y_is_empty=y is empty
 x_contains_reserved_column_name=Column name {} is reserved. Please rename column and try again
 missing_generate_feature=\nWARNING: Feature {} specified in `generate_features` is not present in input columns: {}
+x_unstable_by_date=\nWARNING: Your training sample is unstable in number of rows per date. It is recommended to redesign the training sample.
     # eval set validation
 unsupported_type_eval_set=Unsupported type of eval_set: {}. It should be list of tuples with two elements: X and y
 eval_set_invalid_tuple_size=eval_set contains a tuple of size {}. It should contain only pairs of X and y

{upgini-1.1.264a1 → upgini-1.1.265}/src/upgini/utils/datetime_utils.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 import re
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import numpy as np
 import pandas as pd
@@ -9,7 +9,9 @@ from dateutil.relativedelta import relativedelta
 from pandas.api.types import is_numeric_dtype, is_period_dtype, is_string_dtype
 
 from upgini.errors import ValidationError
+from upgini.metadata import SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils.warning_counter import WarningCounter
 
 DATE_FORMATS = [
     "%Y-%m-%d",
@@ -225,3 +227,49 @@ def is_blocked_time_series(df: pd.DataFrame, date_col: str, search_keys: List[st
 
     is_diff_less_than_two_columns = grouped.apply(check_differences)
     return is_diff_less_than_two_columns.all()
+
+
+def validate_dates_distribution(
+    X: pd.DataFrame,
+    search_keys: Dict[str, SearchKey],
+    logger: Optional[logging.Logger] = None,
+    bundle: Optional[ResourceBundle] = None,
+    warning_counter: Optional[WarningCounter] = None,
+):
+    maybe_date_col = None
+    for key, key_type in search_keys.items():
+        if key_type in [SearchKey.DATE, SearchKey.DATETIME]:
+            maybe_date_col = key
+
+    if maybe_date_col is None:
+        for col in X.columns:
+            if col in search_keys:
+                continue
+            try:
+                pd.to_datetime(X[col])
+                maybe_date_col = col
+                break
+            except Exception:
+                pass
+
+    if maybe_date_col is None:
+        return
+
+    dates = pd.to_datetime(X[maybe_date_col]).dt.date
+
+    date_counts = dates.value_counts().sort_index()
+
+    date_counts_1 = date_counts[: round(len(date_counts) / 2)]
+    date_counts_2 = date_counts[round(len(date_counts) / 2) :]
+    ratio = date_counts_2.mean() / date_counts_1.mean()
+
+    if ratio > 1.2 or ratio < 0.8:
+        if warning_counter is not None:
+            warning_counter.increment()
+        if logger is None:
+            logger = logging.getLogger("muted_logger")
+            logger.setLevel("FATAL")
+        bundle = bundle or get_custom_bundle()
+        msg = bundle.get("x_unstable_by_date")
+        print(msg)
+        logger.warning(msg)

{upgini-1.1.264a1 → upgini-1.1.265}/src/upgini/utils/target_utils.py

@@ -132,9 +132,7 @@ def balance_undersample(
                 class_value = classes[class_idx]
                 class_count = vc[class_value]
                 sample_strategy[class_value] = min(class_count, quantile25_class_cnt * multiclass_bootstrap_loops)
-            sampler = RandomUnderSampler(
-                sampling_strategy=sample_strategy, random_state=random_state
-            )
+            sampler = RandomUnderSampler(sampling_strategy=sample_strategy, random_state=random_state)
             X = df[SYSTEM_RECORD_ID]
             X = X.to_frame(SYSTEM_RECORD_ID)
             new_x, _ = sampler.fit_resample(X, target)  # type: ignore
@@ -153,9 +151,7 @@ def balance_undersample(
         minority_class = df[df[target_column] == min_class_value]
         majority_class = df[df[target_column] != min_class_value]
         sample_size = min(len(majority_class), min_sample_threshold - min_class_count)
-        sampled_majority_class = majority_class.sample(
-            n=sample_size, random_state=random_state
-        )
+        sampled_majority_class = majority_class.sample(n=sample_size, random_state=random_state)
         resampled_data = df[
             (df[SYSTEM_RECORD_ID].isin(minority_class[SYSTEM_RECORD_ID]))
             | (df[SYSTEM_RECORD_ID].isin(sampled_majority_class[SYSTEM_RECORD_ID]))

{upgini-1.1.264a1 → upgini-1.1.265/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.264a1
+Version: 1.1.265
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.264a1 → upgini-1.1.265}/tests/test_datetime_utils.py

@@ -1,7 +1,13 @@
-import pandas as pd
 import numpy as np
+import pandas as pd
 
-from upgini.utils.datetime_utils import is_blocked_time_series, is_time_series
+from upgini.metadata import SearchKey
+from upgini.utils.datetime_utils import (
+    is_blocked_time_series,
+    is_time_series,
+    validate_dates_distribution,
+)
+from upgini.utils.warning_counter import WarningCounter
 
 pd.set_option("mode.chained_assignment", "raise")
 
@@ -183,3 +189,25 @@ def test_multivariate_time_series():
     assert not is_blocked_time_series(df, "date", ["date"])
 
     assert is_blocked_time_series(df, "date", ["date", "feature3"])
+
+
+def test_validate_dates_distribution():
+    df = pd.DataFrame({"date": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {}, warning_counter=warning_counter)
+    assert warning_counter.has_warnings()
+
+    df = pd.DataFrame({"date": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40})
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {}, warning_counter=warning_counter)
+    assert not warning_counter.has_warnings()
+
+    df = pd.DataFrame(
+        {
+            "date2": ["2020-05-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
+            "date1": ["2020-01-01"] * 10 + ["2020-02-01"] * 20 + ["2020-03-01"] * 30 + ["2020-04-01"] * 40,
+        }
+    )
+    warning_counter = WarningCounter()
+    validate_dates_distribution(df, {"date1": SearchKey.DATE}, warning_counter=warning_counter)
+    assert warning_counter.has_warnings()

{upgini-1.1.264a1 → upgini-1.1.265}/tests/test_features_enricher.py

@@ -2164,6 +2164,8 @@ def test_idempotent_order_with_imbalanced_dataset(requests_mock: Mocker):
 
             actual_result_df = result_wrapper.df.sort_values(by="system_record_id").reset_index(drop=True)
             # actual_result_df.to_parquet(expected_result_path)
+            actual_result_df["phone_num_a54a33"] = actual_result_df["phone_num_a54a33"].astype("Int64")
+            actual_result_df["rep_date_f5d6bb"] = actual_result_df["rep_date_f5d6bb"].astype("Int64")
             assert_frame_equal(actual_result_df, expected_result_df)
 
         for i in range(5):