upgini 1.1.269__tar.gz → 1.1.273__tar.gz

{upgini-1.1.269/src/upgini.egg-info → upgini-1.1.273}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.269
+Version: 1.1.273
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers
@@ -26,19 +26,6 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.8,<3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: python-dateutil>=2.8.0
-Requires-Dist: requests>=2.8.0
-Requires-Dist: pandas<2.0.0,>=1.1.0
-Requires-Dist: numpy>=1.19.0
-Requires-Dist: scikit-learn>=1.3.0
-Requires-Dist: pydantic<2.0.0,>=1.8.2
-Requires-Dist: fastparquet>=0.8.1
-Requires-Dist: python-json-logger>=2.0.2
-Requires-Dist: catboost>=1.0.3
-Requires-Dist: lightgbm>=3.3.2
-Requires-Dist: pyjwt>=2.8.0
-Requires-Dist: xhtml2pdf==0.2.11
-Requires-Dist: ipywidgets>=8.1.0
 
 
 <!-- <h2 align="center"> <a href="https://upgini.com/">Upgini</a> : low-code feature search and enrichment library for machine learning </h2> -->

{upgini-1.1.269 → upgini-1.1.273}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.269"
+version = "1.1.273"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.269 → upgini-1.1.273}/src/upgini/autofe/all_operands.py

@@ -1,5 +1,5 @@
 from typing import Dict
-from upgini.autofe.date import DateDiff, DateDiffType2
+from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded
 from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
 from upgini.autofe.unary import Abs, Log, Residual, Sqrt, Square, Sigmoid, Floor, Freq
@@ -38,6 +38,16 @@ ALL_OPERANDS: Dict[str, Operand] = {
         Sim(),
         DateDiff(),
         DateDiffType2(),
+        DateListDiff(aggregation="min"),
+        DateListDiff(aggregation="max"),
+        DateListDiff(aggregation="mean"),
+        DateListDiff(aggregation="nunique"),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=0, upper_bound=18),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=18, upper_bound=23),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=23, upper_bound=30),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=30, upper_bound=45),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=45, upper_bound=60),
+        DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=60),
     ]
 }
 

upgini-1.1.273/src/upgini/autofe/date.py

@@ -0,0 +1,110 @@
+from typing import Any, Optional, Union
+import numpy as np
+import pandas as pd
+from pydantic import BaseModel
+
+from upgini.autofe.operand import PandasOperand
+
+
+class DateDiffMixin(BaseModel):
+    diff_unit: str = "D"
+    left_unit: Optional[str] = None
+    right_unit: Optional[str] = None
+
+    def _convert_to_date(
+        self, x: Union[pd.DataFrame, pd.Series], unit: Optional[str]
+    ) -> Union[pd.DataFrame, pd.Series]:
+        if isinstance(x, pd.DataFrame):
+            return x.apply(lambda y: self._convert_to_date(y, unit), axis=1)
+
+        return pd.to_datetime(x, unit=unit)
+
+
+class DateDiff(PandasOperand, DateDiffMixin):
+    name = "date_diff"
+    is_binary = True
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        left = self._convert_to_date(left, self.left_unit)
+        right = self._convert_to_date(right, self.right_unit)
+        return self.__replace_negative((left - right) / np.timedelta64(1, self.diff_unit))
+
+    def __replace_negative(self, x: Union[pd.DataFrame, pd.Series]):
+        x[x < 0] = None
+        return x
+
+
+class DateDiffType2(PandasOperand, DateDiffMixin):
+    name = "date_diff_type2"
+    is_binary = True
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        left = self._convert_to_date(left, self.left_unit)
+        right = self._convert_to_date(right, self.right_unit)
+        future = right + (left.dt.year - right.dt.year).apply(
+            lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
+        )
+        before = future[future < left]
+        future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
+        diff = (future - left) / np.timedelta64(1, self.diff_unit)
+
+        return diff
+
+
+_ext_aggregations = {"nunique": (lambda x: len(np.unique(x)), 0), "count": (len, 0)}
+
+
+class DateListDiff(PandasOperand, DateDiffMixin):
+    is_binary = True
+    has_symmetry_importance = True
+    aggregation: str
+
+    def __init__(self, **data: Any) -> None:
+        if "name" not in data:
+            data["name"] = f"date_diff_{data.get('aggregation')}"
+        super().__init__(**data)
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        left = self._convert_to_date(left, self.left_unit)
+        right = right.apply(lambda x: pd.arrays.DatetimeArray(self._convert_to_date(x, self.right_unit)))
+
+        return pd.Series(left - right.values).apply(lambda x: self._agg(self._diff(x)))
+
+    def _diff(self, x):
+        x = x / np.timedelta64(1, self.diff_unit)
+        return x[x > 0]
+
+    def _agg(self, x):
+        method = getattr(np, self.aggregation, None)
+        default = np.nan
+        if method is None and self.aggregation in _ext_aggregations:
+            method, default = _ext_aggregations[self.aggregation]
+        elif not callable(method):
+            raise ValueError(f"Unsupported aggregation: {self.aggregation}")
+
+        return method(x) if len(x) > 0 else default
+
+
+class DateListDiffBounded(DateListDiff):
+    lower_bound: Optional[int]
+    upper_bound: Optional[int]
+
+    def __init__(self, **data: Any) -> None:
+        if "name" not in data:
+            lower_bound = data.get("lower_bound")
+            upper_bound = data.get("upper_bound")
+            components = [
+                "date_diff",
+                data.get("diff_unit"),
+                str(lower_bound if lower_bound is not None else "minusinf"),
+                str(upper_bound if upper_bound is not None else "plusinf"),
+            ]
+            components.append(data.get("aggregation"))
+            data["name"] = "_".join(components)
+        super().__init__(**data)
+
+    def _agg(self, x):
+        x = x[(x >= (self.lower_bound or -np.inf)) & (x < (self.upper_bound or np.inf))]
+        return super()._agg(x)

{upgini-1.1.269 → upgini-1.1.273}/src/upgini/data_source/data_source_publisher.py

@@ -72,8 +72,8 @@ class DataSourcePublisher:
                     )
                 if search_keys is None or len(search_keys) == 0:
                     raise ValidationError("Empty search keys")
-                if SearchKey.DATE in search_keys.values() and date_format is None:
-                    raise ValidationError("date_format is required for DATE search key")
+                # if SearchKey.DATE in search_keys.values() and date_format is None:
+                #     raise ValidationError("date_format is required for DATE search key")
                 if update_frequency not in self.ACCEPTABLE_UPDATE_FREQUENCIES:
                     raise ValidationError(
                         f"Invalid update frequency: {update_frequency}. "
@@ -85,11 +85,19 @@ class DataSourcePublisher:
                     or set(search_keys.values()) == {SearchKey.MSISDN_RANGE_FROM, SearchKey.MSISDN_RANGE_TO}
                 ) and sort_column is None:
                     raise ValidationError("Sort column is required for passed search keys")
+                if (
+                    set(search_keys.values()) == {SearchKey.PHONE, SearchKey.DATE}
+                    and snapshot_frequency_days is None
+                    and join_date_abs_limit_days is None
+                ):
+                    raise ValidationError(
+                        "With MSISDN and DATE keys one of the snapshot_frequency_days or"
+                        " join_date_abs_limit_days parameters is required"
+                    )
 
                 request = {
                     "dataTableUri": data_table_uri,
                     "searchKeys": {k: v.value.value for k, v in search_keys.items()},
-                    "dateFormat": date_format,
                     "excludeColumns": exclude_columns,
                     "hashFeatureNames": str(hash_feature_names).lower(),
                     "snapshotFrequencyDays": snapshot_frequency_days,
@@ -98,6 +106,8 @@ class DataSourcePublisher:
                     "featuresForEmbeddings": features_for_embeddings,
                     "forceGeneration": str(_force_generation).lower(),
                 }
+                if date_format is not None:
+                    request["dateFormat"] = date_format
                 if secondary_search_keys is not None:
                     request["secondarySearchKeys"] = {k: v.value.value for k, v in secondary_search_keys.items()}
                 if sort_column is not None:

{upgini-1.1.269 → upgini-1.1.273}/src/upgini/features_enricher.py

@@ -424,7 +424,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.X = X
                 self.y = y
                 self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
-                self.dump_input(trace_id, X, y, eval_set)
+                self.dump_input(trace_id, X, y, self.eval_set)
                 self.__inner_fit(
                     trace_id,
                     X,
@@ -563,7 +563,7 @@ class FeaturesEnricher(TransformerMixin):
                 self.X = X
                 self.y = y
                 self.eval_set = self._check_eval_set(eval_set, X, self.bundle)
-                self.dump_input(trace_id, X, y, eval_set)
+                self.dump_input(trace_id, X, y, self.eval_set)
 
                 if _num_samples(drop_duplicates(X)) > Dataset.MAX_ROWS:
                     raise ValidationError(self.bundle.get("dataset_too_many_rows_registered").format(Dataset.MAX_ROWS))
@@ -823,12 +823,16 @@ class FeaturesEnricher(TransformerMixin):
                 print(msg)
 
             self.__validate_search_keys(self.search_keys, self.search_id)
+            effective_X = X if X is not None else self.X
+            effective_y = y if y is not None else self.y
+            effective_eval_set = eval_set if eval_set is not None else self.eval_set
+            effective_eval_set = self._check_eval_set(effective_eval_set, effective_X, self.bundle)
 
             try:
                 self.__log_debug_information(
-                    X if X is not None else self.X,
-                    y if y is not None else self.y,
-                    eval_set if eval_set is not None else self.eval_set,
+                    effective_X,
+                    effective_y,
+                    effective_eval_set,
                     exclude_features_sources=exclude_features_sources,
                     cv=cv if cv is not None else self.cv,
                     importance_threshold=importance_threshold,
@@ -842,17 +846,14 @@ class FeaturesEnricher(TransformerMixin):
                     self._search_task is None
                     or self._search_task.provider_metadata_v2 is None
                     or len(self._search_task.provider_metadata_v2) == 0
-                    or (self.X is None and X is None)
-                    or (self.y is None and y is None)
+                    or effective_X is None
+                    or effective_y is None
                 ):
                     raise ValidationError(self.bundle.get("metrics_unfitted_enricher"))
 
                 if X is not None and y is None:
                     raise ValidationError("X passed without y")
 
-                effective_X = X if X is not None else self.X
-                effective_eval_set = eval_set if eval_set is not None else self.eval_set
-
                 validate_scoring_argument(scoring)
 
                 self._validate_baseline_score(effective_X, effective_eval_set)
@@ -872,8 +873,7 @@ class FeaturesEnricher(TransformerMixin):
                 ):
                     cat_features = estimator.get_param("cat_features")
                     if len(cat_features) > 0 and isinstance(cat_features[0], int):
-                        effectiveX = X or self.X
-                        cat_features = [effectiveX.columns[i] for i in cat_features]
+                        cat_features = [effective_X.columns[i] for i in cat_features]
                         for cat_feature in cat_features:
                             if cat_feature in self.search_keys:
                                 if self.search_keys[cat_feature] in [SearchKey.COUNTRY, SearchKey.POSTAL_CODE]:
@@ -883,9 +883,9 @@ class FeaturesEnricher(TransformerMixin):
 
                 prepared_data = self._prepare_data_for_metrics(
                     trace_id=trace_id,
-                    X=X,
-                    y=y,
-                    eval_set=eval_set,
+                    X=effective_X,
+                    y=effective_y,
+                    eval_set=effective_eval_set,
                     exclude_features_sources=exclude_features_sources,
                     importance_threshold=importance_threshold,
                     max_features=max_features,
@@ -995,8 +995,6 @@ class FeaturesEnricher(TransformerMixin):
                         enriched_metric = None
                         uplift = None
 
-                    effective_X = X if X is not None else self.X
-                    effective_y = y if y is not None else self.y
                     train_metrics = {
                         self.bundle.get("quality_metrics_segment_header"): self.bundle.get(
                             "quality_metrics_train_segment"
@@ -2823,6 +2821,7 @@ class FeaturesEnricher(TransformerMixin):
 
             maybe_date_col = self._get_date_column(self.search_keys)
             if X is not None and maybe_date_col is not None and maybe_date_col in X.columns:
+                # TODO cast date column to single dtype
                 min_date = X[maybe_date_col].min()
                 max_date = X[maybe_date_col].max()
                 self.logger.info(f"Dates interval is ({min_date}, {max_date})")

{upgini-1.1.269 → upgini-1.1.273}/src/upgini/resource_bundle/strings.properties

@@ -203,7 +203,7 @@ phone_detected_not_registered=\nWARNING: Phone numbers detected in column `{}`.
 target_type_detected=\nDetected task type: {}\n
 # all_ok_community_invite=Chat with us in Slack community:
 all_ok_community_invite=❓ Support request 
-too_small_for_metrics=Your train dataset contains less than 500 rows. For such dataset Upgini will not calculate accuracy metrics. Please increase the number of rows in the training dataset to calculate accuracy metrics
+too_small_for_metrics=Your train dataset or one of eval datasets contains less than 500 rows. For such dataset Upgini will not calculate accuracy metrics. Please increase the number of rows in the training dataset to calculate accuracy metrics
 imbalance_multiclass=Class {0} is on 25% quantile of classes distribution ({1} records in train dataset). \nDownsample classes with records more than {1}.
 loss_selection_info=Using loss `{}` for feature selection
 loss_calc_metrics_info=Using loss `{}` for metrics calculation with default estimator

{upgini-1.1.269 → upgini-1.1.273/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.269
+Version: 1.1.273
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers
@@ -26,19 +26,6 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.8,<3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: python-dateutil>=2.8.0
-Requires-Dist: requests>=2.8.0
-Requires-Dist: pandas<2.0.0,>=1.1.0
-Requires-Dist: numpy>=1.19.0
-Requires-Dist: scikit-learn>=1.3.0
-Requires-Dist: pydantic<2.0.0,>=1.8.2
-Requires-Dist: fastparquet>=0.8.1
-Requires-Dist: python-json-logger>=2.0.2
-Requires-Dist: catboost>=1.0.3
-Requires-Dist: lightgbm>=3.3.2
-Requires-Dist: pyjwt>=2.8.0
-Requires-Dist: xhtml2pdf==0.2.11
-Requires-Dist: ipywidgets>=8.1.0
 
 
 <!-- <h2 align="center"> <a href="https://upgini.com/">Upgini</a> : low-code feature search and enrichment library for machine learning </h2> -->

upgini-1.1.273/tests/test_autofe_operands.py

@@ -0,0 +1,93 @@
+import pandas as pd
+from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded
+
+from datetime import datetime
+from pandas.testing import assert_series_equal
+
+
+def test_date_diff():
+    df = pd.DataFrame(
+        [
+            ["2022-10-10", pd.to_datetime("1993-12-10").timestamp()],
+            ["2022-10-10", pd.to_datetime("2023-10-10").timestamp()],
+        ],
+        columns=["date1", "date2"],
+    )
+
+    operand = DateDiff(right_unit="s")
+    expected_result = pd.Series([10531, None])
+    assert_series_equal(operand.calculate_binary(df.date1, df.date2), expected_result)
+
+
+def test_date_diff_type2():
+    df = pd.DataFrame(
+        [
+            [pd.to_datetime("2022-10-10").timestamp(), datetime(1993, 12, 10)],
+            [pd.to_datetime("2022-10-10").timestamp(), datetime(1993, 4, 10)],
+        ],
+        columns=["date1", "date2"],
+    )
+
+    operand = DateDiffType2(left_unit="s")
+    expected_result = pd.Series([61.0, 182.0])
+    assert_series_equal(operand.calculate_binary(df.date1, df.date2), expected_result)
+
+
+def test_date_diff_list():
+    df = pd.DataFrame(
+        [
+            ["2022-10-10", ["1993-12-10", "1993-12-11"]],
+            ["2022-10-10", ["1993-12-10", "1993-12-10"]],
+            ["2022-10-10", ["2023-10-10"]],
+            ["2022-10-10", []],
+        ],
+        columns=["date1", "date2"],
+    )
+
+    def check(aggregation, expected_name, expected_values):
+        operand = DateListDiff(aggregation=aggregation)
+        assert operand.name == expected_name
+        assert_series_equal(operand.calculate_binary(df.date1, df.date2).rename(None), expected_values)
+
+    check(aggregation="min", expected_name="date_diff_min", expected_values=pd.Series([10530, 10531, None, None]))
+    check(aggregation="max", expected_name="date_diff_max", expected_values=pd.Series([10531, 10531, None, None]))
+    check(aggregation="mean", expected_name="date_diff_mean", expected_values=pd.Series([10530.5, 10531, None, None]))
+    check(aggregation="nunique", expected_name="date_diff_nunique", expected_values=pd.Series([2, 1, 0, 0]))
+
+
+def test_date_diff_list_bounded():
+    df = pd.DataFrame(
+        [
+            ["2022-10-10", ["2013-12-10", "2013-12-11", "1999-12-11"]],
+            [
+                "2022-10-10",
+                [
+                    "2013-12-10",
+                    "2003-12-11",
+                    "1999-12-11",
+                    "1993-12-11",
+                    "1983-12-11",
+                    "1973-12-11",
+                    "1959-12-11",
+                ],
+            ],
+            ["2022-10-10", ["2003-12-10", "2003-12-10"]],
+            ["2022-10-10", ["2023-10-10", "1993-12-10"]],
+            ["2022-10-10", []],
+        ],
+        columns=["date1", "date2"],
+    )
+
+    def check_num_by_years(lower_bound, upper_bound, expected_name, expected_values):
+        operand = DateListDiffBounded(
+            diff_unit="Y", aggregation="count", lower_bound=lower_bound, upper_bound=upper_bound
+        )
+        assert operand.name == expected_name
+        assert_series_equal(operand.calculate_binary(df.date1, df.date2).rename(None), expected_values)
+
+    check_num_by_years(0, 18, "date_diff_Y_0_18_count", pd.Series([2, 1, 0, 0, 0]))
+    check_num_by_years(18, 23, "date_diff_Y_18_23_count", pd.Series([1, 2, 2, 0, 0]))
+    check_num_by_years(23, 30, "date_diff_Y_23_30_count", pd.Series([0, 1, 0, 1, 0]))
+    check_num_by_years(30, 45, "date_diff_Y_30_45_count", pd.Series([0, 1, 0, 0, 0]))
+    check_num_by_years(45, 60, "date_diff_Y_45_60_count", pd.Series([0, 1, 0, 0, 0]))
+    check_num_by_years(60, None, "date_diff_Y_60_plusinf_count", pd.Series([0, 1, 0, 0, 0]))

{upgini-1.1.269 → upgini-1.1.273}/tests/test_metrics.py

@@ -241,6 +241,7 @@ def test_demo_metrics(requests_mock: Mocker):
 
     enricher.X = x_sampled.drop(columns="system_record_id")
     enricher.y = y_sampled
+    enricher.eval_set = []
 
     enricher._FeaturesEnricher__cached_sampled_datasets = (
         x_sampled,

upgini-1.1.269/src/upgini/autofe/date.py

@@ -1,53 +0,0 @@
-from typing import Optional, Union
-import numpy as np
-import pandas as pd
-
-from upgini.autofe.operand import PandasOperand
-
-
-class DateDiffMixin:
-    diff_unit: str = "D"
-    left_unit: Optional[str] = None
-    right_unit: Optional[str] = None
-
-    def _convert_to_date(
-        self, x: Union[pd.DataFrame, pd.Series], unit: Optional[str]
-    ) -> Union[pd.DataFrame, pd.Series]:
-        if isinstance(x, pd.DataFrame):
-            return x.apply(lambda y: self._convert_to_date(y, unit), axis=1)
-
-        return pd.to_datetime(x, unit=unit)
-
-
-class DateDiff(PandasOperand, DateDiffMixin):
-    name = "date_diff"
-    is_binary = True
-    has_symmetry_importance = True
-
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        left = self._convert_to_date(left, self.left_unit)
-        right = self._convert_to_date(right, self.right_unit)
-        return self.__replace_negative((left - right) / np.timedelta64(1, self.diff_unit))
-
-    def __replace_negative(self, x: Union[pd.DataFrame, pd.Series]):
-        x[x < 0] = None
-        return x
-
-
-class DateDiffType2(PandasOperand, DateDiffMixin):
-    name = "date_diff_type2"
-    is_binary = True
-    has_symmetry_importance = True
-    is_vectorizable = False
-
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        left = self._convert_to_date(left, self.left_unit)
-        right = self._convert_to_date(right, self.right_unit)
-        future = right + (left.dt.year - right.dt.year).apply(
-            lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
-        )
-        before = future[future < left]
-        future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
-        diff = (future - left) / np.timedelta64(1, self.diff_unit)
-
-        return diff

upgini-1.1.269/tests/test_autofe_operands.py

@@ -1,27 +0,0 @@
-import pandas as pd
-from upgini.autofe.date import DateDiff, DateDiffType2
-
-from datetime import datetime
-from pandas.testing import assert_series_equal
-
-
-def test_date_diff():
-    df = pd.DataFrame(
-        [[datetime(1993, 12, 10), datetime(2022, 10, 10)], [datetime(2023, 10, 10), datetime(2022, 10, 10)]],
-        columns=["date1", "date2"],
-    )
-
-    operand = DateDiff()
-    expected_result = pd.Series([10531, None])
-    assert_series_equal(operand.calculate_binary(df.date2, df.date1), expected_result)
-
-
-def test_date_diff_future():
-    df = pd.DataFrame(
-        [[datetime(1993, 12, 10), datetime(2022, 10, 10)], [datetime(1993, 4, 10), datetime(2022, 10, 10)]],
-        columns=["date1", "date2"],
-    )
-
-    operand = DateDiffType2()
-    expected_result = pd.Series([61.0, 182.0])
-    assert_series_equal(operand.calculate_binary(df.date2, df.date1), expected_result)