upgini 1.1.280a3418.post10__tar.gz → 1.1.280.dev1__tar.gz

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.280a3418.post10
+Version: 1.1.280.dev1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/pyproject.toml

@@ -72,23 +72,36 @@ packages = [
 type = "virtual"
 python = "3.10"
 
-[tool.hatch.envs.test]
-dependencies = [
-  "coverage[toml]",
-  "pytest",
-  "pytest-cov",
-  "requests-mock",
-]
-
 [tool.hatch.envs.test.scripts]
-cov = 'pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=upgini --cov=tests {args}'
+cov = 'pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=upgini --cov=tests'
 format = "black {args}"
 lint = "ruff check {args}"
-
 test_binary = 'pytest -s -vv tests/test_binary_dataset.py'
 
+[[tool.hatch.envs.test.matrix]]
+python = ["3.8"]
+pandas = ["1.1.0"]
+
 [[tool.hatch.envs.test.matrix]]
 python = ["3.8", "3.9", "3.10"]
+pandas = ["1.2.0", "1.3.0", "1.4.0", "1.5.0", "2.0.0"]
+
+[[tool.hatch.envs.test.matrix]]
+python = ["3.9", "3.10"]
+pandas = ["2.1.0", "2.2.0"]
+
+# from versions: 0.1, 0.2, 0.3.0, 0.4.0, 0.4.1, 0.4.2, 0.4.3, 0.5.0, 0.6.0, 0.6.1, 0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.8.0, 0.8.1, 0.9.0, 0.9.1, 0.10.0, 0.10.1, 0.11.0, 0.12.0, 0.13.0, 0.13.1, 0.14.0, 0.14.1, 0.15.0, 0.15.1, 0.15.2, 0.16.0, 0.16.1, 0.16.2, 0.17.0, 0.17.1, 0.18.0, 0.18.1, 0.19.0, 0.19.1, 0.19.2, 0.20.0, 0.20.1, 0.20.2, 0.20.3, 0.21.0, 0.21.1, 0.22.0, 0.23.0, 0.23.1, 0.23.2, 0.23.3, 0.23.4, 0.24.0, 0.24.1, 0.24.2, 0.25.0, 0.25.1, 0.25.2, 0.25.3, 1.0.0, 1.0.1, 1.0.2, 1.0.3, 1.0.4, 1.0.5, 1.1.0, 1.1.1, 1.1.2, 1.1.3, 1.1.4, 1.1.5, 1.2.0, 1.2.1, 1.2.2, 1.2.3, 1.2.4, 1.2.5, 1.3.0, 1.3.1, 1.3.2, 1.3.3, 1.3.4, 1.3.5, 1.4.0rc0, 1.4.0, 1.4.1, 1.4.2, 1.4.3, 1.4.4, 1.5.0rc0, 1.5.0, 1.5.1, 1.5.2, 1.5.3, 2.0.0rc0, 2.0.0rc1, 2.0.0, 2.0.1, 2.0.2, 2.0.3
+
+[tool.hatch.envs.test]
+dependencies = [
+  "coverage[toml]",
+  "pytest",
+  "pytest-cov",
+#  "pytest-timeout",
+  "requests-mock",
+  "pytest-datafiles",
+  "pandas~={matrix:pandas}.0",
+]
 
 [tool.black]
 line-length = 120

upgini-1.1.280.dev1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.1.280.dev1"

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/autofe/all_operands.py

@@ -1,7 +1,7 @@
 from typing import Dict
 
 from upgini.autofe.binary import Add, Divide, Max, Min, Multiply, Sim, Subtract
-from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded, DatePercentile
+from upgini.autofe.date import DateDiff, DateDiffType2, DateListDiff, DateListDiffBounded
 from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
 from upgini.autofe.operand import Operand
 from upgini.autofe.unary import Abs, Floor, Freq, Log, Residual, Sigmoid, Sqrt, Square
@@ -49,7 +49,6 @@ ALL_OPERANDS: Dict[str, Operand] = {
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=30, upper_bound=45),
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=45, upper_bound=60),
         DateListDiffBounded(diff_unit="Y", aggregation="count", lower_bound=60),
-        DatePercentile(),
     ]
 }
 

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/autofe/date.py

@@ -1,10 +1,9 @@
-from datetime import date
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
 from pandas.core.arrays.timedeltas import TimedeltaArray
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 from upgini.autofe.operand import PandasOperand
 
@@ -28,17 +27,6 @@ class DateDiff(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "diff_unit": self.diff_unit,
-                "left_unit": self.left_unit,
-                "right_unit": self.right_unit,
-            }
-        )
-        return res
-
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
         right = self._convert_to_date(right, self.right_unit)
@@ -54,17 +42,6 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
     is_binary = True
     has_symmetry_importance = True
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "diff_unit": self.diff_unit,
-                "left_unit": self.left_unit,
-                "right_unit": self.right_unit,
-            }
-        )
-        return res
-
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         left = self._convert_to_date(left, self.left_unit)
         right = self._convert_to_date(right, self.right_unit)
@@ -87,15 +64,6 @@ class DateListDiff(PandasOperand, DateDiffMixin):
     has_symmetry_importance = True
     aggregation: str
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "aggregation": self.aggregation,
-            }
-        )
-        return res
-
     def __init__(self, **data: Any) -> None:
         if "name" not in data:
             data["name"] = f"date_diff_{data.get('aggregation')}"
@@ -148,55 +116,3 @@ class DateListDiffBounded(DateListDiff):
     def _agg(self, x):
         x = x[(x >= (self.lower_bound or -np.inf)) & (x < (self.upper_bound or np.inf))]
         return super()._agg(x)
-
-
-class DatePercentile(PandasOperand):
-    name = "date_per"
-    is_binary = True
-    output_type = "float"
-
-    date_unit: Optional[str] = None
-    zero_month: Optional[int]
-    zero_year: Optional[int]
-    zero_bounds: Optional[List[float]]
-    step: int = 30
-
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "date_unit": self.date_unit,
-                "zero_month": self.zero_month,
-                "zero_year": self.zero_year,
-                "zero_bounds": self.zero_bounds,
-                "step": self.step,
-            }
-        )
-        return res
-
-    @validator("zero_bounds", pre=True)
-    def validate_bounds(cls, value):
-        if value is None or isinstance(value, list):
-            return value
-        elif isinstance(value, str):
-            return value[1:-1].split(", ")
-
-    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        # Assuming that left is a date column, right is a feature column
-        left = pd.to_datetime(left, unit=self.date_unit)
-        months = left.dt.month
-        years = left.dt.year
-
-        month_diffs = 12 * (years - (self.zero_year or 0)) + (months - (self.zero_month or 0))
-        bounds = month_diffs.apply(
-            lambda d: np.array(self.zero_bounds if self.zero_bounds is not None else []) + d * 30
-        )
-
-        return right.index.to_series().apply(lambda i: self.__perc(right[i], bounds[i]))
-
-    def __perc(self, f, bounds):
-        hit = np.where(f >= bounds)[0]
-        if hit.size > 0:
-            return np.max(hit) * 10
-        else:
-            return np.nan

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/autofe/feature.py

@@ -16,12 +16,6 @@ class Column:
         self.data = data
         self.calculate_all = calculate_all
 
-    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
-        return self.name
-
-    def set_op_params(self, params: Dict[str, str]) -> "Column":
-        return self
-
     def rename_columns(self, mapping: Dict[str, str]) -> "Column":
         self.name = self._unhash(mapping.get(self.name) or self.name)
         return self
@@ -75,20 +69,14 @@ class Feature:
         self.cached_display_name = cached_display_name
         self.alias = alias
 
-    def set_op_params(self, params: Optional[Dict[str, str]]) -> "Feature":
-        obj_dict = self.op.dict().copy()
-        obj_dict.update(params or {})
-        self.op = self.op.__class__.parse_obj(obj_dict)
+    def set_op_params(self, params: Dict[str, str]) -> "Feature":
         self.op.set_params(params)
-
-        for child in self.children:
-            child.set_op_params(params)
         return self
 
     def get_hash(self) -> str:
-        return hashlib.sha256(
-            "_".join([self.op.name] + [ch.get_display_name() for ch in self.children]).encode("utf-8")
-        ).hexdigest()[:8]
+        return hashlib.sha256("_".join([self.op.name] + [ch.name for ch in self.children]).encode("utf-8")).hexdigest()[
+            :8
+        ]
 
     def set_alias(self, alias: str) -> "Feature":
         self.alias = alias

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/autofe/operand.py

@@ -25,10 +25,8 @@ class Operand(BaseModel):
         self.params = params
         return self
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = {"alias": self.alias}
-        res.update(self.params or {})
-        return res
+    def get_params(self) -> Dict[str, str]:
+        return self.params
 
 
 MAIN_COLUMN = "main_column"

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/dataset.py

@@ -246,7 +246,7 @@ class Dataset:  # (pd.DataFrame):
         if len(columns_to_fix) > 0:
             self.logger.warning(f"Convert strings with decimal comma to float: {columns_to_fix}")
             for col in columns_to_fix:
-                self.data[col] = self.data[col].astype("string").str.replace(",", ".").astype(np.float64)
+                self.data[col] = self.data[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
 
     @staticmethod
     def _ip_to_int(ip: Optional[_BaseAddress]) -> Optional[int]:

{upgini-1.1.280a3418.post10 → upgini-1.1.280.dev1}/src/upgini/features_enricher.py

@@ -1333,6 +1333,9 @@ class FeaturesEnricher(TransformerMixin):
         excluding_search_keys = list(search_keys.keys())
         if search_keys_for_metrics is not None and len(search_keys_for_metrics) > 0:
             excluding_search_keys = [sk for sk in excluding_search_keys if sk not in search_keys_for_metrics]
+        meta = self._search_task.get_all_features_metadata_v2()
+        zero_importance_client_features = [m.name for m in meta if m.source == "etalon" and m.shap_value == 0.0]
+
         client_features = [
             c
             for c in X_sampled.columns.to_list()
@@ -1341,6 +1344,7 @@ class FeaturesEnricher(TransformerMixin):
                 excluding_search_keys
                 + list(self.fit_dropped_features)
                 + [DateTimeSearchKeyConverter.DATETIME_COL, SYSTEM_RECORD_ID]
+                + zero_importance_client_features
             )
         ]
 
@@ -1399,9 +1403,9 @@ class FeaturesEnricher(TransformerMixin):
         if len(decimal_columns_to_fix) > 0:
             self.logger.warning(f"Convert strings with decimal comma to float: {decimal_columns_to_fix}")
             for col in decimal_columns_to_fix:
-                fitting_X[col] = fitting_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                fitting_X[col] = fitting_X[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
                 fitting_enriched_X[col] = (
-                    fitting_enriched_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                    fitting_enriched_X[col].astype("string").str.replace(",", ".", regex=False).astype(np.float64)
                 )
 
         fitting_eval_set_dict = dict()
@@ -1437,9 +1441,17 @@ class FeaturesEnricher(TransformerMixin):
             # Correct string features with decimal commas
             if len(decimal_columns_to_fix) > 0:
                 for col in decimal_columns_to_fix:
-                    fitting_eval_X[col] = fitting_eval_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                    fitting_eval_X[col] = (
+                        fitting_eval_X[col]
+                        .astype("string").str
+                        .replace(",", ".", regex=False)
+                        .astype(np.float64)
+                    )
                     fitting_enriched_eval_X[col] = (
-                        fitting_enriched_eval_X[col].astype("string").str.replace(",", ".").astype(np.float64)
+                        fitting_enriched_eval_X[col]
+                        .astype("string").str
+                        .replace(",", ".", regex=False)
+                        .astype(np.float64)
                     )
 
             fitting_eval_set_dict[idx] = (
@@ -2845,8 +2857,10 @@ class FeaturesEnricher(TransformerMixin):
             maybe_date_col = self._get_date_column(self.search_keys)
             if X is not None and maybe_date_col is not None and maybe_date_col in X.columns:
                 # TODO cast date column to single dtype
-                min_date = X[maybe_date_col].min()
-                max_date = X[maybe_date_col].max()
+                date_converter = DateTimeSearchKeyConverter(maybe_date_col, self.date_format)
+                converted_X = date_converter.convert(X)
+                min_date = converted_X[maybe_date_col].min()
+                max_date = converted_X[maybe_date_col].max()
                 self.logger.info(f"Dates interval is ({min_date}, {max_date})")
 
         except Exception:

upgini-1.1.280a3418.post10/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.1.280a3418-10"