upgini 1.2.13a3__py3-none-any.whl → 1.2.14__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.13a3"
+__version__ = "1.2.14"

upgini/autofe/binary.py

@@ -140,27 +140,21 @@ class Distance(PandasOperand):
     has_symmetry_importance: bool = True
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        # Handle None values by replacing them with 0 in the dot product and norm calculations
-        left = left.apply(lambda x: np.array(x) if x is not None else np.zeros_like(right[0]))
-        right = right.apply(lambda x: np.array(x) if x is not None else np.zeros_like(left[0]))
-
         return pd.Series(
             1 - self.__dot(left, right) / (self.__norm(left) * self.__norm(right)), index=left.index
-        )
+        ).astype(np.float64)
 
     # row-wise dot product, handling None values
     def __dot(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        left = left.apply(lambda x: np.array(x) if x is not None else np.zeros_like(right[0]))
-        right = right.apply(lambda x: np.array(x) if x is not None else np.zeros_like(left[0]))
-
-        # Perform element-wise multiplication and handle missing values
-        res = (left * right).apply(np.sum)
+        left = left.apply(lambda x: np.array(x))
+        right = right.apply(lambda x: np.array(x))
+        res = (left.dropna() * right.dropna()).apply(np.sum)
+        res = res.reindex(left.index.union(right.index))
         return res
 
     # Calculate the norm of a vector, handling None values
     def __norm(self, vector: pd.Series) -> pd.Series:
-        # Replace None with a zero vector
-        vector = vector.apply(lambda x: np.array(x) if x is not None else np.zeros_like(vector[0]))
+        vector = vector.fillna(np.nan)
         return np.sqrt(self.__dot(vector, vector))
 
 

upgini/features_enricher.py

@@ -1577,8 +1577,8 @@ class FeaturesEnricher(TransformerMixin):
             df = generator.generate(df)
             generated_features.extend(generator.generated_features)
 
-        normalizer = Normalizer(search_keys, generated_features, self.bundle, self.logger, self.warning_counter)
-        df = normalizer.normalize(df)
+        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
+        df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
         columns_renaming = normalizer.columns_renaming
 
         df = clean_full_duplicates(df, logger=self.logger, silent=True, bundle=self.bundle)
@@ -2017,10 +2017,8 @@ class FeaturesEnricher(TransformerMixin):
                 df = generator.generate(df)
                 generated_features.extend(generator.generated_features)
 
-            normalizer = Normalizer(
-                search_keys, generated_features, self.bundle, self.logger, self.warning_counter, silent_mode
-            )
-            df = normalizer.normalize(df)
+            normalizer = Normalizer(self.bundle, self.logger, self.warning_counter, silent_mode)
+            df, search_keys, generated_features = normalizer.normalize(df, search_keys, generated_features)
             columns_renaming = normalizer.columns_renaming
 
             # Don't pass all features in backend on transform
@@ -2449,14 +2447,13 @@ class FeaturesEnricher(TransformerMixin):
         if is_numeric_dtype(df[self.TARGET_NAME]) and has_date:
             self._validate_PSI(df.sort_values(by=maybe_date_column))
 
-        self.__adjust_cv(df, maybe_date_column, self.model_task_type)
-
-        normalizer = Normalizer(
-            self.fit_search_keys, self.fit_generated_features, self.bundle, self.logger, self.warning_counter
+        normalizer = Normalizer(self.bundle, self.logger, self.warning_counter)
+        df, self.fit_search_keys, self.fit_generated_features = normalizer.normalize(
+            df, self.fit_search_keys, self.fit_generated_features
         )
-        df = normalizer.normalize(df)
-        columns_renaming = normalizer.columns_renaming
-        self.fit_columns_renaming = columns_renaming
+        self.fit_columns_renaming = normalizer.columns_renaming
+
+        self.__adjust_cv(df)
 
         df = remove_fintech_duplicates(
             df, self.fit_search_keys, date_format=self.date_format, logger=self.logger, bundle=self.bundle
@@ -2470,7 +2467,7 @@ class FeaturesEnricher(TransformerMixin):
         self.df_with_original_index = df.copy()
         # TODO check maybe need to drop _time column from df_with_original_index
 
-        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, columns_renaming)
+        df, unnest_search_keys = self._explode_multiple_search_keys(df, self.fit_search_keys, self.fit_columns_renaming)
 
         # Convert EMAIL to HEM after unnesting to do it only with one column
         email_column = self._get_email_column(self.fit_search_keys)
@@ -2480,7 +2477,7 @@ class FeaturesEnricher(TransformerMixin):
                 email_column,
                 hem_column,
                 self.fit_search_keys,
-                columns_renaming,
+                self.fit_columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.logger,
             )
@@ -2491,7 +2488,7 @@ class FeaturesEnricher(TransformerMixin):
             converter = IpSearchKeyConverter(
                 ip_column,
                 self.fit_search_keys,
-                columns_renaming,
+                self.fit_columns_renaming,
                 list(unnest_search_keys.keys()),
                 self.bundle,
                 self.logger,
@@ -2522,7 +2519,7 @@ class FeaturesEnricher(TransformerMixin):
         features_columns = [c for c in df.columns if c not in non_feature_columns]
 
         features_to_drop = FeaturesValidator(self.logger).validate(
-            df, features_columns, self.generate_features, self.warning_counter, columns_renaming
+            df, features_columns, self.generate_features, self.warning_counter, self.fit_columns_renaming
         )
         self.fit_dropped_features.update(features_to_drop)
         df = df.drop(columns=features_to_drop)
@@ -2563,7 +2560,7 @@ class FeaturesEnricher(TransformerMixin):
             rest_client=self.rest_client,
             logger=self.logger,
         )
-        dataset.columns_renaming = columns_renaming
+        dataset.columns_renaming = self.fit_columns_renaming
 
         self.passed_features = [
             column for column, meaning_type in meaning_types.items() if meaning_type == FileColumnMeaningType.FEATURE
@@ -2710,24 +2707,24 @@ class FeaturesEnricher(TransformerMixin):
         if not self.warning_counter.has_warnings():
             self.__display_support_link(self.bundle.get("all_ok_community_invite"))
 
-    def __adjust_cv(self, df: pd.DataFrame, date_column: pd.Series, model_task_type: ModelTaskType):
+    def __adjust_cv(self, df: pd.DataFrame):
+        date_column = SearchKey.find_key(self.fit_search_keys, [SearchKey.DATE, SearchKey.DATETIME])
         # Check Multivariate time series
         if (
             self.cv is None
             and date_column
-            and model_task_type == ModelTaskType.REGRESSION
+            and self.model_task_type == ModelTaskType.REGRESSION
             and len({SearchKey.PHONE, SearchKey.EMAIL, SearchKey.HEM}.intersection(self.fit_search_keys.keys())) == 0
             and is_blocked_time_series(df, date_column, list(self.fit_search_keys.keys()) + [TARGET])
         ):
             msg = self.bundle.get("multivariate_timeseries_detected")
             self.__override_cv(CVType.blocked_time_series, msg, print_warning=False)
-        elif (
-            self.cv is None
-            and model_task_type != ModelTaskType.REGRESSION
-            and self._get_group_columns(df, self.fit_search_keys)
-        ):
+        elif self.cv is None and self.model_task_type != ModelTaskType.REGRESSION:
             msg = self.bundle.get("group_k_fold_in_classification")
             self.__override_cv(CVType.group_k_fold, msg, print_warning=self.cv is not None)
+            group_columns = self._get_group_columns(df, self.fit_search_keys)
+            self.runtime_parameters.properties["cv_params.group_columns"] = ",".join(group_columns)
+            self.runtime_parameters.properties["cv_params.shuffle_kfold"] = "True"
 
     def __override_cv(self, cv: CVType, msg: str, print_warning: bool = True):
         if print_warning:

upgini/normalizer/normalize_utils.py

@@ -1,6 +1,6 @@
 import hashlib
 from logging import Logger, getLogger
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 import numpy as np
 import pandas as pd
@@ -35,22 +35,25 @@ class Normalizer:
 
     def __init__(
         self,
-        search_keys: Dict[str, SearchKey],
-        generated_features: List[str],
         bundle: ResourceBundle = None,
         logger: Logger = None,
         warnings_counter: WarningCounter = None,
         silent_mode=False,
     ):
-        self.search_keys = search_keys
-        self.generated_features = generated_features
         self.bundle = bundle or get_custom_bundle()
         self.logger = logger or getLogger()
         self.warnings_counter = warnings_counter or WarningCounter()
         self.silent_mode = silent_mode
         self.columns_renaming = {}
+        self.search_keys = {}
+        self.generated_features = []
+
+    def normalize(
+        self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
+    ) -> Tuple[pd.DataFrame, Dict[str, SearchKey], List[str]]:
+        self.search_keys = search_keys.copy()
+        self.generated_features = generated_features.copy()
 
-    def normalize(self, df: pd.DataFrame) -> pd.DataFrame:
         df = df.copy()
         df = self._rename_columns(df)
 
@@ -68,21 +71,25 @@ class Normalizer:
 
         df = self.__convert_features_types(df)
 
-        return df
+        return df, self.search_keys, self.generated_features
 
     def _rename_columns(self, df: pd.DataFrame):
         # logger.info("Replace restricted symbols in column names")
         new_columns = []
         dup_counter = 0
         for column in df.columns:
-            if column in [
-                TARGET,
-                EVAL_SET_INDEX,
-                SYSTEM_RECORD_ID,
-                ENTITY_SYSTEM_RECORD_ID,
-                SEARCH_KEY_UNNEST,
-                DateTimeSearchKeyConverter.DATETIME_COL,
-            ] + self.generated_features:
+            if (
+                column
+                in [
+                    TARGET,
+                    EVAL_SET_INDEX,
+                    SYSTEM_RECORD_ID,
+                    ENTITY_SYSTEM_RECORD_ID,
+                    SEARCH_KEY_UNNEST,
+                    DateTimeSearchKeyConverter.DATETIME_COL,
+                ]
+                + self.generated_features
+            ):
                 self.columns_renaming[column] = column
                 new_columns.append(column)
                 continue

{upgini-1.2.13a3.dist-info → upgini-1.2.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.13a3
+Version: 1.2.14
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.13a3.dist-info → upgini-1.2.14.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=-7DJQLGOhF0r2W7BFtcYxOGA1U0eO5UUMqJzRXTeSrI,25
+upgini/__about__.py,sha256=mxzjGyB-ihJR05pHYcBiUe_XT5X9wj6cBHLxOXBhAeM,23
 upgini/__init__.py,sha256=M64LwQTBa-5Jz24Zm2h8rWwlKQQ1J8nP7gGgIciS0WU,589
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=olZ-OHSfBNoBSCo7R5t7uCLukI2nO7afpx_A-HCiJLk,31067
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=HJJZbZScVrl6ugDBQE71m7om5-ahvMyEnAqZNw-OEJ0,188058
+upgini/features_enricher.py,sha256=vRC7g6n6XQxSrvzXk6NJjP0ZytDQhWR4sTAo4Hp7gmA,188319
 upgini/http.py,sha256=21asexflvavydzCOONJDGQBtQanCElrbnqLXakJ9Cu8,42880
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
@@ -15,7 +15,7 @@ upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9Jvf
 upgini/ads_management/ads_manager.py,sha256=igVbN2jz80Umb2BUJixmJVj-zx8unoKpecVo-R-nGdw,2648
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operands.py,sha256=cCCB44qvkmuWyiRM5Xykx8tkHPIjQthrWyj67STWN80,2578
-upgini/autofe/binary.py,sha256=lb_xRCeJ25X5oJZ-2ER50HcVGpENvTlSoq7bOpcreno,8119
+upgini/autofe/binary.py,sha256=zMhtHVuGUAFLUqem-XiXqJj-GRXxS88tdz8tFuDfSNM,7659
 upgini/autofe/date.py,sha256=OpFc3Al0xO3qlESn2Uokfxw51ArVqmh3xngWwdrsaqE,9762
 upgini/autofe/feature.py,sha256=eL7wABUhDKZzv3E-RPJNcyGwSfB0UptcfU2RbvsOks4,15082
 upgini/autofe/groupby.py,sha256=r-xl_keZZgm_tpiEoDhjYSkT6NHv7a4cRQR4wJ4uCp8,3263
@@ -27,7 +27,7 @@ upgini/data_source/data_source_publisher.py,sha256=X-8aGtVgzGmxyXkMVBoBLIGDMb4lY
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=bHRPWCNrUvt2R9qMX6dZFCJ0i8ENVCQ2Rw3dHH9IJEg,7447
+upgini/normalizer/normalize_utils.py,sha256=Lv75lq7M46z9cAIutwkdKZtPZkWblgoRzToAJ1BwY8A,7709
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
 upgini/resource_bundle/strings.properties,sha256=9kvmcUrsSFUCrzOiN0Ozf-lQ2H8Igz5gATUPoHMOaU4,26456
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=BVtDmrmFMKerSUWaNOIEdzsYHIFiODdpnWbE50QDPDc,7864
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.2.13a3.dist-info/METADATA,sha256=ChCSigDvsmEioMfptnn7ngsRaWHChRyCmVI-vCnXNxA,48579
-upgini-1.2.13a3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.13a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.13a3.dist-info/RECORD,,
+upgini-1.2.14.dist-info/METADATA,sha256=xIbSnwYAnie1HKKRe_6MEUGG0BaZRtvWAsRlc2vzlBw,48577
+upgini-1.2.14.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.14.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.14.dist-info/RECORD,,

{upgini-1.2.13a3.dist-info → upgini-1.2.14.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.24.2
 Root-Is-Purelib: true
 Tag: py3-none-any