upgini 1.1.244a18__tar.gz → 1.1.244a20__tar.gz

{upgini-1.1.244a18/src/upgini.egg-info → upgini-1.1.244a20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.244a18
+Version: 1.1.244a20
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.244a18 → upgini-1.1.244a20}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.244a18"
+version = "1.1.244a20"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.244a18 → upgini-1.1.244a20}/src/upgini/features_enricher.py

@@ -1855,8 +1855,6 @@ class FeaturesEnricher(TransformerMixin):
             )
             meaning_types[SYSTEM_RECORD_ID] = FileColumnMeaningType.SYSTEM_RECORD_ID
 
-            df = clean_full_duplicates(df, self.logger, silent=silent_mode)
-
             df = df.reset_index(drop=True)
             system_columns_with_original_index = [SYSTEM_RECORD_ID] + generated_features
             df_with_original_index = df[system_columns_with_original_index].copy()
@@ -1865,6 +1863,8 @@ class FeaturesEnricher(TransformerMixin):
 
             df_without_features = df.drop(columns=non_keys_columns)
 
+            df_without_features = clean_full_duplicates(df_without_features, self.logger, silent=silent_mode)
+
             del df
             gc.collect()
 
@@ -2092,6 +2092,7 @@ class FeaturesEnricher(TransformerMixin):
                     msg = bundle.get("missing_generate_feature").format(gen_feature, x_columns)
                     print(msg)
                     self.logger.warning(msg)
+            self.runtime_parameters.properties["generate_features"] = ",".join(self.generate_features)
 
         validate_scoring_argument(scoring)
 

{upgini-1.1.244a18 → upgini-1.1.244a20}/src/upgini/metrics.py

@@ -414,6 +414,9 @@ class CatBoostWrapper(EstimatorWrapper):
         self.emb_features = [c for c in X.columns if re.match(emb_pattern, c) and is_numeric_dtype(X[c])]
         embedding_features = []
         if len(self.emb_features) > 3:  # There is no reason to reduce embeddings dimension with less than 4
+            self.logger.info(
+                f"Embedding features count more than 3, so group them into one vector for CatBoost: {self.emb_features}"
+            )
             X, embedding_features = self.group_embeddings(X)
             params["embedding_features"] = embedding_features
         else:
@@ -421,15 +424,13 @@ class CatBoostWrapper(EstimatorWrapper):
 
         # Find text features from passed in generate_features
         if self.text_features is not None:
-            self.text_features = [f for f in self.text_features if not is_numeric_dtype(X[f])]
+            self.logger.info(f"Passed text features for CatBoost: {self.text_features}")
+            self.text_features = [f for f in self.text_features if f in X.columns and not is_numeric_dtype(X[f])]
+            self.logger.info(f"Rest text features after checks: {self.text_features}")
             params["text_features"] = self.text_features
 
         # Find rest categorical features
-        self.cat_features = _get_cat_features(X)
-        if self.text_features is not None:
-            self.cat_features = [
-                f for f in self.cat_features if f not in self.text_features and f not in embedding_features
-            ]
+        self.cat_features = _get_cat_features(X, self.text_features, embedding_features)
         X = fill_na_cat_features(X, self.cat_features)
         unique_cat_features = []
         for name in self.cat_features:
@@ -456,6 +457,7 @@ class CatBoostWrapper(EstimatorWrapper):
 
             del self.estimator._init_params["cat_features"]
 
+        self.logger.info(f"Selected categorical features: {self.cat_features}")
         params["cat_features"] = self.cat_features
 
         return X, y, groups, params
@@ -473,13 +475,11 @@ class CatBoostWrapper(EstimatorWrapper):
         X, y, params = super()._prepare_to_calculate(X, y)
         if self.text_features:
             params["text_features"] = self.text_features
-        # if self.emb_groups:
         if self.emb_features:
             X, emb_columns = self.group_embeddings(X)
             params["embedding_features"] = emb_columns
         if self.cat_features:
             X = fill_na_cat_features(X, self.cat_features)
-        if self.cat_features:
             params["cat_features"] = self.cat_features
 
         return X, y, params
@@ -633,8 +633,13 @@ def _get_scorer(target_type: ModelTaskType, scoring: Union[Callable, str, None])
     return scoring, metric_name, multiplier
 
 
-def _get_cat_features(X: pd.DataFrame) -> List[str]:
-    return [c for c in X.columns if not is_numeric_dtype(X[c])]
+def _get_cat_features(
+    X: pd.DataFrame, text_features: Optional[List[str]] = None, emb_features: Optional[List[str]] = None
+) -> List[str]:
+    text_features = text_features or []
+    emb_features = emb_features or []
+    exclude_features = text_features + emb_features
+    return [c for c in X.columns if c not in exclude_features and not is_numeric_dtype(X[c])]
 
 
 def _get_add_params(input_params, add_params):
@@ -731,11 +736,3 @@ def fill_na_cat_features(df: pd.DataFrame, cat_features: List[str]) -> pd.DataFr
             na_filter = df[c].str.lower().isin(NA_VALUES)
             df.loc[na_filter, c] = NA_REPLACEMENT
     return df
-
-
-def _is_too_many_categorical_values(X: pd.DataFrame) -> bool:
-    many_values_features_count = 0
-    for f in _get_cat_features(X):
-        if X[f].astype("string").nunique() > 100:
-            many_values_features_count += 1
-    return many_values_features_count >= 2

{upgini-1.1.244a18 → upgini-1.1.244a20}/src/upgini/utils/deduplicate_utils.py

@@ -103,6 +103,8 @@ def clean_full_duplicates(
     unique_columns = df.columns.tolist()
     if SYSTEM_RECORD_ID in unique_columns:
         unique_columns.remove(SYSTEM_RECORD_ID)
+    if "sort_id" in unique_columns:
+        unique_columns.remove("sort_id")
     logger.info(f"Dataset shape before clean duplicates: {df.shape}")
     df = df.drop_duplicates(subset=unique_columns)
     logger.info(f"Dataset shape after clean duplicates: {df.shape}")

{upgini-1.1.244a18 → upgini-1.1.244a20/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.244a18
+Version: 1.1.244a20
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.244a18 → upgini-1.1.244a20}/tests/test_blocked_time_series.py

@@ -33,13 +33,20 @@ def test_bts_split_logic():
 def test_bts_metrics():
     X, y, cv, _, model = _prepare_data()
     cv_result = set(cross_val_score(model, X, y, cv=cv, scoring="roc_auc"))
-    assert cv_result == {
-        0.4559664254320743,
-        0.4767320313326982,
-        0.4811855209016638,
-        0.48947924927306374,
-        0.5150543675843606,
+    assert {round(r, 3) for r in cv_result} == {
+        0.456,
+        0.477,
+        0.481,
+        0.489,
+        0.515,
     }
+    # assert cv_result == {
+    #     0.4559664254320743,
+    #     0.4767320313326982,
+    #     0.4811855209016638,
+    #     0.48947924927306374,
+    #     0.5150543675843606,
+    # }
 
 
 def test_bts_exceptions():