upgini 1.2.91a3884.dev3__py3-none-any.whl → 1.2.91a3884.dev5__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.91a3884.dev3"
+__version__ = "1.2.91a3884.dev5"

upgini/dataset.py

@@ -48,17 +48,9 @@ except Exception:
     )
 
 
-class Dataset:  # (pd.DataFrame):
+class Dataset:
     MIN_ROWS_COUNT = 100
     MAX_ROWS = 200_000
-    FIT_SAMPLE_ROWS = 200_000
-    FIT_SAMPLE_THRESHOLD = 200_000
-    FIT_SAMPLE_WITH_EVAL_SET_ROWS = 200_000
-    FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD = 200_000
-    FIT_SAMPLE_THRESHOLD_TS = 54_000
-    FIT_SAMPLE_ROWS_TS = 54_000
-    BINARY_MIN_SAMPLE_THRESHOLD = 5_000
-    MULTICLASS_MIN_SAMPLE_THRESHOLD = 25_000
     IMBALANCE_THESHOLD = 0.6
     BINARY_BOOTSTRAP_LOOPS = 5
     MULTICLASS_BOOTSTRAP_LOOPS = 2
@@ -576,8 +568,8 @@ class Dataset:  # (pd.DataFrame):
     def _set_sample_size(self, runtime_parameters: Optional[RuntimeParameters]) -> Optional[RuntimeParameters]:
         if runtime_parameters is not None and runtime_parameters.properties is not None:
             if self.cv_type is not None and self.cv_type.is_time_series():
-                runtime_parameters.properties["sample_size"] = self.FIT_SAMPLE_ROWS_TS
-                runtime_parameters.properties["iter0_sample_size"] = self.FIT_SAMPLE_ROWS_TS
+                runtime_parameters.properties["sample_size"] = self.sample_config.fit_sample_rows_ts
+                runtime_parameters.properties["iter0_sample_size"] = self.sample_config.fit_sample_rows_ts
         return runtime_parameters
 
     def _clean_generate_features(self, runtime_parameters: Optional[RuntimeParameters]) -> Optional[RuntimeParameters]:

upgini/features_enricher.py

@@ -302,7 +302,7 @@ class FeaturesEnricher(TransformerMixin):
         self._relevant_data_sources_wo_links: pd.DataFrame = self.EMPTY_DATA_SOURCES
         self.metrics: Optional[pd.DataFrame] = None
         self.feature_names_ = []
-        self.zero_shap_client_features = []
+        self.external_source_feature_names = []
         self.feature_importances_ = []
         self.search_id = search_id
         self.disable_force_downsampling = disable_force_downsampling
@@ -981,6 +981,19 @@ class FeaturesEnricher(TransformerMixin):
                 client_cat_features, search_keys_for_metrics = self._get_and_validate_client_cat_features(
                     estimator, validated_X, self.search_keys
                 )
+                if self.id_columns and self.id_columns_encoder is not None:
+                    if cat_features_from_backend:
+                        cat_features_from_backend = [
+                            c
+                            for c in cat_features_from_backend
+                            if self.fit_columns_renaming.get(c, c) not in self.id_columns_encoder.feature_names_in_
+                        ]
+                    if client_cat_features:
+                        client_cat_features = [
+                            c
+                            for c in client_cat_features
+                            if self.fit_columns_renaming.get(c, c) not in self.id_columns_encoder.feature_names_in_
+                        ]
                 for cat_feature in cat_features_from_backend:
                     original_cat_feature = self.fit_columns_renaming.get(cat_feature)
                     if original_cat_feature in self.search_keys:
@@ -2346,7 +2359,9 @@ if response.status_code == 200:
 
             is_demo_dataset = hash_input(df) in DEMO_DATASET_HASHES
 
-            columns_to_drop = [c for c in df.columns if c in self.feature_names_ and c not in (self.id_columns or [])]
+            columns_to_drop = [
+                c for c in df.columns if c in self.feature_names_ and c in self.external_source_feature_names
+            ]
             if len(columns_to_drop) > 0:
                 msg = self.bundle.get("x_contains_enriching_columns").format(columns_to_drop)
                 self.logger.warning(msg)
@@ -2654,7 +2669,7 @@ if response.status_code == 200:
             selecting_columns = [
                 c
                 for c in itertools.chain(validated_Xy.columns.tolist(), selected_generated_features)
-                if c not in self.zero_shap_client_features or c in (self.id_columns or [])
+                if c not in self.external_source_feature_names or c in (self.id_columns or [])
             ]
             selecting_columns.extend(c for c in result.columns if c in filtered_columns and c not in selecting_columns)
             if add_fit_system_record_id:
@@ -4048,7 +4063,7 @@ if response.status_code == 200:
         df = df.rename(columns=original_names_dict)
 
         self.feature_names_ = []
-        self.zero_shap_client_features = []
+        self.external_source_feature_names = []
         self.feature_importances_ = []
         features_info = []
         features_info_without_links = []
@@ -4079,10 +4094,11 @@ if response.status_code == 200:
             original_name = original_names_dict.get(feature_meta.name, feature_meta.name)
             is_client_feature = original_name in df.columns
 
+            if not is_client_feature:
+                self.external_source_feature_names.append(original_name)
+
             # TODO make a decision about selected features based on special flag from mlb
             if original_shaps.get(feature_meta.name, 0.0) == 0.0:
-                if is_client_feature and self.fit_select_features:
-                    self.zero_shap_client_features.append(original_name)
                 continue
 
             # Use only important features

upgini/metrics.py

@@ -815,7 +815,7 @@ class CatBoostWrapper(EstimatorWrapper):
                 else:
                     encoded = cat_encoder.transform(x[self.cat_features])
                     cat_features = encoded.columns.to_list()
-                x[self.cat_features] = encoded
+                x.loc[:, self.cat_features] = encoded
             else:
                 cat_features = self.cat_features
 

upgini/utils/sample_utils.py

@@ -15,7 +15,7 @@ TS_MIN_DIFFERENT_IDS_RATIO = 0.2
 TS_DEFAULT_HIGH_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=2, months=6), pd.DateOffset(years=2, days=7)]
 TS_DEFAULT_LOW_FREQ_TRUNC_LENGTHS = [pd.DateOffset(years=7), pd.DateOffset(years=5)]
 TS_DEFAULT_TIME_UNIT_THRESHOLD = pd.Timedelta(weeks=4)
-FIT_SAMPLE_ROWS_TS = 54_000
+FIT_SAMPLE_ROWS_TS = 100_000
 
 BINARY_MIN_SAMPLE_THRESHOLD = 5_000
 MULTICLASS_MIN_SAMPLE_THRESHOLD = 25_000
@@ -173,6 +173,7 @@ def sample_time_series_train_eval(
                 logger=logger,
                 **kwargs,
             )
+        logger.info(f"Eval set size: {len(eval_df)}")
         df = pd.concat([train_df, eval_df])
 
     elif len(train_df) > max_rows:
@@ -188,6 +189,8 @@ def sample_time_series_train_eval(
     else:
         df = train_df
 
+    logger.info(f"Train set size: {len(df)}")
+
     return df
 
 

{upgini-1.2.91a3884.dev3.dist-info → upgini-1.2.91a3884.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.91a3884.dev3
+Version: 1.2.91a3884.dev5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.91a3884.dev3.dist-info → upgini-1.2.91a3884.dev5.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=55Sg-JLu4aw-5ANNPanS_ciHPSsxXTa8YndbgltGREA,33
+upgini/__about__.py,sha256=tqdF5EG5u2XotiGWCSnsQ61GODo019Lr-s097ItdAHs,33
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=c0rZ-ydrnCdrTzx10WZl4WbO3LdyuF0fUCRD8Ugjitg,33093
+upgini/dataset.py,sha256=MituLJZTDdIwSk-Ia5G1pS52PERNHQ2P99FgCH2kTjQ,32790
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=plU6GbGwc4Pk3eXPzYFezq33yTslX615Ew-HAvdLyVs,217468
+upgini/features_enricher.py,sha256=udQdXpqVO4YAwEHjUzS195k2jxe5_CtZ-KTpWRicjfs,218225
 upgini/http.py,sha256=4i7fQwrwU3WzDUOWzrgR-4C8eJwj_5dBwRAR-UjUtlc,44345
 upgini/metadata.py,sha256=vsbbHyPCP3Rs8WkeDgQg99uAA_zmsbDStAT-NwDYhO4,12455
-upgini/metrics.py,sha256=ju7JPwLUe8vtFUGbBV6w6ecySd952XucrqToc1edVBs,45306
+upgini/metrics.py,sha256=Bc1L9DUmEL8OWwNvIEjPjw5EyHSZbiu3v2hWyBmedis,45313
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -64,14 +64,14 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sample_utils.py,sha256=PpMXRVTPKi6TyAo0gPhF0OmXmutecHdonM7WYUsB1Wo,15249
+upgini/utils/sample_utils.py,sha256=jQ1em2FRnMFLul9ujuBgs5XZ9jAZ4eM4FHT3aDSjOy8,15351
 upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
 upgini/utils/target_utils.py,sha256=i3Xt5l9ybB2_nF_ma5cfPuL3OeFTs2dY2xDI0p4Azpg,9049
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.91a3884.dev3.dist-info/METADATA,sha256=teoc8dCmv4mb2eBV6QruZag3xnwK3YAdlKCHuIKllXw,49546
-upgini-1.2.91a3884.dev3.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.91a3884.dev3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.91a3884.dev3.dist-info/RECORD,,
+upgini-1.2.91a3884.dev5.dist-info/METADATA,sha256=A9MDv_VCFrWDDw2Xyo7Wsx3ps6ECwTul36gmw-wujgI,49546
+upgini-1.2.91a3884.dev5.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.91a3884.dev5.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.91a3884.dev5.dist-info/RECORD,,