upgini 1.2.88a3884.dev1__py3-none-any.whl → 1.2.89a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.88a3884.dev1"
+__version__ = "1.2.89a1"

upgini/data_source/data_source_publisher.py

@@ -5,6 +5,8 @@ from datetime import datetime
 from enum import Enum
 from typing import Dict, List, Literal, Optional, Union
 
+import pandas as pd
+
 from upgini.errors import HttpError, ValidationError
 from upgini.http import LoggerFactory, get_rest_client
 from upgini.mdc import MDC
@@ -33,7 +35,7 @@ class OnlineUploadingType(Enum):
 class DataSourcePublisher:
     FINAL_STATUSES = ["COMPLETED", "FAILED", "TIMED_OUT"]
     ACCEPTABLE_UPDATE_FREQUENCIES = ["Daily", "Weekly", "Monthly", "Quarterly", "Annually"]
-    DEFAULT_GENERATE_EMBEDDINGS = []
+    DEFAULT_GENERATE_EMBEDDINGS = dict()
 
     def __init__(self, api_key: Optional[str] = None, endpoint: Optional[str] = None, logs_enabled=True):
         self._rest_client = get_rest_client(endpoint, api_key)
@@ -58,7 +60,7 @@ class DataSourcePublisher:
         hash_feature_names=False,
         snapshot_frequency_days: Optional[int] = None,
         join_date_abs_limit_days: Optional[int] = None,
-        features_for_embeddings: Optional[List[str]] = DEFAULT_GENERATE_EMBEDDINGS,
+        features_for_embeddings: Optional[Dict[str, str]] = DEFAULT_GENERATE_EMBEDDINGS,
         data_table_id_to_replace: Optional[str] = None,
         keep_features: Optional[List[str]] = None,
         date_features: Optional[List[str]] = None,
@@ -137,6 +139,25 @@ class DataSourcePublisher:
                 ) and not date_format:
                     raise ValidationError("date_format argument is required for PHONE+DATE and HEM+DATE search keys")
 
+                if secondary_search_keys:
+                    response = self._rest_client.get_active_ads_definitions()
+                    definitions = pd.DataFrame(response["adsDefinitions"])
+                    prod_secondary_definitions = definitions.query(
+                        "(secondarySearchKeys.astype('string') != '[]') & (adsDefinitionAccessType == 'PROD')"
+                    )[["name", "searchKeys", "secondarySearchKeys"]]
+                    for _, row in prod_secondary_definitions.iterrows():
+                        existing_secondary_keys = {item for sublist in row["secondarySearchKeys"] for item in sublist}
+                        if existing_secondary_keys == {v.value.name for v in secondary_search_keys.values()}:
+                            existing_search_keys = {item for sublist in row["searchKeys"] for item in sublist}
+                            if (
+                                existing_search_keys == {v.value.name for v in search_keys.values()}
+                                or ("IP" in str(existing_search_keys) and "IP" in str(search_keys.values()))
+                            ):
+                                raise ValidationError(
+                                    "ADS with the same PRIMARY_KEYS -> SECONDARY_KEYS mapping "
+                                    f"already exists: {row['name']}"
+                                )
+
                 request = {
                     "dataTableUri": data_table_uri,
                     "searchKeys": {k: v.value.value for k, v in search_keys.items()},

upgini/features_enricher.py

@@ -7,7 +7,6 @@ import json
 import logging
 import numbers
 import os
-import pickle
 import sys
 import tempfile
 import time
@@ -1671,10 +1670,6 @@ class FeaturesEnricher(TransformerMixin):
                 enriched_eval_y_sorted,
             )
 
-        fitting_X, fitting_enriched_X, fitting_eval_set_dict = self._convert_id_columns_to_int(
-            fitting_X, fitting_enriched_X, fitting_eval_set_dict, columns_renaming
-        )
-
         return (
             validated_X,
             fitting_X,
@@ -1688,38 +1683,6 @@ class FeaturesEnricher(TransformerMixin):
             columns_renaming,
         )
 
-    def _convert_id_columns_to_int(
-        self,
-        fitting_X: pd.DataFrame,
-        fitting_enriched_X: pd.DataFrame,
-        fitting_eval_set_dict: Dict[int, Tuple[pd.DataFrame, pd.Series]],
-        columns_renaming: Dict[str, str] = {},
-    ) -> pd.DataFrame:
-        def _set_encoded(col_name: str, df: pd.DataFrame, slice: Tuple[int, int], combined_col: pd.Series):
-            df[col_name] = combined_col.iloc[slice[0] : slice[1]]
-            return slice[1]
-
-        inverse_columns_renaming = {v: k for k, v in columns_renaming.items()}
-
-        if self.id_columns:
-            self.logger.info(f"Convert id columns to int: {self.id_columns}")
-            for col in self.id_columns:
-                col = inverse_columns_renaming.get(col, col)
-                combined_col = pd.concat(
-                    [fitting_X[col], fitting_enriched_X[col]]
-                    + [eval_set_pair[0][col] for eval_set_pair in fitting_eval_set_dict.values()]
-                )
-                combined_col = combined_col.astype("category").cat.codes
-                slice_end = _set_encoded(col, fitting_X, (0, len(fitting_X)), combined_col)
-                slice_end = _set_encoded(
-                    col, fitting_enriched_X, (slice_end, slice_end + len(fitting_enriched_X)), combined_col
-                )
-                for eval_set_pair in fitting_eval_set_dict.values():
-                    slice_end = _set_encoded(
-                        col, eval_set_pair[0], (slice_end, slice_end + len(eval_set_pair[0])), combined_col
-                    )
-        return fitting_X, fitting_enriched_X, fitting_eval_set_dict
-
     @dataclass
     class _SampledDataForMetrics:
         X_sampled: pd.DataFrame
@@ -3486,6 +3449,11 @@ if response.status_code == 200:
         if self.model_task_type == ModelTaskType.BINARY and eval_y_nunique != 2:
             raise ValidationError(self.bundle.get("binary_target_eval_unique_count_not_2").format(eval_y_nunique))
 
+        # Check for duplicates between train and eval sets by comparing all values
+        train_eval_intersection = pd.merge(X, validated_eval_X, how='inner')
+        if len(train_eval_intersection) > 0:
+            raise ValidationError(self.bundle.get("eval_x_has_train_samples"))
+
         return validated_eval_X, validated_eval_y
 
     def _validate_baseline_score(self, X: pd.DataFrame, eval_set: Optional[List[Tuple]]):
@@ -4012,7 +3980,7 @@ if response.status_code == 200:
         if features_meta is None:
             raise Exception(self.bundle.get("missing_features_meta"))
 
-        return [f.name for f in features_meta if f.type == "categorical" and f.name not in self.id_columns]
+        return [f.name for f in features_meta if f.type == "categorical"]
 
     def __prepare_feature_importances(
         self, trace_id: str, df: pd.DataFrame, updated_shaps: Optional[Dict[str, float]] = None, silent=False
@@ -4605,60 +4573,52 @@ if response.status_code == 200:
         y: Union[pd.DataFrame, pd.Series, None] = None,
         eval_set: Union[Tuple, None] = None,
     ):
-        def dump_task():
+        def dump_task(X_, y_, eval_set_):
             try:
-                random_state = 42
-                rnd = np.random.RandomState(random_state)
-                if _num_samples(X) > 0:
-                    xy_sample_index = rnd.randint(0, _num_samples(X), size=1000)
-                else:
-                    xy_sample_index = []
-
-                def sample(inp, sample_index):
-                    if _num_samples(inp) <= 1000:
-                        return inp
-                    if isinstance(inp, (pd.DataFrame, pd.Series)):
-                        return inp.sample(n=1000, random_state=random_state)
-                    if isinstance(inp, np.ndarray):
-                        return inp[sample_index]
-                    if isinstance(inp, list):
-                        return inp[sample_index]
+                if isinstance(X_, pd.Series):
+                    X_ = X_.to_frame()
+
+                # TODO check that this file was already uploaded
 
                 with tempfile.TemporaryDirectory() as tmp_dir:
-                    with open(f"{tmp_dir}/x.pickle", "wb") as x_file:
-                        pickle.dump(sample(X, xy_sample_index), x_file)
-                    if y is not None:
-                        with open(f"{tmp_dir}/y.pickle", "wb") as y_file:
-                            pickle.dump(sample(y, xy_sample_index), y_file)
-                        if eval_set and _num_samples(eval_set[0][0]) > 0:
-                            eval_xy_sample_index = rnd.randint(0, _num_samples(eval_set[0][0]), size=1000)
-                            with open(f"{tmp_dir}/eval_x.pickle", "wb") as eval_x_file:
-                                pickle.dump(sample(eval_set[0][0], eval_xy_sample_index), eval_x_file)
-                            with open(f"{tmp_dir}/eval_y.pickle", "wb") as eval_y_file:
-                                pickle.dump(sample(eval_set[0][1], eval_xy_sample_index), eval_y_file)
+                    X_.to_parquet(f"{tmp_dir}/x.parquet", compression="zstd")
+
+                    if y_ is not None:
+                        if isinstance(y_, pd.Series):
+                            y_ = y_.to_frame()
+                        y_.to_parquet(f"{tmp_dir}/y.parquet", compression="zstd")
+                        if eval_set_ and _num_samples(eval_set_[0][0]) > 0:
+                            eval_x_ = eval_set_[0][0]
+                            eval_y_ = eval_set_[0][1]
+                            if isinstance(eval_x_, pd.Series):
+                                eval_x_ = eval_x_.to_frame()
+                            eval_x_.to_parquet(f"{tmp_dir}/eval_x.parquet", compression="zstd")
+                            if isinstance(eval_y_, pd.Series):
+                                eval_y_ = eval_y_.to_frame()
+                            eval_y_.to_parquet(f"{tmp_dir}/eval_y.parquet", compression="zstd")
                             self.rest_client.dump_input_files(
                                 trace_id,
-                                f"{tmp_dir}/x.pickle",
-                                f"{tmp_dir}/y.pickle",
-                                f"{tmp_dir}/eval_x.pickle",
-                                f"{tmp_dir}/eval_y.pickle",
+                                f"{tmp_dir}/x.parquet",
+                                f"{tmp_dir}/y.parquet",
+                                f"{tmp_dir}/eval_x.parquet",
+                                f"{tmp_dir}/eval_y.parquet",
                             )
                         else:
                             self.rest_client.dump_input_files(
                                 trace_id,
-                                f"{tmp_dir}/x.pickle",
-                                f"{tmp_dir}/y.pickle",
+                                f"{tmp_dir}/x.parquet",
+                                f"{tmp_dir}/y.parquet",
                             )
                     else:
                         self.rest_client.dump_input_files(
                             trace_id,
-                            f"{tmp_dir}/x.pickle",
+                            f"{tmp_dir}/x.parquet",
                         )
             except Exception:
                 self.logger.warning("Failed to dump input files", exc_info=True)
 
         try:
-            Thread(target=dump_task, daemon=True).start()
+            Thread(target=dump_task, args=(X, y, eval_set), daemon=True).start()
         except Exception:
             self.logger.warning("Failed to dump input files", exc_info=True)
 

upgini/metrics.py

@@ -332,7 +332,7 @@ class EstimatorWrapper:
         self.groups = groups
         self.text_features = text_features
         self.logger = logger or logging.getLogger()
-        self.dropped_features = []
+        self.droped_features = []
         self.converted_to_int = []
         self.converted_to_str = []
         self.converted_to_numeric = []
@@ -381,11 +381,10 @@ class EstimatorWrapper:
         x, y, groups = self._prepare_data(x, y, groups=self.groups)
 
         self.logger.info(f"Before preparing data columns: {x.columns.to_list()}")
-        self.dropped_features = []
+        self.droped_features = []
         self.converted_to_int = []
         self.converted_to_str = []
         self.converted_to_numeric = []
-
         for c in x.columns:
 
             if _get_unique_count(x[c]) < 2:
@@ -393,7 +392,7 @@ class EstimatorWrapper:
                 if c in self.cat_features:
                     self.cat_features.remove(c)
                 x.drop(columns=[c], inplace=True)
-                self.dropped_features.append(c)
+                self.droped_features.append(c)
             elif self.text_features is not None and c in self.text_features:
                 x[c] = x[c].astype(str)
                 self.converted_to_str.append(c)
@@ -428,16 +427,16 @@ class EstimatorWrapper:
                     except (ValueError, TypeError):
                         self.logger.warning(f"Remove feature {c} because it is not numeric and not in cat_features")
                         x.drop(columns=[c], inplace=True)
-                        self.dropped_features.append(c)
+                        self.droped_features.append(c)
 
         return x, y, groups, {}
 
     def _prepare_to_calculate(self, x: pd.DataFrame, y: pd.Series) -> Tuple[pd.DataFrame, np.ndarray, dict]:
         x, y, _ = self._prepare_data(x, y)
 
-        if self.dropped_features:
-            self.logger.info(f"Drop features on calculate metrics: {self.dropped_features}")
-            x = x.drop(columns=self.dropped_features)
+        if self.droped_features:
+            self.logger.info(f"Drop features on calculate metrics: {self.droped_features}")
+            x = x.drop(columns=self.droped_features)
 
         if self.converted_to_int:
             self.logger.info(f"Convert to int features on calculate metrics: {self.converted_to_int}")
@@ -798,7 +797,7 @@ class CatBoostWrapper(EstimatorWrapper):
                     )
                 for f in high_cardinality_features:
                     self.text_features.remove(f)
-                    self.dropped_features.append(f)
+                    self.droped_features.append(f)
                     x = x.drop(columns=f, errors="ignore")
                 return super().cross_val_predict(x, y, baseline_score_column)
             else:
@@ -898,7 +897,7 @@ class LightGBMWrapper(EstimatorWrapper):
         for c in x.columns:
             if x[c].dtype not in ["category", "int64", "float64", "bool"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
-                self.dropped_features.append(c)
+                self.droped_features.append(c)
                 x = x.drop(columns=c, errors="ignore")
         return x, y_numpy, groups, params
 
@@ -989,7 +988,7 @@ class OtherEstimatorWrapper(EstimatorWrapper):
         for c in x.columns:
             if x[c].dtype not in ["category", "int64", "float64", "bool"]:
                 self.logger.warning(f"Feature {c} is not numeric and will be dropped")
-                self.dropped_features.append(c)
+                self.droped_features.append(c)
                 x = x.drop(columns=c, errors="ignore")
         return x, y_numpy, groups, params
 

upgini/resource_bundle/strings.properties

@@ -137,6 +137,8 @@ eval_y_multiindex_unsupported=Multi index in y in eval_set is not supported
 eval_x_is_empty=X in eval_set is empty.
 eval_y_is_empty=y in eval_set is empty.
 x_and_eval_x_diff_types=X and eval_set X has different types: {} and {}
+eval_x_has_train_samples=Eval set X has rows that are present in train set X
+
 baseline_score_column_not_exists=baseline_score_column {} doesn't exist in input dataframe
 baseline_score_column_has_na=baseline_score_column contains NaN. Clear it and and retry
 missing_features_for_transform=Missing some features for transform that were presented on fit: {}

upgini/utils/sklearn_ext.py

@@ -1324,7 +1324,7 @@ def _encode_cat_features(X_train, y_train, X_test, y_test, cat_features, estimat
         else:
             # Shuffle train data
             X_train_shuffled, y_train_shuffled = _shuffle_pair(
-                X_train[cat_features].astype("object"), y_train, random_state
+                X_train[cat_features], y_train, random_state
             )
 
             # Fit encoder on training fold

{upgini-1.2.88a3884.dev1.dist-info → upgini-1.2.89a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.88a3884.dev1
+Version: 1.2.89a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.88a3884.dev1.dist-info → upgini-1.2.89a1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-upgini/__about__.py,sha256=RCAVI4TwhC_It_MBONjiSYbrXtFotET-nMOyORfyw40,33
+upgini/__about__.py,sha256=zOnCJXwGq7WXQ73_SrUBkMqGfV00s4WlXFUPNyejNQ8,25
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=kkNePcLwHKNOLuZLDD8XcIHKVoo_VPUbUM4TSwey60I,218038
+upgini/features_enricher.py,sha256=rieH8wjC1c_q2LYZoju8KZyshokNzFpwVtrCtG88w3s,215940
 upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
-upgini/metrics.py,sha256=ju7JPwLUe8vtFUGbBV6w6ecySd952XucrqToc1edVBs,45306
+upgini/metrics.py,sha256=zIOaiyfQLedU9Fk4877drnlWh-KiImSkZpPeiq6Xr1E,45295
 upgini/search_task.py,sha256=Q5HjBpLIB3OCxAD1zNv5yQ3ZNJx696WCK_-H35_y7Rs,17912
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
 upgini/version_validator.py,sha256=DvbaAvuYFoJqYt0fitpsk6Xcv-H1BYDJYHUMxaKSH_Y,1509
@@ -31,14 +31,14 @@ upgini/autofe/timeseries/roll.py,sha256=zADKXU-eYWQnQ5R3am1yEal8uU6Tm0jLAixwPb_a
 upgini/autofe/timeseries/trend.py,sha256=K1_iw2ko_LIUU8YCUgrvN3n0MkHtsi7-63-8x9er1k4,2129
 upgini/autofe/timeseries/volatility.py,sha256=SvZfhM_ZAWCNpTf87WjSnZsnlblARgruDlu4By4Zvhc,8078
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=4S9qwlAklD8vg9tUU_c1pHE2_glUHAh15-wr5hMwKFw,22879
+upgini/data_source/data_source_publisher.py,sha256=ufL8qK1vg8iUKd5bLWz6hEMGiC3JepUaWYx-nBKVqjA,24294
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=xpHD-3mW1U6Nca0QghC6FSrQLDci9pInuMpOBPPiB8M,28212
+upgini/resource_bundle/strings.properties,sha256=SxO1uWFAc1s7BOFi01OyEI3ajklUKBhs8LkKrstImIg,28290
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -64,13 +64,13 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sklearn_ext.py,sha256=Mdxz0tc-9zT4QyNccA3B86fY4l0MnLDr94POVdYeCT4,49332
+upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
 upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,16832
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.88a3884.dev1.dist-info/METADATA,sha256=KPOdFTBugj7fEYybkMyXP9uABuM75J-eKJmF7V-mEMs,49172
-upgini-1.2.88a3884.dev1.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.88a3884.dev1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.88a3884.dev1.dist-info/RECORD,,
+upgini-1.2.89a1.dist-info/METADATA,sha256=d9XvUcHoqSr2RzIpqLR42x1bffkKnr7PyT6iB6kZGYQ,49164
+upgini-1.2.89a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.89a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.89a1.dist-info/RECORD,,

{upgini-1.2.88a3884.dev1.dist-info → upgini-1.2.89a1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any