upgini 1.2.88a3884.dev0__py3-none-any.whl → 1.2.89a1__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.88a3884.dev0"
+__version__ = "1.2.89a1"

upgini/data_source/data_source_publisher.py

@@ -5,6 +5,8 @@ from datetime import datetime
 from enum import Enum
 from typing import Dict, List, Literal, Optional, Union
 
+import pandas as pd
+
 from upgini.errors import HttpError, ValidationError
 from upgini.http import LoggerFactory, get_rest_client
 from upgini.mdc import MDC
@@ -33,7 +35,7 @@ class OnlineUploadingType(Enum):
 class DataSourcePublisher:
     FINAL_STATUSES = ["COMPLETED", "FAILED", "TIMED_OUT"]
     ACCEPTABLE_UPDATE_FREQUENCIES = ["Daily", "Weekly", "Monthly", "Quarterly", "Annually"]
-    DEFAULT_GENERATE_EMBEDDINGS = []
+    DEFAULT_GENERATE_EMBEDDINGS = dict()
 
     def __init__(self, api_key: Optional[str] = None, endpoint: Optional[str] = None, logs_enabled=True):
         self._rest_client = get_rest_client(endpoint, api_key)
@@ -58,7 +60,7 @@ class DataSourcePublisher:
         hash_feature_names=False,
         snapshot_frequency_days: Optional[int] = None,
         join_date_abs_limit_days: Optional[int] = None,
-        features_for_embeddings: Optional[List[str]] = DEFAULT_GENERATE_EMBEDDINGS,
+        features_for_embeddings: Optional[Dict[str, str]] = DEFAULT_GENERATE_EMBEDDINGS,
         data_table_id_to_replace: Optional[str] = None,
         keep_features: Optional[List[str]] = None,
         date_features: Optional[List[str]] = None,
@@ -137,6 +139,25 @@ class DataSourcePublisher:
                 ) and not date_format:
                     raise ValidationError("date_format argument is required for PHONE+DATE and HEM+DATE search keys")
 
+                if secondary_search_keys:
+                    response = self._rest_client.get_active_ads_definitions()
+                    definitions = pd.DataFrame(response["adsDefinitions"])
+                    prod_secondary_definitions = definitions.query(
+                        "(secondarySearchKeys.astype('string') != '[]') & (adsDefinitionAccessType == 'PROD')"
+                    )[["name", "searchKeys", "secondarySearchKeys"]]
+                    for _, row in prod_secondary_definitions.iterrows():
+                        existing_secondary_keys = {item for sublist in row["secondarySearchKeys"] for item in sublist}
+                        if existing_secondary_keys == {v.value.name for v in secondary_search_keys.values()}:
+                            existing_search_keys = {item for sublist in row["searchKeys"] for item in sublist}
+                            if (
+                                existing_search_keys == {v.value.name for v in search_keys.values()}
+                                or ("IP" in str(existing_search_keys) and "IP" in str(search_keys.values()))
+                            ):
+                                raise ValidationError(
+                                    "ADS with the same PRIMARY_KEYS -> SECONDARY_KEYS mapping "
+                                    f"already exists: {row['name']}"
+                                )
+
                 request = {
                     "dataTableUri": data_table_uri,
                     "searchKeys": {k: v.value.value for k, v in search_keys.items()},

upgini/features_enricher.py

@@ -7,7 +7,6 @@ import json
 import logging
 import numbers
 import os
-import pickle
 import sys
 import tempfile
 import time
@@ -3450,6 +3449,11 @@ if response.status_code == 200:
         if self.model_task_type == ModelTaskType.BINARY and eval_y_nunique != 2:
             raise ValidationError(self.bundle.get("binary_target_eval_unique_count_not_2").format(eval_y_nunique))
 
+        # Check for duplicates between train and eval sets by comparing all values
+        train_eval_intersection = pd.merge(X, validated_eval_X, how='inner')
+        if len(train_eval_intersection) > 0:
+            raise ValidationError(self.bundle.get("eval_x_has_train_samples"))
+
         return validated_eval_X, validated_eval_y
 
     def _validate_baseline_score(self, X: pd.DataFrame, eval_set: Optional[List[Tuple]]):
@@ -4569,60 +4573,52 @@ if response.status_code == 200:
         y: Union[pd.DataFrame, pd.Series, None] = None,
         eval_set: Union[Tuple, None] = None,
     ):
-        def dump_task():
+        def dump_task(X_, y_, eval_set_):
             try:
-                random_state = 42
-                rnd = np.random.RandomState(random_state)
-                if _num_samples(X) > 0:
-                    xy_sample_index = rnd.randint(0, _num_samples(X), size=1000)
-                else:
-                    xy_sample_index = []
-
-                def sample(inp, sample_index):
-                    if _num_samples(inp) <= 1000:
-                        return inp
-                    if isinstance(inp, (pd.DataFrame, pd.Series)):
-                        return inp.sample(n=1000, random_state=random_state)
-                    if isinstance(inp, np.ndarray):
-                        return inp[sample_index]
-                    if isinstance(inp, list):
-                        return inp[sample_index]
+                if isinstance(X_, pd.Series):
+                    X_ = X_.to_frame()
+
+                # TODO check that this file was already uploaded
 
                 with tempfile.TemporaryDirectory() as tmp_dir:
-                    with open(f"{tmp_dir}/x.pickle", "wb") as x_file:
-                        pickle.dump(sample(X, xy_sample_index), x_file)
-                    if y is not None:
-                        with open(f"{tmp_dir}/y.pickle", "wb") as y_file:
-                            pickle.dump(sample(y, xy_sample_index), y_file)
-                        if eval_set and _num_samples(eval_set[0][0]) > 0:
-                            eval_xy_sample_index = rnd.randint(0, _num_samples(eval_set[0][0]), size=1000)
-                            with open(f"{tmp_dir}/eval_x.pickle", "wb") as eval_x_file:
-                                pickle.dump(sample(eval_set[0][0], eval_xy_sample_index), eval_x_file)
-                            with open(f"{tmp_dir}/eval_y.pickle", "wb") as eval_y_file:
-                                pickle.dump(sample(eval_set[0][1], eval_xy_sample_index), eval_y_file)
+                    X_.to_parquet(f"{tmp_dir}/x.parquet", compression="zstd")
+
+                    if y_ is not None:
+                        if isinstance(y_, pd.Series):
+                            y_ = y_.to_frame()
+                        y_.to_parquet(f"{tmp_dir}/y.parquet", compression="zstd")
+                        if eval_set_ and _num_samples(eval_set_[0][0]) > 0:
+                            eval_x_ = eval_set_[0][0]
+                            eval_y_ = eval_set_[0][1]
+                            if isinstance(eval_x_, pd.Series):
+                                eval_x_ = eval_x_.to_frame()
+                            eval_x_.to_parquet(f"{tmp_dir}/eval_x.parquet", compression="zstd")
+                            if isinstance(eval_y_, pd.Series):
+                                eval_y_ = eval_y_.to_frame()
+                            eval_y_.to_parquet(f"{tmp_dir}/eval_y.parquet", compression="zstd")
                             self.rest_client.dump_input_files(
                                 trace_id,
-                                f"{tmp_dir}/x.pickle",
-                                f"{tmp_dir}/y.pickle",
-                                f"{tmp_dir}/eval_x.pickle",
-                                f"{tmp_dir}/eval_y.pickle",
+                                f"{tmp_dir}/x.parquet",
+                                f"{tmp_dir}/y.parquet",
+                                f"{tmp_dir}/eval_x.parquet",
+                                f"{tmp_dir}/eval_y.parquet",
                             )
                         else:
                             self.rest_client.dump_input_files(
                                 trace_id,
-                                f"{tmp_dir}/x.pickle",
-                                f"{tmp_dir}/y.pickle",
+                                f"{tmp_dir}/x.parquet",
+                                f"{tmp_dir}/y.parquet",
                             )
                     else:
                         self.rest_client.dump_input_files(
                             trace_id,
-                            f"{tmp_dir}/x.pickle",
+                            f"{tmp_dir}/x.parquet",
                         )
             except Exception:
                 self.logger.warning("Failed to dump input files", exc_info=True)
 
         try:
-            Thread(target=dump_task, daemon=True).start()
+            Thread(target=dump_task, args=(X, y, eval_set), daemon=True).start()
         except Exception:
             self.logger.warning("Failed to dump input files", exc_info=True)
 

upgini/resource_bundle/strings.properties

@@ -137,6 +137,8 @@ eval_y_multiindex_unsupported=Multi index in y in eval_set is not supported
 eval_x_is_empty=X in eval_set is empty.
 eval_y_is_empty=y in eval_set is empty.
 x_and_eval_x_diff_types=X and eval_set X has different types: {} and {}
+eval_x_has_train_samples=Eval set X has rows that are present in train set X
+
 baseline_score_column_not_exists=baseline_score_column {} doesn't exist in input dataframe
 baseline_score_column_has_na=baseline_score_column contains NaN. Clear it and and retry
 missing_features_for_transform=Missing some features for transform that were presented on fit: {}

upgini/utils/sklearn_ext.py

@@ -1324,7 +1324,7 @@ def _encode_cat_features(X_train, y_train, X_test, y_test, cat_features, estimat
         else:
             # Shuffle train data
             X_train_shuffled, y_train_shuffled = _shuffle_pair(
-                X_train[cat_features].astype("object"), y_train, random_state
+                X_train[cat_features], y_train, random_state
             )
 
             # Fit encoder on training fold

{upgini-1.2.88a3884.dev0.dist-info → upgini-1.2.89a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.88a3884.dev0
+Version: 1.2.89a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.88a3884.dev0.dist-info → upgini-1.2.89a1.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=9UxVEFo0h8LcuPSKD5JSZ_n02IZF15Ksx8d1ITu4M7U,33
+upgini/__about__.py,sha256=zOnCJXwGq7WXQ73_SrUBkMqGfV00s4WlXFUPNyejNQ8,25
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=fRtqSkXNONLnPe6cCL967GMt349FTIpXzy_u8LUKncw,35354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=eFnJVb8jM1INlT-imfjafhWtOfx9EJv2HSvlfyGy0_U,216188
+upgini/features_enricher.py,sha256=rieH8wjC1c_q2LYZoju8KZyshokNzFpwVtrCtG88w3s,215940
 upgini/http.py,sha256=6Qcepv0tDC72mBBJxYHnA2xqw6QwFaKrXN8o4vju8Es,44372
 upgini/metadata.py,sha256=zt_9k0iQbWXuiRZcel4ORNPdQKt6Ou69ucZD_E1Q46o,12341
 upgini/metrics.py,sha256=zIOaiyfQLedU9Fk4877drnlWh-KiImSkZpPeiq6Xr1E,45295
@@ -31,14 +31,14 @@ upgini/autofe/timeseries/roll.py,sha256=zADKXU-eYWQnQ5R3am1yEal8uU6Tm0jLAixwPb_a
 upgini/autofe/timeseries/trend.py,sha256=K1_iw2ko_LIUU8YCUgrvN3n0MkHtsi7-63-8x9er1k4,2129
 upgini/autofe/timeseries/volatility.py,sha256=SvZfhM_ZAWCNpTf87WjSnZsnlblARgruDlu4By4Zvhc,8078
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=4S9qwlAklD8vg9tUU_c1pHE2_glUHAh15-wr5hMwKFw,22879
+upgini/data_source/data_source_publisher.py,sha256=ufL8qK1vg8iUKd5bLWz6hEMGiC3JepUaWYx-nBKVqjA,24294
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/normalizer/normalize_utils.py,sha256=g2TcDXZeJp9kAFO2sTqZ4CAsN4J1qHNgoJHZ8gtzUWo,7376
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=xpHD-3mW1U6Nca0QghC6FSrQLDci9pInuMpOBPPiB8M,28212
+upgini/resource_bundle/strings.properties,sha256=SxO1uWFAc1s7BOFi01OyEI3ajklUKBhs8LkKrstImIg,28290
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=7GpjYqjOp58vYcJLiX__1R5wjUlyQbxvHJ2klFnup_M,6389
@@ -64,13 +64,13 @@ upgini/utils/mstats.py,sha256=u3gQVUtDRbyrOQK6V1UJ2Rx1QbkSNYGjXa6m3Z_dPVs,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
-upgini/utils/sklearn_ext.py,sha256=Mdxz0tc-9zT4QyNccA3B86fY4l0MnLDr94POVdYeCT4,49332
+upgini/utils/sklearn_ext.py,sha256=jLJWAKkqQinV15Z4y1ZnsN3c-fKFwXTsprs00COnyVU,49315
 upgini/utils/sort.py,sha256=8uuHs2nfSMVnz8GgvbOmgMB1PgEIZP1uhmeRFxcwnYw,7039
 upgini/utils/target_utils.py,sha256=LRN840dzx78-wg7ftdxAkp2c1eu8-JDvkACiRThm4HE,16832
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.88a3884.dev0.dist-info/METADATA,sha256=e_lwt9ydR712gQBymukF9Lc2W-5aqj5nrZa-6T-UXA4,49172
-upgini-1.2.88a3884.dev0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.88a3884.dev0.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.88a3884.dev0.dist-info/RECORD,,
+upgini-1.2.89a1.dist-info/METADATA,sha256=d9XvUcHoqSr2RzIpqLR42x1bffkKnr7PyT6iB6kZGYQ,49164
+upgini-1.2.89a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.89a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.89a1.dist-info/RECORD,,

{upgini-1.2.88a3884.dev0.dist-info → upgini-1.2.89a1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.24.2
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any