upgini 1.2.135a3__py3-none-any.whl → 1.2.137__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.135a3"
+__version__ = "1.2.137"

upgini/dataset.py

@@ -7,6 +7,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
@@ -18,6 +20,7 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    CURRENT_DATE_COL,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     SYSTEM_RECORD_ID,
@@ -38,6 +41,7 @@ from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.utils.config import SampleConfig
 from upgini.utils.email_utils import EmailSearchKeyConverter
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.sample_utils import SampleColumns, sample
 
 try:
@@ -71,6 +75,7 @@ class Dataset:
         date_column: Optional[str] = None,
         id_columns: Optional[List[str]] = None,
         is_imbalanced: bool = False,
+        dropped_columns: Optional[List[str]] = None,
         random_state: Optional[int] = None,
         sample_config: Optional[SampleConfig] = None,
         rest_client: Optional[_RestClient] = None,
@@ -118,6 +123,7 @@ class Dataset:
         self.is_imbalanced: bool = False
         self.id_columns = id_columns
         self.is_imbalanced = is_imbalanced
+        self.dropped_columns = dropped_columns
         self.date_column = date_column
         if logger is not None:
             self.logger = logger
@@ -285,7 +291,7 @@ class Dataset:
             for key in search_group
             if key in self.columns_renaming
             and not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
-            and not self.columns_renaming.get(key) == "current_date"
+            and not self.columns_renaming.get(key) == CURRENT_DATE_COL
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (
@@ -467,6 +473,33 @@ class Dataset:
 
             columns.append(column_meta)
 
+        current_date = int(pd.Timestamp(pd.Timestamp.now().date(), tz="UTC").timestamp() * 1000)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            if (
+                self.date_column is not None
+                and self.data[self.date_column].nunique() == 1
+                and self.data[self.date_column].iloc[0] == current_date
+            ):
+                df_without_fake_date = self.data.drop(columns=[self.date_column])
+            else:
+                df_without_fake_date = self.data
+            parquet_file_path = f"{tmp_dir}/{self.dataset_name}.parquet"
+
+            # calculate deterministic digest for any environment
+
+            table = pa.Table.from_pandas(df_without_fake_date, preserve_index=False)
+            table = table.replace_schema_metadata({})  # remove all metadata
+            pq.write_table(
+                table,
+                parquet_file_path,
+                compression=None,  # any compression will make it non-deterministic
+                data_page_size=0,  # optional, to remove page layout variations
+                use_deprecated_int96_timestamps=False,  # fix timestamp format
+                write_statistics=False,  # remove statistics to make it deterministic
+            )
+
+            deterministic_digest = file_hash(parquet_file_path)
+
         return FileMetadata(
             name=self.dataset_name,
             description=self.description,
@@ -476,6 +509,8 @@ class Dataset:
             hierarchicalGroupKeys=self.hierarchical_group_keys,
             hierarchicalSubgroupKeys=self.hierarchical_subgroup_keys,
             taskType=self.task_type,
+            droppedColumns=self.dropped_columns,
+            deterministicDigest=deterministic_digest,
         )
 
     @staticmethod

upgini/features_enricher.py

@@ -44,6 +44,7 @@ from upgini.http import (
 from upgini.mdc import MDC
 from upgini.metadata import (
     COUNTRY,
+    CURRENT_DATE_COL,
     DEFAULT_INDEX,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
@@ -167,7 +168,6 @@ class FeaturesEnricher(TransformerMixin):
     """
 
     TARGET_NAME = "target"
-    CURRENT_DATE = "current_date"
     RANDOM_STATE = 42
     CALCULATE_METRICS_THRESHOLD = 50_000_000
     CALCULATE_METRICS_MIN_THRESHOLD = 500
@@ -1737,6 +1737,10 @@ class FeaturesEnricher(TransformerMixin):
 
         self.logger.info(f"Excluding search keys: {excluding_search_keys}")
 
+        file_meta = self._search_task.get_file_metadata(trace_id)
+        fit_dropped_features = self.fit_dropped_features or file_meta.droppedColumns or []
+        original_dropped_features = [columns_renaming.get(f, f) for f in fit_dropped_features]
+
         client_features = [
             c
             for c in validated_X.columns.to_list()
@@ -1744,7 +1748,7 @@ class FeaturesEnricher(TransformerMixin):
             and c
             not in (
                 excluding_search_keys
-                + list(self.fit_dropped_features)
+                + original_dropped_features
                 + [DateTimeConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
             )
         ]
@@ -2955,8 +2959,10 @@ if response.status_code == 200:
         trace_id: str,
         is_transform: bool = False,
     ):
-        fit_input_columns = [c.originalName for c in self._search_task.get_file_metadata(trace_id).columns]
-        original_dropped_features = [self.fit_columns_renaming.get(c, c) for c in self.fit_dropped_features]
+        file_meta = self._search_task.get_file_metadata(trace_id)
+        fit_dropped_features = self.fit_dropped_features or file_meta.droppedColumns or []
+        fit_input_columns = [c.originalName for c in file_meta.columns]
+        original_dropped_features = [self.fit_columns_renaming.get(c, c) for c in fit_dropped_features]
         new_columns_on_transform = [
             c for c in validated_Xy.columns if c not in fit_input_columns and c not in original_dropped_features
         ]
@@ -2977,6 +2983,9 @@ if response.status_code == 200:
         else:
             selected_input_columns = []
 
+        if DEFAULT_INDEX in selected_input_columns:
+            selected_input_columns.remove(DEFAULT_INDEX)
+
         return selected_input_columns + selected_generated_features
 
     def __validate_search_keys(self, search_keys: dict[str, SearchKey], search_id: str | None = None):
@@ -3161,7 +3170,7 @@ if response.status_code == 200:
 
         if DEFAULT_INDEX in df.columns:
             msg = self.bundle.get("unsupported_index_column")
-            self.logger.info(msg)
+            self.logger.warning(msg)
             print(msg)
             self.fit_dropped_features.add(DEFAULT_INDEX)
             df.drop(columns=DEFAULT_INDEX, inplace=True)
@@ -3380,6 +3389,7 @@ if response.status_code == 200:
             cv_type=self.cv,
             id_columns=self.__get_renamed_id_columns(),
             is_imbalanced=self.imbalanced,
+            dropped_columns=[self.fit_columns_renaming.get(f, f) for f in self.fit_dropped_features],
             date_column=self._get_date_column(self.fit_search_keys),
             date_format=self.date_format,
             random_state=self.random_state,
@@ -4086,9 +4096,10 @@ if response.status_code == 200:
         ):
             if not silent:
                 self.__log_warning(bundle.get("current_date_added"))
-            df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
-            search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
-            converter = DateTimeConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
+            df[CURRENT_DATE_COL] = datetime.date.today()
+            # df[CURRENT_DATE_COL] = datetime.date(2025, 10, 15)
+            search_keys[CURRENT_DATE_COL] = SearchKey.DATE
+            converter = DateTimeConverter(CURRENT_DATE_COL, generate_cyclical_features=False)
             df = converter.convert(df)
         return df
 
@@ -4102,7 +4113,7 @@ if response.status_code == 200:
         return [
             col
             for col, t in search_keys.items()
-            if t not in [SearchKey.DATE, SearchKey.DATETIME] and df[col].dropna().nunique() > 1
+            if t not in [SearchKey.DATE, SearchKey.DATETIME] and col in df.columns and df[col].dropna().nunique() > 1
         ]
 
     @staticmethod

upgini/http.py

@@ -433,8 +433,8 @@ class _RestClient:
             with open(file_path, "rb") as file:
                 content = file.read()
                 md5_hash.update(content)
-                digest = md5_hash.hexdigest()
-                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
+                digest_md5 = md5_hash.hexdigest()
+                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest_md5})
 
             digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})

upgini/metadata.py

@@ -12,10 +12,19 @@ SORT_ID = "sort_id"
 EVAL_SET_INDEX = "eval_set_index"
 TARGET = "target"
 COUNTRY = "country_iso_code"
+CURRENT_DATE_COL = "current_date_"
 RENAMED_INDEX = "index_col"
 DEFAULT_INDEX = "index"
 ORIGINAL_INDEX = "original_index"
-SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
+SYSTEM_COLUMNS = {
+    SYSTEM_RECORD_ID,
+    ENTITY_SYSTEM_RECORD_ID,
+    SEARCH_KEY_UNNEST,
+    EVAL_SET_INDEX,
+    TARGET,
+    COUNTRY,
+    CURRENT_DATE_COL,
+}
 
 
 class FileColumnMeaningType(Enum):
@@ -252,6 +261,8 @@ class FileMetadata(BaseModel):
     rowsCount: Optional[int] = None
     checksumMD5: Optional[str] = None
     digest: Optional[str] = None
+    deterministicDigest: Optional[str] = None
+    droppedColumns: Optional[List[str]] = None
 
     def column_by_name(self, name: str) -> Optional[FileColumnMetadata]:
         for c in self.columns:

upgini/utils/datetime_utils.py

@@ -408,6 +408,10 @@ def is_dates_distribution_valid(
         if maybe_date_col is None:
             return
 
+        # Don't check if date column is constant
+        if X[maybe_date_col].nunique() <= 1:
+            return
+
         if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
             dates = X[maybe_date_col].dt.to_timestamp().dt.date
         elif pd.__version__ >= "2.0.0":

upgini/utils/features_validator.py

@@ -46,7 +46,7 @@ class FeaturesValidator:
 
         columns_renaming = columns_renaming or {}
 
-        if one_hot_encoded_features:
+        if one_hot_encoded_features and len(one_hot_encoded_features) > 1:
             msg = bundle.get("one_hot_encoded_features").format(
                 [columns_renaming.get(f, f) for f in one_hot_encoded_features]
             )

{upgini-1.2.135a3.dist-info → upgini-1.2.137.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.135a3
+Version: 1.2.137
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -34,6 +34,7 @@ Requires-Dist: more-itertools==10.7.0
 Requires-Dist: numpy<3.0.0,>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
 Requires-Dist: psutil>=5.9.0
+Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pydantic<3.0.0,>1.0.0
 Requires-Dist: pyjwt>=2.8.0
 Requires-Dist: python-bidi==0.4.2

{upgini-1.2.135a3.dist-info → upgini-1.2.137.dist-info}/RECORD

@@ -1,11 +1,11 @@
-upgini/__about__.py,sha256=zKar2BSHx-M-ruW4V1DQDr70UfG-vBoiDPsQB4duA-c,26
+upgini/__about__.py,sha256=WAhVusMOc6hw9YR0UCWwVJJi3v2_uHEpPqxnSm9SguM,24
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=qXo1eAUM21jmNMYGa6IeGuRJmjU3jtCieS4sAVVIfXs,31721
+upgini/dataset.py,sha256=D9JzJJkZLPP_dp8GOlGgMhTtrd5pvP-4cHIcqiY3q-E,33354
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=dC71_-18cqymS26EhEnfYnZFGDvRtvW6Pga4ZtJN-BI,235485
-upgini/http.py,sha256=-J_wOpnwVnT0ebPC6sOs6fN3AWtCD0LJLu6nlYmxaqk,44348
-upgini/metadata.py,sha256=H3wiN37k-yqWZgbPD0tJzx8DzaCIkgmX5cybhByQWLg,12619
+upgini/features_enricher.py,sha256=g1daAwtECDpSsM2TtVmozU3crsdMQ_xC5PlBJAafAX0,236099
+upgini/http.py,sha256=y26x4TQVYuEM3jz8JdASxSyBtvBemUkFf-FmX25sx-s,44356
+upgini/metadata.py,sha256=BwUTCY-EUHqPtO0tGazHrk3wqhh-NfjNZhlBHW8bR78,12796
 upgini/metrics.py,sha256=KCPE_apPN-9BIdv6GqASbJVaB_gBcy8wzNApAcyaGo4,46020
 upgini/search_task.py,sha256=5mL_qV5mVtDkIumM9xCOgfa9Lc2B8mxJ1qI21iaScnQ,18656
 upgini/spinner.py,sha256=4iMd-eIe_BnkqFEMIliULTbj6rNI2HkN_VJ4qYe0cUc,1118
@@ -52,13 +52,13 @@ upgini/utils/config.py,sha256=zFdnjchykfp_1Tm3Qep7phLzXBpXIOzr2tIuXchRBLw,1754
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=3_FQoa_ywgEeznaEPN2kuH_ES-LZJWSN2AI39sM9NRg,16988
+upgini/utils/datetime_utils.py,sha256=B9zNaH2ZyV-lbBSTBdCZjc4zq1nVlejci40sf-TYfik,17102
 upgini/utils/deduplicate_utils.py,sha256=CLX0QapRxB-ZVQT7yGvv1vSd2zac5SwRjCJavujdCps,11332
 upgini/utils/display_utils.py,sha256=MoTqXZJvC6pAqgOaI3V0FG-IU_LnMfrn4TDcNvUqsdg,13316
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/feature_info.py,sha256=SQTRbSxJDkh2G2c0KGBmOv8f69gVzWbTtcXn0_2Qb-8,7945
-upgini/utils/features_validator.py,sha256=RdRMisZYeJ8HVCKiKxqSyWjoLf_MsZNXxHIuWf6H2g4,4939
+upgini/utils/features_validator.py,sha256=pAyS57-jYlihMOhweM12GOvCTJC13fTIJ9lQwgoON5c,4977
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/hash_utils.py,sha256=mP2yHyzvDNdpa5g3B4MHzulxBeEz_ZSoGl1YF_VnAyE,5538
 upgini/utils/ip_utils.py,sha256=wmnnwVQdjX9o1cNQw6VQMk6maHhvsq6hNsZBYf9knrw,6585
@@ -74,7 +74,7 @@ upgini/utils/target_utils.py,sha256=CihpV6SC95HwtlMH60rGAUzVDa4Id0Bva8ySprmNHlE,
 upgini/utils/track_info.py,sha256=NDKeQTUlZaYp15UoP-xLKGoDoJQ0drbDMwB0g9R0PUg,6427
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.135a3.dist-info/METADATA,sha256=p6JAaluvHU1B6Al8fHzq5KsgyNQvznWkEDwE1fhHuZI,51135
-upgini-1.2.135a3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-upgini-1.2.135a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.135a3.dist-info/RECORD,,
+upgini-1.2.137.dist-info/METADATA,sha256=qTIcTcJz2tn18BOyAa7SbP14-fOxG4l3rtQyCxG8wDI,51164
+upgini-1.2.137.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+upgini-1.2.137.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.137.dist-info/RECORD,,