upgini 1.2.135a3__tar.gz → 1.2.137__tar.gz

{upgini-1.2.135a3 → upgini-1.2.137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.135a3
+Version: 1.2.137
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -34,6 +34,7 @@ Requires-Dist: more-itertools==10.7.0
 Requires-Dist: numpy<3.0.0,>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
 Requires-Dist: psutil>=5.9.0
+Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pydantic<3.0.0,>1.0.0
 Requires-Dist: pyjwt>=2.8.0
 Requires-Dist: python-bidi==0.4.2

{upgini-1.2.135a3 → upgini-1.2.137}/pyproject.toml

@@ -56,6 +56,7 @@ dependencies = [
     "psutil>=5.9.0",
     "category-encoders>=2.8.1",
     "more_itertools==10.7.0",
+    "pyarrow==18.1.0",
 ]
 
 [project.urls]

upgini-1.2.137/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.137"

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/dataset.py

@@ -7,6 +7,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
@@ -18,6 +20,7 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    CURRENT_DATE_COL,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     SYSTEM_RECORD_ID,
@@ -38,6 +41,7 @@ from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.utils.config import SampleConfig
 from upgini.utils.email_utils import EmailSearchKeyConverter
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.sample_utils import SampleColumns, sample
 
 try:
@@ -71,6 +75,7 @@ class Dataset:
         date_column: Optional[str] = None,
         id_columns: Optional[List[str]] = None,
         is_imbalanced: bool = False,
+        dropped_columns: Optional[List[str]] = None,
         random_state: Optional[int] = None,
         sample_config: Optional[SampleConfig] = None,
         rest_client: Optional[_RestClient] = None,
@@ -118,6 +123,7 @@ class Dataset:
         self.is_imbalanced: bool = False
         self.id_columns = id_columns
         self.is_imbalanced = is_imbalanced
+        self.dropped_columns = dropped_columns
         self.date_column = date_column
         if logger is not None:
             self.logger = logger
@@ -285,7 +291,7 @@ class Dataset:
             for key in search_group
             if key in self.columns_renaming
             and not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
-            and not self.columns_renaming.get(key) == "current_date"
+            and not self.columns_renaming.get(key) == CURRENT_DATE_COL
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (
@@ -467,6 +473,33 @@ class Dataset:
 
             columns.append(column_meta)
 
+        current_date = int(pd.Timestamp(pd.Timestamp.now().date(), tz="UTC").timestamp() * 1000)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            if (
+                self.date_column is not None
+                and self.data[self.date_column].nunique() == 1
+                and self.data[self.date_column].iloc[0] == current_date
+            ):
+                df_without_fake_date = self.data.drop(columns=[self.date_column])
+            else:
+                df_without_fake_date = self.data
+            parquet_file_path = f"{tmp_dir}/{self.dataset_name}.parquet"
+
+            # calculate deterministic digest for any environment
+
+            table = pa.Table.from_pandas(df_without_fake_date, preserve_index=False)
+            table = table.replace_schema_metadata({})  # remove all metadata
+            pq.write_table(
+                table,
+                parquet_file_path,
+                compression=None,  # any compression will make it non-deterministic
+                data_page_size=0,  # optional, to remove page layout variations
+                use_deprecated_int96_timestamps=False,  # fix timestamp format
+                write_statistics=False,  # remove statistics to make it deterministic
+            )
+
+            deterministic_digest = file_hash(parquet_file_path)
+
         return FileMetadata(
             name=self.dataset_name,
             description=self.description,
@@ -476,6 +509,8 @@ class Dataset:
             hierarchicalGroupKeys=self.hierarchical_group_keys,
             hierarchicalSubgroupKeys=self.hierarchical_subgroup_keys,
             taskType=self.task_type,
+            droppedColumns=self.dropped_columns,
+            deterministicDigest=deterministic_digest,
         )
 
     @staticmethod

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/features_enricher.py

@@ -44,6 +44,7 @@ from upgini.http import (
 from upgini.mdc import MDC
 from upgini.metadata import (
     COUNTRY,
+    CURRENT_DATE_COL,
     DEFAULT_INDEX,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
@@ -167,7 +168,6 @@ class FeaturesEnricher(TransformerMixin):
     """
 
     TARGET_NAME = "target"
-    CURRENT_DATE = "current_date"
     RANDOM_STATE = 42
     CALCULATE_METRICS_THRESHOLD = 50_000_000
     CALCULATE_METRICS_MIN_THRESHOLD = 500
@@ -1737,6 +1737,10 @@ class FeaturesEnricher(TransformerMixin):
 
         self.logger.info(f"Excluding search keys: {excluding_search_keys}")
 
+        file_meta = self._search_task.get_file_metadata(trace_id)
+        fit_dropped_features = self.fit_dropped_features or file_meta.droppedColumns or []
+        original_dropped_features = [columns_renaming.get(f, f) for f in fit_dropped_features]
+
         client_features = [
             c
             for c in validated_X.columns.to_list()
@@ -1744,7 +1748,7 @@ class FeaturesEnricher(TransformerMixin):
             and c
             not in (
                 excluding_search_keys
-                + list(self.fit_dropped_features)
+                + original_dropped_features
                 + [DateTimeConverter.DATETIME_COL, SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID]
             )
         ]
@@ -2955,8 +2959,10 @@ if response.status_code == 200:
         trace_id: str,
         is_transform: bool = False,
     ):
-        fit_input_columns = [c.originalName for c in self._search_task.get_file_metadata(trace_id).columns]
-        original_dropped_features = [self.fit_columns_renaming.get(c, c) for c in self.fit_dropped_features]
+        file_meta = self._search_task.get_file_metadata(trace_id)
+        fit_dropped_features = self.fit_dropped_features or file_meta.droppedColumns or []
+        fit_input_columns = [c.originalName for c in file_meta.columns]
+        original_dropped_features = [self.fit_columns_renaming.get(c, c) for c in fit_dropped_features]
         new_columns_on_transform = [
             c for c in validated_Xy.columns if c not in fit_input_columns and c not in original_dropped_features
         ]
@@ -2977,6 +2983,9 @@ if response.status_code == 200:
         else:
             selected_input_columns = []
 
+        if DEFAULT_INDEX in selected_input_columns:
+            selected_input_columns.remove(DEFAULT_INDEX)
+
         return selected_input_columns + selected_generated_features
 
     def __validate_search_keys(self, search_keys: dict[str, SearchKey], search_id: str | None = None):
@@ -3161,7 +3170,7 @@ if response.status_code == 200:
 
         if DEFAULT_INDEX in df.columns:
             msg = self.bundle.get("unsupported_index_column")
-            self.logger.info(msg)
+            self.logger.warning(msg)
             print(msg)
             self.fit_dropped_features.add(DEFAULT_INDEX)
             df.drop(columns=DEFAULT_INDEX, inplace=True)
@@ -3380,6 +3389,7 @@ if response.status_code == 200:
             cv_type=self.cv,
             id_columns=self.__get_renamed_id_columns(),
             is_imbalanced=self.imbalanced,
+            dropped_columns=[self.fit_columns_renaming.get(f, f) for f in self.fit_dropped_features],
             date_column=self._get_date_column(self.fit_search_keys),
             date_format=self.date_format,
             random_state=self.random_state,
@@ -4086,9 +4096,10 @@ if response.status_code == 200:
         ):
             if not silent:
                 self.__log_warning(bundle.get("current_date_added"))
-            df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
-            search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
-            converter = DateTimeConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
+            df[CURRENT_DATE_COL] = datetime.date.today()
+            # df[CURRENT_DATE_COL] = datetime.date(2025, 10, 15)
+            search_keys[CURRENT_DATE_COL] = SearchKey.DATE
+            converter = DateTimeConverter(CURRENT_DATE_COL, generate_cyclical_features=False)
             df = converter.convert(df)
         return df
 
@@ -4102,7 +4113,7 @@ if response.status_code == 200:
         return [
             col
             for col, t in search_keys.items()
-            if t not in [SearchKey.DATE, SearchKey.DATETIME] and df[col].dropna().nunique() > 1
+            if t not in [SearchKey.DATE, SearchKey.DATETIME] and col in df.columns and df[col].dropna().nunique() > 1
         ]
 
     @staticmethod

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/http.py

@@ -433,8 +433,8 @@ class _RestClient:
             with open(file_path, "rb") as file:
                 content = file.read()
                 md5_hash.update(content)
-                digest = md5_hash.hexdigest()
-                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
+                digest_md5 = md5_hash.hexdigest()
+                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest_md5})
 
             digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/metadata.py

@@ -12,10 +12,19 @@ SORT_ID = "sort_id"
 EVAL_SET_INDEX = "eval_set_index"
 TARGET = "target"
 COUNTRY = "country_iso_code"
+CURRENT_DATE_COL = "current_date_"
 RENAMED_INDEX = "index_col"
 DEFAULT_INDEX = "index"
 ORIGINAL_INDEX = "original_index"
-SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
+SYSTEM_COLUMNS = {
+    SYSTEM_RECORD_ID,
+    ENTITY_SYSTEM_RECORD_ID,
+    SEARCH_KEY_UNNEST,
+    EVAL_SET_INDEX,
+    TARGET,
+    COUNTRY,
+    CURRENT_DATE_COL,
+}
 
 
 class FileColumnMeaningType(Enum):
@@ -252,6 +261,8 @@ class FileMetadata(BaseModel):
     rowsCount: Optional[int] = None
     checksumMD5: Optional[str] = None
     digest: Optional[str] = None
+    deterministicDigest: Optional[str] = None
+    droppedColumns: Optional[List[str]] = None
 
     def column_by_name(self, name: str) -> Optional[FileColumnMetadata]:
         for c in self.columns:

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/utils/datetime_utils.py

@@ -408,6 +408,10 @@ def is_dates_distribution_valid(
         if maybe_date_col is None:
             return
 
+        # Don't check if date column is constant
+        if X[maybe_date_col].nunique() <= 1:
+            return
+
         if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
             dates = X[maybe_date_col].dt.to_timestamp().dt.date
         elif pd.__version__ >= "2.0.0":

{upgini-1.2.135a3 → upgini-1.2.137}/src/upgini/utils/features_validator.py

@@ -46,7 +46,7 @@ class FeaturesValidator:
 
         columns_renaming = columns_renaming or {}
 
-        if one_hot_encoded_features:
+        if one_hot_encoded_features and len(one_hot_encoded_features) > 1:
             msg = bundle.get("one_hot_encoded_features").format(
                 [columns_renaming.get(f, f) for f in one_hot_encoded_features]
             )

upgini-1.2.135a3/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.135a3"