upgini 1.2.136__tar.gz → 1.2.138__tar.gz

{upgini-1.2.136 → upgini-1.2.138}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.136
+Version: 1.2.138
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -34,6 +34,7 @@ Requires-Dist: more-itertools==10.7.0
 Requires-Dist: numpy<3.0.0,>=1.19.0
 Requires-Dist: pandas<3.0.0,>=1.1.0
 Requires-Dist: psutil>=5.9.0
+Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pydantic<3.0.0,>1.0.0
 Requires-Dist: pyjwt>=2.8.0
 Requires-Dist: python-bidi==0.4.2

{upgini-1.2.136 → upgini-1.2.138}/pyproject.toml

@@ -56,6 +56,7 @@ dependencies = [
     "psutil>=5.9.0",
     "category-encoders>=2.8.1",
     "more_itertools==10.7.0",
+    "pyarrow==18.1.0",
 ]
 
 [project.urls]

upgini-1.2.138/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.138"

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/dataset.py

@@ -7,6 +7,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 from pandas.api.types import (
     is_float_dtype,
     is_integer_dtype,
@@ -18,6 +20,7 @@ from pandas.api.types import (
 from upgini.errors import ValidationError
 from upgini.http import ProgressStage, SearchProgress, _RestClient
 from upgini.metadata import (
+    CURRENT_DATE_COL,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
     SYSTEM_RECORD_ID,
@@ -38,6 +41,7 @@ from upgini.resource_bundle import ResourceBundle, get_custom_bundle
 from upgini.search_task import SearchTask
 from upgini.utils.config import SampleConfig
 from upgini.utils.email_utils import EmailSearchKeyConverter
+from upgini.utils.hash_utils import file_hash
 from upgini.utils.sample_utils import SampleColumns, sample
 
 try:
@@ -287,7 +291,7 @@ class Dataset:
             for key in search_group
             if key in self.columns_renaming
             and not self.columns_renaming.get(key).endswith(EmailSearchKeyConverter.ONE_DOMAIN_SUFFIX)
-            and not self.columns_renaming.get(key) == "current_date"
+            and not self.columns_renaming.get(key) == CURRENT_DATE_COL
         }
         ipv4_column = self.etalon_def_checked.get(FileColumnMeaningType.IP_ADDRESS.value)
         if (
@@ -469,6 +473,33 @@ class Dataset:
 
             columns.append(column_meta)
 
+        current_date = int(pd.Timestamp(pd.Timestamp.now().date(), tz="UTC").timestamp() * 1000)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            if (
+                self.date_column is not None
+                and self.data[self.date_column].nunique() == 1
+                and self.data[self.date_column].iloc[0] == current_date
+            ):
+                df_without_fake_date = self.data.drop(columns=[self.date_column])
+            else:
+                df_without_fake_date = self.data
+            parquet_file_path = f"{tmp_dir}/{self.dataset_name}.parquet"
+
+            # calculate deterministic digest for any environment
+
+            table = pa.Table.from_pandas(df_without_fake_date, preserve_index=False)
+            table = table.replace_schema_metadata({})  # remove all metadata
+            pq.write_table(
+                table,
+                parquet_file_path,
+                compression=None,  # any compression will make it non-deterministic
+                data_page_size=0,  # optional, to remove page layout variations
+                use_deprecated_int96_timestamps=False,  # fix timestamp format
+                write_statistics=False,  # remove statistics to make it deterministic
+            )
+
+            deterministic_digest = file_hash(parquet_file_path)
+
         return FileMetadata(
             name=self.dataset_name,
             description=self.description,
@@ -479,6 +510,7 @@ class Dataset:
             hierarchicalSubgroupKeys=self.hierarchical_subgroup_keys,
             taskType=self.task_type,
             droppedColumns=self.dropped_columns,
+            deterministicDigest=deterministic_digest,
         )
 
     @staticmethod

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/features_enricher.py

@@ -44,6 +44,7 @@ from upgini.http import (
 from upgini.mdc import MDC
 from upgini.metadata import (
     COUNTRY,
+    CURRENT_DATE_COL,
     DEFAULT_INDEX,
     ENTITY_SYSTEM_RECORD_ID,
     EVAL_SET_INDEX,
@@ -76,6 +77,7 @@ from upgini.utils.custom_loss_utils import (
 )
 from upgini.utils.cv_utils import CVConfig, get_groups
 from upgini.utils.datetime_utils import (
+    DateSearchKeyDetector,
     DateTimeConverter,
     is_blocked_time_series,
     is_dates_distribution_valid,
@@ -167,7 +169,6 @@ class FeaturesEnricher(TransformerMixin):
     """
 
     TARGET_NAME = "target"
-    CURRENT_DATE = "current_date"
     RANDOM_STATE = 42
     CALCULATE_METRICS_THRESHOLD = 50_000_000
     CALCULATE_METRICS_MIN_THRESHOLD = 500
@@ -238,6 +239,7 @@ class FeaturesEnricher(TransformerMixin):
         generate_search_key_features: bool = True,
         sample_config: SampleConfig | None = None,
         print_trace_id: bool = False,
+        print_loaded_report: bool = True,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -284,7 +286,7 @@ class FeaturesEnricher(TransformerMixin):
         self.id_columns = id_columns
         self.id_columns_encoder = None
         self.country_code = country_code
-        self.__validate_search_keys(search_keys, search_id)
+        self.__validate_search_keys(self.search_keys, search_id)
 
         self.model_task_type = ModelTaskType.parse(model_task_type)
         self.endpoint = endpoint
@@ -317,7 +319,8 @@ class FeaturesEnricher(TransformerMixin):
                     self.fit_columns_renaming = {c.name: c.originalName for c in file_metadata.columns}
                     df = pd.DataFrame(columns=x_columns)
                     self.__prepare_feature_importances(trace_id, df, silent=True, update_selected_features=False)
-                    self.__show_selected_features()
+                    if print_loaded_report:
+                        self.__show_selected_features()
                     # TODO validate search_keys with search_keys from file_metadata
                     print(self.bundle.get("search_by_task_id_finish"))
                     self.logger.debug(f"Successfully initialized with search_id: {search_id}")
@@ -2983,6 +2986,9 @@ if response.status_code == 200:
         else:
             selected_input_columns = []
 
+        if DEFAULT_INDEX in selected_input_columns:
+            selected_input_columns.remove(DEFAULT_INDEX)
+
         return selected_input_columns + selected_generated_features
 
     def __validate_search_keys(self, search_keys: dict[str, SearchKey], search_id: str | None = None):
@@ -3167,7 +3173,7 @@ if response.status_code == 200:
 
         if DEFAULT_INDEX in df.columns:
             msg = self.bundle.get("unsupported_index_column")
-            self.logger.info(msg)
+            self.logger.warning(msg)
             print(msg)
             self.fit_dropped_features.add(DEFAULT_INDEX)
             df.drop(columns=DEFAULT_INDEX, inplace=True)
@@ -3223,8 +3229,11 @@ if response.status_code == 200:
             df, self.fit_search_keys, self.fit_generated_features
         )
         self.fit_columns_renaming = normalizer.columns_renaming
-        if normalizer.removed_features:
-            self.__log_warning(self.bundle.get("dataset_date_features").format(normalizer.removed_features))
+        if normalizer.removed_datetime_features:
+            original_removed_datetime_features = [
+                self.fit_columns_renaming.get(f, f) for f in normalizer.removed_datetime_features
+            ]
+            self.__log_warning(self.bundle.get("dataset_date_features").format(original_removed_datetime_features))
 
         non_feature_columns = [
             self.TARGET_NAME,
@@ -4090,12 +4099,14 @@ if response.status_code == 200:
             or set(search_keys.values()) == {SearchKey.EMAIL}
             or set(search_keys.values()) == {SearchKey.HEM}
             or set(search_keys.values()) == {SearchKey.COUNTRY, SearchKey.POSTAL_CODE}
+            or len(search_keys) == 0
+            or set(search_keys.values()) == {SearchKey.CUSTOM_KEY}
         ):
             if not silent:
                 self.__log_warning(bundle.get("current_date_added"))
-            df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
-            search_keys[FeaturesEnricher.CURRENT_DATE] = SearchKey.DATE
-            converter = DateTimeConverter(FeaturesEnricher.CURRENT_DATE, generate_cyclical_features=False)
+            df[CURRENT_DATE_COL] = datetime.date.today()
+            search_keys[CURRENT_DATE_COL] = SearchKey.DATE
+            converter = DateTimeConverter(CURRENT_DATE_COL, generate_cyclical_features=False)
             df = converter.convert(df)
         return df
 
@@ -4109,7 +4120,7 @@ if response.status_code == 200:
         return [
             col
             for col, t in search_keys.items()
-            if t not in [SearchKey.DATE, SearchKey.DATETIME] and df[col].dropna().nunique() > 1
+            if t not in [SearchKey.DATE, SearchKey.DATETIME] and col in df.columns and df[col].dropna().nunique() > 1
         ]
 
     @staticmethod
@@ -4777,7 +4788,8 @@ if response.status_code == 200:
             else:
                 msg = self.bundle.get("unregistered_only_personal_keys")
             self.logger.warning(msg + f" Provided search keys: {search_keys}")
-            raise ValidationError(msg)
+            # Current date will be added later
+            # raise ValidationError(msg)
 
         if (
             len(valid_search_keys.values()) == 1
@@ -4896,6 +4908,16 @@ if response.status_code == 200:
                 search_key in self.fit_search_keys.values() and search_key not in search_keys.values()
             )
 
+        if check_need_detect(SearchKey.DATE) and check_need_detect(SearchKey.DATETIME):
+            maybe_keys = DateSearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if len(maybe_keys) > 0:
+                datetime_key = maybe_keys[0]
+                search_keys[datetime_key] = SearchKey.DATETIME
+                self.autodetected_search_keys[datetime_key] = SearchKey.DATETIME
+                self.logger.info(f"Autodetected search key DATETIME in column {datetime_key}")
+                if not silent_mode:
+                    print(self.bundle.get("datetime_detected").format(datetime_key))
+
         # if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
         if check_need_detect(SearchKey.POSTAL_CODE):
             maybe_keys = PostalCodeSearchKeyDetector().get_search_key_columns(sample, search_keys)

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/http.py

@@ -433,8 +433,8 @@ class _RestClient:
             with open(file_path, "rb") as file:
                 content = file.read()
                 md5_hash.update(content)
-                digest = md5_hash.hexdigest()
-                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest})
+                digest_md5 = md5_hash.hexdigest()
+                metadata_with_md5 = pydantic_copy_method(metadata)(update={"checksumMD5": digest_md5})
 
             digest_sha256 = file_hash(file_path)
             metadata_with_md5 = pydantic_copy_method(metadata_with_md5)(update={"digest": digest_sha256})

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/metadata.py

@@ -12,10 +12,19 @@ SORT_ID = "sort_id"
 EVAL_SET_INDEX = "eval_set_index"
 TARGET = "target"
 COUNTRY = "country_iso_code"
+CURRENT_DATE_COL = "current_date_"
 RENAMED_INDEX = "index_col"
 DEFAULT_INDEX = "index"
 ORIGINAL_INDEX = "original_index"
-SYSTEM_COLUMNS = {SYSTEM_RECORD_ID, ENTITY_SYSTEM_RECORD_ID, SEARCH_KEY_UNNEST, EVAL_SET_INDEX, TARGET, COUNTRY}
+SYSTEM_COLUMNS = {
+    SYSTEM_RECORD_ID,
+    ENTITY_SYSTEM_RECORD_ID,
+    SEARCH_KEY_UNNEST,
+    EVAL_SET_INDEX,
+    TARGET,
+    COUNTRY,
+    CURRENT_DATE_COL,
+}
 
 
 class FileColumnMeaningType(Enum):
@@ -252,6 +261,7 @@ class FileMetadata(BaseModel):
     rowsCount: Optional[int] = None
     checksumMD5: Optional[str] = None
     digest: Optional[str] = None
+    deterministicDigest: Optional[str] = None
     droppedColumns: Optional[List[str]] = None
 
     def column_by_name(self, name: str) -> Optional[FileColumnMetadata]:

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/normalizer/normalize_utils.py

@@ -5,7 +5,6 @@ from typing import Dict, List, Tuple
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_bool_dtype as is_bool
-from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from pandas.api.types import (
     is_float_dtype,
     is_numeric_dtype,
@@ -45,7 +44,7 @@ class Normalizer:
         self.columns_renaming = {}
         self.search_keys = {}
         self.generated_features = []
-        self.removed_features = []
+        self.removed_datetime_features = []
 
     def normalize(
         self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
@@ -134,8 +133,9 @@ class Normalizer:
         features = self._get_features(df)
 
         for f in features:
-            if is_datetime(df[f]) or isinstance(df[f].dtype, pd.PeriodDtype):
-                self.removed_features.append(f)
+            converter = DateTimeConverter(f)
+            if converter.is_datetime(df):
+                self.removed_datetime_features.append(f)
                 df.drop(columns=f, inplace=True)
 
         return df

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/resource_bundle/strings.properties

@@ -210,6 +210,7 @@ features_info_zero_important_features=Oops, we can't find any relevant external
 features_info_zero_hit_rate_search_keys=Oops, looks like values/formats of the search keys {} might be incorrect,\nas we won't be able to match any data source using these values\nPlease check docs https://github.com/upgini/upgini#-search-key-types-we-support-more-to-come or send us a help request in Support:
 features_not_generated=Following features didn't pass checks for automated feature generation: {}
 # Information
+datetime_detected=Datetime detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 postal_code_detected=Postal codes detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 country_detected=Countries detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 country_auto_determined=Search key country_code `{}` was automatically determined by client IP. \nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/utils/base_search_key_detector.py

@@ -24,4 +24,8 @@ class BaseSearchKeyDetector:
         for column_name in other_columns:
             if self._is_search_key_by_values(df[column_name]):
                 columns_by_values.append(column_name)
-        return list(set(columns_by_names + columns_by_values))
+
+        both = [col for col in columns_by_names if col in columns_by_values]
+        only_values = [col for col in columns_by_values if col not in columns_by_names]
+        only_names = [col for col in columns_by_names if col not in columns_by_values]
+        return both + only_values + only_names

{upgini-1.2.136 → upgini-1.2.138}/src/upgini/utils/datetime_utils.py

@@ -10,6 +10,7 @@ from pandas.api.types import is_numeric_dtype
 from upgini.errors import ValidationError
 from upgini.metadata import EVAL_SET_INDEX, SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 DATE_FORMATS = [
     "%Y-%m-%d",
@@ -29,6 +30,15 @@ DATE_FORMATS = [
 DATETIME_PATTERN = r"^[\d\s\.\-:T/+]+$"
 
 
+class DateSearchKeyDetector(BaseSearchKeyDetector):
+    def _is_search_key_by_name(self, column_name: str) -> bool:
+        lower_column_name = str(column_name).lower()
+        return "date" in lower_column_name or "time" in lower_column_name or "timestamp" in lower_column_name
+
+    def _is_search_key_by_values(self, column: pd.Series) -> bool:
+        return DateTimeConverter(column.name).is_datetime(column.to_frame(column.name))
+
+
 class DateTimeConverter:
     DATETIME_COL = "_date_time"
     # MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
@@ -80,7 +90,7 @@ class DateTimeConverter:
             return True
 
         parsed = self.parse_datetime(df, raise_errors=False)
-        return parsed is not None and not parsed.isna().all()
+        return parsed is not None and parsed.isna().mean() <= 0.5
 
     def parse_datetime(self, df: pd.DataFrame, raise_errors=True) -> pd.Series | None:
         if len(df) == 0 or df[self.date_column].isna().all():
@@ -408,6 +418,10 @@ def is_dates_distribution_valid(
         if maybe_date_col is None:
             return
 
+        # Don't check if date column is constant
+        if X[maybe_date_col].nunique() <= 1:
+            return
+
         if isinstance(X[maybe_date_col].dtype, pd.PeriodDtype):
             dates = X[maybe_date_col].dt.to_timestamp().dt.date
         elif pd.__version__ >= "2.0.0":

upgini-1.2.136/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.136"