upgini 1.2.137__py3-none-any.whl → 1.2.139__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.137"
+__version__ = "1.2.139"

upgini/dataset.py

@@ -54,7 +54,7 @@ except Exception:
 
 class Dataset:
     MIN_ROWS_COUNT = 100
-    MAX_ROWS = 200_000
+    MAX_ROWS = 3_000_000
     MIN_SUPPORTED_DATE_TS = 946684800000  # 2000-01-01
     MAX_FEATURES_COUNT = 3500
     MAX_UPLOADING_FILE_SIZE = 268435456  # 256 Mb

upgini/features_enricher.py

@@ -77,6 +77,7 @@ from upgini.utils.custom_loss_utils import (
 )
 from upgini.utils.cv_utils import CVConfig, get_groups
 from upgini.utils.datetime_utils import (
+    DateSearchKeyDetector,
     DateTimeConverter,
     is_blocked_time_series,
     is_dates_distribution_valid,
@@ -238,6 +239,7 @@ class FeaturesEnricher(TransformerMixin):
         generate_search_key_features: bool = True,
         sample_config: SampleConfig | None = None,
         print_trace_id: bool = False,
+        print_loaded_report: bool = True,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -284,7 +286,7 @@ class FeaturesEnricher(TransformerMixin):
         self.id_columns = id_columns
         self.id_columns_encoder = None
         self.country_code = country_code
-        self.__validate_search_keys(search_keys, search_id)
+        self.__validate_search_keys(self.search_keys, search_id)
 
         self.model_task_type = ModelTaskType.parse(model_task_type)
         self.endpoint = endpoint
@@ -317,7 +319,8 @@ class FeaturesEnricher(TransformerMixin):
                     self.fit_columns_renaming = {c.name: c.originalName for c in file_metadata.columns}
                     df = pd.DataFrame(columns=x_columns)
                     self.__prepare_feature_importances(trace_id, df, silent=True, update_selected_features=False)
-                    self.__show_selected_features()
+                    if print_loaded_report:
+                        self.__show_selected_features()
                     # TODO validate search_keys with search_keys from file_metadata
                     print(self.bundle.get("search_by_task_id_finish"))
                     self.logger.debug(f"Successfully initialized with search_id: {search_id}")
@@ -3226,8 +3229,11 @@ if response.status_code == 200:
             df, self.fit_search_keys, self.fit_generated_features
         )
         self.fit_columns_renaming = normalizer.columns_renaming
-        if normalizer.removed_features:
-            self.__log_warning(self.bundle.get("dataset_date_features").format(normalizer.removed_features))
+        if normalizer.removed_datetime_features:
+            original_removed_datetime_features = [
+                self.fit_columns_renaming.get(f, f) for f in normalizer.removed_datetime_features
+            ]
+            self.__log_warning(self.bundle.get("dataset_date_features").format(original_removed_datetime_features))
 
         non_feature_columns = [
             self.TARGET_NAME,
@@ -4093,11 +4099,12 @@ if response.status_code == 200:
             or set(search_keys.values()) == {SearchKey.EMAIL}
             or set(search_keys.values()) == {SearchKey.HEM}
             or set(search_keys.values()) == {SearchKey.COUNTRY, SearchKey.POSTAL_CODE}
+            or len(search_keys) == 0
+            or set(search_keys.values()) == {SearchKey.CUSTOM_KEY}
         ):
             if not silent:
                 self.__log_warning(bundle.get("current_date_added"))
             df[CURRENT_DATE_COL] = datetime.date.today()
-            # df[CURRENT_DATE_COL] = datetime.date(2025, 10, 15)
             search_keys[CURRENT_DATE_COL] = SearchKey.DATE
             converter = DateTimeConverter(CURRENT_DATE_COL, generate_cyclical_features=False)
             df = converter.convert(df)
@@ -4781,7 +4788,8 @@ if response.status_code == 200:
             else:
                 msg = self.bundle.get("unregistered_only_personal_keys")
             self.logger.warning(msg + f" Provided search keys: {search_keys}")
-            raise ValidationError(msg)
+            # Current date will be added later
+            # raise ValidationError(msg)
 
         if (
             len(valid_search_keys.values()) == 1
@@ -4900,6 +4908,21 @@ if response.status_code == 200:
                 search_key in self.fit_search_keys.values() and search_key not in search_keys.values()
             )
 
+        if (
+            SearchKey.DATE not in search_keys.values()
+            and SearchKey.DATETIME not in search_keys.values()
+            and check_need_detect(SearchKey.DATE)
+            and check_need_detect(SearchKey.DATETIME)
+        ):
+            maybe_keys = DateSearchKeyDetector().get_search_key_columns(sample, search_keys)
+            if len(maybe_keys) > 0:
+                datetime_key = maybe_keys[0]
+                search_keys[datetime_key] = SearchKey.DATETIME
+                self.autodetected_search_keys[datetime_key] = SearchKey.DATETIME
+                self.logger.info(f"Autodetected search key DATETIME in column {datetime_key}")
+                if not silent_mode:
+                    print(self.bundle.get("datetime_detected").format(datetime_key))
+
         # if SearchKey.POSTAL_CODE not in search_keys.values() and check_need_detect(SearchKey.POSTAL_CODE):
         if check_need_detect(SearchKey.POSTAL_CODE):
             maybe_keys = PostalCodeSearchKeyDetector().get_search_key_columns(sample, search_keys)

upgini/normalizer/normalize_utils.py

@@ -5,7 +5,6 @@ from typing import Dict, List, Tuple
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_bool_dtype as is_bool
-from pandas.api.types import is_datetime64_any_dtype as is_datetime
 from pandas.api.types import (
     is_float_dtype,
     is_numeric_dtype,
@@ -45,7 +44,7 @@ class Normalizer:
         self.columns_renaming = {}
         self.search_keys = {}
         self.generated_features = []
-        self.removed_features = []
+        self.removed_datetime_features = []
 
     def normalize(
         self, df: pd.DataFrame, search_keys: Dict[str, SearchKey], generated_features: List[str]
@@ -134,8 +133,9 @@ class Normalizer:
         features = self._get_features(df)
 
         for f in features:
-            if is_datetime(df[f]) or isinstance(df[f].dtype, pd.PeriodDtype):
-                self.removed_features.append(f)
+            converter = DateTimeConverter(f)
+            if converter.is_datetime(df):
+                self.removed_datetime_features.append(f)
                 df.drop(columns=f, inplace=True)
 
         return df

upgini/resource_bundle/strings.properties

@@ -210,6 +210,7 @@ features_info_zero_important_features=Oops, we can't find any relevant external
 features_info_zero_hit_rate_search_keys=Oops, looks like values/formats of the search keys {} might be incorrect,\nas we won't be able to match any data source using these values\nPlease check docs https://github.com/upgini/upgini#-search-key-types-we-support-more-to-come or send us a help request in Support:
 features_not_generated=Following features didn't pass checks for automated feature generation: {}
 # Information
+datetime_detected=Datetime detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 postal_code_detected=Postal codes detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 country_detected=Countries detected in column `{}`. It will be used as a search key\nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns
 country_auto_determined=Search key country_code `{}` was automatically determined by client IP. \nSee docs to turn off the automatic detection: https://github.com/upgini/upgini/blob/main/README.md#turn-off-autodetection-for-search-key-columns

upgini/utils/base_search_key_detector.py

@@ -24,4 +24,8 @@ class BaseSearchKeyDetector:
         for column_name in other_columns:
             if self._is_search_key_by_values(df[column_name]):
                 columns_by_values.append(column_name)
-        return list(set(columns_by_names + columns_by_values))
+
+        both = [col for col in columns_by_names if col in columns_by_values]
+        only_values = [col for col in columns_by_values if col not in columns_by_names]
+        only_names = [col for col in columns_by_names if col not in columns_by_values]
+        return both + only_values + only_names

upgini/utils/datetime_utils.py

@@ -10,6 +10,7 @@ from pandas.api.types import is_numeric_dtype
 from upgini.errors import ValidationError
 from upgini.metadata import EVAL_SET_INDEX, SearchKey
 from upgini.resource_bundle import ResourceBundle, get_custom_bundle
+from upgini.utils.base_search_key_detector import BaseSearchKeyDetector
 
 DATE_FORMATS = [
     "%Y-%m-%d",
@@ -29,6 +30,15 @@ DATE_FORMATS = [
 DATETIME_PATTERN = r"^[\d\s\.\-:T/+]+$"
 
 
+class DateSearchKeyDetector(BaseSearchKeyDetector):
+    def _is_search_key_by_name(self, column_name: str) -> bool:
+        lower_column_name = str(column_name).lower()
+        return "date" in lower_column_name or "time" in lower_column_name or "timestamp" in lower_column_name
+
+    def _is_search_key_by_values(self, column: pd.Series) -> bool:
+        return DateTimeConverter(column.name).is_datetime(column.to_frame(column.name))
+
+
 class DateTimeConverter:
     DATETIME_COL = "_date_time"
     # MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
@@ -80,7 +90,7 @@ class DateTimeConverter:
             return True
 
         parsed = self.parse_datetime(df, raise_errors=False)
-        return parsed is not None and not parsed.isna().all()
+        return parsed is not None and parsed.isna().mean() <= 0.5
 
     def parse_datetime(self, df: pd.DataFrame, raise_errors=True) -> pd.Series | None:
         if len(df) == 0 or df[self.date_column].isna().all():

{upgini-1.2.137.dist-info → upgini-1.2.139.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: upgini
-Version: 1.2.137
+Version: 1.2.139
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.137.dist-info → upgini-1.2.139.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=WAhVusMOc6hw9YR0UCWwVJJi3v2_uHEpPqxnSm9SguM,24
+upgini/__about__.py,sha256=OVX5g3MZ0JOT_utGS3DJjpqUC3UrDWMPf-_HJXfHjfo,24
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=D9JzJJkZLPP_dp8GOlGgMhTtrd5pvP-4cHIcqiY3q-E,33354
+upgini/dataset.py,sha256=7qM6O2NnsdnfAVrrPIyHKyUqwlQyuiPsR12ayF0OHsc,33356
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=g1daAwtECDpSsM2TtVmozU3crsdMQ_xC5PlBJAafAX0,236099
+upgini/features_enricher.py,sha256=nddbXsQ27GGcNcS-xl_5IhEk5-7GM7vCxcoRN-kUh2M,237269
 upgini/http.py,sha256=y26x4TQVYuEM3jz8JdASxSyBtvBemUkFf-FmX25sx-s,44356
 upgini/metadata.py,sha256=BwUTCY-EUHqPtO0tGazHrk3wqhh-NfjNZhlBHW8bR78,12796
 upgini/metrics.py,sha256=KCPE_apPN-9BIdv6GqASbJVaB_gBcy8wzNApAcyaGo4,46020
@@ -35,10 +35,10 @@ upgini/data_source/data_source_publisher.py,sha256=CQi3fEukaStV-RiadSEvEFLThOlZJ
 upgini/mdc/__init__.py,sha256=iHJlXQg6xRM1-ZOUtaPSJqw5SpQDszvxp4LyqviNLIQ,1027
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/normalizer/normalize_utils.py,sha256=w9f_9udrwqbhXgFMTs2keuce-6X_j6h3D7EdNo_2X7g,8493
+upgini/normalizer/normalize_utils.py,sha256=oKevieBChYxtugocrbev8Uz2vqSZ_PB4Ibo55fHOylM,8452
 upgini/resource_bundle/__init__.py,sha256=S5F2G47pnJd2LDpmFsjDqEwiKkP8Hm-hcseDbMka6Ko,8345
 upgini/resource_bundle/exceptions.py,sha256=5fRvx0_vWdE1-7HcSgF0tckB4A9AKyf5RiinZkInTsI,621
-upgini/resource_bundle/strings.properties,sha256=3aK2sxXYuvSLuoOyLq8IcyekfINH0Il5nLvVXMsuEpY,29353
+upgini/resource_bundle/strings.properties,sha256=Dym1ymRH2uTeb0CfrO_lccoP9LWj7lxNGvsxmR7vkSw,29583
 upgini/resource_bundle/strings_widget.properties,sha256=gOdqvZWntP2LCza_tyVk1_yRYcG4c04K9sQOAVhF_gw,1577
 upgini/sampler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/sampler/base.py,sha256=Fva2FEhLiNRPZ9Q6uOtJRtRzwsayjv7aphalAZO_4lc,6452
@@ -46,13 +46,13 @@ upgini/sampler/random_under_sampler.py,sha256=4mofmaRTmNwT_HqxecWJyfXdLKK0h9jMBw
 upgini/sampler/utils.py,sha256=PYOk3kKSnFlyxcpdtDNLBEEhTB4lO_iP7pQHqeUcmAc,20211
 upgini/utils/Roboto-Regular.ttf,sha256=kqYnZjMRQMpbyLulIChCLSdgYa1XF8GsUIoRi2Gcauw,168260
 upgini/utils/__init__.py,sha256=O_KgzKiJjW3g4NoqZ7lAxUpoHcBi_gze6r3ndEjCH74,842
-upgini/utils/base_search_key_detector.py,sha256=Inc6iGG-VXQdejWFfbekIkZk2ahC4k7CdGqzOkie6Bs,1021
+upgini/utils/base_search_key_detector.py,sha256=DCmTbfhCdirJaDMpbKicFvhgVv6a9NNf9tXy3AbMCEg,1258
 upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl1UOB4s,3382
 upgini/utils/config.py,sha256=zFdnjchykfp_1Tm3Qep7phLzXBpXIOzr2tIuXchRBLw,1754
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=B9zNaH2ZyV-lbBSTBdCZjc4zq1nVlejci40sf-TYfik,17102
+upgini/utils/datetime_utils.py,sha256=pTi5doYjjjlQgwEVDNrJuKaAEC3Jtx78fjX4W7M_UV4,17615
 upgini/utils/deduplicate_utils.py,sha256=CLX0QapRxB-ZVQT7yGvv1vSd2zac5SwRjCJavujdCps,11332
 upgini/utils/display_utils.py,sha256=MoTqXZJvC6pAqgOaI3V0FG-IU_LnMfrn4TDcNvUqsdg,13316
 upgini/utils/email_utils.py,sha256=pZ2vCfNxLIPUhxr0-OlABNXm12jjU44isBk8kGmqQzA,5277
@@ -74,7 +74,7 @@ upgini/utils/target_utils.py,sha256=CihpV6SC95HwtlMH60rGAUzVDa4Id0Bva8ySprmNHlE,
 upgini/utils/track_info.py,sha256=NDKeQTUlZaYp15UoP-xLKGoDoJQ0drbDMwB0g9R0PUg,6427
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.137.dist-info/METADATA,sha256=qTIcTcJz2tn18BOyAa7SbP14-fOxG4l3rtQyCxG8wDI,51164
-upgini-1.2.137.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-upgini-1.2.137.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.137.dist-info/RECORD,,
+upgini-1.2.139.dist-info/METADATA,sha256=LOVD2PiZDgd7Je06bv_AqVQfyhM_NXYvkMTQYq6aBpg,51164
+upgini-1.2.139.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+upgini-1.2.139.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.139.dist-info/RECORD,,