upgini 1.1.312a3__py3-none-any.whl → 1.1.312a5__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.1.312a3"
+__version__ = "1.1.312a5"

upgini/dataset.py

@@ -440,9 +440,11 @@ class Dataset:  # (pd.DataFrame):
                 FileColumnMeaningType.DATETIME,
                 # FileColumnMeaningType.IP_ADDRESS,
             }:
+                min_value = self.data[column_name].astype("Int64").min()
+                max_value = self.data[column_name].astype("Int64").max()
                 min_max_values = NumericInterval(
-                    minValue=self.data[column_name].astype("Int64").min(),
-                    maxValue=self.data[column_name].astype("Int64").max(),
+                    minValue=min_value,
+                    maxValue=max_value,
                 )
             else:
                 min_max_values = None

upgini/features_enricher.py

@@ -91,7 +91,11 @@ from upgini.utils.display_utils import (
     prepare_and_show_report,
     show_request_quote_button,
 )
-from upgini.utils.email_utils import EmailDomainGenerator, EmailSearchKeyConverter, EmailSearchKeyDetector
+from upgini.utils.email_utils import (
+    EmailDomainGenerator,
+    EmailSearchKeyConverter,
+    EmailSearchKeyDetector,
+)
 from upgini.utils.features_validator import FeaturesValidator
 from upgini.utils.format import Format
 from upgini.utils.ip_utils import IpSearchKeyConverter
@@ -1030,7 +1034,7 @@ class FeaturesEnricher(TransformerMixin):
                         self.bundle.get("quality_metrics_rows_header"): _num_samples(effective_X),
                     }
                     if model_task_type in [ModelTaskType.BINARY, ModelTaskType.REGRESSION] and is_numeric_dtype(
-                        y_sorted
+                        effective_y
                     ):
                         train_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
                             np.mean(effective_y), 4
@@ -1103,7 +1107,7 @@ class FeaturesEnricher(TransformerMixin):
                                 # self.bundle.get("quality_metrics_match_rate_header"): eval_hit_rate,
                             }
                             if model_task_type in [ModelTaskType.BINARY, ModelTaskType.REGRESSION] and is_numeric_dtype(
-                                eval_y_sorted
+                                effective_eval_set[idx][1]
                             ):
                                 eval_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
                                     np.mean(effective_eval_set[idx][1]), 4
@@ -1363,6 +1367,7 @@ class FeaturesEnricher(TransformerMixin):
             importance_threshold,
             max_features,
         )
+        filtered_enriched_features = [c for c in filtered_enriched_features if c not in client_features]
 
         X_sorted, y_sorted = self._sort_by_system_record_id(X_sampled, y_sampled, self.cv)
         enriched_X_sorted, enriched_y_sorted = self._sort_by_system_record_id(enriched_X, y_sampled, self.cv)
@@ -2217,7 +2222,9 @@ class FeaturesEnricher(TransformerMixin):
                 result = enrich()
 
             filtered_columns = self.__filtered_enriched_features(importance_threshold, max_features)
-            existing_filtered_columns = [c for c in filtered_columns if c in result.columns]
+            existing_filtered_columns = [
+                c for c in filtered_columns if c in result.columns and c not in validated_X.columns
+            ]
             selecting_columns = validated_X.columns.tolist() + generated_features + existing_filtered_columns
             if add_fit_system_record_id:
                 selecting_columns.append(SORT_ID)
@@ -2430,9 +2437,7 @@ class FeaturesEnricher(TransformerMixin):
 
         email_columns = SearchKey.find_all_keys(self.fit_search_keys, SearchKey.EMAIL)
         if email_columns:
-            generator = EmailDomainGenerator(
-                email_columns
-            )
+            generator = EmailDomainGenerator(email_columns)
             df = generator.generate(df)
             self.fit_generated_features.extend(generator.generated_features)
 

upgini/utils/datetime_utils.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import re
+import pytz
 from typing import Dict, List, Optional
 
 import numpy as np
@@ -28,12 +29,13 @@ DATE_FORMATS = [
     "%Y-%m-%dT%H:%M:%S.%f",
 ]
 
-DATETIME_PATTERN = r"^[\d\s\.\-:T/]+$"
+DATETIME_PATTERN = r"^[\d\s\.\-:T/+]+$"
 
 
 class DateTimeSearchKeyConverter:
     DATETIME_COL = "_date_time"
-    MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
+    # MIN_SUPPORTED_DATE_TS = datetime.datetime(1999, 12, 31)  # 946684800000  # 2000-01-01
+    MIN_SUPPORTED_DATE_TS = pd.to_datetime(datetime.datetime(1999, 12, 31)).tz_localize(None)
 
     def __init__(
         self,
@@ -106,12 +108,13 @@ class DateTimeSearchKeyConverter:
             df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
             df[self.date_column] = self.parse_date(df)
 
-        df = self.clean_old_dates(df)
-
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features
         seconds = "datetime_seconds"
         df[self.date_column] = df[self.date_column].dt.tz_localize(None)
+
+        df = self.clean_old_dates(df)
+
         df[seconds] = (df[self.date_column] - df[self.date_column].dt.floor("D")).dt.seconds
 
         seconds_without_na = df[seconds].dropna()

upgini/utils/phone_utils.py

@@ -29,21 +29,21 @@ class PhoneSearchKeyConverter:
     def convert(self, df: pd.DataFrame) -> pd.DataFrame:
         df = self.phone_to_int(df)
         if self.country_column is not None:
-            df = df.apply(self.add_prefix, axis=1)
+            df[self.phone_column] = df.apply(self.add_prefix, axis=1)
         df[self.phone_column] = df[self.phone_column].astype("Int64")
         return df
 
     def add_prefix(self, row):
         phone = row[self.phone_column]
         if pd.isna(phone):
-            return row
+            return phone
         country = row[self.country_column]
         country_prefix_tuple = self.COUNTRIES_PREFIXES.get(country)
         if country_prefix_tuple is not None:
             country_prefix, number_of_digits = country_prefix_tuple
             if len(str(phone)) == number_of_digits:
-                row[self.phone_column] = int(country_prefix + str(phone))
-        return row
+                return int(country_prefix + str(phone))
+        return phone
 
     def phone_to_int(self, df: pd.DataFrame) -> pd.DataFrame:
         """

{upgini-1.1.312a3.dist-info → upgini-1.1.312a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.312a3
+Version: 1.1.312a5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.1.312a3.dist-info → upgini-1.1.312a5.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=IaDaRN1MPzK9IEvOazFTrqDhFeyxseC5mkVDu1NRrYc,26
+upgini/__about__.py,sha256=AYWzaD5lVHd4s3vopg53TBKzkKgWzJVi62VrmfO7rRU,26
 upgini/__init__.py,sha256=Xs0YFVBu1KUdtZzbStGRPQtLt3YLzJnjx5nIUBlX8BE,415
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
-upgini/dataset.py,sha256=CdOE1h94E1YgStslQIPfvMp5z_ODt7QfXfxqpmYL5Xs,30758
+upgini/dataset.py,sha256=c6jghh32P9_2CspELYCOsmNIOiShuCADnCCJ8Jj2t50,30834
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=FfFlFW3BArv2rQWGCs-SXrDDDcjQTwwJxzRysZlJfq0,186961
+upgini/features_enricher.py,sha256=LqGOMObkFsAm58sBL3UhTmc7TOnDQmLivxl3jbXh-n0,187132
 upgini/http.py,sha256=a4Epc9YLIJBuYk4t8E_2-QDLBtJFqKO35jn2SnYQZCg,42920
 upgini/lazy_import.py,sha256=EwoM0msNGbSmWBhGbrLDny1DSnOlvTxCjmMKPxYlDms,610
 upgini/metadata.py,sha256=YQ-1HZGyPOksP2iM50ff_pMHXLyzvpChqSfNh8Z0ke4,10833
@@ -42,7 +42,7 @@ upgini/utils/blocked_time_series.py,sha256=Uqr3vp4YqNclj2-PzEYqVy763GSXHn86sbpIl
 upgini/utils/country_utils.py,sha256=lY-eXWwFVegdVENFttbvLcgGDjFO17Sex8hd2PyJaRk,6937
 upgini/utils/custom_loss_utils.py,sha256=kieNZYBYZm5ZGBltF1F_jOSF4ea6C29rYuCyiDcqVNY,3857
 upgini/utils/cv_utils.py,sha256=w6FQb9nO8BWDx88EF83NpjPLarK4eR4ia0Wg0kLBJC4,3525
-upgini/utils/datetime_utils.py,sha256=O-IQbWtWJs6xTAr3m9FMRHyT-fL_28vCMrrt4eqfpa0,12025
+upgini/utils/datetime_utils.py,sha256=JSHCx6kpt7n60i3cphI5yWEatQK729x1coSjC8Gafrg,12135
 upgini/utils/deduplicate_utils.py,sha256=Zvs7zW4QzaERQmJNPrTVf2ZTVBkBLOycFCzyMwtXuV8,8770
 upgini/utils/display_utils.py,sha256=A2ouB5eiZ-Kyt9ykYxkLQwyoRPrdYeJymwNTiajtFXs,10990
 upgini/utils/email_utils.py,sha256=j0Ug1R_0AnCg1Y92zIZ4XMwvKo3G5_pcOlBN1OH_gZs,5191
@@ -50,14 +50,14 @@ upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0-
 upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=ZZj_uQFTHhagzt-MRew__ZBOp2DdnkMrachS7PElkSE,5143
-upgini/utils/phone_utils.py,sha256=c8oNajhT7Z1hXpiRAEH828vX7SoALBJKUun_M5qu9vg,10363
+upgini/utils/phone_utils.py,sha256=PTSRfGAWCuLy8R6I8X6clcc1K7bZXIIrZ_alIB8irC8,10368
 upgini/utils/postal_code_utils.py,sha256=C899tJS8qM_ps4I3g-Ve6qzIa22O_UqwNmGFoyy9sO8,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,44511
 upgini/utils/target_utils.py,sha256=Y96_PJ5cC-WsEbeqg20v9uqywDQobLoTb-xoP7S3o4E,7807
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.1.312a3.dist-info/METADATA,sha256=133zxblkCtyOblc05Qe0mfRcwYYU9qLcPbNNBjHl-mY,48155
-upgini-1.1.312a3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.1.312a3.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.312a3.dist-info/RECORD,,
+upgini-1.1.312a5.dist-info/METADATA,sha256=MdhSWCWMCuajyC7B1bVWfVFOs7b5iIsmD7m3Z48egng,48155
+upgini-1.1.312a5.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.1.312a5.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.312a5.dist-info/RECORD,,