upgini 1.1.274a1__tar.gz → 1.1.274a2__tar.gz

{upgini-1.1.274a1/src/upgini.egg-info → upgini-1.1.274a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.274a1
+Version: 1.1.274a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.274a1 → upgini-1.1.274a2}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.274a1"
+version = "1.1.274a2"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/autofe/date.py

@@ -46,6 +46,7 @@ class DateDiffType2(PandasOperand, DateDiffMixin):
         future = right + (left.dt.year - right.dt.year).apply(
             lambda y: np.datetime64("NaT") if np.isnan(y) else pd.tseries.offsets.DateOffset(years=y)
         )
+        future = pd.to_datetime(future)
         before = future[future < left]
         future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
         diff = (future - left) / np.timedelta64(1, self.diff_unit)

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/data_source/data_source_publisher.py

@@ -48,6 +48,7 @@ class DataSourcePublisher:
         data_table_uri: str,
         search_keys: Dict[str, SearchKey],
         update_frequency: str,
+        exclude_from_autofe_generation: Optional[List[str]],
         secondary_search_keys: Optional[Dict[str, SearchKey]] = None,
         sort_column: Optional[str] = None,
         date_format: Optional[str] = None,
@@ -57,7 +58,6 @@ class DataSourcePublisher:
         join_date_abs_limit_days: Optional[int] = None,
         features_for_embeddings: Optional[List[str]] = DEFAULT_GENERATE_EMBEDDINGS,
         data_table_id_to_replace: Optional[str] = None,
-        exclude_from_autofe_generation: Optional[List[str]] = None,
         _force_generation=False,
         _silent=False,
     ) -> str:

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/features_enricher.py

@@ -1,4 +1,5 @@
 import dataclasses
+import datetime
 import gc
 import hashlib
 import itertools
@@ -146,6 +147,7 @@ class FeaturesEnricher(TransformerMixin):
     """
 
     TARGET_NAME = "target"
+    CURRENT_DATE = "current_date"
     RANDOM_STATE = 42
     CALCULATE_METRICS_THRESHOLD = 50_000_000
     CALCULATE_METRICS_MIN_THRESHOLD = 500
@@ -207,6 +209,7 @@ class FeaturesEnricher(TransformerMixin):
         client_ip: Optional[str] = None,
         client_visitorid: Optional[str] = None,
         custom_bundle_config: Optional[str] = None,
+        add_date_if_missing: bool = True,
         **kwargs,
     ):
         self.bundle = get_custom_bundle(custom_bundle_config)
@@ -317,6 +320,7 @@ class FeaturesEnricher(TransformerMixin):
         self.raise_validation_error = raise_validation_error
         self.exclude_columns = exclude_columns
         self.baseline_score_column = baseline_score_column
+        self.add_date_if_missing = add_date_if_missing
 
     def _get_api_key(self):
         return self._api_key
@@ -1804,10 +1808,13 @@ class FeaturesEnricher(TransformerMixin):
         else:
             features_section = ""
 
-        api_example = f"""curl 'https://inference-upgini.azurewebsites.net/api/http_inference_trigger' \\
+        search_id = self._search_task.search_task_id
+        api_example = (
+            f"""curl 'https://search.upgini.com/online/api/http_inference_trigger?search_id={search_id}' \\
     -H 'Authorization: {self.api_key}' \\
     -H 'Content-Type: application/json' \\
-    -d '{{"search_id": "{self._search_task.search_task_id}", "search_keys": {keys}{features_section}}}'"""
+    -d '{{"search_keys": {keys}{features_section}}}'"""
+        )
         return api_example
 
     def _get_copy_of_runtime_parameters(self) -> RuntimeParameters:
@@ -1902,6 +1909,9 @@ class FeaturesEnricher(TransformerMixin):
                 generated_features.extend(converter.generated_features)
             else:
                 self.logger.info("Input dataset hasn't date column")
+                if self.add_date_if_missing:
+                    df = self._add_current_date_as_key(df)
+                    search_keys[self.CURRENT_DATE] = SearchKey.DATE
             email_column = self._get_email_column(search_keys)
             hem_column = self._get_hem_column(search_keys)
             email_converted_to_hem = False
@@ -2220,9 +2230,7 @@ class FeaturesEnricher(TransformerMixin):
         self.fit_search_keys = self.search_keys.copy()
         self.fit_search_keys = self.__prepare_search_keys(validated_X, self.fit_search_keys, is_demo_dataset)
 
-        validate_dates_distribution(
-            validated_X, self.fit_search_keys, self.logger, self.bundle, self.warning_counter
-        )
+        validate_dates_distribution(validated_X, self.fit_search_keys, self.logger, self.bundle, self.warning_counter)
 
         maybe_date_column = self._get_date_column(self.fit_search_keys)
         has_date = maybe_date_column is not None
@@ -2273,6 +2281,9 @@ class FeaturesEnricher(TransformerMixin):
             self.fit_generated_features.extend(converter.generated_features)
         else:
             self.logger.info("Input dataset hasn't date column")
+            if self.add_date_if_missing:
+                df = self._add_current_date_as_key(df)
+                self.fit_search_keys[self.CURRENT_DATE] = SearchKey.DATE
         email_column = self._get_email_column(self.fit_search_keys)
         hem_column = self._get_hem_column(self.fit_search_keys)
         email_converted_to_hem = False
@@ -2853,6 +2864,10 @@ class FeaturesEnricher(TransformerMixin):
             if t in [SearchKey.DATE, SearchKey.DATETIME]:
                 return col
 
+    @staticmethod
+    def _add_current_date_as_key(df: pd.DataFrame) -> pd.DataFrame:
+        df[FeaturesEnricher.CURRENT_DATE] = datetime.date.today()
+
     @staticmethod
     def _get_group_columns(df: pd.DataFrame, search_keys: Dict[str, SearchKey]) -> List[str]:
         return [
@@ -2903,9 +2918,7 @@ class FeaturesEnricher(TransformerMixin):
                 [
                     c
                     for c in df.columns
-                    if c not in sort_columns
-                    and c not in sort_exclude_columns
-                    and df[c].nunique() > 1
+                    if c not in sort_columns and c not in sort_exclude_columns and df[c].nunique() > 1
                 ]
                 # [
                 #     sk

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/metrics.py

@@ -645,6 +645,7 @@ class OtherEstimatorWrapper(EstimatorWrapper):
 
 
 def validate_scoring_argument(scoring: Union[Callable, str, None]):
+    # TODO validate that if it is Callable then it accepts 3 arguments
     if isinstance(scoring, str) and scoring is not None:
         _get_scorer_by_name(scoring)
 

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/normalizer/phone_normalizer.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 import pandas as pd
-from pandas.api.types import is_float_dtype, is_int64_dtype, is_string_dtype
+from pandas.api.types import is_float_dtype, is_int64_dtype, is_string_dtype, is_object_dtype
 
 from upgini.errors import ValidationError
 
@@ -44,7 +44,7 @@ class PhoneNormalizer:
         Method will remove all non numeric chars from string and convert it to int.
         None will be set for phone numbers that couldn"t be converted to int
         """
-        if is_string_dtype(self.df[self.phone_column_name]):
+        if is_string_dtype(self.df[self.phone_column_name]) or is_object_dtype(self.df[self.phone_column_name]):
             convert_func = self.phone_str_to_int_safe
         elif is_float_dtype(self.df[self.phone_column_name]):
             convert_func = self.phone_float_to_int_safe

{upgini-1.1.274a1 → upgini-1.1.274a2}/src/upgini/utils/datetime_utils.py

@@ -100,6 +100,9 @@ class DateTimeSearchKeyConverter:
                 msg = self.bundle.get("unsupported_date_type").format(self.date_column)
                 self.logger.warning(msg)
                 raise ValidationError(msg)
+        else:
+            df[self.date_column] = df[self.date_column].astype("string").apply(self.clean_date)
+            df[self.date_column] = self.parse_date(df)
 
         # If column with date is datetime then extract seconds of the day and minute of the hour
         # as additional features

{upgini-1.1.274a1 → upgini-1.1.274a2/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.274a1
+Version: 1.1.274a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.274a1 → upgini-1.1.274a2}/tests/test_autofe_operands.py

@@ -30,7 +30,8 @@ def test_date_diff_type2():
 
     operand = DateDiffType2(left_unit="s")
     expected_result = pd.Series([61.0, 182.0])
-    assert_series_equal(operand.calculate_binary(df.date1, df.date2), expected_result)
+    actual = operand.calculate_binary(df.date1, df.date2)
+    assert_series_equal(actual, expected_result)
 
 
 def test_date_diff_list():

{upgini-1.1.274a1 → upgini-1.1.274a2}/tests/test_features_enricher.py

@@ -246,7 +246,7 @@ def test_eval_set_with_diff_order_of_columns(requests_mock: Mocker):
     eval1_df = df[10000:11000].reset_index(drop=True)
     eval1_features = eval1_df.drop(columns="target")
     # shuffle columns
-    eval1_features = eval1_features[set(eval1_features.columns)]
+    eval1_features = eval1_features[list(eval1_features.columns)]
     eval1_target = eval1_df["target"].reset_index(drop=True)
 
     eval2_df = df[11000:12000]
@@ -375,7 +375,7 @@ def test_saved_features_enricher(requests_mock: Mocker):
     url = "http://fake_url2"
 
     path_to_mock_features = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "test_data/binary/validation_features.parquet"
+        os.path.dirname(os.path.realpath(__file__)), "test_data/binary/validation_features_v2.parquet"
     )
 
     mock_default_requests(requests_mock, url)
@@ -462,7 +462,7 @@ def test_saved_features_enricher(requests_mock: Mocker):
             segment_header: [train_segment, eval_1_segment, eval_2_segment],
             rows_header: [10000, 1000, 1000],
             target_mean_header: [0.5044, 0.487, 0.486],
-            enriched_gini: [-0.000136, 0.000000, -0.003728],
+            enriched_gini: [0.021830, -0.006607, -0.018483],
         }
     )
     print("Expected metrics: ")
@@ -496,7 +496,7 @@ def test_saved_features_enricher(requests_mock: Mocker):
             segment_header: [train_segment],
             rows_header: [10000],
             target_mean_header: [0.049],
-            enriched_gini: [0.000985],
+            enriched_gini: [0.054454],
         }
     )
     print("Expected metrics: ")
@@ -2652,5 +2652,4 @@ class DataFrameWrapper:
 
 
 class TestException(Exception):
-    def __init__(self):
-        super().__init__()
+    pass