upgini 1.1.315a1__tar.gz → 1.1.316__tar.gz

{upgini-1.1.315a1 → upgini-1.1.316}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.1.315a1
+Version: 1.1.316
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.1.316/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.1.316"

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/autofe/binary.py

@@ -141,7 +141,7 @@ class Distance(PandasOperand):
 
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         return pd.Series(
-            1 - self.__dot(left, right) / (self.__dot(left, left) * self.__dot(right, right)), index=left.index
+            1 - self.__dot(left, right) / (self.__norm(left) * self.__norm(right)), index=left.index
         )
 
     # row-wise dot product
@@ -152,6 +152,9 @@ class Distance(PandasOperand):
         res = res.reindex(left.index.union(right.index))
         return res
 
+    def __norm(self, vector: pd.Series) -> pd.Series:
+        return np.sqrt(self.__dot(vector, vector))
+
 
 # Left for backward compatibility
 class Sim(Distance):

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/autofe/unary.py

@@ -121,6 +121,9 @@ class Norm(PandasOperand):
 
     def calculate_unary(self, data: pd.Series) -> pd.Series:
         data_dropna = data.dropna()
+        if data_dropna.empty:
+            return data
+
         normalized_data = Normalizer().transform(data_dropna.to_frame().T).T
         normalized_data = pd.Series(normalized_data[:, 0], index=data_dropna.index, name=data.name)
         normalized_data = normalized_data.reindex(data.index)

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/data_source/data_source_publisher.py

@@ -63,6 +63,7 @@ class DataSourcePublisher:
         keep_features: Optional[List[str]] = None,
         date_features: Optional[List[str]] = None,
         date_vector_features: Optional[List[str]] = None,
+        generate_runtime_embeddings: Optional[List[str]] = None,
         _force_generation=False,
         _silent=False,
     ) -> str:
@@ -163,6 +164,8 @@ class DataSourcePublisher:
                     if date_format is None:
                         raise ValidationError("date_format should be presented if you use date vector features")
                     request["dateVectorFeatures"] = date_vector_features
+                if generate_runtime_embeddings is not None:
+                    request["generateRuntimeEmbeddingsFeatures"] = generate_runtime_embeddings
                 self.logger.info(f"Start registering data table {request}")
 
                 task_id = self._rest_client.register_ads(request, trace_id)
@@ -276,6 +279,8 @@ class DataSourcePublisher:
         client_emails: Optional[List[str]] = None,
         date_features: Optional[List[str]] = None,
         date_vector_features: Optional[List[str]] = None,
+        exclude_from_autofe_generation: Optional[List[str]] = None,
+        generate_runtime_embeddings: Optional[List[str]] = None,
     ):
         trace_id = str(uuid.uuid4())
         with MDC(trace_id=trace_id):
@@ -327,6 +332,10 @@ class DataSourcePublisher:
                     request["dateFeatures"] = date_features
                 if date_vector_features is not None:
                     request["dateVectorFeatures"] = date_vector_features
+                if exclude_from_autofe_generation is not None:
+                    request["excludeFromGenerationFeatures"] = exclude_from_autofe_generation
+                if generate_runtime_embeddings is not None:
+                    request["generateRuntimeEmbeddingsFeatures"] = generate_runtime_embeddings
                 self.logger.info(f"Activating data tables with request {request}")
 
                 self._rest_client.activate_datatables(request, trace_id)

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/dataset.py

@@ -692,7 +692,7 @@ class Dataset:  # (pd.DataFrame):
         parquet_file_path = f"{base_path}/{self.dataset_name}.parquet"
         self.data.to_parquet(path=parquet_file_path, index=False, compression="gzip", engine="fastparquet")
         uploading_file_size = Path(parquet_file_path).stat().st_size
-        self.logger.info(f"Size of prepared uploading file: {uploading_file_size}")
+        self.logger.info(f"Size of prepared uploading file: {uploading_file_size}. {len(self.data)} rows")
         if uploading_file_size > self.MAX_UPLOADING_FILE_SIZE:
             raise ValidationError(self.bundle.get("dataset_too_big_file"))
         return parquet_file_path

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/features_enricher.py

@@ -846,17 +846,37 @@ class FeaturesEnricher(TransformerMixin):
                 self.logger.warning(msg)
                 print(msg)
 
+            if X is not None and y is None:
+                raise ValidationError("X passed without y")
+
             self.__validate_search_keys(self.search_keys, self.search_id)
             effective_X = X if X is not None else self.X
             effective_y = y if y is not None else self.y
             effective_eval_set = eval_set if eval_set is not None else self.eval_set
             effective_eval_set = self._check_eval_set(effective_eval_set, effective_X, self.bundle)
 
+            if (
+                self._search_task is None
+                or self._search_task.provider_metadata_v2 is None
+                or len(self._search_task.provider_metadata_v2) == 0
+                or effective_X is None
+                or effective_y is None
+            ):
+                raise ValidationError(self.bundle.get("metrics_unfitted_enricher"))
+
+            validated_X = self._validate_X(effective_X)
+            validated_y = self._validate_y(validated_X, effective_y)
+            validated_eval_set = (
+                [self._validate_eval_set_pair(validated_X, eval_pair) for eval_pair in effective_eval_set]
+                if effective_eval_set is not None
+                else None
+            )
+
             try:
                 self.__log_debug_information(
-                    effective_X,
-                    effective_y,
-                    effective_eval_set,
+                    validated_X,
+                    validated_y,
+                    validated_eval_set,
                     exclude_features_sources=exclude_features_sources,
                     cv=cv if cv is not None else self.cv,
                     importance_threshold=importance_threshold,
@@ -866,21 +886,9 @@ class FeaturesEnricher(TransformerMixin):
                     remove_outliers_calc_metrics=remove_outliers_calc_metrics,
                 )
 
-                if (
-                    self._search_task is None
-                    or self._search_task.provider_metadata_v2 is None
-                    or len(self._search_task.provider_metadata_v2) == 0
-                    or effective_X is None
-                    or effective_y is None
-                ):
-                    raise ValidationError(self.bundle.get("metrics_unfitted_enricher"))
-
-                if X is not None and y is None:
-                    raise ValidationError("X passed without y")
-
                 validate_scoring_argument(scoring)
 
-                self._validate_baseline_score(effective_X, effective_eval_set)
+                self._validate_baseline_score(validated_X, validated_eval_set)
 
                 if self._has_paid_features(exclude_features_sources):
                     msg = self.bundle.get("metrics_with_paid_features")
@@ -889,7 +897,7 @@ class FeaturesEnricher(TransformerMixin):
                     return None
 
                 cat_features, search_keys_for_metrics = self._get_client_cat_features(
-                    estimator, effective_X, self.search_keys
+                    estimator, validated_X, self.search_keys
                 )
 
                 prepared_data = self._prepare_data_for_metrics(
@@ -1034,10 +1042,10 @@ class FeaturesEnricher(TransformerMixin):
                         self.bundle.get("quality_metrics_rows_header"): _num_samples(effective_X),
                     }
                     if model_task_type in [ModelTaskType.BINARY, ModelTaskType.REGRESSION] and is_numeric_dtype(
-                        effective_y
+                        validated_y
                     ):
                         train_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
-                            np.mean(effective_y), 4
+                            np.mean(validated_y), 4
                         )
                     if etalon_metric is not None:
                         train_metrics[self.bundle.get("quality_metrics_baseline_header").format(metric)] = etalon_metric
@@ -1107,10 +1115,10 @@ class FeaturesEnricher(TransformerMixin):
                                 # self.bundle.get("quality_metrics_match_rate_header"): eval_hit_rate,
                             }
                             if model_task_type in [ModelTaskType.BINARY, ModelTaskType.REGRESSION] and is_numeric_dtype(
-                                effective_eval_set[idx][1]
+                                validated_eval_set[idx][1]
                             ):
                                 eval_metrics[self.bundle.get("quality_metrics_mean_target_header")] = round(
-                                    np.mean(effective_eval_set[idx][1]), 4
+                                    np.mean(validated_eval_set[idx][1]), 4
                                 )
                             if etalon_eval_metric is not None:
                                 eval_metrics[self.bundle.get("quality_metrics_baseline_header").format(metric)] = (
@@ -3158,6 +3166,7 @@ class FeaturesEnricher(TransformerMixin):
         if len(search_key_names_by_type) == 0:
             return df, {}
 
+        self.logger.info(f"Start exploding dataset by {search_key_names_by_type}. Size before: {len(df)}")
         multiple_keys_columns = [col for cols in search_key_names_by_type.values() for col in cols]
         other_columns = [col for col in df.columns if col not in multiple_keys_columns]
         exploded_dfs = []
@@ -3176,6 +3185,7 @@ class FeaturesEnricher(TransformerMixin):
             columns_renaming[new_search_key] = new_search_key
 
         df = pd.concat(exploded_dfs, ignore_index=True)
+        self.logger.info(f"Finished explosion. Size after: {len(df)}")
         return df, unnest_search_keys
 
     def __add_fit_system_record_id(
@@ -3209,18 +3219,26 @@ class FeaturesEnricher(TransformerMixin):
                 date_column = SearchKey.find_key(search_keys, [SearchKey.DATE, SearchKey.DATETIME])
             sort_columns = [date_column] if date_column is not None else []
 
+            sorted_other_keys = sorted(search_keys, key=lambda x: str(search_keys.get(x)))
+            sorted_other_keys = [k for k in sorted_other_keys if k not in sort_exclude_columns]
+
             other_columns = sorted(
                 [
                     c
                     for c in df.columns
-                    if c not in sort_columns and c not in sort_exclude_columns and df[c].nunique() > 1
+                    if c not in sort_columns
+                    and c not in sorted_other_keys
+                    and c not in sort_exclude_columns
+                    and df[c].nunique() > 1
                 ]
             )
 
+            all_other_columns = sorted_other_keys + other_columns
+
             search_keys_hash = "search_keys_hash"
-            if len(other_columns) > 0:
+            if len(all_other_columns) > 0:
                 sort_columns.append(search_keys_hash)
-                df[search_keys_hash] = pd.util.hash_pandas_object(df[other_columns], index=False)
+                df[search_keys_hash] = pd.util.hash_pandas_object(df[all_other_columns], index=False)
 
             df = df.sort_values(by=sort_columns)
 

{upgini-1.1.315a1 → upgini-1.1.316}/src/upgini/utils/datetime_utils.py

@@ -1,7 +1,6 @@
 import datetime
 import logging
 import re
-import pytz
 from typing import Dict, List, Optional
 
 import numpy as np

upgini-1.1.315a1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.1.315a1"