upgini 1.2.32__tar.gz → 1.2.34a1__tar.gz

{upgini-1.2.32 → upgini-1.2.34a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.32
+Version: 1.2.34a1
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -110,7 +110,7 @@ Description-Content-Type: text/markdown
   </tr>
 </table>   
 
-⭐️ [Simple Drag & Drop Search UI](https://upgini.com/upgini-widget):   
+⭐️ [Simple Drag & Drop Search UI](https://www.upgini.com/data-search-widget):   
 <a href="https://upgini.com/upgini-widget">
 <img width="710" alt="Drag & Drop Search UI" src="https://github.com/upgini/upgini/assets/95645411/36b6460c-51f3-400e-9f04-445b938bf45e">
 </a>

{upgini-1.2.32 → upgini-1.2.34a1}/README.md

@@ -68,7 +68,7 @@
   </tr>
 </table>   
 
-⭐️ [Simple Drag & Drop Search UI](https://upgini.com/upgini-widget):   
+⭐️ [Simple Drag & Drop Search UI](https://www.upgini.com/data-search-widget):   
 <a href="https://upgini.com/upgini-widget">
 <img width="710" alt="Drag & Drop Search UI" src="https://github.com/upgini/upgini/assets/95645411/36b6460c-51f3-400e-9f04-445b938bf45e">
 </a>

upgini-1.2.34a1/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.34a1"

{upgini-1.2.32 → upgini-1.2.34a1}/src/upgini/dataset.py

@@ -422,11 +422,11 @@ class Dataset:  # (pd.DataFrame):
                     + "".join("<tr>" + "".join(map(map_color, row[1:])) + "</tr>" for row in df_stats.itertuples())
                     + "</table>"
                 )
-                print()
                 display(HTML(html_stats))
-            except (ImportError, NameError):
                 print()
+            except (ImportError, NameError):
                 print(df_stats)
+                print()
 
         if len(self.data) == 0:
             raise ValidationError(self.bundle.get("all_search_keys_invalid"))
@@ -494,11 +494,17 @@ class Dataset:  # (pd.DataFrame):
             taskType=self.task_type,
         )
 
+    @staticmethod
+    def is_column_binary_type(column):
+        return column.apply(lambda x: x is None or isinstance(x, (bytes, bytearray))).all()
+
     def __get_data_type(self, pandas_data_type, column_name: str) -> DataType:
         if is_integer_dtype(pandas_data_type):
             return DataType.INT
         elif is_float_dtype(pandas_data_type):
             return DataType.DECIMAL
+        elif self.is_column_binary_type(self.data[column_name]):
+            return DataType.BYTES
         elif is_string_dtype(pandas_data_type) or is_object_dtype(pandas_data_type):
             return DataType.STRING
         else:

{upgini-1.2.32 → upgini-1.2.34a1}/src/upgini/features_enricher.py

@@ -111,7 +111,11 @@ try:
 except Exception:
     from upgini.utils.fallback_progress_bar import CustomFallbackProgressBar as ProgressBar
 
-from upgini.utils.target_utils import calculate_psi, define_task
+from upgini.utils.target_utils import (
+    balance_undersample_forced,
+    calculate_psi,
+    define_task,
+)
 from upgini.utils.warning_counter import WarningCounter
 from upgini.version_validator import validate_version
 
@@ -967,6 +971,13 @@ class FeaturesEnricher(TransformerMixin):
                     self.__log_warning(self.bundle.get("metrics_no_important_free_features"))
                     return None
 
+                maybe_phone_column = self._get_phone_column(self.search_keys)
+                text_features = (
+                    [f for f in self.generate_features if f != maybe_phone_column]
+                    if self.generate_features is not None
+                    else None
+                )
+
                 print(self.bundle.get("metrics_start"))
                 with Spinner():
                     self._check_train_and_eval_target_distribution(y_sorted, fitting_eval_set_dict)
@@ -982,7 +993,7 @@ class FeaturesEnricher(TransformerMixin):
                         fitting_enriched_X,
                         scoring,
                         groups=groups,
-                        text_features=self.generate_features,
+                        text_features=text_features,
                         has_date=has_date,
                     )
                     metric = wrapper.metric_name
@@ -1009,7 +1020,7 @@ class FeaturesEnricher(TransformerMixin):
                             cat_features,
                             add_params=custom_loss_add_params,
                             groups=groups,
-                            text_features=self.generate_features,
+                            text_features=text_features,
                             has_date=has_date,
                         )
                         etalon_cv_result = baseline_estimator.cross_val_predict(
@@ -1044,7 +1055,7 @@ class FeaturesEnricher(TransformerMixin):
                             cat_features,
                             add_params=custom_loss_add_params,
                             groups=groups,
-                            text_features=self.generate_features,
+                            text_features=text_features,
                             has_date=has_date,
                         )
                         enriched_cv_result = enriched_estimator.cross_val_predict(fitting_enriched_X, enriched_y_sorted)
@@ -1827,7 +1838,27 @@ class FeaturesEnricher(TransformerMixin):
 
             # downsample if need to eval_set threshold
             num_samples = _num_samples(df)
-            if num_samples > Dataset.FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD:
+            phone_column = self._get_phone_column(self.search_keys)
+            force_downsampling = (
+                not self.disable_force_downsampling
+                and self.generate_features is not None
+                and phone_column is not None
+                and self.fit_columns_renaming[phone_column] in self.generate_features
+                and num_samples > Dataset.FORCE_SAMPLE_SIZE
+            )
+            if force_downsampling:
+                self.logger.info(f"Force downsampling from {num_samples} to {Dataset.FORCE_SAMPLE_SIZE}")
+                df = balance_undersample_forced(
+                    df=df,
+                    target_column=TARGET,
+                    task_type=self.model_task_type,
+                    random_state=self.random_state,
+                    sample_size=Dataset.FORCE_SAMPLE_SIZE,
+                    logger=self.logger,
+                    bundle=self.bundle,
+                    warning_callback=self.__log_warning,
+                )
+            elif num_samples > Dataset.FIT_SAMPLE_WITH_EVAL_SET_THRESHOLD:
                 self.logger.info(f"Downsampling from {num_samples} to {Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS}")
                 df = df.sample(n=Dataset.FIT_SAMPLE_WITH_EVAL_SET_ROWS, random_state=self.random_state)
 
@@ -2063,6 +2094,15 @@ class FeaturesEnricher(TransformerMixin):
                 self.__display_support_link(msg)
                 return None, {c: c for c in X.columns}, []
 
+            features_meta = self._search_task.get_all_features_metadata_v2()
+            online_api_features = [fm.name for fm in features_meta if fm.from_online_api]
+            if len(online_api_features) > 0:
+                self.logger.warning(
+                    f"There are important features for transform, that generated by online API: {online_api_features}"
+                )
+                # TODO
+                raise Exception("There are features selected that are paid. Contact support (sales@upgini.com)")
+
             if not metrics_calculation:
                 transform_usage = self.rest_client.get_current_transform_usage(trace_id)
                 self.logger.info(f"Current transform usage: {transform_usage}. Transforming {len(X)} rows")
@@ -2708,8 +2748,9 @@ class FeaturesEnricher(TransformerMixin):
             and self.generate_features is not None
             and phone_column is not None
             and self.fit_columns_renaming[phone_column] in self.generate_features
+            and len(df) > Dataset.FORCE_SAMPLE_SIZE
         )
-        if force_downsampling and len(df) > Dataset.FORCE_SAMPLE_SIZE:
+        if force_downsampling:
             runtime_parameters.properties["fast_fit"] = True
 
         dataset = Dataset(

{upgini-1.2.32 → upgini-1.2.34a1}/src/upgini/metadata.py

@@ -43,6 +43,9 @@ class FileColumnMeaningType(Enum):
     EVAL_SET_INDEX = "EVAL_SET_INDEX"
     ENTITY_SYSTEM_RECORD_ID = "ENTITY_SYSTEM_RECORD_ID"
     UNNEST_KEY = "UNNEST_KEY"
+    IP_BINARY = "IP_BINARY"
+    IP_RANGE_FROM_BINARY = "IP_RANGE_FROM_BINARY"
+    IP_RANGE_TO_BINARY = "IP_RANGE_TO_BINARY"
 
 
 class SearchKey(Enum):
@@ -60,6 +63,9 @@ class SearchKey(Enum):
     IPV6_ADDRESS = FileColumnMeaningType.IPV6_ADDRESS
     IPV6_RANGE_FROM = FileColumnMeaningType.IPV6_RANGE_FROM
     IPV6_RANGE_TO = FileColumnMeaningType.IPV6_RANGE_TO
+    IP_BINARY = FileColumnMeaningType.IP_BINARY
+    IP_RANGE_FROM_BINARY = FileColumnMeaningType.IP_RANGE_FROM_BINARY
+    IP_RANGE_TO_BINARY = FileColumnMeaningType.IP_RANGE_TO_BINARY
 
     # For data source registration. Don't use it for FeaturesEnricher
     EMAIL_ONE_DOMAIN = FileColumnMeaningType.EMAIL_ONE_DOMAIN
@@ -112,6 +118,12 @@ class SearchKey(Enum):
             return SearchKey.MSISDN_RANGE_FROM
         if meaning_type == FileColumnMeaningType.MSISDN_RANGE_TO:
             return SearchKey.MSISDN_RANGE_TO
+        if meaning_type == FileColumnMeaningType.IP_BINARY:
+            return SearchKey.IP_BINARY
+        if meaning_type == FileColumnMeaningType.IP_RANGE_FROM_BINARY:
+            return SearchKey.IP_RANGE_FROM_BINARY
+        if meaning_type == FileColumnMeaningType.IP_RANGE_TO_BINARY:
+            return SearchKey.IP_RANGE_TO_BINARY
 
     @staticmethod
     def find_key(search_keys: Dict[str, SearchKey], keys: Union[SearchKey, List[SearchKey]]) -> Optional[SearchKey]:
@@ -136,6 +148,7 @@ class DataType(Enum):
     DATE_TIME = "DATE_TIME"
     STRING = "STRING"
     BOOLEAN = "BOOLEAN"
+    BYTES = "BYTES"
 
 
 class ModelTaskType(Enum):
@@ -255,6 +268,7 @@ class FeaturesMetadataV2(BaseModel):
     data_source_links: Optional[List[str]] = None
     doc_link: Optional[str] = None
     update_frequency: Optional[str] = None
+    from_online_api: Optional[bool] = None
 
 
 class HitRateMetrics(BaseModel):

{upgini-1.2.32 → upgini-1.2.34a1}/src/upgini/utils/ip_utils.py

@@ -42,6 +42,16 @@ class IpSearchKeyConverter:
         except Exception:
             pass
 
+    @staticmethod
+    def _ip_to_binary(ip: Optional[_BaseAddress]) -> Optional[bytes]:
+        try:
+            if isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None:
+                return ip.ipv4_mapped.packed
+            else:
+                return ip.packed
+        except Exception:
+            pass
+
     @staticmethod
     def _ip_to_int_str(ip: Optional[_BaseAddress]) -> Optional[str]:
         try:
@@ -100,11 +110,16 @@ class IpSearchKeyConverter:
             .astype("string")
             # .str.replace(".0", "", regex=False)
         )
+        ip_binary = self.ip_column + "_binary"
+        df[ip_binary] = df[self.ip_column].apply(self._ip_to_binary)
+
         df = df.drop(columns=self.ip_column)
         del self.search_keys[self.ip_column]
         del self.columns_renaming[self.ip_column]
         self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
+        self.search_keys[ip_binary] = SearchKey.IP_BINARY
         self.columns_renaming[ipv6] = original_ip  # could be __unnest_ip...
+        self.columns_renaming[ip_binary] = original_ip
 
         return df
 

upgini-1.2.32/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.32"