upgini 1.2.6a1__py3-none-any.whl → 1.2.7__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.6a1"
+__version__ = "1.2.7"

upgini/autofe/all_operands.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Dict
 
 from upgini.autofe.binary import (
@@ -83,4 +84,4 @@ ALL_OPERANDS: Dict[str, Operand] = {
 
 
 def find_op(name):
-    return ALL_OPERANDS.get(name)
+    return deepcopy(ALL_OPERANDS.get(name))

upgini/autofe/feature.py

@@ -22,6 +22,9 @@ class Column:
     def set_op_params(self, params: Dict[str, str]) -> "Column":
         return self
 
+    def get_op_params(self, **kwargs):
+        return dict()
+
     def rename_columns(self, mapping: Dict[str, str]) -> "Column":
         self.name = self._unhash(mapping.get(self.name) or self.name)
         return self
@@ -44,6 +47,10 @@ class Column:
     def get_columns(self, **kwargs) -> List[str]:
         return [self.name]
 
+    @property
+    def children(self) -> List[Union["Feature", "Column"]]:
+        return []
+
     def infer_type(self, data: pd.DataFrame) -> DtypeObj:
         return data[self.name].dtype
 
@@ -88,9 +95,30 @@ class Feature:
         self.op.set_params(params)
 
         for child in self.children:
-            child.set_op_params(params)
+            child_params = {
+                k[len(child.get_display_name()) + 1 :]: v
+                for k, v in params.items()
+                if k.startswith(child.get_display_name())
+            }
+            if not child_params:
+                child_params = params
+            child.set_op_params(child_params)
         return self
 
+    def get_op_params(self, **kwargs) -> Dict[str, str]:
+        return {
+            k: str(v)
+            for k, v in dict(
+                (
+                    (f"{child.get_display_name(**kwargs)}_{k}", v)
+                    for child in self.children
+                    for k, v in child.get_op_params(**kwargs).items()
+                ),
+                **(self.op.get_params() or {}),
+            ).items()
+            if v is not None
+        }
+
     def get_hash(self) -> str:
         return hashlib.sha256(
             "_".join([self.op.name] + [ch.get_display_name() for ch in self.children]).encode("utf-8")
@@ -326,24 +354,26 @@ class FeatureGroup:
         return names
 
     def calculate(self, data: pd.DataFrame, is_root=False) -> pd.DataFrame:
-        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
         if isinstance(self.op, PandasOperand):
-            columns = self.get_columns()
-            lower_order_children = [
+            main_column = None if self.main_column_node is None else self.main_column_node.get_display_name()
+            lower_order_children = []
+            if self.main_column_node is not None:
+                lower_order_children.append(self.main_column_node)
+            lower_order_children.extend(
                 ch for f in self.children for ch in f.children if ch.get_display_name() != main_column
-            ]
+            )
             lower_order_names = [ch.get_display_name() for ch in lower_order_children]
-            if any(isinstance(f, Feature) for f in lower_order_children):
-                child_data = pd.concat(
-                    [data[main_column or []]] + [ch.calculate(data) for ch in lower_order_children],
-                    axis=1,
-                )
-                child_data.columns = ([main_column] if main_column is not None else []) + lower_order_names
-            else:
-                child_data = data[columns]
+            child_data = pd.concat(
+                [ch.calculate(data) for ch in lower_order_children],
+                axis=1,
+            )
+            child_data.columns = lower_order_names
 
             new_data = self.op.calculate_group(child_data, main_column=main_column)
-            new_data.rename(columns=dict(zip(lower_order_names, self.get_display_names())), inplace=True)
+            new_data.rename(
+                columns=dict(zip((n for n in lower_order_names if n != main_column), self.get_display_names())),
+                inplace=True,
+            )
         else:
             raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 

upgini/data_source/data_source_publisher.py

@@ -64,6 +64,7 @@ class DataSourcePublisher:
         date_features: Optional[List[str]] = None,
         date_vector_features: Optional[List[str]] = None,
         generate_runtime_embeddings: Optional[List[str]] = None,
+        exclude_raw: Optional[List[str]] = None,
         _force_generation=False,
         _silent=False,
     ) -> str:
@@ -88,6 +89,8 @@ class DataSourcePublisher:
         features_for_embeddings - optional list of str - list of features that should be used for GPT features
             generation
 
+        exclude_raw - optional list of str - list of features that should NOT be used as raw features
+
         ...
 
         data_table_id_to_replace - optional str - id of registered ADS that should be replaced by new table
@@ -166,6 +169,8 @@ class DataSourcePublisher:
                     request["dateVectorFeatures"] = date_vector_features
                 if generate_runtime_embeddings is not None:
                     request["generateRuntimeEmbeddingsFeatures"] = generate_runtime_embeddings
+                if exclude_raw is not None:
+                    request["excludeRaw"] = exclude_raw
                 self.logger.info(f"Start registering data table {request}")
 
                 task_id = self._rest_client.register_ads(request, trace_id)
@@ -281,6 +286,7 @@ class DataSourcePublisher:
         date_vector_features: Optional[List[str]] = None,
         exclude_from_autofe_generation: Optional[List[str]] = None,
         generate_runtime_embeddings: Optional[List[str]] = None,
+        exclude_raw: Optional[List[str]] = None,
     ):
         trace_id = str(uuid.uuid4())
         with MDC(trace_id=trace_id):
@@ -336,6 +342,8 @@ class DataSourcePublisher:
                     request["excludeFromGenerationFeatures"] = exclude_from_autofe_generation
                 if generate_runtime_embeddings is not None:
                     request["generateRuntimeEmbeddingsFeatures"] = generate_runtime_embeddings
+                if exclude_raw is not None:
+                    request["excludeRaw"] = exclude_raw
                 self.logger.info(f"Activating data tables with request {request}")
 
                 self._rest_client.activate_datatables(request, trace_id)
@@ -378,7 +386,6 @@ class DataSourcePublisher:
                 search_keys = [k.value.value for k in search_keys] if search_keys else None
                 request = {"bqTableId": bq_table_id, "searchKeys": search_keys}
                 task_id = self._rest_client.upload_online(request, trace_id)
-                print(f"Start polling management task_id={task_id} with trace_id={trace_id}")
                 with Spinner():
                     status_response = self._rest_client.poll_ads_management_task_status(task_id, trace_id)
                     while status_response["status"] not in self.FINAL_STATUSES:

upgini/features_enricher.py

@@ -3235,7 +3235,6 @@ class FeaturesEnricher(TransformerMixin):
                 ]
             )
 
-            # TODO some columns not exists
             all_other_columns = sorted_other_keys + other_columns
 
             search_keys_hash = "search_keys_hash"

upgini/utils/ip_utils.py

@@ -57,17 +57,17 @@ class IpSearchKeyConverter:
         except ValueError:
             pass
 
-    # @staticmethod
-    # def _is_ipv4(ip: Optional[_BaseAddress]):
-    #     return ip is not None and (
-    #         isinstance(ip, IPv4Address) or (isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None)
-    #     )
-
-    # @staticmethod
-    # def _to_ipv4(ip: Optional[_BaseAddress]) -> Optional[IPv4Address]:
-    #     if isinstance(ip, IPv4Address):
-    #         return ip
-    #     return None
+    @staticmethod
+    def _is_ipv4(ip: Optional[_BaseAddress]):
+        return ip is not None and (
+            isinstance(ip, IPv4Address) or (isinstance(ip, IPv6Address) and ip.ipv4_mapped is not None)
+        )
+
+    @staticmethod
+    def _to_ipv4(ip: Optional[_BaseAddress]) -> Optional[IPv4Address]:
+        if isinstance(ip, IPv4Address):
+            return ip
+        return None
 
     @staticmethod
     def _to_ipv6(ip: Optional[_BaseAddress]) -> Optional[IPv6Address]:
@@ -87,10 +87,10 @@ class IpSearchKeyConverter:
             raise ValidationError(self.bundle.get("invalid_ip").format(self.ip_column))
 
         # legacy support
-        # ipv4 = self.ip_column + "_v4"
-        # df[ipv4] = df[self.ip_column].apply(self._to_ipv4).apply(self._ip_to_int).astype("Int64")
-        # self.search_keys[ipv4] = SearchKey.IP
-        # self.columns_renaming[ipv4] = original_ip
+        ipv4 = self.ip_column + "_v4"
+        df[ipv4] = df[self.ip_column].apply(self._to_ipv4).apply(self._ip_to_int).astype("Int64")
+        self.search_keys[ipv4] = SearchKey.IP
+        self.columns_renaming[ipv4] = original_ip
 
         ipv6 = self.ip_column + "_v6"
         df[ipv6] = (
@@ -104,7 +104,7 @@ class IpSearchKeyConverter:
         del self.search_keys[self.ip_column]
         del self.columns_renaming[self.ip_column]
         self.search_keys[ipv6] = SearchKey.IPV6_ADDRESS
-        self.columns_renaming[ipv6] = original_ip  # could be upgini_ip_unnest...
+        self.columns_renaming[ipv6] = original_ip  # could be __unnest_ip...
 
         return df
 

{upgini-1.2.6a1.dist-info → upgini-1.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.6a1
+Version: 1.2.7
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

{upgini-1.2.6a1.dist-info → upgini-1.2.7.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=iKaF643F-CtlunSR-fjGeg8d8_wX6p_P6BtFkRK6sEA,24
+upgini/__about__.py,sha256=49prCLbE3fFzLfxem5rd2dr1iV4_L-bN0N4J7jxU5yA,22
 upgini/__init__.py,sha256=M64LwQTBa-5Jz24Zm2h8rWwlKQQ1J8nP7gGgIciS0WU,589
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=olZ-OHSfBNoBSCo7R5t7uCLukI2nO7afpx_A-HCiJLk,31067
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=NHlf_ib70UiaDTzWvA30Gz2HANzpf61Ql5EDFZqQzk0,188120
+upgini/features_enricher.py,sha256=twH4qdl91iHZF_AraLk0aIbRDw61S_DYtCWCZ34Yjjg,188077
 upgini/http.py,sha256=21asexflvavydzCOONJDGQBtQanCElrbnqLXakJ9Cu8,42880
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=osmzdNESeh7yP3BZday6N9Q3eaIHfzhhRM1d6NSgcf0,11223
@@ -14,16 +14,16 @@ upgini/version_validator.py,sha256=ddSKUK_-eGJB3NgrqOMoWJU-OxQ253WsNLp8aqJkaIM,1
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
 upgini/ads_management/ads_manager.py,sha256=igVbN2jz80Umb2BUJixmJVj-zx8unoKpecVo-R-nGdw,2648
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/autofe/all_operands.py,sha256=3LiH9iU-ArGmYpS8FHWH7yCFx40ILfvlSXJlKIa75BQ,2542
+upgini/autofe/all_operands.py,sha256=cCCB44qvkmuWyiRM5Xykx8tkHPIjQthrWyj67STWN80,2578
 upgini/autofe/binary.py,sha256=TRjEdxsfyPY5E8ksYfdKMmU6GtvALfGFPNVIG7DBhzM,7520
 upgini/autofe/date.py,sha256=OpFc3Al0xO3qlESn2Uokfxw51ArVqmh3xngWwdrsaqE,9762
-upgini/autofe/feature.py,sha256=gwGWY2UcX_0wHAvfEiu1rRU7GFZyzMWZIaPVcf6kD80,14223
+upgini/autofe/feature.py,sha256=eL7wABUhDKZzv3E-RPJNcyGwSfB0UptcfU2RbvsOks4,15082
 upgini/autofe/groupby.py,sha256=r-xl_keZZgm_tpiEoDhjYSkT6NHv7a4cRQR4wJ4uCp8,3263
 upgini/autofe/operand.py,sha256=uk883RaNqgXqtkaRqA1re1d9OFnnpv0JVvelYx09Yw0,2943
 upgini/autofe/unary.py,sha256=T3E7F3dA_7o_rkdCFq7JV6nHLzcoHLHQTcxO7y5Opa4,4646
 upgini/autofe/vector.py,sha256=ehcZUDqV71TfbU8EmKfdYp603gS2dJY_-fpr10ho5sI,663
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-upgini/data_source/data_source_publisher.py,sha256=6ZiT36flJHioh1J3F9lv6vSyqjRnQ_enZ6Mmb3GI2T8,22209
+upgini/data_source/data_source_publisher.py,sha256=X-8aGtVgzGmxyXkMVBoBLIGDMb4lYQaGZbxDnOd4A3Q,22516
 upgini/mdc/__init__.py,sha256=aM08nIWFc2gWdWUa3_IuEnNND0cQPkBGnYpRMnfFN8k,1019
 upgini/mdc/context.py,sha256=3u1B-jXt7tXEvNcV3qmR9SDCseudnY7KYsLclBdwVLk,1405
 upgini/normalizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -49,7 +49,7 @@ upgini/utils/email_utils.py,sha256=j0Ug1R_0AnCg1Y92zIZ4XMwvKo3G5_pcOlBN1OH_gZs,5
 upgini/utils/fallback_progress_bar.py,sha256=PDaKb8dYpVZaWMroNcOHsTc3pSjgi9mOm0--cOFTwJ0,1074
 upgini/utils/features_validator.py,sha256=PgKNt5dyqfErTvjtRNNUS9g7GFqHBtAtnsfA-V5UO1A,3307
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
-upgini/utils/ip_utils.py,sha256=fB9jhabRr8mtZzNNscXIHQ5-QJqIZkAw3FO06eQ9jO8,5176
+upgini/utils/ip_utils.py,sha256=ZZj_uQFTHhagzt-MRew__ZBOp2DdnkMrachS7PElkSE,5143
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
@@ -57,7 +57,7 @@ upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,4
 upgini/utils/target_utils.py,sha256=BVtDmrmFMKerSUWaNOIEdzsYHIFiODdpnWbE50QDPDc,7864
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/warning_counter.py,sha256=dIWBB4dI5XRRJZudvIlqlIYKEiwLLPcXarsZuYRt338,227
-upgini-1.2.6a1.dist-info/METADATA,sha256=h6NV4SfVmBdj8hXvweTrWrOCid84grVTAGPVFukvqiI,48609
-upgini-1.2.6a1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-upgini-1.2.6a1.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.6a1.dist-info/RECORD,,
+upgini-1.2.7.dist-info/METADATA,sha256=WvOelDedurBhIekxyFlENU8uZxq7ZcQhhEQCrc2TL04,48607
+upgini-1.2.7.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+upgini-1.2.7.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.7.dist-info/RECORD,,