upgini 1.1.231a2__py3-none-any.whl → 1.1.232a2__py3-none-any.whl

upgini/autofe/feature.py

@@ -1,78 +1,61 @@
 import hashlib
-from typing import Dict
+import itertools
+from typing import Dict, List, Optional, Tuple, Union
+
 import numpy as np
 import pandas as pd
-import itertools
-from upgini.autofe.operand import PandasOperand
-from upgini.autofe.all_operands import (
-    find_op,
-)
+from pandas._typing import DtypeObj
 
+from upgini.autofe.all_operands import find_op
+from upgini.autofe.operand import Operand, PandasOperand
 
-class FeatureGroup(object):
-    def __init__(self, op, main_column, children):
-        self.op = op
-        self.main_column_node = main_column
-        self.children = children
-        self.data = None
-
-    def get_columns(self, **kwargs):
-        column_list = []
-        seen = set()
-        for child in self.children:
-            columns = child.get_columns(**kwargs)
-            column_list.extend([f for f in columns if f not in seen])
-            seen.update(columns)
-        return column_list
 
-    def get_display_names(self, **kwargs):
-        names = [f.get_display_name(**kwargs) for f in self.children]
-        return names
+class Column:
+    def __init__(self, name: str, data: Optional[pd.Series] = None, calculate_all=False):
+        self.name = name
+        self.data = data
+        self.calculate_all = calculate_all
 
-    def calculate(self, data: pd.DataFrame, is_root=False):
-        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
-        if isinstance(self.op, PandasOperand):
-            columns = self.get_columns()
-            new_data = self.op.calculate_group(data[columns], main_column=main_column)
-            new_data.rename(columns=dict(zip(columns, self.get_display_names())), inplace=True)
+    def rename_columns(self, mapping: Dict[str, str]) -> "Column":
+        self.name = self._unhash(mapping.get(self.name) or self.name)
+        return self
 
+    def _unhash(self, feature_name: str) -> str:
+        last_component_idx = feature_name.rfind("_")
+        if not feature_name.startswith("f_"):
+            return feature_name  # etalon feature
+        elif last_component_idx == 1:
+            return feature_name[2:]  # fully hashed name, cannot unhash
         else:
-            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
+            return feature_name[2:last_component_idx]
 
-        new_data.replace([-np.inf, np.inf], np.nan, inplace=True)
+    def delete_data(self):
+        self.data = None
 
-        if is_root:
-            self.data = new_data
-        return new_data
+    def get_column_nodes(self) -> List["Column"]:
+        return [self]
 
-    @staticmethod
-    def make_groups(candidates):
-        grouped_features = []
-        for op_child, features in itertools.groupby(
-            candidates, lambda f: (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
-        ):
-            op, main_child = op_child
-            feature_list = list(features)
-            if op.is_vectorizable:
-                if op.is_unary:
-                    group = FeatureGroup(op, main_column=None, children=feature_list)
-                else:
-                    group = FeatureGroup(op, main_column=main_child, children=feature_list)
-                grouped_features.append(group)
-            else:
-                grouped_features.extend(feature_list)
-        return grouped_features
+    def get_columns(self) -> List[str]:
+        return [self.name]
 
-    def delete_data(self):
-        self.data = None
-        if self.main_column_node:
-            self.main_column_node.delete_data()
-        for child in self.children:
-            child.delete_data()
+    def infer_type(self, data: pd.DataFrame) -> DtypeObj:
+        return data[self.name].dtype
 
+    def calculate(self, data: pd.DataFrame) -> pd.Series:
+        self.data = data[self.name]
+        return self.data
+
+    def to_formula(self, **kwargs) -> str:
+        return str(self.get_columns(**kwargs)[0])
 
-class Feature(object):
-    def __init__(self, op, children, data=None, display_index=None, cached_display_name=None, alias=None):
+    def to_pretty_formula(self) -> str:
+        return self.to_formula()
+
+
+class Feature:
+    def __init__(self, op: Operand, children: List[Union[Column, "Feature"]], data: Optional[pd.DataFrame] = None,
+                 display_index: Optional[str] = None, cached_display_name: Optional[str] = None,
+                 alias: Optional[str] = None):
         self.op = op
         self.children = children
         self.data = data
@@ -80,32 +63,32 @@ class Feature(object):
         self.cached_display_name = cached_display_name
         self.alias = alias
 
-    def set_op_params(self, params: Dict):
+    def set_op_params(self, params: Dict[str, str]) -> "Feature":
         self.op.set_params(params)
         return self
 
-    def get_hash(self):
+    def get_hash(self) -> str:
         return hashlib.sha256("_".join([self.op.name] + [ch.name for ch in self.children]).encode("utf-8")).hexdigest()[
             :8
         ]
 
-    def set_alias(self, alias):
+    def set_alias(self, alias: str) -> "Feature":
         self.alias = alias
         return self
 
-    def rename_columns(self, mapping: Dict):
+    def rename_columns(self, mapping: Dict[str, str]) -> "Feature":
         for child in self.children:
             child.rename_columns(mapping)
         self.cached_display_name = None
         return self
 
-    def get_column_nodes(self):
+    def get_column_nodes(self) -> List[Union[Column, "Feature"]]:
         res = []
         for child in self.children:
             res.extend(child.get_column_nodes())
         return res
 
-    def get_columns(self, **kwargs):
+    def get_columns(self, **kwargs) -> List[str]:
         column_list = []
         seen = set()
         for child in self.children:
@@ -119,7 +102,7 @@ class Feature(object):
         for child in self.children:
             child.delete_data()
 
-    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs):
+    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
         if self.cached_display_name is not None and cache:
             return self.cached_display_name
 
@@ -139,27 +122,27 @@ class Feature(object):
             self.cached_display_name = display_name
         return display_name
 
-    def set_display_index(self, index):
+    def set_display_index(self, index) -> "Feature":
         self.display_index = index
         self.cached_display_name = None
         return self
 
-    def infer_type(self, data):
+    def infer_type(self, data: pd.DataFrame) -> Union[str, DtypeObj]:
         if self.op.output_type:
             return self.op.output_type
         else:
             # either a symmetrical operator or group by
             return self.children[0].infer_type(data)
 
-    def calculate(self, data, is_root=False):
+    def calculate(self, data: pd.DataFrame, is_root=False) -> Union[pd.Series, pd.DataFrame]:
         if isinstance(self.op, PandasOperand) and self.op.is_vector:
-            ds = [child.calculate(data) for child in self.children]
-            new_data = self.op.calculate(data=ds)
-
-        elif isinstance(self.op, PandasOperand):
-            d1 = self.children[0].calculate(data)
-            d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
-            new_data = self.op.calculate(data=d1, left=d1, right=d2)
+            if self.op.is_vector:
+                ds = [child.calculate(data) for child in self.children]
+                new_data = self.op.calculate(data=ds)
+            else:
+                d1 = self.children[0].calculate(data)
+                d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
+                new_data = self.op.calculate(data=d1, left=d1, right=d2)
         else:
             raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 
@@ -173,8 +156,8 @@ class Feature(object):
         return new_data
 
     @staticmethod
-    def check_xor(left, right):
-        def _get_all_columns(feature):
+    def check_xor(left: Union[Column, "Feature"], right: Union[Column, "Feature"]) -> bool:
+        def _get_all_columns(feature: Union[Column, "Feature"]) -> List[str]:
             if isinstance(feature, Column):
                 return [feature.name]
             else:
@@ -190,7 +173,7 @@ class Feature(object):
         else:
             return True
 
-    def to_formula(self, **kwargs):
+    def to_formula(self, **kwargs) -> str:
         if self.op.name in ["+", "-", "*", "/"]:
             left = self.children[0].to_formula(**kwargs)
             right = self.children[1].to_formula(**kwargs)
@@ -205,15 +188,30 @@ class Feature(object):
             result.append(")")
             return "".join(result)
 
+    def to_pretty_formula(self) -> str:
+        if self.op.name in ["+", "-", "*", "/"]:
+            left = self.children[0].to_pretty_formula()
+            right = self.children[1].to_pretty_formula()
+            return f"{left} {self.op.name} {right}"
+        else:
+            result = [self.op.name, "("]
+            for i in range(len(self.children)):
+                string_i = self.children[i].to_pretty_formula()
+                result.append(string_i)
+                result.append(", ")
+            result.pop()
+            result.append(")")
+            return "".join(result)
+
     @staticmethod
-    def from_formula(string):
+    def from_formula(string: str) -> Union[Column, "Feature"]:
         if string[-1] != ")":
             return Column(string)
 
-        def is_trivial_char(c):
+        def is_trivial_char(c: str) -> bool:
             return not (c in "()+-*/,")
 
-        def find_prev(string):
+        def find_prev(string: str) -> int:
             if string[-1] != ")":
                 return max([(0 if is_trivial_char(c) else i + 1) for i, c in enumerate(string)])
             level, pos = 0, -1
@@ -259,40 +257,65 @@ class Feature(object):
                 return Feature(op, base_features)
 
 
-class Column(object):
-    def __init__(self, name, data=None, calculate_all=False):
-        self.name = name
-        self.data = data
-        self.calculate_all = calculate_all
+class FeatureGroup:
+    def __init__(self, op: Operand, main_column: Optional[Union[Column, Feature]],
+                 children: List[Union[Column, Feature]]):
+        self.op = op
+        self.main_column_node = main_column
+        self.children = children
+        self.data: Optional[pd.DataFrame] = None
 
-    def rename_columns(self, mapping: Dict):
-        self.name = self._unhash(mapping.get(self.name) or self.name)
-        return self
+    def get_columns(self, **kwargs) -> List[str]:
+        column_list = []
+        seen = set()
+        for child in self.children:
+            columns = child.get_columns(**kwargs)
+            column_list.extend([f for f in columns if f not in seen])
+            seen.update(columns)
+        return column_list
 
-    def _unhash(self, feature_name):
-        last_component_idx = feature_name.rfind("_")
-        if not feature_name.startswith("f_"):
-            return feature_name  # etalon feature
-        elif last_component_idx == 1:
-            return feature_name[2:]  # fully hashed name, cannot unhash
+    def get_display_names(self, **kwargs) -> List[str]:
+        names = [f.get_display_name(**kwargs) for f in self.children]
+        return names
+
+    def calculate(self, data: pd.DataFrame, is_root=False) -> pd.DataFrame:
+        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
+        if isinstance(self.op, PandasOperand):
+            columns = self.get_columns()
+            new_data = self.op.calculate_group(data[columns], main_column=main_column)
+            new_data.rename(columns=dict(zip(columns, self.get_display_names())), inplace=True)
         else:
-            return feature_name[2:last_component_idx]
+            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 
-    def delete_data(self):
-        self.data = None
+        new_data.replace([-np.inf, np.inf], np.nan, inplace=True)
 
-    def get_column_nodes(self):
-        return [self]
+        if is_root:
+            self.data = new_data
+        return new_data
 
-    def get_columns(self):
-        return [self.name]
+    @staticmethod
+    def make_groups(candidates: List[Feature]) -> List[Union[Feature, "FeatureGroup"]]:
+        grouped_features = []
 
-    def infer_type(self, data):
-        return data[self.name].dtype
+        def groupby_func(f: Feature) -> Tuple[Operand, Union[Column, Feature]]:
+            return (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
 
-    def calculate(self, data):
-        self.data = data[self.name]
-        return self.data
+        for op_child, features in itertools.groupby(candidates, groupby_func):
+            op, main_child = op_child
+            feature_list = list(features)
+            if op.is_vectorizable:
+                if op.is_unary:
+                    group = FeatureGroup(op, main_column=None, children=feature_list)
+                else:
+                    group = FeatureGroup(op, main_column=main_child, children=feature_list)
+                grouped_features.append(group)
+            else:
+                grouped_features.extend(feature_list)
+        return grouped_features
 
-    def to_formula(self, **kwargs):
-        return str(self.get_columns(**kwargs)[0])
+    def delete_data(self):
+        self.data = None
+        if self.main_column_node:
+            self.main_column_node.delete_data()
+        for child in self.children:
+            child.delete_data()

upgini/features_enricher.py

@@ -2921,6 +2921,7 @@ class FeaturesEnricher(TransformerMixin):
                 if feature_meta is None:
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
+                description["shap"] = feature_meta.shap_value
                 description["Sources"] = feature_meta.data_source\
                     .replace("AutoFE: features from ", "")\
                     .replace("AutoFE: feature from ", "")
@@ -2940,6 +2941,8 @@ class FeaturesEnricher(TransformerMixin):
 
             descriptions_df = pd.DataFrame(descriptions)
             descriptions_df.fillna("", inplace=True)
+            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
+            descriptions_df.drop(columns="shap", inplace=True)
             return descriptions_df
         except Exception:
             self.logger.exception("Failed to generate AutoFE features description")

{upgini-1.1.231a2.dist-info → upgini-1.1.232a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.231a2
+Version: 1.1.232a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.231a2.dist-info → upgini-1.1.232a2.dist-info}/RECORD

@@ -2,7 +2,7 @@ upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
 upgini/dataset.py,sha256=7z9zbVvd1_MiufmoZlCwEHwQ25Q2DX_0g9PFcSMlqMY,49764
 upgini/errors.py,sha256=BqpvfhW2jJW5fa5KXj0alhXatGl-WK4xTl309-QNLp8,959
-upgini/features_enricher.py,sha256=vPol2Oi_Mm-4F_iQBA9L_bm_bWdv_SU0Jmu9sVJK9YM,158650
+upgini/features_enricher.py,sha256=2B9rk_8QNMV7o1khbgZX8A1T6vJqyfki4F4UAYoR0po,158857
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=HzUSZudCdISJGUqHC1gAT1v_x1n_dIFVDJW4z3Q7DCs,41204
 upgini/metadata.py,sha256=FZ5CQluLLWrfrBVThSIes1SW6wcs7n50aNZwzYnHiF0,9584
@@ -15,7 +15,7 @@ upgini/ads_management/ads_manager.py,sha256=O6Pcl_y5e_ULfQ-xmGGn_qBP4z7EtV7TP9et
 upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/autofe/all_operands.py,sha256=du44N6ISWe3ikb0y9ZzSOHNbLiyEYrJPwoBo0Z6xp2s,1487
 upgini/autofe/binary.py,sha256=f8LQqZi9zyaMUAv-jASMmWNA_vT05ncYCjZq0qx3USs,3972
-upgini/autofe/feature.py,sha256=cElNcLfw9BeBVUkkaFzWWXrnyWNUCXiw0FGqsitorbE,10133
+upgini/autofe/feature.py,sha256=iDB_cL49w7AYl-96AkVqWBynrE_ZqK0fxgTfuJJoruA,11847
 upgini/autofe/groupby.py,sha256=iXRfOmOc84ooSzRhsh9GmmG7rTafX0-ekXko8s9Qs68,3089
 upgini/autofe/operand.py,sha256=8WqEoSIA5rEWCK1xuC303E4NW5a72GZ5jUMAEj4skII,2291
 upgini/autofe/unary.py,sha256=7TBe7PCt7l_XQEqu_G5g_TC2cW3tppL7uPDcX8xsqz0,2731
@@ -53,8 +53,8 @@ upgini/utils/sklearn_ext.py,sha256=IMx2La70AXAggApVpT7sMEjWqVWon5AMZt4MARDsIMQ,4
 upgini/utils/target_utils.py,sha256=cu52icjhDIPpEStHYMXrD2hIl9gzvfnxZr0Ra5osV0k,1616
 upgini/utils/track_info.py,sha256=DVNVZmXUb4f25DSPEuUNEFx49hNEBfmuY9iSW5jkMnI,5708
 upgini/utils/warning_counter.py,sha256=vnmdFo5-7GBkU2bK9h_uC0K0Y_wtfcYstxOdeRfacO0,228
-upgini-1.1.231a2.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.231a2.dist-info/METADATA,sha256=RRZJq05KPtLkixg7feODcdrsAdwioqWdEM-D9aJnffY,48400
-upgini-1.1.231a2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-upgini-1.1.231a2.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.231a2.dist-info/RECORD,,
+upgini-1.1.232a2.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.232a2.dist-info/METADATA,sha256=s6aUWidVUESHbFanlUtLlmWr4izBNGLTeQj1O1bH82A,48400
+upgini-1.1.232a2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+upgini-1.1.232a2.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.232a2.dist-info/RECORD,,