upgini 1.1.231a2__tar.gz → 1.1.232a2__tar.gz

{upgini-1.1.231a2/src/upgini.egg-info → upgini-1.1.232a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.231a2
+Version: 1.1.232a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.231a2 → upgini-1.1.232a2}/setup.py

@@ -40,7 +40,7 @@ def send_log(msg: str):
 
 
 here = Path(__file__).parent.resolve()
-version = "1.1.231a2"
+version = "1.1.232a2"
 try:
     send_log(f"Start setup PyLib version {version}")
     setup(

{upgini-1.1.231a2 → upgini-1.1.232a2}/src/upgini/autofe/feature.py

@@ -1,78 +1,61 @@
 import hashlib
-from typing import Dict
+import itertools
+from typing import Dict, List, Optional, Tuple, Union
+
 import numpy as np
 import pandas as pd
-import itertools
-from upgini.autofe.operand import PandasOperand
-from upgini.autofe.all_operands import (
-    find_op,
-)
+from pandas._typing import DtypeObj
 
+from upgini.autofe.all_operands import find_op
+from upgini.autofe.operand import Operand, PandasOperand
 
-class FeatureGroup(object):
-    def __init__(self, op, main_column, children):
-        self.op = op
-        self.main_column_node = main_column
-        self.children = children
-        self.data = None
-
-    def get_columns(self, **kwargs):
-        column_list = []
-        seen = set()
-        for child in self.children:
-            columns = child.get_columns(**kwargs)
-            column_list.extend([f for f in columns if f not in seen])
-            seen.update(columns)
-        return column_list
 
-    def get_display_names(self, **kwargs):
-        names = [f.get_display_name(**kwargs) for f in self.children]
-        return names
+class Column:
+    def __init__(self, name: str, data: Optional[pd.Series] = None, calculate_all=False):
+        self.name = name
+        self.data = data
+        self.calculate_all = calculate_all
 
-    def calculate(self, data: pd.DataFrame, is_root=False):
-        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
-        if isinstance(self.op, PandasOperand):
-            columns = self.get_columns()
-            new_data = self.op.calculate_group(data[columns], main_column=main_column)
-            new_data.rename(columns=dict(zip(columns, self.get_display_names())), inplace=True)
+    def rename_columns(self, mapping: Dict[str, str]) -> "Column":
+        self.name = self._unhash(mapping.get(self.name) or self.name)
+        return self
 
+    def _unhash(self, feature_name: str) -> str:
+        last_component_idx = feature_name.rfind("_")
+        if not feature_name.startswith("f_"):
+            return feature_name  # etalon feature
+        elif last_component_idx == 1:
+            return feature_name[2:]  # fully hashed name, cannot unhash
         else:
-            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
+            return feature_name[2:last_component_idx]
 
-        new_data.replace([-np.inf, np.inf], np.nan, inplace=True)
+    def delete_data(self):
+        self.data = None
 
-        if is_root:
-            self.data = new_data
-        return new_data
+    def get_column_nodes(self) -> List["Column"]:
+        return [self]
 
-    @staticmethod
-    def make_groups(candidates):
-        grouped_features = []
-        for op_child, features in itertools.groupby(
-            candidates, lambda f: (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
-        ):
-            op, main_child = op_child
-            feature_list = list(features)
-            if op.is_vectorizable:
-                if op.is_unary:
-                    group = FeatureGroup(op, main_column=None, children=feature_list)
-                else:
-                    group = FeatureGroup(op, main_column=main_child, children=feature_list)
-                grouped_features.append(group)
-            else:
-                grouped_features.extend(feature_list)
-        return grouped_features
+    def get_columns(self) -> List[str]:
+        return [self.name]
 
-    def delete_data(self):
-        self.data = None
-        if self.main_column_node:
-            self.main_column_node.delete_data()
-        for child in self.children:
-            child.delete_data()
+    def infer_type(self, data: pd.DataFrame) -> DtypeObj:
+        return data[self.name].dtype
 
+    def calculate(self, data: pd.DataFrame) -> pd.Series:
+        self.data = data[self.name]
+        return self.data
+
+    def to_formula(self, **kwargs) -> str:
+        return str(self.get_columns(**kwargs)[0])
 
-class Feature(object):
-    def __init__(self, op, children, data=None, display_index=None, cached_display_name=None, alias=None):
+    def to_pretty_formula(self) -> str:
+        return self.to_formula()
+
+
+class Feature:
+    def __init__(self, op: Operand, children: List[Union[Column, "Feature"]], data: Optional[pd.DataFrame] = None,
+                 display_index: Optional[str] = None, cached_display_name: Optional[str] = None,
+                 alias: Optional[str] = None):
         self.op = op
         self.children = children
         self.data = data
@@ -80,32 +63,32 @@ class Feature(object):
         self.cached_display_name = cached_display_name
         self.alias = alias
 
-    def set_op_params(self, params: Dict):
+    def set_op_params(self, params: Dict[str, str]) -> "Feature":
         self.op.set_params(params)
         return self
 
-    def get_hash(self):
+    def get_hash(self) -> str:
         return hashlib.sha256("_".join([self.op.name] + [ch.name for ch in self.children]).encode("utf-8")).hexdigest()[
             :8
         ]
 
-    def set_alias(self, alias):
+    def set_alias(self, alias: str) -> "Feature":
         self.alias = alias
         return self
 
-    def rename_columns(self, mapping: Dict):
+    def rename_columns(self, mapping: Dict[str, str]) -> "Feature":
         for child in self.children:
             child.rename_columns(mapping)
         self.cached_display_name = None
         return self
 
-    def get_column_nodes(self):
+    def get_column_nodes(self) -> List[Union[Column, "Feature"]]:
         res = []
         for child in self.children:
             res.extend(child.get_column_nodes())
         return res
 
-    def get_columns(self, **kwargs):
+    def get_columns(self, **kwargs) -> List[str]:
         column_list = []
         seen = set()
         for child in self.children:
@@ -119,7 +102,7 @@ class Feature(object):
         for child in self.children:
             child.delete_data()
 
-    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs):
+    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs) -> str:
         if self.cached_display_name is not None and cache:
             return self.cached_display_name
 
@@ -139,27 +122,27 @@ class Feature(object):
             self.cached_display_name = display_name
         return display_name
 
-    def set_display_index(self, index):
+    def set_display_index(self, index) -> "Feature":
         self.display_index = index
         self.cached_display_name = None
         return self
 
-    def infer_type(self, data):
+    def infer_type(self, data: pd.DataFrame) -> Union[str, DtypeObj]:
         if self.op.output_type:
             return self.op.output_type
         else:
             # either a symmetrical operator or group by
             return self.children[0].infer_type(data)
 
-    def calculate(self, data, is_root=False):
+    def calculate(self, data: pd.DataFrame, is_root=False) -> Union[pd.Series, pd.DataFrame]:
         if isinstance(self.op, PandasOperand) and self.op.is_vector:
-            ds = [child.calculate(data) for child in self.children]
-            new_data = self.op.calculate(data=ds)
-
-        elif isinstance(self.op, PandasOperand):
-            d1 = self.children[0].calculate(data)
-            d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
-            new_data = self.op.calculate(data=d1, left=d1, right=d2)
+            if self.op.is_vector:
+                ds = [child.calculate(data) for child in self.children]
+                new_data = self.op.calculate(data=ds)
+            else:
+                d1 = self.children[0].calculate(data)
+                d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
+                new_data = self.op.calculate(data=d1, left=d1, right=d2)
         else:
             raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 
@@ -173,8 +156,8 @@ class Feature(object):
         return new_data
 
     @staticmethod
-    def check_xor(left, right):
-        def _get_all_columns(feature):
+    def check_xor(left: Union[Column, "Feature"], right: Union[Column, "Feature"]) -> bool:
+        def _get_all_columns(feature: Union[Column, "Feature"]) -> List[str]:
             if isinstance(feature, Column):
                 return [feature.name]
             else:
@@ -190,7 +173,7 @@ class Feature(object):
         else:
             return True
 
-    def to_formula(self, **kwargs):
+    def to_formula(self, **kwargs) -> str:
         if self.op.name in ["+", "-", "*", "/"]:
             left = self.children[0].to_formula(**kwargs)
             right = self.children[1].to_formula(**kwargs)
@@ -205,15 +188,30 @@ class Feature(object):
             result.append(")")
             return "".join(result)
 
+    def to_pretty_formula(self) -> str:
+        if self.op.name in ["+", "-", "*", "/"]:
+            left = self.children[0].to_pretty_formula()
+            right = self.children[1].to_pretty_formula()
+            return f"{left} {self.op.name} {right}"
+        else:
+            result = [self.op.name, "("]
+            for i in range(len(self.children)):
+                string_i = self.children[i].to_pretty_formula()
+                result.append(string_i)
+                result.append(", ")
+            result.pop()
+            result.append(")")
+            return "".join(result)
+
     @staticmethod
-    def from_formula(string):
+    def from_formula(string: str) -> Union[Column, "Feature"]:
         if string[-1] != ")":
             return Column(string)
 
-        def is_trivial_char(c):
+        def is_trivial_char(c: str) -> bool:
             return not (c in "()+-*/,")
 
-        def find_prev(string):
+        def find_prev(string: str) -> int:
             if string[-1] != ")":
                 return max([(0 if is_trivial_char(c) else i + 1) for i, c in enumerate(string)])
             level, pos = 0, -1
@@ -259,40 +257,65 @@ class Feature(object):
                 return Feature(op, base_features)
 
 
-class Column(object):
-    def __init__(self, name, data=None, calculate_all=False):
-        self.name = name
-        self.data = data
-        self.calculate_all = calculate_all
+class FeatureGroup:
+    def __init__(self, op: Operand, main_column: Optional[Union[Column, Feature]],
+                 children: List[Union[Column, Feature]]):
+        self.op = op
+        self.main_column_node = main_column
+        self.children = children
+        self.data: Optional[pd.DataFrame] = None
 
-    def rename_columns(self, mapping: Dict):
-        self.name = self._unhash(mapping.get(self.name) or self.name)
-        return self
+    def get_columns(self, **kwargs) -> List[str]:
+        column_list = []
+        seen = set()
+        for child in self.children:
+            columns = child.get_columns(**kwargs)
+            column_list.extend([f for f in columns if f not in seen])
+            seen.update(columns)
+        return column_list
 
-    def _unhash(self, feature_name):
-        last_component_idx = feature_name.rfind("_")
-        if not feature_name.startswith("f_"):
-            return feature_name  # etalon feature
-        elif last_component_idx == 1:
-            return feature_name[2:]  # fully hashed name, cannot unhash
+    def get_display_names(self, **kwargs) -> List[str]:
+        names = [f.get_display_name(**kwargs) for f in self.children]
+        return names
+
+    def calculate(self, data: pd.DataFrame, is_root=False) -> pd.DataFrame:
+        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
+        if isinstance(self.op, PandasOperand):
+            columns = self.get_columns()
+            new_data = self.op.calculate_group(data[columns], main_column=main_column)
+            new_data.rename(columns=dict(zip(columns, self.get_display_names())), inplace=True)
         else:
-            return feature_name[2:last_component_idx]
+            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 
-    def delete_data(self):
-        self.data = None
+        new_data.replace([-np.inf, np.inf], np.nan, inplace=True)
 
-    def get_column_nodes(self):
-        return [self]
+        if is_root:
+            self.data = new_data
+        return new_data
 
-    def get_columns(self):
-        return [self.name]
+    @staticmethod
+    def make_groups(candidates: List[Feature]) -> List[Union[Feature, "FeatureGroup"]]:
+        grouped_features = []
 
-    def infer_type(self, data):
-        return data[self.name].dtype
+        def groupby_func(f: Feature) -> Tuple[Operand, Union[Column, Feature]]:
+            return (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
 
-    def calculate(self, data):
-        self.data = data[self.name]
-        return self.data
+        for op_child, features in itertools.groupby(candidates, groupby_func):
+            op, main_child = op_child
+            feature_list = list(features)
+            if op.is_vectorizable:
+                if op.is_unary:
+                    group = FeatureGroup(op, main_column=None, children=feature_list)
+                else:
+                    group = FeatureGroup(op, main_column=main_child, children=feature_list)
+                grouped_features.append(group)
+            else:
+                grouped_features.extend(feature_list)
+        return grouped_features
 
-    def to_formula(self, **kwargs):
-        return str(self.get_columns(**kwargs)[0])
+    def delete_data(self):
+        self.data = None
+        if self.main_column_node:
+            self.main_column_node.delete_data()
+        for child in self.children:
+            child.delete_data()

{upgini-1.1.231a2 → upgini-1.1.232a2}/src/upgini/features_enricher.py

@@ -2921,6 +2921,7 @@ class FeaturesEnricher(TransformerMixin):
                 if feature_meta is None:
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
+                description["shap"] = feature_meta.shap_value
                 description["Sources"] = feature_meta.data_source\
                     .replace("AutoFE: features from ", "")\
                     .replace("AutoFE: feature from ", "")
@@ -2940,6 +2941,8 @@ class FeaturesEnricher(TransformerMixin):
 
             descriptions_df = pd.DataFrame(descriptions)
             descriptions_df.fillna("", inplace=True)
+            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
+            descriptions_df.drop(columns="shap", inplace=True)
             return descriptions_df
         except Exception:
             self.logger.exception("Failed to generate AutoFE features description")

{upgini-1.1.231a2 → upgini-1.1.232a2/src/upgini.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.231a2
+Version: 1.1.232a2
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers