upgini 1.1.228__py3-none-any.whl → 1.1.231__py3-none-any.whl

upgini/autofe/all_operands.py

@@ -0,0 +1,43 @@
+from typing import Dict
+from upgini.autofe.groupby import GroupByThenAgg, GroupByThenRank
+from upgini.autofe.operand import Operand
+from upgini.autofe.unary import Abs, Log, Residual, Sqrt, Square, Sigmoid, Floor, Freq
+from upgini.autofe.binary import Min, Max, Add, Subtract, Multiply, Divide, Sim
+from upgini.autofe.vector import Mean, Sum
+
+ALL_OPERANDS: Dict[str, Operand] = {
+    op.name: op
+    for op in [
+        Freq(),
+        Mean(),
+        Sum(),
+        Abs(),
+        Log(),
+        Sqrt(),
+        Square(),
+        Sigmoid(),
+        Floor(),
+        Residual(),
+        Min(),
+        Max(),
+        Add(),
+        Subtract(),
+        Multiply(),
+        Divide(),
+        GroupByThenAgg(name="GroupByThenMin", agg="min"),
+        GroupByThenAgg(name="GroupByThenMax", agg="max"),
+        GroupByThenAgg(name="GroupByThenMean", agg="mean"),
+        GroupByThenAgg(name="GroupByThenMedian", agg="median"),
+        GroupByThenAgg(name="GroupByThenStd", output_type="float", agg="std"),
+        GroupByThenRank(),
+        Operand(name="Combine", has_symmetry_importance=True, output_type="object", is_categorical=True),
+        Operand(name="CombineThenFreq", has_symmetry_importance=True, output_type="float"),
+        Operand(name="GroupByThenNUnique", output_type="int", is_vectorizable=True, is_grouping=True),
+        Operand(name="GroupByThenFreq", output_type="float", is_grouping=True),
+        Sim(),
+    ]
+}
+
+
+def find_op(name):
+    return ALL_OPERANDS.get(name)

upgini/autofe/binary.py

@@ -0,0 +1,133 @@
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
+import numpy as np
+import pandas as pd
+from numpy import dot
+from numpy.linalg import norm
+
+
+class Min(PandasOperand):
+    name = "min"
+    is_binary = True
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return np.minimum(left, right)
+
+
+class Max(PandasOperand):
+    name = "max"
+    is_binary = True
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return np.maximum(left, right)
+
+
+class Add(PandasOperand, VectorizableMixin):
+    name = "+"
+    alias = "add"
+    is_binary = True
+    has_symmetry_importance = True
+    is_vectorizable = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return left + right
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+
+        return d1.add(d2, axis=0)
+
+
+class Subtract(PandasOperand, VectorizableMixin):
+    name = "-"
+    alias = "sub"
+    is_binary = True
+    has_symmetry_importance = True
+    is_vectorizable = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return left - right
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+
+        return d1.sub(d2, axis=0)
+
+
+class Multiply(PandasOperand, VectorizableMixin):
+    name = "*"
+    alias = "mul"
+    is_binary = True
+    has_symmetry_importance = True
+    is_vectorizable = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return left * right
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+
+        return d1.mul(d2, axis=0)
+
+
+class Divide(PandasOperand, VectorizableMixin):
+    name = "/"
+    alias = "div"
+    is_binary = True
+    has_symmetry_importance = True
+    is_vectorizable = True
+    output_type = "float"
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return left / right.replace(0, np.nan)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+
+        return d1.div(d2.replace(0, np.nan), axis=0)
+
+
+class Combine(PandasOperand):
+    name = "Combine"
+    is_binary = True
+    has_symmetry_importance = True
+    output_type = "object"
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        temp = left.astype(str) + "_" + right.astype(str)
+        temp[left.isna() | right.isna()] = np.nan
+        return pd.Series(temp, index=left.index)
+
+
+class CombineThenFreq(PandasOperand):
+    name = "CombineThenFreq"
+    is_binary = True
+    has_symmetry_importance = True
+    output_type = "float"
+    is_distribution_dependent = True
+    input_type = "discrete"
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        temp = left.astype(str) + "_" + right.astype(str)
+        temp[left.isna() | right.isna()] = np.nan
+        value_counts = temp.value_counts(normalize=True)
+        self._loc(temp, value_counts)
+
+
+class Sim(PandasOperand):
+    name = "sim"
+    is_binary = True
+    output_type = "float"
+    has_symmetry_importance = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return dot(left, right) / (norm(left) * norm(right))

upgini/autofe/feature.py

@@ -0,0 +1,298 @@
+import hashlib
+from typing import Dict
+import numpy as np
+import pandas as pd
+import itertools
+from upgini.autofe.operand import PandasOperand
+from upgini.autofe.all_operands import (
+    find_op,
+)
+
+
+class FeatureGroup(object):
+    def __init__(self, op, main_column, children):
+        self.op = op
+        self.main_column_node = main_column
+        self.children = children
+        self.data = None
+
+    def get_columns(self, **kwargs):
+        column_list = []
+        seen = set()
+        for child in self.children:
+            columns = child.get_columns(**kwargs)
+            column_list.extend([f for f in columns if f not in seen])
+            seen.update(columns)
+        return column_list
+
+    def get_display_names(self, **kwargs):
+        names = [f.get_display_name(**kwargs) for f in self.children]
+        return names
+
+    def calculate(self, data: pd.DataFrame, is_root=False):
+        main_column = None if self.main_column_node is None else self.main_column_node.get_columns()[0]
+        if isinstance(self.op, PandasOperand):
+            columns = self.get_columns()
+            new_data = self.op.calculate_group(data[columns], main_column=main_column)
+            new_data.rename(columns=dict(zip(columns, self.get_display_names())), inplace=True)
+
+        else:
+            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
+
+        new_data.replace([-np.inf, np.inf], np.nan, inplace=True)
+
+        if is_root:
+            self.data = new_data
+        return new_data
+
+    @staticmethod
+    def make_groups(candidates):
+        grouped_features = []
+        for op_child, features in itertools.groupby(
+            candidates, lambda f: (f.op, f.children[0] if f.op.is_unary or f.op.is_vector else f.children[1])
+        ):
+            op, main_child = op_child
+            feature_list = list(features)
+            if op.is_vectorizable:
+                if op.is_unary:
+                    group = FeatureGroup(op, main_column=None, children=feature_list)
+                else:
+                    group = FeatureGroup(op, main_column=main_child, children=feature_list)
+                grouped_features.append(group)
+            else:
+                grouped_features.extend(feature_list)
+        return grouped_features
+
+    def delete_data(self):
+        self.data = None
+        if self.main_column_node:
+            self.main_column_node.delete_data()
+        for child in self.children:
+            child.delete_data()
+
+
+class Feature(object):
+    def __init__(self, op, children, data=None, display_index=None, cached_display_name=None, alias=None):
+        self.op = op
+        self.children = children
+        self.data = data
+        self.display_index = display_index
+        self.cached_display_name = cached_display_name
+        self.alias = alias
+
+    def set_op_params(self, params: Dict):
+        self.op.set_params(params)
+        return self
+
+    def get_hash(self):
+        return hashlib.sha256("_".join([self.op.name] + [ch.name for ch in self.children]).encode("utf-8")).hexdigest()[
+            :8
+        ]
+
+    def set_alias(self, alias):
+        self.alias = alias
+        return self
+
+    def rename_columns(self, mapping: Dict):
+        for child in self.children:
+            child.rename_columns(mapping)
+        self.cached_display_name = None
+        return self
+
+    def get_column_nodes(self):
+        res = []
+        for child in self.children:
+            res.extend(child.get_column_nodes())
+        return res
+
+    def get_columns(self, **kwargs):
+        column_list = []
+        seen = set()
+        for child in self.children:
+            columns = child.get_columns(**kwargs)
+            column_list.extend([f for f in columns if f not in seen])
+            seen.update(columns)
+        return column_list
+
+    def delete_data(self):
+        self.data = None
+        for child in self.children:
+            child.delete_data()
+
+    def get_display_name(self, cache: bool = True, shorten: bool = False, **kwargs):
+        if self.cached_display_name is not None and cache:
+            return self.cached_display_name
+
+        if self.alias:
+            components = ["f_autofe", self.alias]
+        elif shorten and not self.op.is_unary:
+            components = ["f_autofe", self.op.alias or self.op.name.lower()]
+        else:
+            components = ["f_" + "_f_".join(self.get_columns(**kwargs))] + [
+                "autofe",
+                self.op.alias or self.op.name.lower(),
+            ]
+        components.extend([str(self.display_index)] if self.display_index is not None else [])
+        display_name = "_".join(components)
+
+        if cache:
+            self.cached_display_name = display_name
+        return display_name
+
+    def set_display_index(self, index):
+        self.display_index = index
+        self.cached_display_name = None
+        return self
+
+    def infer_type(self, data):
+        if self.op.output_type:
+            return self.op.output_type
+        else:
+            # either a symmetrical operator or group by
+            return self.children[0].infer_type(data)
+
+    def calculate(self, data, is_root=False):
+        if isinstance(self.op, PandasOperand) and self.op.is_vector:
+            ds = [child.calculate(data) for child in self.children]
+            new_data = self.op.calculate(data=ds)
+
+        elif isinstance(self.op, PandasOperand):
+            d1 = self.children[0].calculate(data)
+            d2 = None if len(self.children) < 2 else self.children[1].calculate(data)
+            new_data = self.op.calculate(data=d1, left=d1, right=d2)
+        else:
+            raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
+
+        if (str(new_data.dtype) == "category") | (str(new_data.dtype) == "object"):
+            pass
+        else:
+            new_data = new_data.replace([-np.inf, np.inf], np.nan)
+
+        if is_root:
+            self.data = new_data
+        return new_data
+
+    @staticmethod
+    def check_xor(left, right):
+        def _get_all_columns(feature):
+            if isinstance(feature, Column):
+                return [feature.name]
+            else:
+                res = []
+                for child in feature.children:
+                    res.extend(_get_all_columns(child))
+                return res
+
+        column1 = set(_get_all_columns(left))
+        column2 = set(_get_all_columns(right))
+        if len(column1 ^ column2) == 0:
+            return False
+        else:
+            return True
+
+    def to_formula(self, **kwargs):
+        if self.op.name in ["+", "-", "*", "/"]:
+            left = self.children[0].to_formula(**kwargs)
+            right = self.children[1].to_formula(**kwargs)
+            return f"({left}{self.op.name}{right})"
+        else:
+            result = [self.op.name, "("]
+            for i in range(len(self.children)):
+                string_i = self.children[i].to_formula(**kwargs)
+                result.append(string_i)
+                result.append(",")
+            result.pop()
+            result.append(")")
+            return "".join(result)
+
+    @staticmethod
+    def from_formula(string):
+        if string[-1] != ")":
+            return Column(string)
+
+        def is_trivial_char(c):
+            return not (c in "()+-*/,")
+
+        def find_prev(string):
+            if string[-1] != ")":
+                return max([(0 if is_trivial_char(c) else i + 1) for i, c in enumerate(string)])
+            level, pos = 0, -1
+            for i in range(len(string) - 1, -1, -1):
+                if string[i] == ")":
+                    level += 1
+                if string[i] == "(":
+                    level -= 1
+                if level == 0:
+                    pos = i
+                    break
+            while (pos > 0) and is_trivial_char(string[pos - 1]):
+                pos -= 1
+            return pos
+
+        p2 = find_prev(string[:-1])
+        if string[p2 - 1] == "(":
+            return Feature(find_op(string[: p2 - 1]), [Feature.from_formula(string[p2:-1])])
+        p1 = find_prev(string[: p2 - 1])
+        if string[0] == "(":
+            return Feature(
+                find_op(string[p2 - 1]),
+                [Feature.from_formula(string[p1 : p2 - 1]), Feature.from_formula(string[p2:-1])],
+            )
+        else:
+            op = find_op(string[: p1 - 1])
+            if op is not None:
+                return Feature(
+                    op,
+                    [Feature.from_formula(string[p1 : p2 - 1]), Feature.from_formula(string[p2:-1])],
+                )
+            else:
+                base_features = [
+                    Feature.from_formula(string[p2:-1]),
+                    Feature.from_formula(string[p1 : p2 - 1]),
+                ]
+                while op is None:
+                    p2 = p1
+                    p1 = find_prev(string[: p1 - 1])
+                    base_features.append(Feature.from_formula(string[p1 : p2 - 1]))
+                    op = find_op(string[: p1 - 1])
+                base_features.reverse()
+                return Feature(op, base_features)
+
+
+class Column(object):
+    def __init__(self, name, data=None, calculate_all=False):
+        self.name = name
+        self.data = data
+        self.calculate_all = calculate_all
+
+    def rename_columns(self, mapping: Dict):
+        self.name = self._unhash(mapping.get(self.name) or self.name)
+        return self
+
+    def _unhash(self, feature_name):
+        last_component_idx = feature_name.rfind("_")
+        if not feature_name.startswith("f_"):
+            return feature_name  # etalon feature
+        elif last_component_idx == 1:
+            return feature_name[2:]  # fully hashed name, cannot unhash
+        else:
+            return feature_name[2:last_component_idx]
+
+    def delete_data(self):
+        self.data = None
+
+    def get_column_nodes(self):
+        return [self]
+
+    def get_columns(self):
+        return [self.name]
+
+    def infer_type(self, data):
+        return data[self.name].dtype
+
+    def calculate(self, data):
+        self.data = data[self.name]
+        return self.data
+
+    def to_formula(self, **kwargs):
+        return str(self.get_columns(**kwargs)[0])

upgini/autofe/groupby.py

@@ -0,0 +1,82 @@
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
+from typing import Optional
+import pandas as pd
+
+
+class GroupByThenAgg(PandasOperand, VectorizableMixin):
+    agg: Optional[str]
+    is_vectorizable = True
+    is_grouping = True
+    is_distribution_dependent = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        temp = left.groupby(right).agg(self.agg)
+        return self._loc(right, temp)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+        temp = d1.groupby(d2).agg(self.agg)
+        return temp.merge(d2, how="right", on=[group_column])[value_columns]
+
+
+class GroupByThenMedian(GroupByThenAgg):
+    name = "GroupByThenMedian"
+    pandas_agg = "median"
+    is_distribution_dependent = True
+
+
+class GroupByThenRank(PandasOperand, VectorizableMixin):
+    name = "GroupByThenRank"
+    is_vectorizable = True
+    is_grouping = True
+    output_type = "float"
+    is_distribution_dependent = True
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        temp = pd.DataFrame(left[~right.isna()].groupby(right).rank(ascending=True, pct=True)).reset_index()
+        return temp.merge(pd.DataFrame(right).reset_index(), how="right", on=["index"])[left.name]
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+        temp = d1[~d2.isna()].groupby(d2).rank(ascending=True, pct=True)[value_columns].reset_index()
+        return temp.merge(d2.reset_index(), how="right", on=["index"])[value_columns]
+
+
+class GroupByThenNUnique(PandasOperand, VectorizableMixin):
+    name = "GroupByThenNUnique"
+    is_vectorizable = True
+    is_grouping = True
+    output_type = "int"
+    is_distribution_dependent = True
+    input_type = "discrete"
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        nunique = left.groupby(right).nunique()
+        return self._loc(right, nunique)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        group_column, value_columns = self.validate_calculation(data.columns, **kwargs)
+        d1 = data[value_columns]
+        d2 = data[group_column]
+        nunique = d1.groupby(d2).nunique()
+        return nunique.merge(d2, how="right", on=[group_column])[value_columns]
+
+
+class GroupByThenFreq(PandasOperand):
+    name = "GroupByThenFreq"
+    is_grouping = True
+    output_type = "float"
+    is_distribution_dependent = True
+    input_type = "discrete"
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        def _f(x):
+            value_counts = x.value_counts(normalize=True)
+            return self._loc(x, value_counts)
+
+        freq = left.groupby(right).apply(_f)
+        return pd.Series(freq, index=right.index)

upgini/autofe/operand.py

@@ -0,0 +1,70 @@
+from pydantic import BaseModel
+from typing import Dict, List, Optional, Tuple
+import abc
+import pandas as pd
+import numpy as np
+
+
+class Operand(BaseModel):
+    name: str
+    alias: Optional[str]
+    is_unary: bool = False
+    has_symmetry_importance: bool = False
+    input_type: Optional[str]
+    output_type: Optional[str]
+    is_categorical: bool = False
+    is_vectorizable: bool = False
+    is_grouping: bool = False
+    is_binary: bool = False
+    is_vector: bool = False
+    is_distribution_dependent: bool = False
+    params: Optional[Dict[str, str]]
+
+    def set_params(self, params: Dict[str, str]):
+        self.params = params
+        return self
+
+    def get_params(self) -> Dict[str, str]:
+        return self.params
+
+
+MAIN_COLUMN = "main_column"
+
+
+class PandasOperand(Operand, abc.ABC):
+    def calculate(self, **kwargs) -> pd.Series:
+        if self.is_unary:
+            return self.calculate_unary(kwargs["data"])
+        elif self.is_binary or self.is_grouping:
+            return self.calculate_binary(kwargs["left"], kwargs["right"])
+        else:
+            return self.calculate_vector(kwargs["data"])
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        pass
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        pass
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        pass
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        if not self.is_vectorizable:
+            raise RuntimeError(f"Cannot apply calculate_group: operator {self.name} is not vectorizable")
+        else:
+            raise RuntimeError(f"Unimplemented calculate_group for operator {self.name}")
+
+    def _loc(self, df_to, df_from):
+        df_from.loc[np.nan] = np.nan
+        return df_to.fillna(np.nan).apply(lambda x: df_from.loc[x])
+
+
+class VectorizableMixin(Operand):
+    def validate_calculation(self, input_columns: List[str], **kwargs) -> Tuple[str, List[str]]:
+        if not kwargs.get(MAIN_COLUMN):
+            raise ValueError(f"Expected argument {MAIN_COLUMN} for grouping operator {self.name} not found")
+        group_column = kwargs[MAIN_COLUMN]
+        value_columns = [col for col in input_columns if col != group_column]
+
+        return group_column, value_columns

upgini/autofe/unary.py

@@ -0,0 +1,105 @@
+from upgini.autofe.operand import PandasOperand
+import numpy as np
+import pandas as pd
+
+
+class Abs(PandasOperand):
+    name = "abs"
+    is_unary = True
+    is_vectorizable = True
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return data.abs()
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return data.abs()
+
+
+class Log(PandasOperand):
+    name = "log"
+    is_unary = True
+    is_vectorizable = True
+    output_type = "float"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return np.log(np.abs(data.replace(0, np.nan)))
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return np.log(data.replace(0, np.nan).abs())
+
+
+class Sqrt(PandasOperand):
+    name = "sqrt"
+    is_unary = True
+    is_vectorizable = True
+    output_type = "float"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return np.sqrt(np.abs(data))
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return np.sqrt(data.abs())
+
+
+class Square(PandasOperand):
+    name = "square"
+    is_unary = True
+    is_vectorizable = True
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return np.square(data)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return np.square(data)
+
+
+class Sigmoid(PandasOperand):
+    name = "sigmoid"
+    is_unary = True
+    is_vectorizable = True
+    output_type = "float"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return 1 / (1 + np.exp(-data))
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return 1 / (1 + np.exp(-data))
+
+
+class Floor(PandasOperand):
+    name = "floor"
+    is_unary = True
+    is_vectorizable = True
+    output_type = "int"
+    input_type = "continuous"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return np.floor(data)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return np.floor(data)
+
+
+class Residual(PandasOperand):
+    name = "residual"
+    is_unary = True
+    is_vectorizable = True
+    input_type = "continuous"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        return data - np.floor(data)
+
+    def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return data - np.floor(data)
+
+
+class Freq(PandasOperand):
+    name = "freq"
+    is_unary = True
+    output_type = "float"
+    is_distribution_dependent = True
+    input_type = "discrete"
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        value_counts = data.value_counts(normalize=True)
+        return self._loc(data, value_counts)

upgini/autofe/vector.py

@@ -0,0 +1,20 @@
+from typing import List
+import pandas as pd
+from upgini.autofe.operand import PandasOperand
+
+
+class Mean(PandasOperand):
+    name = "mean"
+    output_type = "float"
+    is_vector = True
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).mean(axis=1)
+
+
+class Sum(PandasOperand):
+    name = "sum"
+    is_vector = True
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).sum(axis=1)

upgini/features_enricher.py

@@ -5,7 +5,6 @@ import logging
 import numbers
 import os
 import pickle
-import re
 import sys
 import tempfile
 import time
@@ -22,6 +21,7 @@ from scipy.stats import ks_2samp
 from sklearn.base import TransformerMixin
 from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import BaseCrossValidator
+from upgini.autofe.feature import Feature
 
 from upgini.data_source.data_source_publisher import CommercialSchema
 from upgini.dataset import Dataset
@@ -251,6 +251,7 @@ class FeaturesEnricher(TransformerMixin):
                     raise e
 
         self.runtime_parameters = runtime_parameters or RuntimeParameters()
+        self.runtime_parameters.properties["feature_generation_params.hash_index"] = True
         self.date_format = date_format
         self.random_state = random_state
         self.detect_missing_search_keys = detect_missing_search_keys
@@ -904,6 +905,9 @@ class FeaturesEnricher(TransformerMixin):
 
                     model_task_type = self.model_task_type or define_task(y_sorted, self.logger, silent=True)
                     _cv = cv or self.cv
+                    if groups is None and _cv == CVType.group_k_fold:
+                        self.logger.info("Replacing group_k_fold with k_fold as no groups were found")
+                        _cv = CVType.k_fold
                     if not isinstance(_cv, BaseCrossValidator):
                         date_column = self._get_date_column(search_keys)
                         date_series = validated_X[date_column] if date_column is not None else None
@@ -1629,9 +1633,9 @@ class FeaturesEnricher(TransformerMixin):
                 c.originalName or c.name for c in file_metadata.columns if c.name in features_for_transform
             ]
             features_section = (
-                ', "features": {' +
-                ", ".join([f'"{feature}": "test_value"' for feature in original_features_for_transform]) +
-                "}"
+                ', "features": {'
+                + ", ".join([f'"{feature}": "test_value"' for feature in original_features_for_transform])
+                + "}"
             )
         else:
             features_section = ""
@@ -2269,7 +2273,7 @@ class FeaturesEnricher(TransformerMixin):
             msg = bundle.get("multivariate_timeseries_detected")
             self.__override_cv(CVType.blocked_time_series, msg, print_warning=False)
         elif (
-            (self.cv is None or self.cv == CVType.k_fold)
+            self.cv is None
             and model_task_type != ModelTaskType.REGRESSION
             and self._get_group_columns(self.fit_search_keys)
         ):
@@ -2907,13 +2911,20 @@ class FeaturesEnricher(TransformerMixin):
 
             descriptions = []
             for m in autofe_meta:
+                autofe_feature = Feature.from_formula(m.formula)
+                if autofe_feature.op.is_vector:
+                    continue
+
                 description = dict()
 
                 feature_meta = get_feature_by_display_index(m.display_index)
                 if feature_meta is None:
                     self.logger.warning(f"Feature meta for display index {m.display_index} not found")
                     continue
-                description["Sources"] = feature_meta.data_source.replace("AutoFE: features from ", "")
+                description["shap"] = feature_meta.shap_value
+                description["Sources"] = feature_meta.data_source\
+                    .replace("AutoFE: features from ", "")\
+                    .replace("AutoFE: feature from ", "")
                 description["Feature name"] = feature_meta.name
 
                 feature_idx = 1
@@ -2921,11 +2932,7 @@ class FeaturesEnricher(TransformerMixin):
                     description[f"Feature {feature_idx}"] = bc.hashed_name
                     feature_idx += 1
 
-                match = re.match(f"f_autofe_(.+)_{m.display_index}", feature_meta.name)
-                if match is None:
-                    self.logger.warning(f"Failed to infer autofe function from name {feature_meta.name}")
-                else:
-                    description["Function"] = match.group(1)
+                description["Function"] = autofe_feature.op.name
 
                 descriptions.append(description)
 
@@ -2934,6 +2941,8 @@ class FeaturesEnricher(TransformerMixin):
 
             descriptions_df = pd.DataFrame(descriptions)
             descriptions_df.fillna("", inplace=True)
+            descriptions_df.sort_values(by="shap", ascending=False, inplace=True)
+            descriptions_df.drop(columns="shap", inplace=True)
             return descriptions_df
         except Exception:
             self.logger.exception("Failed to generate AutoFE features description")

upgini/metadata.py

@@ -68,7 +68,7 @@ class SearchKey(Enum):
     @staticmethod
     def personal_keys() -> List["SearchKey"]:
         return [SearchKey.EMAIL, SearchKey.HEM, SearchKey.IP, SearchKey.PHONE]
-    
+
     @staticmethod
     def from_meaning_type(meaning_type: FileColumnMeaningType) -> "SearchKey":
         if meaning_type == FileColumnMeaningType.EMAIL:
@@ -247,11 +247,12 @@ class BaseColumnMetadata(BaseModel):
     original_name: str
     hashed_name: str
     ads_definition_id: Optional[str]
+    is_augmented: bool
 
 
 class GeneratedFeatureMetadata(BaseModel):
     formula: str  # on hashed names
-    display_index: int
+    display_index: str
     base_columns: List[BaseColumnMetadata]
 
 

upgini/utils/cv_utils.py

@@ -22,7 +22,7 @@ class CVConfig:
         elif isinstance(cv_type, CVType):
             self.cv_type = cv_type
         else:
-            raise Exception(f"Unexcpected type of cv_type: {type(cv_type)}")
+            raise Exception(f"Unexpected type of cv_type: {type(cv_type)}")
 
         self.shuffle_kfold: Optional[bool] = shuffle_kfold
         self.test_size = 0.2

{upgini-1.1.228.dist-info → upgini-1.1.231.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: upgini
-Version: 1.1.228
+Version: 1.1.231
 Summary: Intelligent data search & enrichment for Machine Learning
 Home-page: https://upgini.com/
 Author: Upgini Developers

{upgini-1.1.228.dist-info → upgini-1.1.231.dist-info}/RECORD

@@ -2,16 +2,24 @@ upgini/__init__.py,sha256=asENHgEVHQBIkV-e_0IhE_ZWqkCG6398U3ZLrNzAH6k,407
 upgini/ads.py,sha256=mre6xn44wcC_fg63iLT_kTh4mViZqR9AKRJZAtpQz8Y,2592
 upgini/dataset.py,sha256=7z9zbVvd1_MiufmoZlCwEHwQ25Q2DX_0g9PFcSMlqMY,49764
 upgini/errors.py,sha256=BqpvfhW2jJW5fa5KXj0alhXatGl-WK4xTl309-QNLp8,959
-upgini/features_enricher.py,sha256=Zk-zVTVdGK5Wlywda7e7W43UYNFLtMIiNg7hzwJlzf0,158363
+upgini/features_enricher.py,sha256=2B9rk_8QNMV7o1khbgZX8A1T6vJqyfki4F4UAYoR0po,158857
 upgini/fingerprint.js,sha256=VygVIQlN1v4NGZfjHqtRogOw8zjTnnMNJg_f7M5iGQU,33442
 upgini/http.py,sha256=HzUSZudCdISJGUqHC1gAT1v_x1n_dIFVDJW4z3Q7DCs,41204
-upgini/metadata.py,sha256=050EPfvwDRv676e-ZFr3OQ1qyR0AFLEFfhqD_iqxBQE,9565
+upgini/metadata.py,sha256=FZ5CQluLLWrfrBVThSIes1SW6wcs7n50aNZwzYnHiF0,9584
 upgini/metrics.py,sha256=YeYHJtEIs8OG-EzidG-nbSYB919pjZ4MMbdcZ_jfV2s,23639
 upgini/search_task.py,sha256=7YxH1zrUHMmePO0VbPBBCJjeoer7jAC0Gltc9EVAOIg,17126
 upgini/spinner.py,sha256=yhakBaydMNS8E8TRAwTdCMdnWrHeWT0cR1M8c9hP6jA,1157
 upgini/version_validator.py,sha256=rDIncP6BEko4J2F2hUcMOtKm_vZbI4ICWcNcw8hrwM4,1400
 upgini/ads_management/__init__.py,sha256=qzyisOToVRP-tquAJD1PblZhNtMrOB8FiyF9JvfkvgE,50
 upgini/ads_management/ads_manager.py,sha256=O6Pcl_y5e_ULfQ-xmGGn_qBP4z7EtV7TP9etjrsLkLE,2647
+upgini/autofe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+upgini/autofe/all_operands.py,sha256=du44N6ISWe3ikb0y9ZzSOHNbLiyEYrJPwoBo0Z6xp2s,1487
+upgini/autofe/binary.py,sha256=f8LQqZi9zyaMUAv-jASMmWNA_vT05ncYCjZq0qx3USs,3972
+upgini/autofe/feature.py,sha256=cElNcLfw9BeBVUkkaFzWWXrnyWNUCXiw0FGqsitorbE,10133
+upgini/autofe/groupby.py,sha256=iXRfOmOc84ooSzRhsh9GmmG7rTafX0-ekXko8s9Qs68,3089
+upgini/autofe/operand.py,sha256=8WqEoSIA5rEWCK1xuC303E4NW5a72GZ5jUMAEj4skII,2291
+upgini/autofe/unary.py,sha256=7TBe7PCt7l_XQEqu_G5g_TC2cW3tppL7uPDcX8xsqz0,2731
+upgini/autofe/vector.py,sha256=Qk7VmdwURNwVw7fIMEspWEo7HTiyUWCYIqu3hcWQQio,507
 upgini/data_source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 upgini/data_source/data_source_publisher.py,sha256=zFu0WMKwPM11gPZHq8dpsBP7s4wmTtBqYoDEakgNxoY,13725
 upgini/mdc/__init__.py,sha256=CuKmWYCqAnmiq1S7wgMzJhSCTsXuoeiZWXSfzw0lyig,1152
@@ -30,7 +38,7 @@ upgini/utils/base_search_key_detector.py,sha256=DGwhXLvc8i5VZWMDr0rncFfV5GEHdsCS
 upgini/utils/blocked_time_series.py,sha256=dMz5ewk3PsoeOrc3lDzInCVPS9u_2XQkV0W6PuMMjPg,3380
 upgini/utils/country_utils.py,sha256=9BXSXoGm3nVoOZE_bRENY-KMkwMUFvAF3Au0zxUNA1o,6436
 upgini/utils/custom_loss_utils.py,sha256=DBslpjWGPt7xTeypt78baR59012SYphbPsO_YLKdilo,3972
-upgini/utils/cv_utils.py,sha256=NTVd7itVWiyrEiM2LPXHGXIzMCncdGR4SRxrOu57Euc,2492
+upgini/utils/cv_utils.py,sha256=6pSSL_Ft_8C6n6aInJeiyeSBD7McjsMxKZpHqSBV0uY,2491
 upgini/utils/datetime_utils.py,sha256=P56e7gcgAogJYfs2Blzk1uypxb9yrFzNaeJpMCRm6Zc,7716
 upgini/utils/display_utils.py,sha256=tiq5sFOfMwkKCjQ7OGdyK_twe0Qdr9F3mzkW1QXSDog,10664
 upgini/utils/email_utils.py,sha256=MhCLUAWqbp81xRyKizauNhVx6t_MFeJQRQ8pFM7EpFo,3480
@@ -45,8 +53,8 @@ upgini/utils/sklearn_ext.py,sha256=IMx2La70AXAggApVpT7sMEjWqVWon5AMZt4MARDsIMQ,4
 upgini/utils/target_utils.py,sha256=cu52icjhDIPpEStHYMXrD2hIl9gzvfnxZr0Ra5osV0k,1616
 upgini/utils/track_info.py,sha256=DVNVZmXUb4f25DSPEuUNEFx49hNEBfmuY9iSW5jkMnI,5708
 upgini/utils/warning_counter.py,sha256=vnmdFo5-7GBkU2bK9h_uC0K0Y_wtfcYstxOdeRfacO0,228
-upgini-1.1.228.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.1.228.dist-info/METADATA,sha256=efz_h9E6ySqjajEYyIDs_C60RESULmo11iPnhd04FL0,48398
-upgini-1.1.228.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-upgini-1.1.228.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
-upgini-1.1.228.dist-info/RECORD,,
+upgini-1.1.231.dist-info/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.1.231.dist-info/METADATA,sha256=itfTK0u6dBS8WjaITVm9TcsXC2mgQ_W0LRBqB_Nialk,48398
+upgini-1.1.231.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+upgini-1.1.231.dist-info/top_level.txt,sha256=OFhTGiDIWKl5gFI49qvWq1R9IKflPaE2PekcbDXDtx4,7
+upgini-1.1.231.dist-info/RECORD,,