upgini 1.2.157.dev1__tar.gz → 1.2.157.dev3__tar.gz

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.157.dev1
+Version: 1.2.157.dev3
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.157.dev3/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.157.dev3"

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/src/upgini/autofe/binary.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from jarowinkler import jarowinkler_similarity
 
-from upgini.autofe.operand import OperandValue
+from upgini.autofe.operand import OperandKind, OperandValue
 from upgini.autofe.operator import PandasOperator, VectorizableMixin
 
 
@@ -158,8 +158,23 @@ class Distance(PandasOperator):
     has_symmetry_importance: bool = True
 
     def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
-        left = left.as_series()
-        right = right.as_series()
+        if left.kind == OperandKind.MATRIX and right.kind == OperandKind.MATRIX:
+            return self._calculate_binary_matrices(left, right)
+        return self._calculate_binary_series(left.as_series(), right.as_series())
+
+    def _calculate_binary_matrices(self, left: OperandValue, right: OperandValue) -> pd.Series:
+        left_mat = left.as_matrix()
+        right_mat = right.as_matrix()
+        dot = np.sum(left_mat * right_mat, axis=1)
+        left_norm = np.sqrt(np.sum(left_mat * left_mat, axis=1))
+        right_norm = np.sqrt(np.sum(right_mat * right_mat, axis=1))
+        denom = left_norm * right_norm
+        with np.errstate(invalid="ignore", divide="ignore"):
+            result = 1 - dot / denom
+        result[denom == 0] = np.nan
+        return pd.Series(result, index=left.index, dtype=np.float64)
+
+    def _calculate_binary_series(self, left: pd.Series, right: pd.Series) -> pd.Series:
         return pd.Series(
             1 - self.__dot(left, right) / (self.__norm(left) * self.__norm(right)), index=left.index
         ).astype(np.float64)

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/src/upgini/autofe/date.py

@@ -341,7 +341,7 @@ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
 
         right_mask = DateListDiffLists._non_empty_list_mask(right)
         diff_lists = self._lists_op().calculate(left=left, right=right)
-        result = self._agg_op().calculate(data=diff_lists)
+        result = self._agg_op().calculate(data=diff_lists).as_series()
         if self.aggregation in _count_aggregations:
             result[~right_mask] = 0.0
         return result.astype(np.float64)

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/src/upgini/autofe/feature.py

@@ -8,7 +8,7 @@ import pandas as pd
 from pandas._typing import DtypeObj
 
 from upgini.autofe.all_operators import find_op
-from upgini.autofe.operand import CalculationContext, OperandValue, wrap_operand
+from upgini.autofe.operand import CalculationContext, OperandValue, operand_to_output
 from upgini.autofe.operator import Operator, PandasOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
 from upgini.autofe.utils import pydantic_dump_method, pydantic_parse_method
@@ -62,9 +62,14 @@ class Column:
     def infer_type(self, data: pd.DataFrame) -> DtypeObj:
         return data[self.name].dtype
 
-    def calculate(self, data: Union[pd.DataFrame, CalculationContext]) -> pd.Series:
+    def calculate(
+        self,
+        data: Union[pd.DataFrame, CalculationContext],
+        preserve_kind: bool = False,
+    ) -> Union[pd.Series, OperandValue]:
         ctx = data if isinstance(data, CalculationContext) else CalculationContext.from_dataframe(data)
-        return self._eval(ctx).as_series()
+        operand = self._eval(ctx)
+        return operand if preserve_kind else operand.as_series()
 
     def _eval(self, ctx: CalculationContext) -> OperandValue:
         self.data = ctx.resolve_operand(self.name)
@@ -245,10 +250,11 @@ class Feature:
         self,
         data: Union[pd.DataFrame, CalculationContext],
         is_root: bool = False,
-    ) -> Union[pd.Series, pd.DataFrame]:
+        preserve_kind: bool = False,
+    ) -> Union[pd.Series, pd.DataFrame, OperandValue]:
         ctx = data if isinstance(data, CalculationContext) else CalculationContext.from_dataframe(data)
         result = self._eval(ctx)
-        new_data = self._operand_to_output(result)
+        new_data = operand_to_output(result, preserve_kind=preserve_kind)
 
         if is_root:
             self.data = new_data
@@ -258,24 +264,14 @@ class Feature:
         if isinstance(self.op, PandasOperator):
             if self.op.is_vector:
                 operands = [child._eval(ctx) for child in self.children]
-                result = self.op.calculate(data=operands)
-                return wrap_operand(result)
+                return self.op.calculate(data=operands)
             left = self.children[0]._eval(ctx)
             if len(self.children) < 2:
-                result = self.op.calculate(data=left)
-                return wrap_operand(result, source=left.source)
+                return self.op.calculate(data=left)
             right = self.children[1]._eval(ctx)
-            result = self.op.calculate(left=left, right=right)
-            return wrap_operand(result, source=left.source)
+            return self.op.calculate(left=left, right=right)
         raise NotImplementedError(f"Unrecognized operator {self.op.name}.")
 
-    @staticmethod
-    def _operand_to_output(operand: OperandValue) -> Union[pd.Series, pd.DataFrame]:
-        new_data = operand.as_series()
-        if (str(new_data.dtype) == "category") | (str(new_data.dtype) == "object"):
-            return new_data
-        return new_data.replace([-np.inf, np.inf], np.nan)
-
     @staticmethod
     def check_xor(left: Union[Column, "Feature"], right: Union[Column, "Feature"]) -> bool:
         def _get_all_columns(feature: Union[Column, "Feature"]) -> List[str]:

upgini-1.2.157.dev3/src/upgini/autofe/operand.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+
+class OperandKind(str, Enum):
+    SERIES = "series"
+    MATRIX = "matrix"
+    ARRAY = "array"
+
+
+@dataclass
+class OperandValue:
+    kind: OperandKind
+    data: Union[pd.Series, np.ndarray]
+    index: pd.Index
+    source: Optional[str] = None
+
+    @classmethod
+    def from_series(cls, series: pd.Series, source: Optional[str] = None) -> "OperandValue":
+        return cls(kind=OperandKind.SERIES, data=series, index=series.index, source=source)
+
+    @classmethod
+    def from_matrix(cls, matrix: np.ndarray, index: pd.Index, source: Optional[str] = None) -> "OperandValue":
+        return cls(kind=OperandKind.MATRIX, data=matrix, index=index, source=source)
+
+    @classmethod
+    def from_array(cls, array: np.ndarray, index: pd.Index, source: Optional[str] = None) -> "OperandValue":
+        return cls(kind=OperandKind.ARRAY, data=array, index=index, source=source)
+
+    def as_series(self) -> pd.Series:
+        if self.kind == OperandKind.SERIES:
+            return self.data  # type: ignore[return-value]
+        if self.kind == OperandKind.ARRAY:
+            return pd.Series(self.data, index=self.index, dtype=np.float64)
+        if self.kind == OperandKind.MATRIX:
+            return pd.Series(list(self.as_matrix()), index=self.index)
+        raise TypeError(f"Cannot convert operand kind {self.kind.value} to series")
+
+    def as_matrix(self) -> np.ndarray:
+        if self.kind != OperandKind.MATRIX:
+            raise TypeError(f"Cannot convert operand kind {self.kind.value} to matrix")
+        return self.data  # type: ignore[return-value]
+
+
+@dataclass
+class CalculationContext:
+    data: Optional[pd.DataFrame] = None
+    matrices: Dict[str, np.ndarray] = field(default_factory=dict)
+    row_index: Optional[pd.Index] = None
+
+    def __post_init__(self) -> None:
+        if self.data is None and not self.matrices:
+            raise ValueError("CalculationContext requires data and/or matrices")
+        if self.matrices:
+            lengths = {name: matrix.shape[0] for name, matrix in self.matrices.items()}
+            if len(set(lengths.values())) > 1:
+                raise ValueError(f"Matrix row counts mismatch: {lengths}")
+
+    @property
+    def index(self) -> pd.Index:
+        if self.data is not None:
+            return self.data.index
+        if self.row_index is not None:
+            return self.row_index
+        if self.matrices:
+            nrows = next(iter(self.matrices.values())).shape[0]
+            return pd.RangeIndex(nrows)
+        raise ValueError("CalculationContext has no index source")
+
+    @classmethod
+    def from_dataframe(cls, data: pd.DataFrame) -> "CalculationContext":
+        return cls(data=data)
+
+    @classmethod
+    def from_matrices(
+        cls,
+        matrices: Dict[str, np.ndarray],
+        index: Optional[pd.Index] = None,
+    ) -> "CalculationContext":
+        return cls(matrices=matrices, row_index=index)
+
+    def resolve_operand(self, name: str) -> OperandValue:
+        matrix = self.matrices.get(name)
+        if matrix is not None:
+            return OperandValue.from_matrix(matrix, index=self.index, source=name)
+        if self.data is not None and name in self.data:
+            return OperandValue.from_series(self.data[name], source=name)
+        raise KeyError(name)
+
+
+def finalize_operand(operand: OperandValue) -> OperandValue:
+    if operand.kind == OperandKind.MATRIX:
+        matrix = np.where(np.isinf(operand.as_matrix()), np.nan, operand.as_matrix())
+        return OperandValue.from_matrix(matrix, index=operand.index, source=operand.source)
+    if operand.kind == OperandKind.ARRAY:
+        array = operand.data  # type: ignore[assignment]
+        array = np.where(np.isinf(array), np.nan, array)
+        return OperandValue.from_array(array, index=operand.index, source=operand.source)
+    new_data = operand.as_series()
+    if (str(new_data.dtype) == "category") | (str(new_data.dtype) == "object"):
+        return OperandValue.from_series(new_data, source=operand.source)
+    return OperandValue.from_series(new_data.replace([-np.inf, np.inf], np.nan), source=operand.source)
+
+
+def operand_to_output(operand: OperandValue, preserve_kind: bool = False) -> Union[pd.Series, OperandValue]:
+    finalized = finalize_operand(operand)
+    if preserve_kind:
+        return finalized
+    return finalized.as_series()
+
+
+def wrap_operand(
+    result: Union[pd.Series, np.ndarray, "OperandValue"],
+    source: Optional[str] = None,
+    index: Optional[pd.Index] = None,
+) -> "OperandValue":
+    if isinstance(result, OperandValue):
+        return result
+    if isinstance(result, np.ndarray):
+        if index is None:
+            raise TypeError("index is required when wrapping ndarray result")
+        if result.ndim == 1:
+            return OperandValue.from_array(result, index=index, source=source)
+        if result.ndim == 2:
+            return OperandValue.from_matrix(result, index=index, source=source)
+        raise TypeError(f"Cannot wrap ndarray with {result.ndim} dimensions")
+    return OperandValue.from_series(result, source=source)

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/src/upgini/autofe/operator.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from pydantic import BaseModel, PrivateAttr
 
-from upgini.autofe.operand import OperandValue
+from upgini.autofe.operand import OperandValue, wrap_operand
 
 
 class OperatorRegistry(type(BaseModel)):
@@ -140,26 +140,29 @@ class PandasOperator(Operator, abc.ABC):
             return value
         return OperandValue.from_series(value)
 
-    def calculate(self, **kwargs) -> Union[pd.Series, OperandValue]:
+    def calculate(self, **kwargs) -> OperandValue:
         if self.is_unary:
-            return self.calculate_unary(self._operand(kwargs["data"]))
-        elif self.is_binary or self.is_grouping:
-            return self.calculate_binary(
-                self._operand(kwargs["left"]),
-                self._operand(kwargs["right"]),
-            )
-        else:
-            return self.calculate_vector([self._operand(value) for value in kwargs["data"]])
-
-    def calculate_unary(self, data: OperandValue) -> Union[pd.Series, OperandValue]:
+            data = self._operand(kwargs["data"])
+            return wrap_operand(self.calculate_unary(data), source=data.source, index=data.index)
+        if self.is_binary or self.is_grouping:
+            left = self._operand(kwargs["left"])
+            right = self._operand(kwargs["right"])
+            return wrap_operand(self.calculate_binary(left, right), source=left.source, index=left.index)
+        operands = [self._operand(value) for value in kwargs["data"]]
+        index = operands[0].index if operands else None
+        return wrap_operand(self.calculate_vector(operands), index=index)
+
+    def calculate_unary(self, data: OperandValue) -> Union[pd.Series, np.ndarray, OperandValue]:
         pass
 
     def calculate_binary(
         self, left: OperandValue, right: OperandValue
-    ) -> Union[pd.Series, OperandValue]:
+    ) -> Union[pd.Series, np.ndarray, OperandValue]:
         pass
 
-    def calculate_vector(self, data: List[OperandValue]) -> Union[pd.Series, OperandValue]:
+    def calculate_vector(
+        self, data: List[OperandValue]
+    ) -> Union[pd.Series, np.ndarray, OperandValue]:
         pass
 
     def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:

{upgini-1.2.157.dev1 → upgini-1.2.157.dev3}/src/upgini/autofe/timeseries/cross.py

@@ -126,7 +126,7 @@ class CrossSeriesInteraction(TimeSeriesBase, ParametrizedOperator):
         right_mask = self._get_mask(data, self.right_descriptor)
         right = self._extract_series(data, right_mask)
 
-        interaction: pd.Series = self.interaction_op.calculate(left=left, right=right)
+        interaction: pd.Series = self.interaction_op.calculate(left=left, right=right).as_series()
         interaction = interaction.reindex(self._get_index(data))
         res = pd.Series(np.nan, index=data[-1].index, name=data[-1].name)
         res.loc[left_mask] = interaction[left_mask].values

upgini-1.2.157.dev1/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.157.dev1"

upgini-1.2.157.dev1/src/upgini/autofe/operand.py

@@ -1,73 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from enum import Enum
-from typing import Optional, Union
-
-import numpy as np
-import pandas as pd
-
-
-class OperandKind(str, Enum):
-    SERIES = "series"
-    MATRIX = "matrix"
-    ARRAY = "array"
-
-
-@dataclass
-class OperandValue:
-    kind: OperandKind
-    data: Union[pd.Series, np.ndarray]
-    index: pd.Index
-    source: Optional[str] = None
-
-    @classmethod
-    def from_series(cls, series: pd.Series, source: Optional[str] = None) -> "OperandValue":
-        return cls(kind=OperandKind.SERIES, data=series, index=series.index, source=source)
-
-    @classmethod
-    def from_matrix(cls, matrix: np.ndarray, index: pd.Index, source: Optional[str] = None) -> "OperandValue":
-        return cls(kind=OperandKind.MATRIX, data=matrix, index=index, source=source)
-
-    @classmethod
-    def from_array(cls, array: np.ndarray, index: pd.Index, source: Optional[str] = None) -> "OperandValue":
-        return cls(kind=OperandKind.ARRAY, data=array, index=index, source=source)
-
-    def as_series(self) -> pd.Series:
-        if self.kind == OperandKind.SERIES:
-            return self.data  # type: ignore[return-value]
-        if self.kind == OperandKind.ARRAY:
-            return pd.Series(self.data, index=self.index, dtype=np.float64)
-        if self.kind == OperandKind.MATRIX:
-            return pd.Series(list(self.as_matrix()), index=self.index)
-        raise TypeError(f"Cannot convert operand kind {self.kind.value} to series")
-
-    def as_matrix(self) -> np.ndarray:
-        if self.kind != OperandKind.MATRIX:
-            raise TypeError(f"Cannot convert operand kind {self.kind.value} to matrix")
-        return self.data  # type: ignore[return-value]
-
-
-@dataclass
-class CalculationContext:
-    data: pd.DataFrame
-
-    @property
-    def index(self) -> pd.Index:
-        return self.data.index
-
-    @classmethod
-    def from_dataframe(cls, data: pd.DataFrame) -> "CalculationContext":
-        return cls(data=data)
-
-    def resolve_operand(self, name: str) -> OperandValue:
-        return OperandValue.from_series(self.data[name], source=name)
-
-
-def wrap_operand(
-    result: Union[pd.Series, "OperandValue"],
-    source: Optional[str] = None,
-) -> "OperandValue":
-    if isinstance(result, OperandValue):
-        return result
-    return OperandValue.from_series(result, source=source)