upgini 1.2.62a3818.dev3__tar.gz → 1.2.63__tar.gz

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.62a3818.dev3
+Version: 1.2.63
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -38,6 +38,7 @@ Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: python-json-logger>=3.3.0
 Requires-Dist: requests>=2.8.0
 Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: scipy>=1.10.0
 Requires-Dist: xhtml2pdf<0.3.0,>=0.2.11
 Description-Content-Type: text/markdown
 

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
     "python-json-logger>=3.3.0",
     "requests>=2.8.0",
     "scikit-learn>=1.3.0",
+    "scipy>=1.10.0",
     "python-bidi==0.4.2",
     "xhtml2pdf>=0.2.11,<0.3.0",
     "jarowinkler>=2.0.0",

upgini-1.2.63/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.63"

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/all_operands.py

@@ -1,4 +1,4 @@
-from upgini.autofe.operator import OperatorRegistry
+from upgini.autofe.operand import OperandRegistry
 from upgini.autofe.unary import *  # noqa
 from upgini.autofe.binary import *  # noqa
 from upgini.autofe.groupby import *  # noqa
@@ -7,4 +7,4 @@ from upgini.autofe.vector import *  # noqa
 
 
 def find_op(name):
-    return OperatorRegistry.get_operand(name)
+    return OperandRegistry.get_operand(name)

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/binary.py

@@ -5,10 +5,10 @@ import numpy as np
 import pandas as pd
 from jarowinkler import jarowinkler_similarity
 
-from upgini.autofe.operator import PandasOperator, VectorizableMixin
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
 
-class Min(PandasOperator):
+class Min(PandasOperand):
     name: str = "min"
     is_binary: bool = True
     is_symmetrical: bool = True
@@ -18,7 +18,7 @@ class Min(PandasOperator):
         return np.minimum(left, right)
 
 
-class Max(PandasOperator):
+class Max(PandasOperand):
     name: str = "max"
     is_binary: bool = True
     is_symmetrical: bool = True
@@ -28,7 +28,7 @@ class Max(PandasOperator):
         return np.maximum(left, right)
 
 
-class Add(PandasOperator, VectorizableMixin):
+class Add(PandasOperand, VectorizableMixin):
     name: str = "+"
     alias: str = "add"
     is_binary: bool = True
@@ -47,7 +47,7 @@ class Add(PandasOperator, VectorizableMixin):
         return d1.add(d2, axis=0)
 
 
-class Subtract(PandasOperator, VectorizableMixin):
+class Subtract(PandasOperand, VectorizableMixin):
     name: str = "-"
     alias: str = "sub"
     is_binary: bool = True
@@ -66,7 +66,7 @@ class Subtract(PandasOperator, VectorizableMixin):
         return d1.sub(d2, axis=0)
 
 
-class Multiply(PandasOperator, VectorizableMixin):
+class Multiply(PandasOperand, VectorizableMixin):
     name: str = "*"
     alias: str = "mul"
     is_binary: bool = True
@@ -85,7 +85,7 @@ class Multiply(PandasOperator, VectorizableMixin):
         return d1.mul(d2, axis=0)
 
 
-class Divide(PandasOperator, VectorizableMixin):
+class Divide(PandasOperand, VectorizableMixin):
     name: str = "/"
     alias: str = "div"
     is_binary: bool = True
@@ -104,7 +104,7 @@ class Divide(PandasOperator, VectorizableMixin):
         return d1.div(d2.replace(0, np.nan), axis=0)
 
 
-class Combine(PandasOperator):
+class Combine(PandasOperand):
     name: str = "Combine"
     is_binary: bool = True
     has_symmetry_importance: bool = True
@@ -116,7 +116,7 @@ class Combine(PandasOperator):
         return pd.Series(temp, index=left.index)
 
 
-class CombineThenFreq(PandasOperator):
+class CombineThenFreq(PandasOperand):
     name: str = "CombineThenFreq"
     is_binary: bool = True
     is_symmetrical: bool = True
@@ -132,7 +132,7 @@ class CombineThenFreq(PandasOperator):
         self._loc(temp, value_counts)
 
 
-class Distance(PandasOperator):
+class Distance(PandasOperand):
     name: str = "dist"
     is_binary: bool = True
     output_type: Optional[str] = "float"
@@ -170,7 +170,7 @@ class Sim(Distance):
         return 1 - super().calculate_binary(left, right)
 
 
-class StringSim(PandasOperator, abc.ABC):
+class StringSim(PandasOperand, abc.ABC):
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         sims = []
         for i in left.index:

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/date.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas.core.arrays.timedeltas import TimedeltaArray
 from pydantic import BaseModel, __version__ as pydantic_version
 
-from upgini.autofe.operator import PandasOperator, ParametrizedOperator
+from upgini.autofe.operand import PandasOperand, ParametrizedOperand
 
 
 def get_pydantic_version():
@@ -43,7 +43,7 @@ class DateDiffMixin(BaseModel):
             raise Exception(f"Unsupported difference unit: {self.diff_unit}")
 
 
-class DateDiff(PandasOperator, DateDiffMixin):
+class DateDiff(PandasOperand, DateDiffMixin):
     name: str = "date_diff"
     alias: Optional[str] = "date_diff_type1"
     is_binary: bool = True
@@ -78,7 +78,7 @@ class DateDiff(PandasOperator, DateDiffMixin):
         return x
 
 
-class DateDiffType2(PandasOperator, DateDiffMixin):
+class DateDiffType2(PandasOperand, DateDiffMixin):
     name: str = "date_diff_type2"
     is_binary: bool = True
     has_symmetry_importance: bool = True
@@ -112,7 +112,7 @@ _ext_aggregations = {"nunique": (lambda x: len(np.unique(x)), 0), "count": (len,
 _count_aggregations = ["nunique", "count"]
 
 
-class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
+class DateListDiff(PandasOperand, DateDiffMixin, ParametrizedOperand):
     is_binary: bool = True
     has_symmetry_importance: bool = True
 
@@ -183,7 +183,7 @@ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
         return method(x) if len(x) > 0 else default
 
 
-class DateListDiffBounded(DateListDiff, ParametrizedOperator):
+class DateListDiffBounded(DateListDiff, ParametrizedOperand):
     lower_bound: Optional[int] = None
     upper_bound: Optional[int] = None
 
@@ -217,7 +217,7 @@ class DateListDiffBounded(DateListDiff, ParametrizedOperator):
         return super()._agg(x)
 
 
-class DatePercentileBase(PandasOperator, abc.ABC):
+class DatePercentileBase(PandasOperand, abc.ABC):
     is_binary: bool = True
     output_type: Optional[str] = "float"
 

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/feature.py

@@ -7,7 +7,7 @@ import pandas as pd
 from pandas._typing import DtypeObj
 
 from upgini.autofe.all_operands import find_op
-from upgini.autofe.operator import Operator, PandasOperator
+from upgini.autofe.operand import Operand, PandasOperand
 
 
 class Column:
@@ -65,7 +65,7 @@ class Column:
 class Feature:
     def __init__(
         self,
-        op: Operator,
+        op: Operand,
         children: List[Union[Column, "Feature"]],
         data: Optional[pd.DataFrame] = None,
         display_index: Optional[str] = None,
@@ -188,7 +188,7 @@ class Feature:
             return self.children[0].infer_type(data)
 
     def calculate(self, data: pd.DataFrame, is_root=False) -> Union[pd.Series, pd.DataFrame]:
-        if isinstance(self.op, PandasOperator):
+        if isinstance(self.op, PandasOperand):
             if self.op.is_vector:
                 ds = [child.calculate(data) for child in self.children]
                 new_data = self.op.calculate(data=ds)
@@ -324,7 +324,7 @@ class Feature:
 
 class FeatureGroup:
     def __init__(
-        self, op: Operator, main_column: Optional[Union[Column, Feature]], children: List[Union[Column, Feature]]
+        self, op: Operand, main_column: Optional[Union[Column, Feature]], children: List[Union[Column, Feature]]
     ):
         self.op = op
         self.main_column_node = main_column
@@ -345,7 +345,7 @@ class FeatureGroup:
         return names
 
     def calculate(self, data: pd.DataFrame, is_root=False) -> pd.DataFrame:
-        if isinstance(self.op, PandasOperator):
+        if isinstance(self.op, PandasOperand):
             main_column = None if self.main_column_node is None else self.main_column_node.get_display_name()
             lower_order_children = []
             if self.main_column_node is not None:
@@ -378,7 +378,7 @@ class FeatureGroup:
     def make_groups(candidates: List[Feature]) -> List[Union[Feature, "FeatureGroup"]]:
         grouped_features = []
 
-        def groupby_func(f: Feature) -> Tuple[Operator, Union[Column, Feature]]:
+        def groupby_func(f: Feature) -> Tuple[Operand, Union[Column, Feature]]:
             return (f.op, f.children[0 if not f.op.is_vectorizable else f.op.group_index])
 
         for op_child, features in itertools.groupby(candidates, groupby_func):

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/groupby.py

@@ -2,13 +2,13 @@ from typing import Optional
 
 import pandas as pd
 
-from upgini.autofe.operator import PandasOperator, ParametrizedOperator, VectorizableMixin
+from upgini.autofe.operand import PandasOperand, ParametrizedOperand, VectorizableMixin
 
 
 class GroupByThenAgg(
-    PandasOperator,
+    PandasOperand,
     VectorizableMixin,
-    ParametrizedOperator,
+    ParametrizedOperand,
 ):
     agg: Optional[str]
     is_vectorizable: bool = True
@@ -39,7 +39,7 @@ class GroupByThenAgg(
         return temp.merge(d2, how="right", on=[group_column])[value_columns]
 
 
-class GroupByThenRank(PandasOperator, VectorizableMixin):
+class GroupByThenRank(PandasOperand, VectorizableMixin):
     name: str = "GroupByThenRank"
     is_vectorizable: bool = True
     is_grouping: bool = True
@@ -58,7 +58,7 @@ class GroupByThenRank(PandasOperator, VectorizableMixin):
         return temp.merge(d2.reset_index(), how="right", on=["index"])[value_columns]
 
 
-class GroupByThenNUnique(PandasOperator, VectorizableMixin):
+class GroupByThenNUnique(PandasOperand, VectorizableMixin):
     name: str = "GroupByThenNUnique"
     is_vectorizable: bool = True
     is_grouping: bool = True
@@ -78,7 +78,7 @@ class GroupByThenNUnique(PandasOperator, VectorizableMixin):
         return nunique.merge(d2, how="right", on=[group_column])[value_columns]
 
 
-class GroupByThenFreq(PandasOperator):
+class GroupByThenFreq(PandasOperand):
     name: str = "GroupByThenFreq"
     is_grouping: bool = True
     output_type: Optional[str] = "float"

upgini-1.2.62a3818.dev3/src/upgini/autofe/operator.py → upgini-1.2.63/src/upgini/autofe/operand.py

@@ -6,7 +6,7 @@ import pandas as pd
 from pydantic import BaseModel
 
 
-class OperatorRegistry(type(BaseModel)):
+class OperandRegistry(type(BaseModel)):
     _registry = {}
     _parametrized_registry = []
 
@@ -20,9 +20,9 @@ class OperatorRegistry(type(BaseModel)):
             base_names.update(b.__name__ for b in base.__bases__)
             base_classes.extend(base.__bases__)
 
-        if "Operator" in base_names:
+        if "Operand" in base_names:
             # Track parametrized operands separately
-            if "ParametrizedOperator" in base_names:
+            if "ParametrizedOperand" in base_names:
                 cls._parametrized_registry.append(new_class)
             else:
                 try:
@@ -33,7 +33,7 @@ class OperatorRegistry(type(BaseModel)):
         return new_class
 
     @classmethod
-    def get_operand(cls, name: str) -> Optional["Operator"]:
+    def get_operand(cls, name: str) -> Optional["Operand"]:
         # First try to resolve as a parametrized operand formula
         for operand_cls in cls._parametrized_registry:
             resolved = operand_cls.from_formula(name)
@@ -46,7 +46,7 @@ class OperatorRegistry(type(BaseModel)):
         return None
 
 
-class Operator(BaseModel, metaclass=OperatorRegistry):
+class Operand(BaseModel, metaclass=OperandRegistry):
     name: Optional[str] = None
     alias: Optional[str] = None
     is_unary: bool = False
@@ -75,7 +75,7 @@ class Operator(BaseModel, metaclass=OperatorRegistry):
         return self.name
 
 
-class ParametrizedOperator(Operator, abc.ABC):
+class ParametrizedOperand(Operand, abc.ABC):
 
     @abc.abstractmethod
     def to_formula(self) -> str:
@@ -83,14 +83,14 @@ class ParametrizedOperator(Operator, abc.ABC):
 
     @classmethod
     @abc.abstractmethod
-    def from_formula(cls, formula: str) -> Optional["Operator"]:
+    def from_formula(cls, formula: str) -> Optional["Operand"]:
         pass
 
 
 MAIN_COLUMN = "main_column"
 
 
-class PandasOperator(Operator, abc.ABC):
+class PandasOperand(Operand, abc.ABC):
     def calculate(self, **kwargs) -> pd.Series:
         if self.is_unary:
             return self.calculate_unary(kwargs["data"])
@@ -131,7 +131,7 @@ class PandasOperator(Operator, abc.ABC):
             return value
 
 
-class VectorizableMixin(Operator):
+class VectorizableMixin(Operand):
     group_index: int = 1
 
     def validate_calculation(self, input_columns: List[str], **kwargs) -> Tuple[str, List[str]]:

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/autofe/unary.py

@@ -2,10 +2,10 @@ from typing import Dict, Optional
 import numpy as np
 import pandas as pd
 
-from upgini.autofe.operator import PandasOperator, VectorizableMixin
+from upgini.autofe.operand import PandasOperand, VectorizableMixin
 
 
-class Abs(PandasOperator, VectorizableMixin):
+class Abs(PandasOperand, VectorizableMixin):
     name: str = "abs"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -20,7 +20,7 @@ class Abs(PandasOperator, VectorizableMixin):
         # return data.abs()
 
 
-class Log(PandasOperator, VectorizableMixin):
+class Log(PandasOperand, VectorizableMixin):
     name: str = "log"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -34,7 +34,7 @@ class Log(PandasOperator, VectorizableMixin):
         return self._round_value(np.log(data.replace(0, np.nan).abs()), 10)
 
 
-class Sqrt(PandasOperator, VectorizableMixin):
+class Sqrt(PandasOperand, VectorizableMixin):
     name: str = "sqrt"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -48,7 +48,7 @@ class Sqrt(PandasOperator, VectorizableMixin):
         return self._round_value(np.sqrt(data.abs()))
 
 
-class Square(PandasOperator, VectorizableMixin):
+class Square(PandasOperand, VectorizableMixin):
     name: str = "square"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -61,7 +61,7 @@ class Square(PandasOperator, VectorizableMixin):
         return np.square(data)
 
 
-class Sigmoid(PandasOperator, VectorizableMixin):
+class Sigmoid(PandasOperand, VectorizableMixin):
     name: str = "sigmoid"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -75,7 +75,7 @@ class Sigmoid(PandasOperator, VectorizableMixin):
         return self._round_value(1 / (1 + np.exp(-data)))
 
 
-class Floor(PandasOperator, VectorizableMixin):
+class Floor(PandasOperand, VectorizableMixin):
     name: str = "floor"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -90,7 +90,7 @@ class Floor(PandasOperator, VectorizableMixin):
         return np.floor(data)
 
 
-class Residual(PandasOperator, VectorizableMixin):
+class Residual(PandasOperand, VectorizableMixin):
     name: str = "residual"
     is_unary: bool = True
     is_vectorizable: bool = True
@@ -104,7 +104,7 @@ class Residual(PandasOperator, VectorizableMixin):
         return data - np.floor(data)
 
 
-class Freq(PandasOperator):
+class Freq(PandasOperand):
     name: str = "freq"
     is_unary: bool = True
     output_type: Optional[str] = "float"
@@ -116,7 +116,7 @@ class Freq(PandasOperator):
         return self._loc(data, value_counts)
 
 
-class Norm(PandasOperator):
+class Norm(PandasOperand):
     name: str = "norm"
     is_unary: bool = True
     output_type: Optional[str] = "float"
@@ -148,7 +148,7 @@ class Norm(PandasOperator):
         return res
 
 
-class Embeddings(PandasOperator):
+class Embeddings(PandasOperand):
     name: str = "emb"
     is_unary: bool = True
     input_type: Optional[str] = "string"

upgini-1.2.62a3818.dev3/src/upgini/autofe/timeseries.py → upgini-1.2.63/src/upgini/autofe/vector.py

@@ -2,15 +2,35 @@ import abc
 from typing import Dict, List, Optional
 
 import pandas as pd
-from upgini.autofe.operator import PandasOperator, ParametrizedOperator
 
 try:
     from pydantic import field_validator as validator  # V2
 except ImportError:
     from pydantic import validator  # V1
 
+from upgini.autofe.operand import PandasOperand, ParametrizedOperand, VectorizableMixin
 
-class TimeSeriesBase(PandasOperator, abc.ABC):
+
+class Mean(PandasOperand, VectorizableMixin):
+    name: str = "mean"
+    output_type: Optional[str] = "float"
+    is_vector: bool = True
+    group_index: int = 0
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).mean(axis=1)
+
+
+class Sum(PandasOperand, VectorizableMixin):
+    name: str = "sum"
+    is_vector: bool = True
+    group_index: int = 0
+
+    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
+        return pd.DataFrame(data).T.fillna(0).sum(axis=1)
+
+
+class TimeSeriesBase(PandasOperand, abc.ABC):
     is_vector: bool = True
     date_unit: Optional[str] = None
     offset_size: int = 0
@@ -35,7 +55,7 @@ class TimeSeriesBase(PandasOperator, abc.ABC):
         ts.set_index(date.name, inplace=True)
         ts = ts[ts.index.notna()].sort_index()
         ts = (
-            ts.groupby([c.name for c in data[1:-1]], group_keys=True)
+            ts.groupby([c.name for c in data[1:-1]])
             .apply(self._shift)[data[-1].name]
             .to_frame()
             .reset_index()
@@ -64,10 +84,10 @@ class TimeSeriesBase(PandasOperator, abc.ABC):
         pass
 
 
-_roll_aggregations = {"norm_mean": lambda x: x[-1] / x.mean(), "last": lambda x: x[-1]}
+_roll_aggregations = {"norm_mean": lambda x: x[-1] / x.mean()}
 
 
-class Roll(TimeSeriesBase, ParametrizedOperator):
+class Roll(TimeSeriesBase, ParametrizedOperand):
     aggregation: str
     window_size: int = 1
     window_unit: str = "D"
@@ -142,7 +162,7 @@ class Roll(TimeSeriesBase, ParametrizedOperator):
         )
 
 
-class Lag(TimeSeriesBase, ParametrizedOperator):
+class Lag(TimeSeriesBase, ParametrizedOperand):
     lag_size: int
     lag_unit: str = "D"
 

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/features_enricher.py

@@ -31,7 +31,7 @@ from sklearn.exceptions import NotFittedError
 from sklearn.model_selection import BaseCrossValidator
 
 from upgini.autofe.feature import Feature
-from upgini.autofe.timeseries import TimeSeriesBase
+from upgini.autofe.vector import TimeSeriesBase
 from upgini.data_source.data_source_publisher import CommercialSchema
 from upgini.dataset import Dataset
 from upgini.errors import HttpError, ValidationError

{upgini-1.2.62a3818.dev3 → upgini-1.2.63}/src/upgini/utils/mstats.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 
 import numpy as np
 import numpy.ma as ma
-import scipy
 from joblib import Parallel, delayed
 from numpy import ndarray
 from psutil import cpu_count
@@ -116,6 +115,22 @@ def spearmanr(
     if nan_policy == "omit":
         x = mask_fn(x)
 
+    # - dof: degrees of freedom
+    # - t_stat: t-statistic
+    # - alternative: 'two-sided', 'greater', 'less'
+    def compute_t_pvalue(t_stat, dof, alternative='two-sided'):
+        from scipy.stats import t
+
+        if alternative == "two-sided":
+            prob = 2 * t.sf(abs(t_stat), dof)
+        elif alternative == "greater":
+            prob = t.sf(t_stat, dof)
+        elif alternative == "less":
+            prob = t.cdf(t_stat, dof)
+        else:
+            raise ValueError(f"Unknown alternative: {alternative}")
+        return t_stat, prob
+
     def _spearmanr_2cols(x):
         # Mask the same observations for all variables, and then drop those
         # observations (can't leave them masked, rankdata is weird).
@@ -142,7 +157,7 @@ def spearmanr(
             # errors before taking the square root
             t = rs * np.sqrt((dof / ((rs + 1.0) * (1.0 - rs))).clip(0))
 
-        t, prob = scipy.stats._mstats_basic._ttest_finish(dof, t, alternative)
+        t, prob = compute_t_pvalue(dof, t, alternative)
 
         # For backwards compatibility, return scalars when comparing 2 columns
         if rs.shape == (2, 2):

upgini-1.2.62a3818.dev3/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.62a3818.dev3"

upgini-1.2.62a3818.dev3/src/upgini/autofe/vector.py

@@ -1,24 +0,0 @@
-from typing import List, Optional
-
-import pandas as pd
-
-from upgini.autofe.operator import PandasOperator, VectorizableMixin
-
-
-class Mean(PandasOperator, VectorizableMixin):
-    name: str = "mean"
-    output_type: Optional[str] = "float"
-    is_vector: bool = True
-    group_index: int = 0
-
-    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
-        return pd.DataFrame(data).T.fillna(0).mean(axis=1)
-
-
-class Sum(PandasOperator, VectorizableMixin):
-    name: str = "sum"
-    is_vector: bool = True
-    group_index: int = 0
-
-    def calculate_vector(self, data: List[pd.Series]) -> pd.Series:
-        return pd.DataFrame(data).T.fillna(0).sum(axis=1)