upgini 1.2.62a3818.dev4__tar.gz → 1.2.65a3818.dev5__tar.gz

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.62a3818.dev4
+Version: 1.2.65a3818.dev5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -38,6 +38,7 @@ Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: python-json-logger>=3.3.0
 Requires-Dist: requests>=2.8.0
 Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: scipy>=1.10.0
 Requires-Dist: xhtml2pdf<0.3.0,>=0.2.11
 Description-Content-Type: text/markdown
 

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
     "python-json-logger>=3.3.0",
     "requests>=2.8.0",
     "scikit-learn>=1.3.0",
+    "scipy>=1.10.0",
     "python-bidi==0.4.2",
     "xhtml2pdf>=0.2.11,<0.3.0",
     "jarowinkler>=2.0.0",

upgini-1.2.65a3818.dev5/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.65a3818.dev5"

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/src/upgini/autofe/timeseries/delta.py

@@ -1,15 +1,34 @@
 import pandas as pd
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
 from upgini.autofe.timeseries.lag import Lag
 
 
-class Delta(TimeSeriesBase, ParametrizedOperator):
+class DeltaBase(TimeSeriesBase):
     delta_size: int
     delta_unit: str = "D"
 
+    def get_params(self) -> Dict[str, Optional[str]]:
+        res = super().get_params()
+        res.update(
+            {
+                "delta_size": self.delta_size,
+                "delta_unit": self.delta_unit,
+            }
+        )
+        return res
+
+    def _calculate_delta(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
+        return_series = isinstance(x, pd.Series)
+        x = pd.DataFrame(x)
+        lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
+        x.iloc[:, -1] = x.iloc[:, -1] - lag._aggregate(x.iloc[:, -1])
+        return x.iloc[:, -1] if return_series else x
+
+
+class Delta(DeltaBase, ParametrizedOperator):
     def to_formula(self) -> str:
         base_formula = f"delta_{self.delta_size}{self.delta_unit}"
         return self._add_offset_to_formula(base_formula)
@@ -47,28 +66,11 @@ class Delta(TimeSeriesBase, ParametrizedOperator):
 
         return cls(**params)
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "delta_size": self.delta_size,
-                "delta_unit": self.delta_unit,
-                "offset_size": self.offset_size,
-                "offset_unit": self.offset_unit,
-            }
-        )
-        return res
-
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        lag0 = Lag(lag_size=0, lag_unit=self.delta_unit)
-        lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
-        return lag0._aggregate(ts) - lag._aggregate(ts)
+        return ts.apply(self._calculate_delta).iloc[:, [-1]]
 
 
-class Delta2(TimeSeriesBase, ParametrizedOperator):
-    delta_size: int
-    delta_unit: str = "D"
-
+class Delta2(DeltaBase, ParametrizedOperator):
     def to_formula(self) -> str:
         base_formula = f"delta2_{self.delta_size}{self.delta_unit}"
         return self._add_offset_to_formula(base_formula)
@@ -106,20 +108,12 @@ class Delta2(TimeSeriesBase, ParametrizedOperator):
 
         return cls(**params)
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "delta_size": self.delta_size,
-                "delta_unit": self.delta_unit,
-            }
-        )
-        return res
-
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
+        return ts.apply(self._calculate_delta2).iloc[:, [-1]]
+
+    def _calculate_delta2(self, x):
         # Calculate first delta
-        delta1 = Delta(delta_size=self.delta_size, delta_unit=self.delta_unit)
-        first_delta = delta1._aggregate(ts)
+        first_delta = self._calculate_delta(x)
 
         # Calculate delta of delta (second derivative)
-        return delta1._aggregate(first_delta)
+        return self._calculate_delta(first_delta)

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/src/upgini/features_enricher.py

@@ -3632,7 +3632,7 @@ if response.status_code == 200:
                     )
                 do_sorting = False
             else:
-                columns_to_hash = list(search_keys.keys()) + renamed_id_columns + [target_name]
+                columns_to_hash = list(set(list(search_keys.keys()) + renamed_id_columns + [target_name]))
                 columns_to_hash = sort_columns(
                     df[columns_to_hash],
                     target_name,

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/src/upgini/utils/mstats.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 
 import numpy as np
 import numpy.ma as ma
-import scipy
 from joblib import Parallel, delayed
 from numpy import ndarray
 from psutil import cpu_count
@@ -116,6 +115,22 @@ def spearmanr(
     if nan_policy == "omit":
         x = mask_fn(x)
 
+    # - dof: degrees of freedom
+    # - t_stat: t-statistic
+    # - alternative: 'two-sided', 'greater', 'less'
+    def compute_t_pvalue(t_stat, dof, alternative='two-sided'):
+        from scipy.stats import t
+
+        if alternative == "two-sided":
+            prob = 2 * t.sf(abs(t_stat), dof)
+        elif alternative == "greater":
+            prob = t.sf(t_stat, dof)
+        elif alternative == "less":
+            prob = t.cdf(t_stat, dof)
+        else:
+            raise ValueError(f"Unknown alternative: {alternative}")
+        return t_stat, prob
+
     def _spearmanr_2cols(x):
         # Mask the same observations for all variables, and then drop those
         # observations (can't leave them masked, rankdata is weird).
@@ -142,7 +157,7 @@ def spearmanr(
             # errors before taking the square root
             t = rs * np.sqrt((dof / ((rs + 1.0) * (1.0 - rs))).clip(0))
 
-        t, prob = scipy.stats._mstats_basic._ttest_finish(dof, t, alternative)
+        t, prob = compute_t_pvalue(dof, t, alternative)
 
         # For backwards compatibility, return scalars when comparing 2 columns
         if rs.shape == (2, 2):

{upgini-1.2.62a3818.dev4 → upgini-1.2.65a3818.dev5}/src/upgini/utils/sort.py

@@ -28,12 +28,13 @@ def sort_columns(
         logger = logging.getLogger(__name__)
         logger.setLevel(logging.FATAL)
     df = df.copy()  # avoid side effects
+    search_keys = {k: v for k, v in search_keys.items() if v != SearchKey.CUSTOM_KEY}
 
     # Check multiple search keys
     search_key_values = list(search_keys.values())
     has_duplicate_search_keys = len(search_key_values) != len(set(search_key_values))
     if has_duplicate_search_keys:
-        logging.warning(f"WARNING: Found duplicate SearchKey values in search_keys: {search_keys}")
+        logger.warning(f"WARNING: Found duplicate SearchKey values in search_keys: {search_keys}")
 
     sorted_keys = sorted(search_keys.keys(), key=lambda x: str(search_keys.get(x)))
     sorted_keys = [k for k in sorted_keys if k in df.columns and k not in exclude_columns]
@@ -68,8 +69,9 @@ def get_sort_columns_dict(
     if len(string_features) > 0:
         if len(df) > len(df.drop(columns=string_features).drop_duplicates()) or sort_all_columns:
             # factorize string features
+            df = df.copy()
             for c in string_features:
-                df.loc[:, c] = pd.Series(df[c].factorize(sort=True)[0], index=df.index, dtype="int")
+                df = df.assign(**{c: pd.factorize(df[c], sort=True)[0].astype(int)})
             columns_for_sort.extend(string_features)
 
     if len(columns_for_sort) == 0:

upgini-1.2.62a3818.dev4/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.62a3818.dev4"