upgini 1.2.62a3818.dev4__py3-none-any.whl → 1.2.65a3818.dev5__py3-none-any.whl

upgini/__about__.py

@@ -1 +1 @@
-__version__ = "1.2.62a3818.dev4"
+__version__ = "1.2.65a3818.dev5"

upgini/autofe/timeseries/delta.py

@@ -1,15 +1,34 @@
 import pandas as pd
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from upgini.autofe.operator import ParametrizedOperator
 from upgini.autofe.timeseries.base import TimeSeriesBase
 from upgini.autofe.timeseries.lag import Lag
 
 
-class Delta(TimeSeriesBase, ParametrizedOperator):
+class DeltaBase(TimeSeriesBase):
     delta_size: int
     delta_unit: str = "D"
 
+    def get_params(self) -> Dict[str, Optional[str]]:
+        res = super().get_params()
+        res.update(
+            {
+                "delta_size": self.delta_size,
+                "delta_unit": self.delta_unit,
+            }
+        )
+        return res
+
+    def _calculate_delta(self, x: Union[pd.DataFrame, pd.Series]) -> Union[pd.DataFrame, pd.Series]:
+        return_series = isinstance(x, pd.Series)
+        x = pd.DataFrame(x)
+        lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
+        x.iloc[:, -1] = x.iloc[:, -1] - lag._aggregate(x.iloc[:, -1])
+        return x.iloc[:, -1] if return_series else x
+
+
+class Delta(DeltaBase, ParametrizedOperator):
     def to_formula(self) -> str:
         base_formula = f"delta_{self.delta_size}{self.delta_unit}"
         return self._add_offset_to_formula(base_formula)
@@ -47,28 +66,11 @@ class Delta(TimeSeriesBase, ParametrizedOperator):
 
         return cls(**params)
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "delta_size": self.delta_size,
-                "delta_unit": self.delta_unit,
-                "offset_size": self.offset_size,
-                "offset_unit": self.offset_unit,
-            }
-        )
-        return res
-
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
-        lag0 = Lag(lag_size=0, lag_unit=self.delta_unit)
-        lag = Lag(lag_size=self.delta_size, lag_unit=self.delta_unit)
-        return lag0._aggregate(ts) - lag._aggregate(ts)
+        return ts.apply(self._calculate_delta).iloc[:, [-1]]
 
 
-class Delta2(TimeSeriesBase, ParametrizedOperator):
-    delta_size: int
-    delta_unit: str = "D"
-
+class Delta2(DeltaBase, ParametrizedOperator):
     def to_formula(self) -> str:
         base_formula = f"delta2_{self.delta_size}{self.delta_unit}"
         return self._add_offset_to_formula(base_formula)
@@ -106,20 +108,12 @@ class Delta2(TimeSeriesBase, ParametrizedOperator):
 
         return cls(**params)
 
-    def get_params(self) -> Dict[str, Optional[str]]:
-        res = super().get_params()
-        res.update(
-            {
-                "delta_size": self.delta_size,
-                "delta_unit": self.delta_unit,
-            }
-        )
-        return res
-
     def _aggregate(self, ts: pd.DataFrame) -> pd.DataFrame:
+        return ts.apply(self._calculate_delta2).iloc[:, [-1]]
+
+    def _calculate_delta2(self, x):
         # Calculate first delta
-        delta1 = Delta(delta_size=self.delta_size, delta_unit=self.delta_unit)
-        first_delta = delta1._aggregate(ts)
+        first_delta = self._calculate_delta(x)
 
         # Calculate delta of delta (second derivative)
-        return delta1._aggregate(first_delta)
+        return self._calculate_delta(first_delta)

upgini/features_enricher.py

@@ -3632,7 +3632,7 @@ if response.status_code == 200:
                     )
                 do_sorting = False
             else:
-                columns_to_hash = list(search_keys.keys()) + renamed_id_columns + [target_name]
+                columns_to_hash = list(set(list(search_keys.keys()) + renamed_id_columns + [target_name]))
                 columns_to_hash = sort_columns(
                     df[columns_to_hash],
                     target_name,

upgini/utils/mstats.py

@@ -3,7 +3,6 @@ from collections import namedtuple
 
 import numpy as np
 import numpy.ma as ma
-import scipy
 from joblib import Parallel, delayed
 from numpy import ndarray
 from psutil import cpu_count
@@ -116,6 +115,22 @@ def spearmanr(
     if nan_policy == "omit":
         x = mask_fn(x)
 
+    # - dof: degrees of freedom
+    # - t_stat: t-statistic
+    # - alternative: 'two-sided', 'greater', 'less'
+    def compute_t_pvalue(t_stat, dof, alternative='two-sided'):
+        from scipy.stats import t
+
+        if alternative == "two-sided":
+            prob = 2 * t.sf(abs(t_stat), dof)
+        elif alternative == "greater":
+            prob = t.sf(t_stat, dof)
+        elif alternative == "less":
+            prob = t.cdf(t_stat, dof)
+        else:
+            raise ValueError(f"Unknown alternative: {alternative}")
+        return t_stat, prob
+
     def _spearmanr_2cols(x):
         # Mask the same observations for all variables, and then drop those
         # observations (can't leave them masked, rankdata is weird).
@@ -142,7 +157,7 @@ def spearmanr(
             # errors before taking the square root
             t = rs * np.sqrt((dof / ((rs + 1.0) * (1.0 - rs))).clip(0))
 
-        t, prob = scipy.stats._mstats_basic._ttest_finish(dof, t, alternative)
+        t, prob = compute_t_pvalue(dof, t, alternative)
 
         # For backwards compatibility, return scalars when comparing 2 columns
         if rs.shape == (2, 2):

upgini/utils/sort.py

@@ -28,12 +28,13 @@ def sort_columns(
         logger = logging.getLogger(__name__)
         logger.setLevel(logging.FATAL)
     df = df.copy()  # avoid side effects
+    search_keys = {k: v for k, v in search_keys.items() if v != SearchKey.CUSTOM_KEY}
 
     # Check multiple search keys
     search_key_values = list(search_keys.values())
     has_duplicate_search_keys = len(search_key_values) != len(set(search_key_values))
     if has_duplicate_search_keys:
-        logging.warning(f"WARNING: Found duplicate SearchKey values in search_keys: {search_keys}")
+        logger.warning(f"WARNING: Found duplicate SearchKey values in search_keys: {search_keys}")
 
     sorted_keys = sorted(search_keys.keys(), key=lambda x: str(search_keys.get(x)))
     sorted_keys = [k for k in sorted_keys if k in df.columns and k not in exclude_columns]
@@ -68,8 +69,9 @@ def get_sort_columns_dict(
     if len(string_features) > 0:
         if len(df) > len(df.drop(columns=string_features).drop_duplicates()) or sort_all_columns:
             # factorize string features
+            df = df.copy()
             for c in string_features:
-                df.loc[:, c] = pd.Series(df[c].factorize(sort=True)[0], index=df.index, dtype="int")
+                df = df.assign(**{c: pd.factorize(df[c], sort=True)[0].astype(int)})
             columns_for_sort.extend(string_features)
 
     if len(columns_for_sort) == 0:

{upgini-1.2.62a3818.dev4.dist-info → upgini-1.2.65a3818.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.62a3818.dev4
+Version: 1.2.65a3818.dev5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/
@@ -38,6 +38,7 @@ Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: python-json-logger>=3.3.0
 Requires-Dist: requests>=2.8.0
 Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: scipy>=1.10.0
 Requires-Dist: xhtml2pdf<0.3.0,>=0.2.11
 Description-Content-Type: text/markdown
 

{upgini-1.2.62a3818.dev4.dist-info → upgini-1.2.65a3818.dev5.dist-info}/RECORD

@@ -1,9 +1,9 @@
-upgini/__about__.py,sha256=DRRGQ9hjWuzUUDq0H9hZpymmoGVeS9BXeeOQ2XoHmjc,33
+upgini/__about__.py,sha256=WOd200tkoJAB3TJo-txEZDg-U6YHThQugeQsJTyxs64,33
 upgini/__init__.py,sha256=LXSfTNU0HnlOkE69VCxkgIKDhWP-JFo_eBQ71OxTr5Y,261
 upgini/ads.py,sha256=nvuRxRx5MHDMgPr9SiU-fsqRdFaBv8p4_v1oqiysKpc,2714
 upgini/dataset.py,sha256=OGjpeFHbj3lWiZTOHTpWEoMMDmFY1FlNC44FKktoZvU,34956
 upgini/errors.py,sha256=2b_Wbo0OYhLUbrZqdLIx5jBnAsiD1Mcenh-VjR4HCTw,950
-upgini/features_enricher.py,sha256=cB2I5rNpbztjkYEEW5aJuKj2fCMnfxp40X4Eo63oyuQ,205340
+upgini/features_enricher.py,sha256=nXGBMC42VPAmqQKXbEqZJFIHiGj6F_G2AwhurA8LuQs,205351
 upgini/http.py,sha256=ud0Cp7h0jNeHuuZGpU_1dAAEiabGoJjGxc1X5oeBQr4,43496
 upgini/lazy_import.py,sha256=74gQ8JuA48BGRLxAo7lNHNKY2D2emMxrUxKGdxVGhuY,1012
 upgini/metadata.py,sha256=Jh6YTaS00m_nbaOY_owvlSyn9zgkErkqu8iTr9ZjKI8,12279
@@ -25,7 +25,7 @@ upgini/autofe/vector.py,sha256=l0KdKg-txlZxDSE4hPPfCtfGQofYbl7oaABPr830sPI,667
 upgini/autofe/timeseries/__init__.py,sha256=PGwwDAMwvkXl3el12tXVEmZUgDUvlmIPlXtROm6bD18,738
 upgini/autofe/timeseries/base.py,sha256=T9Ec8LKJbiwTUGGsd_xhM0U0NUJblqmKchkzUI1sK88,3755
 upgini/autofe/timeseries/cross.py,sha256=8ggDhsvwdxHkrWKRPl2fcFt7wamTYhkVzQcOWvIIyvU,4612
-upgini/autofe/timeseries/delta.py,sha256=hXEiFWHdZndz8I7Ef5zhTHLJac9illhZOZITwpL9ppw,3618
+upgini/autofe/timeseries/delta.py,sha256=h0YhmI1TlPJnjwFpN_GQxLb6r59DQuucnG5tQAXSgjU,3520
 upgini/autofe/timeseries/lag.py,sha256=LfQtg484vuqM0mgY4Wft1swHX_Srq7OKKgZswCXoiXI,1882
 upgini/autofe/timeseries/roll.py,sha256=bNFMDszSYTWvB7EyhHbRY1DJqzSURvHlPAcBebt0y0Y,2878
 upgini/autofe/timeseries/trend.py,sha256=eP0q1fBW4MYPrjfy7vr88tTG8qk0xypClaGHaVv1hAs,1962
@@ -60,17 +60,17 @@ upgini/utils/feature_info.py,sha256=0rOXSyCj-sw-8migWP0ge8qrOzGU50dQvH0JUJUrDfQ,
 upgini/utils/features_validator.py,sha256=lEfmk4DoxZ4ooOE1HC0ZXtUb_lFKRFHIrnFULZ4_rL8,3746
 upgini/utils/format.py,sha256=Yv5cvvSs2bOLUzzNu96Pu33VMDNbabio92QepUj41jU,243
 upgini/utils/ip_utils.py,sha256=TSQ_qDsLlVnm09X1HacpabEf_HNqSWpxBF4Sdc2xs08,6580
-upgini/utils/mstats.py,sha256=GjBAUacgfAoVQVFUrMiRYdVkmx93CIThLRNvYLLiV48,5765
+upgini/utils/mstats.py,sha256=dLJQr5Ak5BAoV-pDPpnfvMURZVkZ3_v250QzAsSlqY4,6286
 upgini/utils/phone_utils.py,sha256=IrbztLuOJBiePqqxllfABWfYlfAjYevPhXKipl95wUI,10432
 upgini/utils/postal_code_utils.py,sha256=5M0sUqH2DAr33kARWCTXR-ACyzWbjDq_-0mmEml6ZcU,1716
 upgini/utils/progress_bar.py,sha256=N-Sfdah2Hg8lXP_fV9EfUTXz_PyRt4lo9fAHoUDOoLc,1550
 upgini/utils/sklearn_ext.py,sha256=13jQS_k7v0aUtudXV6nGUEWjttPQzAW9AFYL5wgEz9k,44511
-upgini/utils/sort.py,sha256=w-CoT33W_53ekOROpKI_VRsRmiyWNr2b3IpE5_4MLLA,6395
+upgini/utils/sort.py,sha256=GfWfCIbfK7e7BvSPZZNJD-PEtiN19DnTCEQkeefHHxI,6491
 upgini/utils/target_utils.py,sha256=b1GzO8_gMcwXSZ2v98CY50MJJBzKbWHId_BJGybXfkM,16579
 upgini/utils/track_info.py,sha256=G5Lu1xxakg2_TQjKZk4b5SvrHsATTXNVV3NbvWtT8k8,5663
 upgini/utils/ts_utils.py,sha256=26vhC0pN7vLXK6R09EEkMK3Lwb9IVPH7LRdqFIQ3kPs,1383
 upgini/utils/warning_counter.py,sha256=-GRY8EUggEBKODPSuXAkHn9KnEQwAORC0mmz_tim-PM,254
-upgini-1.2.62a3818.dev4.dist-info/METADATA,sha256=_sL9eQLnB5X1kyhbUiMzXIB5HUgK0KFfmuwgp3Su59c,49094
-upgini-1.2.62a3818.dev4.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-upgini-1.2.62a3818.dev4.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
-upgini-1.2.62a3818.dev4.dist-info/RECORD,,
+upgini-1.2.65a3818.dev5.dist-info/METADATA,sha256=ThM2VKSF3JJw-jQz1wSWOQh81T7FsB4FvnjB5r9bcKk,49123
+upgini-1.2.65a3818.dev5.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+upgini-1.2.65a3818.dev5.dist-info/licenses/LICENSE,sha256=5RRzgvdJUu3BUDfv4bzVU6FqKgwHlIay63pPCSmSgzw,1514
+upgini-1.2.65a3818.dev5.dist-info/RECORD,,