upgini 1.2.156.dev3__tar.gz → 1.2.156.dev5__tar.gz

{upgini-1.2.156.dev3 → upgini-1.2.156.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: upgini
-Version: 1.2.156.dev3
+Version: 1.2.156.dev5
 Summary: Intelligent data search & enrichment for Machine Learning
 Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
 Project-URL: Homepage, https://upgini.com/

upgini-1.2.156.dev5/src/upgini/__about__.py

@@ -0,0 +1 @@
+__version__ = "1.2.156.dev5"

{upgini-1.2.156.dev3 → upgini-1.2.156.dev5}/src/upgini/autofe/binary.py

@@ -140,23 +140,120 @@ class Distance(PandasOperator):
     is_symmetrical: bool = True
     has_symmetry_importance: bool = True
 
+    def _to_float_vector(self, value) -> Optional[np.ndarray]:
+        if value is None or (isinstance(value, float) and np.isnan(value)):
+            return None
+        return np.atleast_1d(np.asarray(value, dtype=np.float64))
+
+    def _stack_vectors(self, arr: np.ndarray, indices: np.ndarray) -> Optional[np.ndarray]:
+        try:
+            stacked = np.vstack(arr[indices])
+        except (ValueError, TypeError):
+            return None
+        if stacked.ndim != 2 or stacked.shape[1] == 0:
+            return None
+        if stacked.dtype != np.float64:
+            stacked = stacked.astype(np.float64, copy=False)
+        return stacked
+
+    def _cosine_distance_stacked(self, left_mat: np.ndarray, right_mat: np.ndarray) -> np.ndarray:
+        if left_mat.shape != right_mat.shape:
+            raise ValueError("Left and right matrices must have the same shape")
+        dots = np.sum(left_mat * right_mat, axis=1)
+        denom = np.sqrt(np.sum(left_mat * left_mat, axis=1)) * np.sqrt(np.sum(right_mat * right_mat, axis=1))
+        results = np.full(len(left_mat), np.nan, dtype=np.float64)
+        valid = denom > 0
+        results[valid] = 1 - dots[valid] / denom[valid]
+        return results
+
+    def _try_fast_distance(
+        self, left_arr: np.ndarray, right_arr: np.ndarray, row_present: np.ndarray, n: int
+    ) -> Optional[np.ndarray]:
+        valid_idx = np.flatnonzero(row_present)
+        if len(valid_idx) == 0:
+            return None
+
+        left_mat = self._stack_vectors(left_arr, valid_idx)
+        right_mat = self._stack_vectors(right_arr, valid_idx)
+        if left_mat is None or right_mat is None:
+            return None
+
+        stacked_results = self._cosine_distance_stacked(left_mat, right_mat)
+        if row_present.all():
+            return stacked_results
+
+        results = np.full(n, np.nan, dtype=np.float64)
+        results[valid_idx] = stacked_results
+        return results
+
+    def _parse_vectors(self, left_arr: np.ndarray, right_arr: np.ndarray) -> tuple[list, list, np.ndarray]:
+        n = len(left_arr)
+        left_vecs: list[Optional[np.ndarray]] = [None] * n
+        right_vecs: list[Optional[np.ndarray]] = [None] * n
+        row_valid = np.zeros(n, dtype=bool)
+        for i, (left_value, right_value) in enumerate(zip(left_arr, right_arr)):
+            left_vec = self._to_float_vector(left_value)
+            right_vec = self._to_float_vector(right_value)
+            left_vecs[i] = left_vec
+            right_vecs[i] = right_vec
+            if left_vec is not None and right_vec is not None and left_vec.size > 0 and right_vec.size > 0:
+                row_valid[i] = True
+        return left_vecs, right_vecs, row_valid
+
+    def _masked_cosine_distance(
+        self, left_mat: np.ndarray, right_mat: np.ndarray, row_valid: np.ndarray
+    ) -> np.ndarray:
+        results = self._cosine_distance_stacked(left_mat, right_mat)
+        if len(results) == len(row_valid):
+            results[~row_valid] = np.nan
+        return results
+
+    def _calculate_rowwise(
+        self, results: np.ndarray, indices: np.ndarray, left_vecs: list, right_vecs: list
+    ) -> None:
+        for index in indices:
+            left_vec = left_vecs[index]
+            right_vec = right_vecs[index]
+            dot = np.sum(left_vec * right_vec)
+            denom = np.sqrt(np.sum(left_vec * left_vec)) * np.sqrt(np.sum(right_vec * right_vec))
+            if denom != 0:
+                results[index] = 1 - dot / denom
+
+    def _calculate_fallback_distance(
+        self, left_arr: np.ndarray, right_arr: np.ndarray, n: int, index: pd.Index
+    ) -> pd.Series:
+        results = np.full(n, np.nan, dtype=np.float64)
+        left_vecs, right_vecs, row_valid = self._parse_vectors(left_arr, right_arr)
+        if not row_valid.any():
+            return pd.Series(results, index=index, dtype=np.float64)
+
+        valid_idx = np.flatnonzero(row_valid)
+        dim = left_vecs[valid_idx[0]].size
+        if not all(left_vecs[i].size == dim and right_vecs[i].size == dim for i in valid_idx):
+            self._calculate_rowwise(results, valid_idx, left_vecs, right_vecs)
+            return pd.Series(results, index=index, dtype=np.float64)
+
+        left_mat = np.zeros((n, dim), dtype=np.float64)
+        right_mat = np.zeros((n, dim), dtype=np.float64)
+        left_mat[valid_idx] = np.stack([left_vecs[i] for i in valid_idx])
+        right_mat[valid_idx] = np.stack([right_vecs[i] for i in valid_idx])
+
+        return pd.Series(self._masked_cosine_distance(left_mat, right_mat, row_valid), index=index, dtype=np.float64)
+
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        return pd.Series(
-            1 - self.__dot(left, right) / (self.__norm(left) * self.__norm(right)), index=left.index
-        ).astype(np.float64)
-
-    # row-wise dot product, handling None values
-    def __dot(self, left: pd.Series, right: pd.Series) -> pd.Series:
-        left = left.apply(lambda x: np.array(x).astype(np.float64))
-        right = right.apply(lambda x: np.array(x).astype(np.float64))
-        res = (left.dropna() * right.dropna()).apply(np.sum)
-        res = res.reindex(left.index.union(right.index))
-        return res
-
-    # Calculate the norm of a vector, handling None values
-    def __norm(self, vector: pd.Series) -> pd.Series:
-        vector = vector.fillna(np.nan)
-        return np.sqrt(self.__dot(vector, vector))
+        n = len(left)
+        if n == 0:
+            return pd.Series(np.full(n, np.nan), index=left.index, dtype=np.float64)
+
+        row_present = left.notna().to_numpy() & right.notna().to_numpy()
+        left_arr = left.to_numpy()
+        right_arr = right.to_numpy()
+
+        fast_results = self._try_fast_distance(left_arr, right_arr, row_present, n)
+        if fast_results is not None:
+            return pd.Series(fast_results, index=left.index, dtype=np.float64)
+
+        return self._calculate_fallback_distance(left_arr, right_arr, n, left.index)
 
 
 # Left for backward compatibility

{upgini-1.2.156.dev3 → upgini-1.2.156.dev5}/src/upgini/autofe/date.py

@@ -113,18 +113,30 @@ _ext_aggregations = {"nunique": (lambda x: len(np.unique(x)), 0), "count": (len,
 _count_aggregations = ["nunique", "count"]
 
 
-class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
+def _aggregate_diffs(values: np.ndarray, aggregation: str) -> float:
+    values = np.atleast_1d(np.asarray(values, dtype=np.float64))
+    method = getattr(np, aggregation, None)
+    default = np.nan
+    if method is None and aggregation in _ext_aggregations:
+        method, default = _ext_aggregations[aggregation]
+    elif not callable(method):
+        raise ValueError(f"Unsupported aggregation: {aggregation}")
+
+    return method(values) if len(values) > 0 else default
+
+
+class DateListDiffLists(PandasOperator, DateDiffMixin, ParametrizedOperator):
+    name: str = "date_diff_lists"
     is_binary: bool = True
     has_symmetry_importance: bool = True
+    output_type: Optional[str] = "vector"
 
-    aggregation: str
     replace_negative: bool = False
 
     def get_params(self) -> Dict[str, Optional[str]]:
         res = super().get_params()
         res.update(
             {
-                "aggregation": self.aggregation,
                 "diff_unit": self.diff_unit,
                 "left_unit": self.left_unit,
                 "right_unit": self.right_unit,
@@ -134,16 +146,17 @@ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
         return res
 
     def to_formula(self) -> str:
-        return f"date_diff_{self.aggregation}"
+        return f"date_diff_lists_{self.diff_unit}"
 
     @classmethod
-    def from_formula(cls, formula: str) -> Optional["DateListDiff"]:
-        if not formula.startswith("date_diff_") or formula.startswith("date_diff_type"):
-            return None
-        aggregation = formula.replace("date_diff_", "")
-        if "_" in aggregation:
-            return None
-        return cls(aggregation=aggregation)
+    def from_formula(cls, formula: str) -> Optional["DateListDiffLists"]:
+        if formula == "date_diff_lists":
+            return cls()
+        if formula.startswith("date_diff_lists_"):
+            diff_unit = formula.replace("date_diff_lists_", "")
+            if diff_unit in {"D", "Y"}:
+                return cls(diff_unit=diff_unit)
+        return None
 
     @staticmethod
     def _non_empty_list_mask(right: pd.Series) -> pd.Series:
@@ -168,48 +181,165 @@ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
             }
         )
 
+    def _row_diffs(self, left_date, right_dates: pd.arrays.DatetimeArray) -> List[float]:
+        diffs = self._convert_diff_to_unit(left_date - right_dates)
+        if self.replace_negative:
+            diffs = diffs[diffs > 0]
+        return np.atleast_1d(np.asarray(diffs, dtype=np.float64)).tolist()
+
     def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
         if left.isna().all() or right.isna().all():
-            return pd.Series([None] * len(left), index=left.index, dtype=np.float64)
+            return pd.Series([None] * len(left), index=left.index, dtype=object)
 
         left = self._convert_to_date(left, self.left_unit)
         right_mask = self._non_empty_list_mask(right)
         mask = left.notna() & right.notna() & right_mask
 
+        results = pd.Series([None] * len(left), index=left.index, dtype=object)
         if not mask.any():
-            res = pd.Series(np.nan, index=left.index, dtype=np.float64)
-            if self.aggregation in _count_aggregations:
-                res[~right_mask] = 0.0
-            return res
+            return results
 
         masked_left = left[mask]
         converted_lists = self._convert_date_lists(right[mask])
-        results = np.empty(len(masked_left), dtype=np.float64)
+        for idx, left_date in masked_left.items():
+            results.loc[idx] = self._row_diffs(left_date, converted_lists[idx])
+        return results
+
+
+class DateListDiffAggWithinBounds(PandasOperator, ParametrizedOperator):
+    name: str = "date_diff_list_agg"
+    is_unary: bool = True
+    output_type: Optional[str] = "float"
+
+    lower_bound: Optional[int] = None
+    upper_bound: Optional[int] = None
+    aggregation: str
+    normalize: Optional[bool] = None
+
+    def get_params(self) -> Dict[str, Optional[str]]:
+        res = super().get_params()
+        res.update(
+            {
+                "aggregation": self.aggregation,
+                "lower_bound": str(self.lower_bound) if self.lower_bound is not None else None,
+                "upper_bound": str(self.upper_bound) if self.upper_bound is not None else None,
+                "normalize": str(self.normalize) if self.normalize is not None else None,
+            }
+        )
+        return res
+
+    def to_formula(self) -> str:
+        lower_bound = "minusinf" if self.lower_bound is None else self.lower_bound
+        upper_bound = "plusinf" if self.upper_bound is None else self.upper_bound
+        norm = "_norm" if self.normalize else ""
+        return f"date_diff_list_agg_{lower_bound}_{upper_bound}_{self.aggregation}{norm}"
+
+    @classmethod
+    def from_formula(cls, formula: str) -> Optional["DateListDiffAggWithinBounds"]:
+        import re
+
+        normalize = formula.endswith("_norm")
+        formula = formula.replace("_norm", "")
+
+        pattern = r"^date_diff_list_agg_((minusinf|\d+))_((plusinf|\d+))_(\w+)$"
+        match = re.match(pattern, formula)
+        if not match:
+            return None
+
+        lower_bound = None if match.group(1) == "minusinf" else int(match.group(1))
+        upper_bound = None if match.group(3) == "plusinf" else int(match.group(3))
+        aggregation = match.group(5)
+        return cls(
+            lower_bound=lower_bound,
+            upper_bound=upper_bound,
+            aggregation=aggregation,
+            normalize=normalize,
+        )
+
+    def _aggregate_row(self, diffs) -> float:
+        if diffs is None or (isinstance(diffs, float) and np.isnan(diffs)):
+            return np.nan
+
+        diffs = np.atleast_1d(np.asarray(diffs, dtype=np.float64))
+        orig_len = len(diffs)
+        if self.lower_bound is not None or self.upper_bound is not None:
+            lower = self.lower_bound if self.lower_bound is not None else -np.inf
+            upper = self.upper_bound if self.upper_bound is not None else np.inf
+            diffs = diffs[(diffs >= lower) & (diffs < upper)]
+        agg_res = _aggregate_diffs(diffs, self.aggregation)
+        if self.normalize and orig_len > 0:
+            return agg_res / orig_len
+        return agg_res
+
+    def calculate_unary(self, data: pd.Series) -> pd.Series:
+        results = np.empty(len(data), dtype=np.float64)
         results[:] = np.nan
-        for i, (idx, left_date) in enumerate(masked_left.items()):
-            results[i] = self._agg(self._diff(left_date - converted_lists[idx]))
+        for i, diffs in enumerate(data.to_numpy()):
+            results[i] = self._aggregate_row(diffs)
+        return pd.Series(results, index=data.index, dtype=np.float64)
 
-        res_masked = pd.Series(results, index=masked_left.index)
-        res = res_masked.reindex(left.index.union(right.index))
-        if self.aggregation in _count_aggregations:
-            res[~right_mask] = 0.0
-        res = res.astype(np.float64)
 
+class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
+    is_binary: bool = True
+    has_symmetry_importance: bool = True
+
+    aggregation: str
+    replace_negative: bool = False
+
+    def get_params(self) -> Dict[str, Optional[str]]:
+        res = super().get_params()
+        res.update(
+            {
+                "aggregation": self.aggregation,
+                "diff_unit": self.diff_unit,
+                "left_unit": self.left_unit,
+                "right_unit": self.right_unit,
+                "replace_negative": self.replace_negative,
+            }
+        )
         return res
 
-    def _diff(self, x: TimedeltaArray):
-        x = self._convert_diff_to_unit(x)
-        return x[x > 0] if self.replace_negative else x
+    def to_formula(self) -> str:
+        return f"date_diff_{self.aggregation}"
 
-    def _agg(self, x):
-        method = getattr(np, self.aggregation, None)
-        default = np.nan
-        if method is None and self.aggregation in _ext_aggregations:
-            method, default = _ext_aggregations[self.aggregation]
-        elif not callable(method):
-            raise ValueError(f"Unsupported aggregation: {self.aggregation}")
+    @classmethod
+    def from_formula(cls, formula: str) -> Optional["DateListDiff"]:
+        if not formula.startswith("date_diff_") or formula.startswith("date_diff_type"):
+            return None
+        aggregation = formula.replace("date_diff_", "")
+        if "_" in aggregation:
+            return None
+        return cls(aggregation=aggregation)
 
-        return method(x) if len(x) > 0 else default
+    def _lists_op(self) -> DateListDiffLists:
+        return DateListDiffLists(
+            diff_unit=self.diff_unit,
+            left_unit=self.left_unit,
+            right_unit=self.right_unit,
+            replace_negative=self.replace_negative,
+        )
+
+    def _agg_op(self) -> DateListDiffAggWithinBounds:
+        return DateListDiffAggWithinBounds(
+            lower_bound=None,
+            upper_bound=None,
+            aggregation=self.aggregation,
+            normalize=False,
+        )
+
+    def _compose_list_diff(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        if left.isna().all() or right.isna().all():
+            return pd.Series([None] * len(left), index=left.index, dtype=np.float64)
+
+        right_mask = DateListDiffLists._non_empty_list_mask(right)
+        diff_lists = self._lists_op().calculate_binary(left, right)
+        result = self._agg_op().calculate_unary(diff_lists)
+        if self.aggregation in _count_aggregations:
+            result[~right_mask] = 0.0
+        return result.astype(np.float64)
+
+    def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
+        return self._compose_list_diff(left, right)
 
 
 class DateListDiffBounded(DateListDiff, ParametrizedOperator):
@@ -258,16 +388,13 @@ class DateListDiffBounded(DateListDiff, ParametrizedOperator):
             res["normalize"] = str(self.normalize)
         return res
 
-    def _agg(self, x):
-        orig_len = len(x)
-        x = x[
-            (x >= (self.lower_bound if self.lower_bound is not None else -np.inf))
-            & (x < (self.upper_bound if self.upper_bound is not None else np.inf))
-        ]
-        agg_res = super()._agg(x)
-        if self.normalize and orig_len > 0:
-            return agg_res / orig_len
-        return agg_res
+    def _agg_op(self) -> DateListDiffAggWithinBounds:
+        return DateListDiffAggWithinBounds(
+            lower_bound=self.lower_bound,
+            upper_bound=self.upper_bound,
+            aggregation=self.aggregation,
+            normalize=self.normalize or False,
+        )
 
 
 class DatePercentileBase(PandasOperator, abc.ABC):

upgini-1.2.156.dev3/src/upgini/__about__.py

@@ -1 +0,0 @@
-__version__ = "1.2.156.dev3"