upgini 1.2.157a1.dev1__tar.gz → 1.2.157a4180.dev2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/PKG-INFO +1 -1
  2. upgini-1.2.157a4180.dev2/src/upgini/__about__.py +1 -0
  3. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/binary.py +49 -11
  4. upgini-1.2.157a4180.dev2/src/upgini/autofe/date.py +527 -0
  5. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/feature.py +34 -19
  6. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/groupby.py +13 -4
  7. upgini-1.2.157a4180.dev2/src/upgini/autofe/operand.py +133 -0
  8. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/operator.py +26 -10
  9. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/base.py +3 -1
  10. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/cross.py +5 -3
  11. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/unary.py +27 -19
  12. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/utils.py +43 -1
  13. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/vector.py +10 -6
  14. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/search_task.py +6 -37
  15. upgini-1.2.157a1.dev1/src/upgini/__about__.py +0 -1
  16. upgini-1.2.157a1.dev1/src/upgini/autofe/date.py +0 -329
  17. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/.gitignore +0 -0
  18. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/LICENSE +0 -0
  19. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/README.md +0 -0
  20. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/pyproject.toml +0 -0
  21. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/__init__.py +0 -0
  22. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/ads.py +0 -0
  23. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/ads_management/__init__.py +0 -0
  24. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/ads_management/ads_manager.py +0 -0
  25. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/__init__.py +0 -0
  26. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/all_operators.py +0 -0
  27. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/__init__.py +0 -0
  28. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/delta.py +0 -0
  29. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/lag.py +0 -0
  30. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/roll.py +0 -0
  31. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/trend.py +0 -0
  32. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/autofe/timeseries/volatility.py +0 -0
  33. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/data_source/__init__.py +0 -0
  34. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/data_source/data_source_publisher.py +0 -0
  35. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/dataset.py +0 -0
  36. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/errors.py +0 -0
  37. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/features_enricher.py +0 -0
  38. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/http.py +0 -0
  39. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/mdc/__init__.py +0 -0
  40. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/mdc/context.py +0 -0
  41. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/metadata.py +0 -0
  42. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/metrics.py +0 -0
  43. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/normalizer/__init__.py +0 -0
  44. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/normalizer/normalize_utils.py +0 -0
  45. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/resource_bundle/__init__.py +0 -0
  46. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/resource_bundle/exceptions.py +0 -0
  47. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/resource_bundle/strings.properties +0 -0
  48. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/resource_bundle/strings_widget.properties +0 -0
  49. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/sampler/__init__.py +0 -0
  50. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/sampler/base.py +0 -0
  51. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/sampler/random_under_sampler.py +0 -0
  52. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/sampler/utils.py +0 -0
  53. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/spinner.py +0 -0
  54. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/Roboto-Regular.ttf +0 -0
  55. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/__init__.py +0 -0
  56. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/base_search_key_detector.py +0 -0
  57. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/blocked_time_series.py +0 -0
  58. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/config.py +0 -0
  59. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/country_utils.py +0 -0
  60. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/custom_loss_utils.py +0 -0
  61. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/cv_utils.py +0 -0
  62. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/datetime_utils.py +0 -0
  63. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/deduplicate_utils.py +0 -0
  64. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/display_utils.py +0 -0
  65. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/email_utils.py +0 -0
  66. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/fallback_progress_bar.py +0 -0
  67. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/feature_info.py +0 -0
  68. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/features_validator.py +0 -0
  69. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/format.py +0 -0
  70. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/hash_utils.py +0 -0
  71. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/ip_utils.py +0 -0
  72. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/mstats.py +0 -0
  73. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/one_hot_encoder.py +0 -0
  74. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/phone_utils.py +0 -0
  75. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/postal_code_utils.py +0 -0
  76. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/progress_bar.py +0 -0
  77. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/psi.py +0 -0
  78. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/sample_utils.py +0 -0
  79. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/sklearn_ext.py +0 -0
  80. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/sort.py +0 -0
  81. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/target_utils.py +0 -0
  82. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/track_info.py +0 -0
  83. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/ts_utils.py +0 -0
  84. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/utils/warning_counter.py +0 -0
  85. {upgini-1.2.157a1.dev1 → upgini-1.2.157a4180.dev2}/src/upgini/version_validator.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: upgini
3
- Version: 1.2.157a1.dev1
3
+ Version: 1.2.157a4180.dev2
4
4
  Summary: Intelligent data search & enrichment for Machine Learning
5
5
  Project-URL: Bug Reports, https://github.com/upgini/upgini/issues
6
6
  Project-URL: Homepage, https://upgini.com/
@@ -0,0 +1 @@
1
+ __version__ = "1.2.157.a4180.dev2"
@@ -6,6 +6,7 @@ import numpy as np
6
6
  import pandas as pd
7
7
  from jarowinkler import jarowinkler_similarity
8
8
 
9
+ from upgini.autofe.operand import OperandKind, OperandValue
9
10
  from upgini.autofe.operator import PandasOperator, VectorizableMixin
10
11
 
11
12
 
@@ -15,7 +16,9 @@ class Min(PandasOperator):
15
16
  is_symmetrical: bool = True
16
17
  has_symmetry_importance: bool = True
17
18
 
18
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
19
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
20
+ left = left.as_series()
21
+ right = right.as_series()
19
22
  return np.minimum(left, right)
20
23
 
21
24
 
@@ -25,7 +28,9 @@ class Max(PandasOperator):
25
28
  is_symmetrical: bool = True
26
29
  has_symmetry_importance: bool = True
27
30
 
28
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
31
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
32
+ left = left.as_series()
33
+ right = right.as_series()
29
34
  return np.maximum(left, right)
30
35
 
31
36
 
@@ -37,7 +42,9 @@ class Add(PandasOperator, VectorizableMixin):
37
42
  has_symmetry_importance: bool = True
38
43
  is_vectorizable: bool = True
39
44
 
40
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
45
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
46
+ left = left.as_series()
47
+ right = right.as_series()
41
48
  return left + right
42
49
 
43
50
  def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
@@ -56,7 +63,9 @@ class Subtract(PandasOperator, VectorizableMixin):
56
63
  has_symmetry_importance: bool = True
57
64
  is_vectorizable: bool = True
58
65
 
59
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
66
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
67
+ left = left.as_series()
68
+ right = right.as_series()
60
69
  return left - right
61
70
 
62
71
  def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
@@ -75,7 +84,9 @@ class Multiply(PandasOperator, VectorizableMixin):
75
84
  has_symmetry_importance: bool = True
76
85
  is_vectorizable: bool = True
77
86
 
78
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
87
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
88
+ left = left.as_series()
89
+ right = right.as_series()
79
90
  return left * right
80
91
 
81
92
  def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
@@ -94,7 +105,9 @@ class Divide(PandasOperator, VectorizableMixin):
94
105
  is_vectorizable: bool = True
95
106
  output_type: Optional[str] = "float"
96
107
 
97
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
108
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
109
+ left = left.as_series()
110
+ right = right.as_series()
98
111
  return left / right.replace(0, np.nan)
99
112
 
100
113
  def calculate_group(self, data: pd.DataFrame, **kwargs) -> pd.DataFrame:
@@ -111,7 +124,9 @@ class Combine(PandasOperator):
111
124
  has_symmetry_importance: bool = True
112
125
  output_type: Optional[str] = "object"
113
126
 
114
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
127
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
128
+ left = left.as_series()
129
+ right = right.as_series()
115
130
  temp = left.astype(str) + "_" + right.astype(str)
116
131
  temp[left.isna() | right.isna()] = np.nan
117
132
  return pd.Series(temp, index=left.index)
@@ -126,7 +141,9 @@ class CombineThenFreq(PandasOperator):
126
141
  is_distribution_dependent: bool = True
127
142
  input_type: Optional[str] = "discrete"
128
143
 
129
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
144
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
145
+ left = left.as_series()
146
+ right = right.as_series()
130
147
  temp = left.astype(str) + "_" + right.astype(str)
131
148
  temp[left.isna() | right.isna()] = np.nan
132
149
  value_counts = temp.value_counts(normalize=True)
@@ -140,7 +157,26 @@ class Distance(PandasOperator):
140
157
  is_symmetrical: bool = True
141
158
  has_symmetry_importance: bool = True
142
159
 
143
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
160
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
161
+ if left.kind == OperandKind.MATRIX and right.kind == OperandKind.MATRIX:
162
+ return self._calculate_binary_matrices(left, right)
163
+ return self._calculate_binary_series(left.as_series(), right.as_series())
164
+
165
+ def _calculate_binary_matrices(self, left: OperandValue, right: OperandValue) -> pd.Series:
166
+ left_mat = left.as_matrix()
167
+ right_mat = right.as_matrix()
168
+ if left_mat.shape[1] != right_mat.shape[1]:
169
+ return pd.Series(np.full(len(left_mat), np.nan), index=left.index, dtype=np.float64)
170
+ dot = np.sum(left_mat * right_mat, axis=1)
171
+ left_norm = np.sqrt(np.sum(left_mat * left_mat, axis=1))
172
+ right_norm = np.sqrt(np.sum(right_mat * right_mat, axis=1))
173
+ denom = left_norm * right_norm
174
+ with np.errstate(invalid="ignore", divide="ignore"):
175
+ result = 1 - dot / denom
176
+ result[denom == 0] = np.nan
177
+ return pd.Series(result, index=left.index, dtype=np.float64)
178
+
179
+ def _calculate_binary_series(self, left: pd.Series, right: pd.Series) -> pd.Series:
144
180
  return pd.Series(
145
181
  1 - self.__dot(left, right) / (self.__norm(left) * self.__norm(right)), index=left.index
146
182
  ).astype(np.float64)
@@ -167,12 +203,14 @@ class Sim(Distance):
167
203
  is_symmetrical: bool = True
168
204
  has_symmetry_importance: bool = True
169
205
 
170
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
206
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
171
207
  return 1 - super().calculate_binary(left, right)
172
208
 
173
209
 
174
210
  class StringSim(PandasOperator, abc.ABC):
175
- def calculate_binary(self, left: pd.Series, right: pd.Series) -> pd.Series:
211
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
212
+ left = left.as_series()
213
+ right = right.as_series()
176
214
  sims = []
177
215
  for i in left.index:
178
216
  left_i = self._prepare_value(left.get(i))
@@ -0,0 +1,527 @@
1
+ import abc
2
+ import json
3
+ from typing import Dict, List, Optional, Union
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+ from pandas.core.arrays.timedeltas import TimedeltaArray
8
+ from pydantic import BaseModel, __version__ as pydantic_version
9
+
10
+ from upgini.autofe.operand import OperandValue
11
+ from upgini.autofe.operator import PandasOperator, ParametrizedOperator
12
+ from upgini.autofe.utils import bin_index, bin_index_many, bin_index_vectorized, pydantic_validator
13
+
14
+
15
+ def get_pydantic_version():
16
+ major_version = int(pydantic_version.split(".")[0])
17
+ return major_version
18
+
19
+
20
+ class DateDiffMixin(BaseModel):
21
+ diff_unit: str = "D"
22
+ left_unit: Optional[str] = None
23
+ right_unit: Optional[str] = None
24
+
25
+ def _convert_to_date(
26
+ self, x: Union[pd.DataFrame, pd.Series], unit: Optional[str]
27
+ ) -> Union[pd.DataFrame, pd.Series]:
28
+ if isinstance(x, pd.DataFrame):
29
+ return x.apply(lambda y: self._convert_to_date(y, unit), axis=1)
30
+
31
+ return pd.to_datetime(x, unit=unit, errors="coerce")
32
+
33
+ def _convert_diff_to_unit(self, diff: Union[pd.Series, TimedeltaArray]) -> Union[pd.Series, TimedeltaArray]:
34
+ if self.diff_unit == "D":
35
+ if isinstance(diff, pd.Series) and diff.dtype == "object":
36
+ return diff.apply(lambda x: None if isinstance(x, float) and np.isnan(x) else x.days)
37
+ else:
38
+ return diff / np.timedelta64(1, self.diff_unit)
39
+ elif self.diff_unit == "Y":
40
+ if isinstance(diff, TimedeltaArray):
41
+ return (diff / 365 / 24 / 60 / 60 / 10**9).astype(int)
42
+ else:
43
+ return (diff / 365 / 24 / 60 / 60 / 10**9).dt.nanoseconds
44
+ else:
45
+ raise Exception(f"Unsupported difference unit: {self.diff_unit}")
46
+
47
+
48
+ class DateDiff(PandasOperator, DateDiffMixin):
49
+ name: str = "date_diff"
50
+ alias: Optional[str] = "date_diff_type1"
51
+ is_binary: bool = True
52
+ has_symmetry_importance: bool = True
53
+
54
+ replace_negative: bool = False
55
+
56
+ def get_params(self) -> Dict[str, Optional[str]]:
57
+ res = super().get_params()
58
+ res.update(
59
+ {
60
+ "diff_unit": self.diff_unit,
61
+ "left_unit": self.left_unit,
62
+ "right_unit": self.right_unit,
63
+ "replace_negative": self.replace_negative,
64
+ }
65
+ )
66
+ return res
67
+
68
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
69
+ left = left.as_series()
70
+ right = right.as_series()
71
+ if left.isna().all() or right.isna().all():
72
+ return pd.Series([None] * len(left), index=left.index, dtype=np.float64)
73
+
74
+ left = self._convert_to_date(left, self.left_unit)
75
+ right = self._convert_to_date(right, self.right_unit)
76
+ diff = self._convert_diff_to_unit(left.dt.date - right.dt.date)
77
+ return self.__replace_negative(diff)
78
+
79
+ def __replace_negative(self, x: Union[pd.DataFrame, pd.Series]):
80
+ if self.replace_negative:
81
+ x[x < 0] = None
82
+ return x
83
+
84
+
85
+ class DateDiffType2(PandasOperator, DateDiffMixin):
86
+ name: str = "date_diff_type2"
87
+ is_binary: bool = True
88
+ has_symmetry_importance: bool = True
89
+
90
+ def get_params(self) -> Dict[str, Optional[str]]:
91
+ res = super().get_params()
92
+ res.update(
93
+ {
94
+ "diff_unit": self.diff_unit,
95
+ "left_unit": self.left_unit,
96
+ "right_unit": self.right_unit,
97
+ }
98
+ )
99
+ return res
100
+
101
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
102
+ left = left.as_series()
103
+ right = right.as_series()
104
+ left = self._convert_to_date(left, self.left_unit)
105
+ right = self._convert_to_date(right, self.right_unit)
106
+ future = right + (left.dt.year - right.dt.year).apply(
107
+ lambda y: pd.tseries.offsets.DateOffset(years=0 if np.isnan(y) else y)
108
+ )
109
+ future = pd.to_datetime(future)
110
+ before = future[future < left]
111
+ future[future < left] = before + pd.tseries.offsets.DateOffset(years=1)
112
+ diff = (future - left) / np.timedelta64(1, self.diff_unit)
113
+
114
+ return diff
115
+
116
+
117
+ _ext_aggregations = {"nunique": (lambda x: float(np.unique(x).size), 0), "count": (len, 0)}
118
+ _count_aggregations = ["nunique", "count"]
119
+
120
+
121
+ def _aggregate_diffs(values: np.ndarray, aggregation: str) -> float:
122
+ values = np.atleast_1d(np.asarray(values, dtype=np.float64))
123
+ method = getattr(np, aggregation, None)
124
+ default = np.nan
125
+ if method is None and aggregation in _ext_aggregations:
126
+ method, default = _ext_aggregations[aggregation]
127
+ elif not callable(method):
128
+ raise ValueError(f"Unsupported aggregation: {aggregation}")
129
+
130
+ return method(values) if len(values) > 0 else default
131
+
132
+
133
+ class DateListDiffLists(PandasOperator, DateDiffMixin, ParametrizedOperator):
134
+ name: str = "date_diff_lists"
135
+ is_binary: bool = True
136
+ has_symmetry_importance: bool = True
137
+ output_type: Optional[str] = "vector"
138
+
139
+ replace_negative: bool = False
140
+
141
+ def get_params(self) -> Dict[str, Optional[str]]:
142
+ res = super().get_params()
143
+ res.update(
144
+ {
145
+ "diff_unit": self.diff_unit,
146
+ "left_unit": self.left_unit,
147
+ "right_unit": self.right_unit,
148
+ "replace_negative": self.replace_negative,
149
+ }
150
+ )
151
+ return res
152
+
153
+ def to_formula(self) -> str:
154
+ return f"date_diff_lists_{self.diff_unit}"
155
+
156
+ @classmethod
157
+ def from_formula(cls, formula: str) -> Optional["DateListDiffLists"]:
158
+ if formula == "date_diff_lists":
159
+ return cls()
160
+ if formula.startswith("date_diff_lists_"):
161
+ diff_unit = formula.replace("date_diff_lists_", "")
162
+ if diff_unit in {"D", "Y"}:
163
+ return cls(diff_unit=diff_unit)
164
+ return None
165
+
166
+ @staticmethod
167
+ def _non_empty_list_mask(right: pd.Series) -> pd.Series:
168
+ values = right.to_numpy()
169
+ mask = np.empty(len(values), dtype=bool)
170
+ for i, value in enumerate(values):
171
+ if value is None or (isinstance(value, float) and np.isnan(value)):
172
+ mask[i] = False
173
+ elif isinstance(value, (list, tuple, np.ndarray)):
174
+ mask[i] = len(value) > 0
175
+ else:
176
+ mask[i] = False
177
+ return pd.Series(mask, index=right.index)
178
+
179
+ def _convert_date_lists(self, lists: pd.Series) -> pd.Series:
180
+ exploded = lists.explode()
181
+ converted = pd.to_datetime(exploded, unit=self.right_unit, errors="coerce")
182
+ return pd.Series(
183
+ {
184
+ idx: pd.arrays.DatetimeArray(values.to_numpy())
185
+ for idx, values in converted.groupby(converted.index, sort=False)
186
+ }
187
+ )
188
+
189
+ def _row_diffs(self, left_date, right_dates: pd.arrays.DatetimeArray) -> List[float]:
190
+ diffs = self._convert_diff_to_unit(left_date - right_dates)
191
+ if self.replace_negative:
192
+ diffs = diffs[diffs > 0]
193
+ return np.atleast_1d(np.asarray(diffs, dtype=np.float64)).tolist()
194
+
195
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
196
+ left = left.as_series()
197
+ right = right.as_series()
198
+ if left.isna().all() or right.isna().all():
199
+ return pd.Series([None] * len(left), index=left.index, dtype=object)
200
+
201
+ left = self._convert_to_date(left, self.left_unit)
202
+ right_mask = self._non_empty_list_mask(right)
203
+ mask = left.notna() & right.notna() & right_mask
204
+
205
+ results = pd.Series([None] * len(left), index=left.index, dtype=object)
206
+ if not mask.any():
207
+ return results
208
+
209
+ masked_left = left[mask]
210
+ converted_lists = self._convert_date_lists(right[mask])
211
+ for idx, left_date in masked_left.items():
212
+ results.loc[idx] = self._row_diffs(left_date, converted_lists[idx])
213
+ empty_right = right.notna() & ~right_mask
214
+ for idx in right.index[empty_right]:
215
+ results.at[idx] = []
216
+ return results
217
+
218
+
219
+ class DateListDiffAggWithinBounds(PandasOperator, ParametrizedOperator):
220
+ name: str = "date_diff_list_agg"
221
+ is_unary: bool = True
222
+ output_type: Optional[str] = "float"
223
+
224
+ lower_bound: Optional[int] = None
225
+ upper_bound: Optional[int] = None
226
+ aggregation: str
227
+ normalize: Optional[bool] = None
228
+
229
+ def get_params(self) -> Dict[str, Optional[str]]:
230
+ res = super().get_params()
231
+ res.update(
232
+ {
233
+ "aggregation": self.aggregation,
234
+ "lower_bound": str(self.lower_bound) if self.lower_bound is not None else None,
235
+ "upper_bound": str(self.upper_bound) if self.upper_bound is not None else None,
236
+ "normalize": str(self.normalize) if self.normalize is not None else None,
237
+ }
238
+ )
239
+ return res
240
+
241
+ def to_formula(self) -> str:
242
+ lower_bound = "minusinf" if self.lower_bound is None else self.lower_bound
243
+ upper_bound = "plusinf" if self.upper_bound is None else self.upper_bound
244
+ norm = "_norm" if self.normalize else ""
245
+ return f"date_diff_list_agg_{lower_bound}_{upper_bound}_{self.aggregation}{norm}"
246
+
247
+ @classmethod
248
+ def from_formula(cls, formula: str) -> Optional["DateListDiffAggWithinBounds"]:
249
+ import re
250
+
251
+ normalize = formula.endswith("_norm")
252
+ formula = formula.replace("_norm", "")
253
+
254
+ pattern = r"^date_diff_list_agg_((minusinf|\d+))_((plusinf|\d+))_(\w+)$"
255
+ match = re.match(pattern, formula)
256
+ if not match:
257
+ return None
258
+
259
+ lower_bound = None if match.group(1) == "minusinf" else int(match.group(1))
260
+ upper_bound = None if match.group(3) == "plusinf" else int(match.group(3))
261
+ aggregation = match.group(5)
262
+ return cls(
263
+ lower_bound=lower_bound,
264
+ upper_bound=upper_bound,
265
+ aggregation=aggregation,
266
+ normalize=normalize,
267
+ )
268
+
269
+ def _uses_unbounded_aggregation(self) -> bool:
270
+ return (
271
+ self.lower_bound is None
272
+ and self.upper_bound is None
273
+ and not self.normalize
274
+ )
275
+
276
+ def _calculate_unary_nunique(self, data: pd.Series) -> pd.Series:
277
+ results = np.full(len(data), np.nan, dtype=np.float64)
278
+ for i, diffs in enumerate(data.to_numpy()):
279
+ if diffs is None or (isinstance(diffs, float) and np.isnan(diffs)):
280
+ continue
281
+ values = np.asarray(diffs, dtype=np.float64)
282
+ results[i] = len(np.unique(values)) if values.size > 0 else 0.0
283
+ return pd.Series(results, index=data.index, dtype=np.float64)
284
+
285
+ def _aggregate_row(self, diffs) -> float:
286
+ if diffs is None or (isinstance(diffs, float) and np.isnan(diffs)):
287
+ return np.nan
288
+
289
+ diffs = np.atleast_1d(np.asarray(diffs, dtype=np.float64))
290
+ orig_len = len(diffs)
291
+ if self.lower_bound is not None or self.upper_bound is not None:
292
+ lower = self.lower_bound if self.lower_bound is not None else -np.inf
293
+ upper = self.upper_bound if self.upper_bound is not None else np.inf
294
+ diffs = diffs[(diffs >= lower) & (diffs < upper)]
295
+ agg_res = _aggregate_diffs(diffs, self.aggregation)
296
+ if self.normalize and orig_len > 0:
297
+ return agg_res / orig_len
298
+ return agg_res
299
+
300
+ def calculate_unary(self, data: OperandValue) -> pd.Series:
301
+ data = data.as_series()
302
+ if self.aggregation == "nunique" and self._uses_unbounded_aggregation():
303
+ return self._calculate_unary_nunique(data)
304
+
305
+ results = np.empty(len(data), dtype=np.float64)
306
+ results[:] = np.nan
307
+ for i, diffs in enumerate(data.to_numpy()):
308
+ results[i] = self._aggregate_row(diffs)
309
+ return pd.Series(results, index=data.index, dtype=np.float64)
310
+
311
+
312
+ class DateListDiff(PandasOperator, DateDiffMixin, ParametrizedOperator):
313
+ is_binary: bool = True
314
+ has_symmetry_importance: bool = True
315
+
316
+ aggregation: str
317
+ replace_negative: bool = False
318
+
319
+ def get_params(self) -> Dict[str, Optional[str]]:
320
+ res = super().get_params()
321
+ res.update(
322
+ {
323
+ "aggregation": self.aggregation,
324
+ "diff_unit": self.diff_unit,
325
+ "left_unit": self.left_unit,
326
+ "right_unit": self.right_unit,
327
+ "replace_negative": self.replace_negative,
328
+ }
329
+ )
330
+ return res
331
+
332
+ def to_formula(self) -> str:
333
+ return f"date_diff_{self.aggregation}"
334
+
335
+ @classmethod
336
+ def from_formula(cls, formula: str) -> Optional["DateListDiff"]:
337
+ if not formula.startswith("date_diff_") or formula.startswith("date_diff_type"):
338
+ return None
339
+ aggregation = formula.replace("date_diff_", "")
340
+ if "_" in aggregation:
341
+ return None
342
+ return cls(aggregation=aggregation)
343
+
344
+ def _lists_op(self) -> DateListDiffLists:
345
+ return DateListDiffLists(
346
+ diff_unit=self.diff_unit,
347
+ left_unit=self.left_unit,
348
+ right_unit=self.right_unit,
349
+ replace_negative=self.replace_negative,
350
+ )
351
+
352
+ def _agg_op(self) -> DateListDiffAggWithinBounds:
353
+ return DateListDiffAggWithinBounds(
354
+ lower_bound=None,
355
+ upper_bound=None,
356
+ aggregation=self.aggregation,
357
+ normalize=False,
358
+ )
359
+
360
+ def _compose_list_diff(self, left: pd.Series, right: pd.Series) -> pd.Series:
361
+ if left.isna().all() or right.isna().all():
362
+ return pd.Series([None] * len(left), index=left.index, dtype=np.float64)
363
+
364
+ right_mask = DateListDiffLists._non_empty_list_mask(right)
365
+ diff_lists = self._lists_op().calculate(left=left, right=right)
366
+ result = self._agg_op().calculate(data=diff_lists).as_series()
367
+ if self.aggregation in _count_aggregations:
368
+ result[~right_mask] = 0.0
369
+ return result.astype(np.float64)
370
+
371
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
372
+ left = left.as_series()
373
+ right = right.as_series()
374
+ return self._compose_list_diff(left, right)
375
+
376
+
377
+ class DateListDiffBounded(DateListDiff, ParametrizedOperator):
378
+ lower_bound: Optional[int] = None
379
+ upper_bound: Optional[int] = None
380
+ normalize: Optional[bool] = None
381
+
382
+ def to_formula(self) -> str:
383
+ lower_bound = "minusinf" if self.lower_bound is None else self.lower_bound
384
+ upper_bound = "plusinf" if self.upper_bound is None else self.upper_bound
385
+ norm = "_norm" if self.normalize else ""
386
+ return f"date_diff_{self.diff_unit}_{lower_bound}_{upper_bound}_{self.aggregation}{norm}"
387
+
388
+ @classmethod
389
+ def from_formula(cls, formula: str) -> Optional["DateListDiffBounded"]:
390
+ import re
391
+
392
+ normalize = formula.endswith("_norm")
393
+ formula = formula.replace("_norm", "")
394
+
395
+ pattern = r"^date_diff_([^_]+)_((minusinf|\d+))_((plusinf|\d+))_(\w+)$"
396
+ match = re.match(pattern, formula)
397
+
398
+ if not match:
399
+ return None
400
+
401
+ diff_unit = match.group(1)
402
+ lower_bound = None if match.group(2) == "minusinf" else int(match.group(2))
403
+ upper_bound = None if match.group(4) == "plusinf" else int(match.group(4))
404
+ aggregation = match.group(6)
405
+ return cls(
406
+ diff_unit=diff_unit,
407
+ lower_bound=lower_bound,
408
+ upper_bound=upper_bound,
409
+ aggregation=aggregation,
410
+ normalize=normalize,
411
+ )
412
+
413
+ def get_params(self) -> Dict[str, Optional[str]]:
414
+ res = super().get_params()
415
+ if self.lower_bound is not None:
416
+ res["lower_bound"] = str(self.lower_bound)
417
+ if self.upper_bound is not None:
418
+ res["upper_bound"] = str(self.upper_bound)
419
+ if self.normalize is not None:
420
+ res["normalize"] = str(self.normalize)
421
+ return res
422
+
423
+ def _agg_op(self) -> DateListDiffAggWithinBounds:
424
+ return DateListDiffAggWithinBounds(
425
+ lower_bound=self.lower_bound,
426
+ upper_bound=self.upper_bound,
427
+ aggregation=self.aggregation,
428
+ normalize=self.normalize or False,
429
+ )
430
+
431
+
432
+ class DatePercentileBase(PandasOperator, abc.ABC):
433
+ is_binary: bool = True
434
+ is_categorical: bool = True
435
+ output_type: Optional[str] = "category"
436
+
437
+ date_unit: Optional[str] = None
438
+
439
+ def calculate_binary(self, left: OperandValue, right: OperandValue) -> pd.Series:
440
+ left = left.as_series()
441
+ right = right.as_series()
442
+ # Assuming that left is a date column, right is a feature column
443
+ left = pd.to_datetime(left, unit=self.date_unit)
444
+
445
+ bounds = self._get_bounds(left)
446
+ values = pd.to_numeric(right, errors="coerce").to_numpy(dtype=np.float64, copy=False)
447
+ bounds_list = bounds.tolist()
448
+ result = np.full(len(values), np.nan)
449
+
450
+ if not bounds_list:
451
+ return pd.Series(result, index=right.index).astype(pd.Int64Dtype()).astype("category")
452
+
453
+ bounds_lengths = {len(b) for b in bounds_list if isinstance(b, (list, np.ndarray))}
454
+ if len(bounds_lengths) == 1 and all(isinstance(b, (list, np.ndarray)) for b in bounds_list):
455
+ bounds_2d = np.asarray(bounds_list, dtype=np.float64)
456
+ if bounds_2d.ndim == 1:
457
+ result = bin_index_vectorized(values, bounds_2d)
458
+ else:
459
+ result = bin_index_many(values, bounds_2d)
460
+ else:
461
+ for i, row_bounds in enumerate(bounds_list):
462
+ if isinstance(row_bounds, (list, np.ndarray)) and len(row_bounds) > 0:
463
+ result[i] = bin_index(values[i], row_bounds)
464
+
465
+ return pd.Series(result, index=right.index).astype(pd.Int64Dtype()).astype("category")
466
+
467
+ @abc.abstractmethod
468
+ def _get_bounds(self, date_col: pd.Series) -> pd.Series:
469
+ pass
470
+
471
+ def _perc(self, f, bounds):
472
+ return bin_index(f, bounds)
473
+
474
+ def get_params(self) -> Dict[str, Optional[str]]:
475
+ res = super().get_params()
476
+ res.update(
477
+ {
478
+ "date_unit": self.date_unit,
479
+ }
480
+ )
481
+ return res
482
+
483
+
484
+ class DatePercentile(DatePercentileBase):
485
+ name: str = "date_per"
486
+ alias: Optional[str] = "date_per_method1"
487
+
488
+ zero_month: Optional[int] = None
489
+ zero_year: Optional[int] = None
490
+ zero_bounds: Optional[List[float]] = None
491
+ step: int = 30
492
+
493
+ def get_params(self) -> Dict[str, Optional[str]]:
494
+ res = super().get_params()
495
+ res.update(
496
+ {
497
+ "zero_month": self.zero_month,
498
+ "zero_year": self.zero_year,
499
+ "zero_bounds": json.dumps(self.zero_bounds),
500
+ "step": self.step,
501
+ }
502
+ )
503
+ return res
504
+
505
+ @pydantic_validator("zero_bounds", mode="before")
506
+ def parse_zero_bounds(cls, value):
507
+ if isinstance(value, str):
508
+ return json.loads(value)
509
+ return value
510
+
511
+ def _get_bounds(self, date_col: pd.Series) -> pd.Series:
512
+ zero_bounds = self.zero_bounds if self.zero_bounds is not None else []
513
+ if not zero_bounds:
514
+ return pd.Series([[] for _ in range(len(date_col))], index=date_col.index)
515
+
516
+ month_diffs = (
517
+ 12 * (date_col.dt.year - (self.zero_year or 0)) + (date_col.dt.month - (self.zero_month or 0))
518
+ ).to_numpy()
519
+ bounds_2d = np.asarray(zero_bounds, dtype=np.float64) + month_diffs[:, None] * self.step
520
+ return pd.Series(list(bounds_2d), index=date_col.index)
521
+
522
+
523
+ class DatePercentileMethod2(DatePercentileBase):
524
+ name: str = "date_per_method2"
525
+
526
+ def _get_bounds(self, date_col: pd.Series) -> pd.Series:
527
+ pass