rdt 1.12.4.dev0__tar.gz → 1.13.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rdt-1.12.4.dev0/rdt.egg-info → rdt-1.13.0.dev0}/PKG-INFO +1 -1
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/pyproject.toml +3 -2
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/__init__.py +1 -1
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/__init__.py +3 -1
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/base.py +1 -1
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/boolean.py +6 -6
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/categorical.py +14 -14
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/datetime.py +6 -6
- rdt-1.13.0.dev0/rdt/performance/datasets/id.py +57 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/numerical.py +10 -10
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/pii.py +2 -2
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/text.py +4 -4
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/__init__.py +12 -10
- rdt-1.12.4.dev0/rdt/transformers/text.py → rdt-1.13.0.dev0/rdt/transformers/id.py +5 -3
- rdt-1.13.0.dev0/rdt/transformers/text.py +12 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/utils.py +35 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0/rdt.egg-info}/PKG-INFO +1 -1
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt.egg-info/SOURCES.txt +2 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/LICENSE +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/README.md +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/errors.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/hyper_transformer.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/__init__.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/datasets/utils.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/performance.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/performance/profiling.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/_validators.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/base.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/boolean.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/categorical.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/datetime.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/null.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/numerical.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/pii/__init__.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/pii/anonymization.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/pii/anonymizer.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt/transformers/pii/utils.py +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt.egg-info/dependency_links.txt +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt.egg-info/entry_points.txt +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt.egg-info/requires.txt +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/rdt.egg-info/top_level.txt +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/setup.cfg +0 -0
- {rdt-1.12.4.dev0 → rdt-1.13.0.dev0}/tests/test_tasks.py +0 -0
|
@@ -137,7 +137,7 @@ collect_ignore = ['pyproject.toml']
|
|
|
137
137
|
exclude_lines = ['NotImplementedError()']
|
|
138
138
|
|
|
139
139
|
[tool.bumpversion]
|
|
140
|
-
current_version = "1.
|
|
140
|
+
current_version = "1.13.0.dev0"
|
|
141
141
|
parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
|
|
142
142
|
serialize = [
|
|
143
143
|
'{major}.{minor}.{patch}.{release}{candidate}',
|
|
@@ -183,6 +183,7 @@ exclude = [
|
|
|
183
183
|
'.tox',
|
|
184
184
|
'.git',
|
|
185
185
|
'__pycache__',
|
|
186
|
+
'*.ipynb',
|
|
186
187
|
'.ipynb_checkpoints',
|
|
187
188
|
'tasks.py',
|
|
188
189
|
'tests/contributing.py'
|
|
@@ -235,4 +236,4 @@ convention = "google"
|
|
|
235
236
|
|
|
236
237
|
[tool.ruff.lint.pycodestyle]
|
|
237
238
|
max-doc-length = 100
|
|
238
|
-
max-line-length = 100
|
|
239
|
+
max-line-length = 100
|
|
@@ -6,6 +6,7 @@ from rdt.performance.datasets import (
|
|
|
6
6
|
boolean,
|
|
7
7
|
categorical,
|
|
8
8
|
datetime,
|
|
9
|
+
id,
|
|
9
10
|
numerical,
|
|
10
11
|
pii,
|
|
11
12
|
text,
|
|
@@ -16,9 +17,10 @@ __all__ = [
|
|
|
16
17
|
'boolean',
|
|
17
18
|
'categorical',
|
|
18
19
|
'datetime',
|
|
20
|
+
'id',
|
|
19
21
|
'numerical',
|
|
20
|
-
'text',
|
|
21
22
|
'pii',
|
|
23
|
+
'text',
|
|
22
24
|
'BaseDatasetGenerator',
|
|
23
25
|
]
|
|
24
26
|
|
|
@@ -34,7 +34,7 @@ class RandomBooleanGenerator(BooleanGenerator):
|
|
|
34
34
|
|
|
35
35
|
@staticmethod
|
|
36
36
|
def get_performance_thresholds():
|
|
37
|
-
"""Return the expected
|
|
37
|
+
"""Return the expected thresholds."""
|
|
38
38
|
return {
|
|
39
39
|
'fit': {'time': 2e-5, 'memory': 400.0},
|
|
40
40
|
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
@@ -63,7 +63,7 @@ class RandomBooleanNaNsGenerator(BooleanGenerator):
|
|
|
63
63
|
|
|
64
64
|
@staticmethod
|
|
65
65
|
def get_performance_thresholds():
|
|
66
|
-
"""Return the expected
|
|
66
|
+
"""Return the expected thresholds."""
|
|
67
67
|
return {
|
|
68
68
|
'fit': {'time': 2e-5, 'memory': 400.0},
|
|
69
69
|
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
@@ -90,7 +90,7 @@ class RandomSkewedBooleanGenerator(BooleanGenerator):
|
|
|
90
90
|
|
|
91
91
|
@staticmethod
|
|
92
92
|
def get_performance_thresholds():
|
|
93
|
-
"""Return the expected
|
|
93
|
+
"""Return the expected thresholds."""
|
|
94
94
|
return {
|
|
95
95
|
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
96
96
|
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
@@ -119,7 +119,7 @@ class RandomSkewedBooleanNaNsGenerator(BooleanGenerator):
|
|
|
119
119
|
|
|
120
120
|
@staticmethod
|
|
121
121
|
def get_performance_thresholds():
|
|
122
|
-
"""Return the expected
|
|
122
|
+
"""Return the expected thresholds."""
|
|
123
123
|
return {
|
|
124
124
|
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
125
125
|
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
@@ -141,7 +141,7 @@ class ConstantBooleanGenerator(BooleanGenerator):
|
|
|
141
141
|
|
|
142
142
|
@staticmethod
|
|
143
143
|
def get_performance_thresholds():
|
|
144
|
-
"""Return the expected
|
|
144
|
+
"""Return the expected thresholds."""
|
|
145
145
|
return {
|
|
146
146
|
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
147
147
|
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
@@ -169,7 +169,7 @@ class ConstantBooleanNaNsGenerator(BooleanGenerator):
|
|
|
169
169
|
|
|
170
170
|
@staticmethod
|
|
171
171
|
def get_performance_thresholds():
|
|
172
|
-
"""Return the expected
|
|
172
|
+
"""Return the expected thresholds."""
|
|
173
173
|
return {
|
|
174
174
|
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
175
175
|
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
@@ -26,7 +26,7 @@ class RandomIntegerGenerator(CategoricalGenerator):
|
|
|
26
26
|
|
|
27
27
|
@staticmethod
|
|
28
28
|
def get_performance_thresholds():
|
|
29
|
-
"""Return the expected
|
|
29
|
+
"""Return the expected thresholds."""
|
|
30
30
|
return {
|
|
31
31
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
32
32
|
'transform': {'time': 5e-05, 'memory': 400.0},
|
|
@@ -47,7 +47,7 @@ class RandomIntegerNaNsGenerator(CategoricalGenerator):
|
|
|
47
47
|
|
|
48
48
|
@staticmethod
|
|
49
49
|
def get_performance_thresholds():
|
|
50
|
-
"""Return the expected
|
|
50
|
+
"""Return the expected thresholds."""
|
|
51
51
|
return {
|
|
52
52
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
53
53
|
'transform': {'time': 5e-05, 'memory': 1000.0},
|
|
@@ -69,7 +69,7 @@ class RandomStringGenerator(CategoricalGenerator):
|
|
|
69
69
|
|
|
70
70
|
@staticmethod
|
|
71
71
|
def get_performance_thresholds():
|
|
72
|
-
"""Return the expected
|
|
72
|
+
"""Return the expected thresholds."""
|
|
73
73
|
return {
|
|
74
74
|
'fit': {'time': 2e-05, 'memory': 500.0},
|
|
75
75
|
'transform': {'time': 1e-05, 'memory': 500.0},
|
|
@@ -90,7 +90,7 @@ class RandomStringNaNsGenerator(CategoricalGenerator):
|
|
|
90
90
|
|
|
91
91
|
@staticmethod
|
|
92
92
|
def get_performance_thresholds():
|
|
93
|
-
"""Return the expected
|
|
93
|
+
"""Return the expected thresholds."""
|
|
94
94
|
return {
|
|
95
95
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
96
96
|
'transform': {'time': 1e-05, 'memory': 1000.0},
|
|
@@ -126,7 +126,7 @@ class RandomMixedGenerator(CategoricalGenerator):
|
|
|
126
126
|
|
|
127
127
|
@staticmethod
|
|
128
128
|
def get_performance_thresholds():
|
|
129
|
-
"""Return the expected
|
|
129
|
+
"""Return the expected thresholds."""
|
|
130
130
|
return {
|
|
131
131
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
132
132
|
'transform': {'time': 1e-05, 'memory': 1000.0},
|
|
@@ -158,7 +158,7 @@ class RandomMixedNaNsGenerator(CategoricalGenerator):
|
|
|
158
158
|
|
|
159
159
|
@staticmethod
|
|
160
160
|
def get_performance_thresholds():
|
|
161
|
-
"""Return the expected
|
|
161
|
+
"""Return the expected thresholds."""
|
|
162
162
|
return {
|
|
163
163
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
164
164
|
'transform': {'time': 1e-05, 'memory': 2000.0},
|
|
@@ -180,7 +180,7 @@ class SingleIntegerGenerator(CategoricalGenerator):
|
|
|
180
180
|
|
|
181
181
|
@staticmethod
|
|
182
182
|
def get_performance_thresholds():
|
|
183
|
-
"""Return the expected
|
|
183
|
+
"""Return the expected thresholds."""
|
|
184
184
|
return {
|
|
185
185
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
186
186
|
'transform': {'time': 3e-05, 'memory': 400.0},
|
|
@@ -201,7 +201,7 @@ class SingleIntegerNaNsGenerator(CategoricalGenerator):
|
|
|
201
201
|
|
|
202
202
|
@staticmethod
|
|
203
203
|
def get_performance_thresholds():
|
|
204
|
-
"""Return the expected
|
|
204
|
+
"""Return the expected thresholds."""
|
|
205
205
|
return {
|
|
206
206
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
207
207
|
'transform': {'time': 3e-05, 'memory': 400.0},
|
|
@@ -223,7 +223,7 @@ class SingleStringGenerator(CategoricalGenerator):
|
|
|
223
223
|
|
|
224
224
|
@staticmethod
|
|
225
225
|
def get_performance_thresholds():
|
|
226
|
-
"""Return the expected
|
|
226
|
+
"""Return the expected thresholds."""
|
|
227
227
|
return {
|
|
228
228
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
229
229
|
'transform': {'time': 4e-05, 'memory': 400.0},
|
|
@@ -244,7 +244,7 @@ class SingleStringNaNsGenerator(CategoricalGenerator):
|
|
|
244
244
|
|
|
245
245
|
@staticmethod
|
|
246
246
|
def get_performance_thresholds():
|
|
247
|
-
"""Return the expected
|
|
247
|
+
"""Return the expected thresholds."""
|
|
248
248
|
return {
|
|
249
249
|
'fit': {'time': 2e-05, 'memory': 400.0},
|
|
250
250
|
'transform': {'time': 3e-05, 'memory': 400.0},
|
|
@@ -265,7 +265,7 @@ class UniqueIntegerGenerator(CategoricalGenerator):
|
|
|
265
265
|
|
|
266
266
|
@staticmethod
|
|
267
267
|
def get_performance_thresholds():
|
|
268
|
-
"""Return the expected
|
|
268
|
+
"""Return the expected thresholds."""
|
|
269
269
|
return {
|
|
270
270
|
'fit': {'time': 0.0004, 'memory': 2000.0},
|
|
271
271
|
'transform': {'time': 0.0004, 'memory': 500000.0},
|
|
@@ -286,7 +286,7 @@ class UniqueIntegerNaNsGenerator(CategoricalGenerator):
|
|
|
286
286
|
|
|
287
287
|
@staticmethod
|
|
288
288
|
def get_performance_thresholds():
|
|
289
|
-
"""Return the expected
|
|
289
|
+
"""Return the expected thresholds."""
|
|
290
290
|
return {
|
|
291
291
|
'fit': {'time': 0.0004, 'memory': 1000.0},
|
|
292
292
|
'transform': {'time': 0.0004, 'memory': 1000000.0},
|
|
@@ -307,7 +307,7 @@ class UniqueStringGenerator(CategoricalGenerator):
|
|
|
307
307
|
|
|
308
308
|
@staticmethod
|
|
309
309
|
def get_performance_thresholds():
|
|
310
|
-
"""Return the expected
|
|
310
|
+
"""Return the expected thresholds."""
|
|
311
311
|
return {
|
|
312
312
|
'fit': {'time': 0.002, 'memory': 2000.0},
|
|
313
313
|
'transform': {'time': 0.0004, 'memory': 500000.0},
|
|
@@ -328,7 +328,7 @@ class UniqueStringNaNsGenerator(CategoricalGenerator):
|
|
|
328
328
|
|
|
329
329
|
@staticmethod
|
|
330
330
|
def get_performance_thresholds():
|
|
331
|
-
"""Return the expected
|
|
331
|
+
"""Return the expected thresholds."""
|
|
332
332
|
return {
|
|
333
333
|
'fit': {'time': 0.001, 'memory': 1000.0},
|
|
334
334
|
'transform': {'time': 0.0005, 'memory': 1000000.0},
|
|
@@ -29,7 +29,7 @@ class RandomGapDatetimeGenerator(DatetimeGenerator):
|
|
|
29
29
|
|
|
30
30
|
@staticmethod
|
|
31
31
|
def get_performance_thresholds():
|
|
32
|
-
"""Return the expected
|
|
32
|
+
"""Return the expected thresholds."""
|
|
33
33
|
return {
|
|
34
34
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
35
35
|
'transform': {'time': 5e-05, 'memory': 350.0},
|
|
@@ -53,7 +53,7 @@ class RandomGapSecondsDatetimeGenerator(DatetimeGenerator):
|
|
|
53
53
|
|
|
54
54
|
@staticmethod
|
|
55
55
|
def get_performance_thresholds():
|
|
56
|
-
"""Return the expected
|
|
56
|
+
"""Return the expected thresholds."""
|
|
57
57
|
return {
|
|
58
58
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
59
59
|
'transform': {'time': 5e-05, 'memory': 350.0},
|
|
@@ -75,7 +75,7 @@ class RandomGapDatetimeNaNsGenerator(DatetimeGenerator):
|
|
|
75
75
|
|
|
76
76
|
@staticmethod
|
|
77
77
|
def get_performance_thresholds():
|
|
78
|
-
"""Return the expected
|
|
78
|
+
"""Return the expected thresholds."""
|
|
79
79
|
return {
|
|
80
80
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
81
81
|
'transform': {'time': 5e-05, 'memory': 1000.0},
|
|
@@ -99,7 +99,7 @@ class EqualGapHoursDatetimeGenerator(DatetimeGenerator):
|
|
|
99
99
|
|
|
100
100
|
@staticmethod
|
|
101
101
|
def get_performance_thresholds():
|
|
102
|
-
"""Return the expected
|
|
102
|
+
"""Return the expected thresholds."""
|
|
103
103
|
return {
|
|
104
104
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
105
105
|
'transform': {'time': 5e-05, 'memory': 350.0},
|
|
@@ -126,7 +126,7 @@ class EqualGapDaysDatetimeGenerator(DatetimeGenerator):
|
|
|
126
126
|
|
|
127
127
|
@staticmethod
|
|
128
128
|
def get_performance_thresholds():
|
|
129
|
-
"""Return the expected
|
|
129
|
+
"""Return the expected thresholds."""
|
|
130
130
|
return {
|
|
131
131
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
132
132
|
'transform': {'time': 5e-05, 'memory': 350.0},
|
|
@@ -153,7 +153,7 @@ class EqualGapWeeksDatetimeGenerator(DatetimeGenerator):
|
|
|
153
153
|
|
|
154
154
|
@staticmethod
|
|
155
155
|
def get_performance_thresholds():
|
|
156
|
-
"""Return the expected
|
|
156
|
+
"""Return the expected thresholds."""
|
|
157
157
|
return {
|
|
158
158
|
'fit': {'time': 5e-05, 'memory': 500.0},
|
|
159
159
|
'transform': {'time': 5e-05, 'memory': 350.0},
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Dataset Generators for ID transformers."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from rdt.performance.datasets.base import BaseDatasetGenerator
|
|
8
|
+
from rdt.performance.datasets.utils import add_nans
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RegexGeneratorGenerator(BaseDatasetGenerator, ABC):
|
|
12
|
+
"""Base class for generators that generate ID data."""
|
|
13
|
+
|
|
14
|
+
SDTYPE = 'id'
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class RandomStringGenerator(RegexGeneratorGenerator):
|
|
18
|
+
"""Generator that creates an array of random strings."""
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
def generate(num_rows):
|
|
22
|
+
"""Generate a ``num_rows`` number of rows."""
|
|
23
|
+
categories = ['Alice', 'Bob', 'Charlie', 'Dave', 'Eve']
|
|
24
|
+
return np.random.choice(a=categories, size=num_rows)
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
def get_performance_thresholds():
|
|
28
|
+
"""Return the expected thresholds."""
|
|
29
|
+
return {
|
|
30
|
+
'fit': {'time': 1e-05, 'memory': 500.0},
|
|
31
|
+
'transform': {'time': 1e-05, 'memory': 500.0},
|
|
32
|
+
'reverse_transform': {
|
|
33
|
+
'time': 2e-05,
|
|
34
|
+
'memory': 1000.0,
|
|
35
|
+
},
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RandomStringNaNsGenerator(RegexGeneratorGenerator):
|
|
40
|
+
"""Generator that creates an array of random strings with nans."""
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def generate(num_rows):
|
|
44
|
+
"""Generate a ``num_rows`` number of rows."""
|
|
45
|
+
return add_nans(RandomStringGenerator.generate(num_rows).astype('O'))
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def get_performance_thresholds():
|
|
49
|
+
"""Return the expected thresholds."""
|
|
50
|
+
return {
|
|
51
|
+
'fit': {'time': 1e-05, 'memory': 400.0},
|
|
52
|
+
'transform': {'time': 1e-05, 'memory': 1000.0},
|
|
53
|
+
'reverse_transform': {
|
|
54
|
+
'time': 2e-05,
|
|
55
|
+
'memory': 1000.0,
|
|
56
|
+
},
|
|
57
|
+
}
|
|
@@ -25,7 +25,7 @@ class RandomIntegerGenerator(NumericalGenerator):
|
|
|
25
25
|
|
|
26
26
|
@staticmethod
|
|
27
27
|
def get_performance_thresholds():
|
|
28
|
-
"""Return the expected
|
|
28
|
+
"""Return the expected thresholds."""
|
|
29
29
|
return {
|
|
30
30
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
31
31
|
'transform': {'time': 5e-05, 'memory': 400.0},
|
|
@@ -46,7 +46,7 @@ class RandomIntegerNaNsGenerator(NumericalGenerator):
|
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
48
|
def get_performance_thresholds():
|
|
49
|
-
"""Return the expected
|
|
49
|
+
"""Return the expected thresholds."""
|
|
50
50
|
return {
|
|
51
51
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
52
52
|
'transform': {'time': 4e-05, 'memory': 400.0},
|
|
@@ -69,7 +69,7 @@ class ConstantIntegerGenerator(NumericalGenerator):
|
|
|
69
69
|
|
|
70
70
|
@staticmethod
|
|
71
71
|
def get_performance_thresholds():
|
|
72
|
-
"""Return the expected
|
|
72
|
+
"""Return the expected thresholds."""
|
|
73
73
|
return {
|
|
74
74
|
'fit': {'time': 1e-03, 'memory': 400.0},
|
|
75
75
|
'transform': {'time': 1e-05, 'memory': 400.0},
|
|
@@ -90,7 +90,7 @@ class ConstantIntegerNaNsGenerator(NumericalGenerator):
|
|
|
90
90
|
|
|
91
91
|
@staticmethod
|
|
92
92
|
def get_performance_thresholds():
|
|
93
|
-
"""Return the expected
|
|
93
|
+
"""Return the expected thresholds."""
|
|
94
94
|
return {
|
|
95
95
|
'fit': {'time': 1e-03, 'memory': 600.0},
|
|
96
96
|
'transform': {'time': 3e-05, 'memory': 400.0},
|
|
@@ -116,7 +116,7 @@ class AlmostConstantIntegerGenerator(NumericalGenerator):
|
|
|
116
116
|
|
|
117
117
|
@staticmethod
|
|
118
118
|
def get_performance_thresholds():
|
|
119
|
-
"""Return the expected
|
|
119
|
+
"""Return the expected thresholds."""
|
|
120
120
|
return {
|
|
121
121
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
122
122
|
'transform': {'time': 1e-05, 'memory': 2000.0},
|
|
@@ -142,7 +142,7 @@ class AlmostConstantIntegerNaNsGenerator(NumericalGenerator):
|
|
|
142
142
|
|
|
143
143
|
@staticmethod
|
|
144
144
|
def get_performance_thresholds():
|
|
145
|
-
"""Return the expected
|
|
145
|
+
"""Return the expected thresholds."""
|
|
146
146
|
return {
|
|
147
147
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
148
148
|
'transform': {'time': 3e-05, 'memory': 1000.0},
|
|
@@ -163,7 +163,7 @@ class NormalGenerator(NumericalGenerator):
|
|
|
163
163
|
|
|
164
164
|
@staticmethod
|
|
165
165
|
def get_performance_thresholds():
|
|
166
|
-
"""Return the expected
|
|
166
|
+
"""Return the expected thresholds."""
|
|
167
167
|
return {
|
|
168
168
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
169
169
|
'transform': {'time': 1e-05, 'memory': 400.0},
|
|
@@ -184,7 +184,7 @@ class NormalNaNsGenerator(NumericalGenerator):
|
|
|
184
184
|
|
|
185
185
|
@staticmethod
|
|
186
186
|
def get_performance_thresholds():
|
|
187
|
-
"""Return the expected
|
|
187
|
+
"""Return the expected thresholds."""
|
|
188
188
|
return {
|
|
189
189
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
190
190
|
'transform': {'time': 4e-05, 'memory': 400.0},
|
|
@@ -205,7 +205,7 @@ class BigNormalGenerator(NumericalGenerator):
|
|
|
205
205
|
|
|
206
206
|
@staticmethod
|
|
207
207
|
def get_performance_thresholds():
|
|
208
|
-
"""Return the expected
|
|
208
|
+
"""Return the expected thresholds."""
|
|
209
209
|
return {
|
|
210
210
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
211
211
|
'transform': {'time': 5e-05, 'memory': 400.0},
|
|
@@ -226,7 +226,7 @@ class BigNormalNaNsGenerator(NumericalGenerator):
|
|
|
226
226
|
|
|
227
227
|
@staticmethod
|
|
228
228
|
def get_performance_thresholds():
|
|
229
|
-
"""Return the expected
|
|
229
|
+
"""Return the expected thresholds."""
|
|
230
230
|
return {
|
|
231
231
|
'fit': {'time': 1e-03, 'memory': 2500.0},
|
|
232
232
|
'transform': {'time': 3e-05, 'memory': 400.0},
|
|
@@ -25,7 +25,7 @@ class RandomStringGenerator(PIIGenerator):
|
|
|
25
25
|
|
|
26
26
|
@staticmethod
|
|
27
27
|
def get_performance_thresholds():
|
|
28
|
-
"""Return the expected
|
|
28
|
+
"""Return the expected thresholds."""
|
|
29
29
|
return {
|
|
30
30
|
'fit': {'time': 1e-05, 'memory': 500.0},
|
|
31
31
|
'transform': {'time': 1e-05, 'memory': 500.0},
|
|
@@ -46,7 +46,7 @@ class RandomStringNaNsGenerator(PIIGenerator):
|
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
48
|
def get_performance_thresholds():
|
|
49
|
-
"""Return the expected
|
|
49
|
+
"""Return the expected thresholds."""
|
|
50
50
|
return {
|
|
51
51
|
'fit': {'time': 1e-05, 'memory': 400.0},
|
|
52
52
|
'transform': {'time': 1e-05, 'memory': 1000.0},
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Dataset Generators for
|
|
1
|
+
"""Dataset Generators for 'text' transformers."""
|
|
2
2
|
|
|
3
3
|
from abc import ABC
|
|
4
4
|
|
|
@@ -9,7 +9,7 @@ from rdt.performance.datasets.utils import add_nans
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class RegexGeneratorGenerator(BaseDatasetGenerator, ABC):
|
|
12
|
-
"""Base class for generators that generate
|
|
12
|
+
"""Base class for generators that generate ID data."""
|
|
13
13
|
|
|
14
14
|
SDTYPE = 'text'
|
|
15
15
|
|
|
@@ -25,7 +25,7 @@ class RandomStringGenerator(RegexGeneratorGenerator):
|
|
|
25
25
|
|
|
26
26
|
@staticmethod
|
|
27
27
|
def get_performance_thresholds():
|
|
28
|
-
"""Return the expected
|
|
28
|
+
"""Return the expected thresholds."""
|
|
29
29
|
return {
|
|
30
30
|
'fit': {'time': 1e-05, 'memory': 500.0},
|
|
31
31
|
'transform': {'time': 1e-05, 'memory': 500.0},
|
|
@@ -46,7 +46,7 @@ class RandomStringNaNsGenerator(RegexGeneratorGenerator):
|
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
48
|
def get_performance_thresholds():
|
|
49
|
-
"""Return the expected
|
|
49
|
+
"""Return the expected thresholds."""
|
|
50
50
|
return {
|
|
51
51
|
'fit': {'time': 1e-05, 'memory': 400.0},
|
|
52
52
|
'transform': {'time': 1e-05, 'memory': 1000.0},
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import importlib
|
|
4
4
|
import inspect
|
|
5
|
+
import warnings
|
|
5
6
|
from collections import defaultdict
|
|
6
7
|
from copy import deepcopy
|
|
7
8
|
from functools import lru_cache
|
|
@@ -21,6 +22,7 @@ from rdt.transformers.datetime import (
|
|
|
21
22
|
OptimizedTimestampEncoder,
|
|
22
23
|
UnixTimestampEncoder,
|
|
23
24
|
)
|
|
25
|
+
from rdt.transformers.id import IDGenerator, RegexGenerator
|
|
24
26
|
from rdt.transformers.null import NullTransformer
|
|
25
27
|
from rdt.transformers.numerical import (
|
|
26
28
|
ClusterBasedNormalizer,
|
|
@@ -31,7 +33,7 @@ from rdt.transformers.pii.anonymizer import (
|
|
|
31
33
|
AnonymizedFaker,
|
|
32
34
|
PseudoAnonymizedFaker,
|
|
33
35
|
)
|
|
34
|
-
from rdt.transformers.
|
|
36
|
+
from rdt.transformers.utils import WarnDict
|
|
35
37
|
|
|
36
38
|
__all__ = [
|
|
37
39
|
'BaseTransformer',
|
|
@@ -88,15 +90,15 @@ TRANSFORMERS = {
|
|
|
88
90
|
for transformer in BaseTransformer.get_subclasses()
|
|
89
91
|
}
|
|
90
92
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
93
|
+
DEFAULT_TRANSFORMERS = WarnDict(
|
|
94
|
+
boolean=UniformEncoder(),
|
|
95
|
+
categorical=UniformEncoder(),
|
|
96
|
+
datetime=UnixTimestampEncoder(),
|
|
97
|
+
id=RegexGenerator(),
|
|
98
|
+
numerical=FloatFormatter(),
|
|
99
|
+
pii=AnonymizedFaker(),
|
|
100
|
+
text=RegexGenerator(),
|
|
101
|
+
)
|
|
100
102
|
|
|
101
103
|
|
|
102
104
|
@lru_cache()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Transformers for
|
|
1
|
+
"""Transformers for ID data."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import warnings
|
|
@@ -30,7 +30,8 @@ class IDGenerator(BaseTransformer):
|
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
32
|
IS_GENERATOR = True
|
|
33
|
-
INPUT_SDTYPE = '
|
|
33
|
+
INPUT_SDTYPE = 'id'
|
|
34
|
+
SUPPORTED_SDTYPES = ['id', 'text']
|
|
34
35
|
|
|
35
36
|
def __init__(self, prefix=None, starting_value=0, suffix=None):
|
|
36
37
|
super().__init__()
|
|
@@ -92,7 +93,8 @@ class RegexGenerator(BaseTransformer):
|
|
|
92
93
|
"""
|
|
93
94
|
|
|
94
95
|
IS_GENERATOR = True
|
|
95
|
-
INPUT_SDTYPE = '
|
|
96
|
+
INPUT_SDTYPE = 'id'
|
|
97
|
+
SUPPORTED_SDTYPES = ['id', 'text']
|
|
96
98
|
|
|
97
99
|
def __getstate__(self):
|
|
98
100
|
"""Remove the generator when pickling."""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Transformers for text data."""
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
from rdt.transformers.id import IDGenerator, RegexGenerator # noqa: F401
|
|
6
|
+
|
|
7
|
+
warnings.warn(
|
|
8
|
+
"Importing 'IDGenerator' or 'RegexGenerator' for ID columns from 'rdt.transformers.text' "
|
|
9
|
+
"is deprecated. Please use 'rdt.transformers.id' instead.",
|
|
10
|
+
DeprecationWarning,
|
|
11
|
+
stacklevel=2,
|
|
12
|
+
)
|
|
@@ -5,6 +5,7 @@ import re
|
|
|
5
5
|
import string
|
|
6
6
|
import sys
|
|
7
7
|
import warnings
|
|
8
|
+
from collections import defaultdict
|
|
8
9
|
|
|
9
10
|
import numpy as np
|
|
10
11
|
import pandas as pd
|
|
@@ -14,6 +15,7 @@ import sre_parse # isort:skip
|
|
|
14
15
|
LOGGER = logging.getLogger(__name__)
|
|
15
16
|
|
|
16
17
|
MAX_DECIMALS = sys.float_info.dig - 1
|
|
18
|
+
DEPRECATED_SDTYPES_MAPPING = {'text': 'id'}
|
|
17
19
|
|
|
18
20
|
|
|
19
21
|
def _literal(character, max_repeat):
|
|
@@ -274,3 +276,36 @@ def learn_rounding_digits(data):
|
|
|
274
276
|
name,
|
|
275
277
|
)
|
|
276
278
|
return None
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class WarnDict(dict):
|
|
282
|
+
"""Custom dictionary to raise a deprecation warning."""
|
|
283
|
+
|
|
284
|
+
def __init__(self, *args, **kwargs):
|
|
285
|
+
super().__init__(*args, **kwargs)
|
|
286
|
+
self._warned = defaultdict()
|
|
287
|
+
|
|
288
|
+
def get(self, sdtype):
|
|
289
|
+
"""Return the value for sdtype if sdtype is in the dictionary, else default.
|
|
290
|
+
|
|
291
|
+
If the sdtype is `text` raises a `DeprecationWarning` stating that it will be
|
|
292
|
+
phased out.
|
|
293
|
+
"""
|
|
294
|
+
if sdtype in DEPRECATED_SDTYPES_MAPPING and not self._warned.get(sdtype):
|
|
295
|
+
new_sdtype = DEPRECATED_SDTYPES_MAPPING.get(sdtype)
|
|
296
|
+
warnings.warn(
|
|
297
|
+
f"The sdtype '{sdtype}' is deprecated and will be phased out. "
|
|
298
|
+
f"Please use '{new_sdtype}' instead.",
|
|
299
|
+
DeprecationWarning,
|
|
300
|
+
)
|
|
301
|
+
self._warned[sdtype] = True
|
|
302
|
+
|
|
303
|
+
return super().get(sdtype)
|
|
304
|
+
|
|
305
|
+
def __getitem__(self, sdtype):
|
|
306
|
+
"""Return the value for sdtype if sdtype is in the dictionary.
|
|
307
|
+
|
|
308
|
+
If the sdtype is `text` raises a `DeprecationWarning` stating that it will be
|
|
309
|
+
phased out.
|
|
310
|
+
"""
|
|
311
|
+
return self.get(sdtype)
|
|
@@ -18,6 +18,7 @@ rdt/performance/datasets/base.py
|
|
|
18
18
|
rdt/performance/datasets/boolean.py
|
|
19
19
|
rdt/performance/datasets/categorical.py
|
|
20
20
|
rdt/performance/datasets/datetime.py
|
|
21
|
+
rdt/performance/datasets/id.py
|
|
21
22
|
rdt/performance/datasets/numerical.py
|
|
22
23
|
rdt/performance/datasets/pii.py
|
|
23
24
|
rdt/performance/datasets/text.py
|
|
@@ -28,6 +29,7 @@ rdt/transformers/base.py
|
|
|
28
29
|
rdt/transformers/boolean.py
|
|
29
30
|
rdt/transformers/categorical.py
|
|
30
31
|
rdt/transformers/datetime.py
|
|
32
|
+
rdt/transformers/id.py
|
|
31
33
|
rdt/transformers/null.py
|
|
32
34
|
rdt/transformers/numerical.py
|
|
33
35
|
rdt/transformers/text.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|