rdt 1.12.0.dev1__tar.gz → 1.12.1.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rdt-1.12.0.dev1/rdt.egg-info → rdt-1.12.1.dev0}/PKG-INFO +6 -33
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/pyproject.toml +55 -35
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/__init__.py +30 -26
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/hyper_transformer.py +51 -34
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/__init__.py +8 -1
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/boolean.py +18 -54
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/categorical.py +45 -128
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/datetime.py +18 -54
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/numerical.py +30 -90
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/pii.py +6 -18
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/text.py +6 -18
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/performance.py +7 -17
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/profiling.py +2 -2
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/__init__.py +22 -6
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/_validators.py +15 -6
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/base.py +12 -8
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/boolean.py +11 -5
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/categorical.py +21 -21
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/datetime.py +29 -14
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/null.py +5 -7
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/numerical.py +58 -30
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/pii/__init__.py +4 -1
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/pii/anonymization.py +10 -28
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/pii/anonymizer.py +53 -21
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/text.py +28 -15
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/utils.py +10 -10
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0/rdt.egg-info}/PKG-INFO +6 -33
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt.egg-info/SOURCES.txt +0 -1
- rdt-1.12.1.dev0/rdt.egg-info/requires.txt +53 -0
- rdt-1.12.1.dev0/setup.cfg +4 -0
- rdt-1.12.0.dev1/rdt.egg-info/requires.txt +0 -78
- rdt-1.12.0.dev1/setup.cfg +0 -35
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/LICENSE +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/README.md +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/errors.py +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/__init__.py +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/base.py +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/performance/datasets/utils.py +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt/transformers/pii/utils.py +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt.egg-info/dependency_links.txt +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt.egg-info/entry_points.txt +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/rdt.egg-info/top_level.txt +0 -0
- {rdt-1.12.0.dev1 → rdt-1.12.1.dev0}/tests/test_tasks.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rdt
|
|
3
|
-
Version: 1.12.
|
|
3
|
+
Version: 1.12.1.dev0
|
|
4
4
|
Summary: Reversible Data Transforms
|
|
5
5
|
Author-email: "DataCebo, Inc." <info@sdv.dev>
|
|
6
6
|
License: BSL-1.1
|
|
@@ -24,17 +24,16 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
|
24
24
|
Requires-Python: <3.13,>=3.8
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
-
Requires-Dist: numpy>=1.
|
|
27
|
+
Requires-Dist: numpy>=1.21.0; python_version < "3.10"
|
|
28
28
|
Requires-Dist: numpy>=1.23.3; python_version >= "3.10" and python_version < "3.12"
|
|
29
29
|
Requires-Dist: numpy>=1.26.0; python_version >= "3.12"
|
|
30
|
-
Requires-Dist: pandas>=1.
|
|
31
|
-
Requires-Dist: pandas>=1.3.4; python_version >= "3.10" and python_version < "3.11"
|
|
30
|
+
Requires-Dist: pandas>=1.4.0; python_version < "3.11"
|
|
32
31
|
Requires-Dist: pandas>=1.5.0; python_version >= "3.11" and python_version < "3.12"
|
|
33
32
|
Requires-Dist: pandas>=2.1.1; python_version >= "3.12"
|
|
34
|
-
Requires-Dist: scipy>=1.
|
|
33
|
+
Requires-Dist: scipy>=1.7.3; python_version < "3.10"
|
|
35
34
|
Requires-Dist: scipy>=1.9.2; python_version >= "3.10" and python_version < "3.12"
|
|
36
35
|
Requires-Dist: scipy>=1.12.0; python_version >= "3.12"
|
|
37
|
-
Requires-Dist: scikit-learn>=0.
|
|
36
|
+
Requires-Dist: scikit-learn>=1.0.2; python_version < "3.10"
|
|
38
37
|
Requires-Dist: scikit-learn>=1.1.0; python_version >= "3.10" and python_version < "3.11"
|
|
39
38
|
Requires-Dist: scikit-learn>=1.1.3; python_version >= "3.11" and python_version < "3.12"
|
|
40
39
|
Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12"
|
|
@@ -56,33 +55,7 @@ Requires-Dist: build<2,>=1.0.0; extra == "dev"
|
|
|
56
55
|
Requires-Dist: bump-my-version<1,>=0.18.3; extra == "dev"
|
|
57
56
|
Requires-Dist: pip>=9.0.1; extra == "dev"
|
|
58
57
|
Requires-Dist: watchdog<5,>=1.0.1; extra == "dev"
|
|
59
|
-
Requires-Dist:
|
|
60
|
-
Requires-Dist: pyflakes<3.3,>=2.3.0; extra == "dev"
|
|
61
|
-
Requires-Dist: flake8<8,>=3.7.7; extra == "dev"
|
|
62
|
-
Requires-Dist: flake8-absolute-import<2,>=1.0; extra == "dev"
|
|
63
|
-
Requires-Dist: flake8-builtins<3,>=1.5.3; extra == "dev"
|
|
64
|
-
Requires-Dist: flake8-comprehensions<4,>=3.6.1; extra == "dev"
|
|
65
|
-
Requires-Dist: flake8-debugger<5,>=4.0.0; extra == "dev"
|
|
66
|
-
Requires-Dist: flake8-docstrings<2,>=1.5.0; extra == "dev"
|
|
67
|
-
Requires-Dist: flake8-eradicate<2,>=1.1.0; extra == "dev"
|
|
68
|
-
Requires-Dist: flake8-fixme<1.2,>=1.1.1; extra == "dev"
|
|
69
|
-
Requires-Dist: flake8-mock<1,>=0.3; extra == "dev"
|
|
70
|
-
Requires-Dist: flake8-multiline-containers<0.1,>=0.0.18; extra == "dev"
|
|
71
|
-
Requires-Dist: flake8-mutable<1.3,>=1.2.0; extra == "dev"
|
|
72
|
-
Requires-Dist: flake8-expression-complexity<0.1,>=0.0.9; extra == "dev"
|
|
73
|
-
Requires-Dist: flake8-print<4.1,>=4.0.0; extra == "dev"
|
|
74
|
-
Requires-Dist: flake8-pytest-style<3,>=2.0.0; extra == "dev"
|
|
75
|
-
Requires-Dist: flake8-quotes<4,>=3.3.0; extra == "dev"
|
|
76
|
-
Requires-Dist: flake8-sfs<2,>=0.0.3; extra == "dev"
|
|
77
|
-
Requires-Dist: flake8-variables-names<0.1,>=0.0.4; extra == "dev"
|
|
78
|
-
Requires-Dist: dlint<1,>=0.11.0; extra == "dev"
|
|
79
|
-
Requires-Dist: isort<6,>=5.13.2; extra == "dev"
|
|
80
|
-
Requires-Dist: pandas-vet<2024,>=0.2.3; extra == "dev"
|
|
81
|
-
Requires-Dist: pep8-naming<1,>=0.12.1; extra == "dev"
|
|
82
|
-
Requires-Dist: pydocstyle<7,>=6.1.1; extra == "dev"
|
|
83
|
-
Requires-Dist: pylint<4,>=2.5.3; extra == "dev"
|
|
84
|
-
Requires-Dist: autoflake<3,>=1.1; extra == "dev"
|
|
85
|
-
Requires-Dist: autopep8<3,>=1.4.3; extra == "dev"
|
|
58
|
+
Requires-Dist: ruff<1,>=0.3.2; extra == "dev"
|
|
86
59
|
Requires-Dist: twine<6,>=1.10.0; extra == "dev"
|
|
87
60
|
Requires-Dist: wheel>=0.30.0; extra == "dev"
|
|
88
61
|
Requires-Dist: coverage<8,>=4.5.12; extra == "dev"
|
|
@@ -21,17 +21,16 @@ license = { text = 'BSL-1.1' }
|
|
|
21
21
|
requires-python = '>=3.8,<3.13'
|
|
22
22
|
readme = 'README.md'
|
|
23
23
|
dependencies = [
|
|
24
|
-
"numpy>=1.
|
|
24
|
+
"numpy>=1.21.0;python_version<'3.10'",
|
|
25
25
|
"numpy>=1.23.3;python_version>='3.10' and python_version<'3.12'",
|
|
26
26
|
"numpy>=1.26.0;python_version>='3.12'",
|
|
27
|
-
"pandas>=1.
|
|
28
|
-
"pandas>=1.3.4;python_version>='3.10' and python_version<'3.11'",
|
|
27
|
+
"pandas>=1.4.0;python_version<'3.11'",
|
|
29
28
|
"pandas>=1.5.0;python_version>='3.11' and python_version<'3.12'",
|
|
30
29
|
"pandas>=2.1.1;python_version>='3.12'",
|
|
31
|
-
"scipy>=1.
|
|
30
|
+
"scipy>=1.7.3;python_version<'3.10'",
|
|
32
31
|
"scipy>=1.9.2;python_version>='3.10' and python_version<'3.12'",
|
|
33
32
|
"scipy>=1.12.0;python_version>='3.12'",
|
|
34
|
-
"scikit-learn>=0.
|
|
33
|
+
"scikit-learn>=1.0.2;python_version<'3.10'",
|
|
35
34
|
"scikit-learn>=1.1.0;python_version>='3.10' and python_version<'3.11'",
|
|
36
35
|
"scikit-learn>=1.1.3;python_version>='3.11' and python_version<'3.12'",
|
|
37
36
|
"scikit-learn>=1.3.1;python_version>='3.12'",
|
|
@@ -71,35 +70,7 @@ dev = [
|
|
|
71
70
|
'watchdog>=1.0.1,<5',
|
|
72
71
|
|
|
73
72
|
# style check
|
|
74
|
-
'
|
|
75
|
-
'pyflakes>=2.3.0,<3.3',
|
|
76
|
-
'flake8>=3.7.7,<8',
|
|
77
|
-
'flake8-absolute-import>=1.0,<2',
|
|
78
|
-
'flake8-builtins>=1.5.3,<3',
|
|
79
|
-
'flake8-comprehensions>=3.6.1,<4',
|
|
80
|
-
'flake8-debugger>=4.0.0,<5',
|
|
81
|
-
'flake8-docstrings>=1.5.0,<2',
|
|
82
|
-
'flake8-eradicate>=1.1.0,<2',
|
|
83
|
-
'flake8-fixme>=1.1.1,<1.2',
|
|
84
|
-
'flake8-mock>=0.3,<1',
|
|
85
|
-
'flake8-multiline-containers>=0.0.18,<0.1',
|
|
86
|
-
'flake8-mutable>=1.2.0,<1.3',
|
|
87
|
-
'flake8-expression-complexity>=0.0.9,<0.1',
|
|
88
|
-
'flake8-print>=4.0.0,<4.1',
|
|
89
|
-
'flake8-pytest-style>=2.0.0,<3',
|
|
90
|
-
'flake8-quotes>=3.3.0,<4',
|
|
91
|
-
'flake8-sfs>=0.0.3,<2',
|
|
92
|
-
'flake8-variables-names>=0.0.4,<0.1',
|
|
93
|
-
'dlint>=0.11.0,<1',
|
|
94
|
-
'isort>=5.13.2,<6',
|
|
95
|
-
'pandas-vet>=0.2.3,<2024',
|
|
96
|
-
'pep8-naming>=0.12.1,<1',
|
|
97
|
-
'pydocstyle>=6.1.1,<7',
|
|
98
|
-
'pylint>=2.5.3,<4',
|
|
99
|
-
|
|
100
|
-
# fix style issues
|
|
101
|
-
'autoflake>=1.1,<3',
|
|
102
|
-
'autopep8>=1.4.3,<3',
|
|
73
|
+
'ruff>=0.3.2,<1',
|
|
103
74
|
|
|
104
75
|
# distribute on PyPI
|
|
105
76
|
'twine>=1.10.0,<6',
|
|
@@ -166,7 +137,7 @@ collect_ignore = ['pyproject.toml']
|
|
|
166
137
|
exclude_lines = ['NotImplementedError()']
|
|
167
138
|
|
|
168
139
|
[tool.bumpversion]
|
|
169
|
-
current_version = "1.12.
|
|
140
|
+
current_version = "1.12.1.dev0"
|
|
170
141
|
parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
|
|
171
142
|
serialize = [
|
|
172
143
|
'{major}.{minor}.{patch}.{release}{candidate}',
|
|
@@ -201,3 +172,52 @@ replace = "__version__ = '{new_version}'"
|
|
|
201
172
|
[build-system]
|
|
202
173
|
requires = ['setuptools', 'wheel']
|
|
203
174
|
build-backend = 'setuptools.build_meta'
|
|
175
|
+
|
|
176
|
+
[tool.ruff]
|
|
177
|
+
preview = true
|
|
178
|
+
line-length = 100
|
|
179
|
+
indent-width = 4
|
|
180
|
+
src = ["rdt"]
|
|
181
|
+
target-version = "py312"
|
|
182
|
+
exclude = [
|
|
183
|
+
"docs",
|
|
184
|
+
".tox",
|
|
185
|
+
".git",
|
|
186
|
+
"__pycache__",
|
|
187
|
+
".ipynb_checkpoints"
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
[tool.ruff.lint]
|
|
191
|
+
select = [
|
|
192
|
+
# Pyflakes
|
|
193
|
+
"F",
|
|
194
|
+
# Pycodestyle
|
|
195
|
+
"E",
|
|
196
|
+
"W",
|
|
197
|
+
# isort
|
|
198
|
+
"I001"
|
|
199
|
+
]
|
|
200
|
+
ignore = [
|
|
201
|
+
"E501",
|
|
202
|
+
"D107", # Missing docstring in __init__
|
|
203
|
+
"D417", # Missing argument descriptions in the docstring, this is a bug from pydocstyle: https://github.com/PyCQA/pydocstyle/issues/449
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
[tool.ruff.format]
|
|
207
|
+
quote-style = "single"
|
|
208
|
+
indent-style = "space"
|
|
209
|
+
preview = true
|
|
210
|
+
docstring-code-format = true
|
|
211
|
+
docstring-code-line-length = "dynamic"
|
|
212
|
+
|
|
213
|
+
[tool.ruff.lint.pep8-naming]
|
|
214
|
+
extend-ignore-names = ["X", "C", "X_padded", "Y", "Y_padded"]
|
|
215
|
+
|
|
216
|
+
[tool.ruff.lint.isort]
|
|
217
|
+
known-first-party = ["rdt"]
|
|
218
|
+
|
|
219
|
+
[tool.ruff.lint.per-file-ignores]
|
|
220
|
+
"__init__.py" = ["F401", "E402", "F403", "F405", "E501", "I001"]
|
|
221
|
+
|
|
222
|
+
[tool.ruff.lint.pydocstyle]
|
|
223
|
+
convention = "google"
|
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
"""Top-level package for RDT."""
|
|
4
4
|
|
|
5
|
-
|
|
6
5
|
__author__ = 'DataCebo, Inc.'
|
|
7
6
|
__email__ = 'info@sdv.dev'
|
|
8
|
-
__version__ = '1.12.
|
|
7
|
+
__version__ = '1.12.1.dev0'
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
import sys
|
|
@@ -20,10 +19,7 @@ import pandas as pd
|
|
|
20
19
|
from rdt import transformers
|
|
21
20
|
from rdt.hyper_transformer import HyperTransformer
|
|
22
21
|
|
|
23
|
-
__all__ = [
|
|
24
|
-
'HyperTransformer',
|
|
25
|
-
'transformers'
|
|
26
|
-
]
|
|
22
|
+
__all__ = ['HyperTransformer', 'transformers']
|
|
27
23
|
|
|
28
24
|
RANDOM_SEED = 42
|
|
29
25
|
|
|
@@ -42,9 +38,10 @@ def get_demo(num_rows=5):
|
|
|
42
38
|
pd.DataFrame
|
|
43
39
|
"""
|
|
44
40
|
# Hard code first five rows
|
|
45
|
-
login_dates = pd.Series(
|
|
46
|
-
'2021-06-26', '2021-02-10', 'NAT', '2020-09-26', '2020-12-22'
|
|
47
|
-
|
|
41
|
+
login_dates = pd.Series(
|
|
42
|
+
['2021-06-26', '2021-02-10', 'NAT', '2020-09-26', '2020-12-22'],
|
|
43
|
+
dtype='datetime64[ns]',
|
|
44
|
+
)
|
|
48
45
|
email_optin = pd.Series([False, False, False, True, np.nan], dtype='object')
|
|
49
46
|
credit_card = ['VISA', 'VISA', 'AMEX', np.nan, 'DISCOVER']
|
|
50
47
|
age = [29, 18, 21, 45, 32]
|
|
@@ -55,7 +52,7 @@ def get_demo(num_rows=5):
|
|
|
55
52
|
'email_optin': email_optin,
|
|
56
53
|
'credit_card': credit_card,
|
|
57
54
|
'age': age,
|
|
58
|
-
'dollars_spent': dollars_spent
|
|
55
|
+
'dollars_spent': dollars_spent,
|
|
59
56
|
})
|
|
60
57
|
|
|
61
58
|
if num_rows <= 5:
|
|
@@ -67,14 +64,18 @@ def get_demo(num_rows=5):
|
|
|
67
64
|
try:
|
|
68
65
|
num_rows -= 5
|
|
69
66
|
|
|
70
|
-
login_dates = np.array(
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
login_dates = np.array(
|
|
68
|
+
[
|
|
69
|
+
np.datetime64('2000-01-01') + np.timedelta64(np.random.randint(0, 10000), 'D')
|
|
70
|
+
for _ in range(num_rows)
|
|
71
|
+
],
|
|
72
|
+
dtype='datetime64[ns]',
|
|
73
|
+
)
|
|
74
74
|
login_dates[np.random.random(size=num_rows) > 0.8] = np.datetime64('NaT')
|
|
75
75
|
|
|
76
76
|
email_optin = pd.Series([True, False, np.nan], dtype='object').sample(
|
|
77
|
-
num_rows, replace=True
|
|
77
|
+
num_rows, replace=True
|
|
78
|
+
)
|
|
78
79
|
credit_card = np.random.choice(['VISA', 'AMEX', np.nan, 'DISCOVER'], size=num_rows)
|
|
79
80
|
age = np.random.randint(18, 100, size=num_rows)
|
|
80
81
|
|
|
@@ -84,16 +85,19 @@ def get_demo(num_rows=5):
|
|
|
84
85
|
finally:
|
|
85
86
|
np.random.set_state(random_state)
|
|
86
87
|
|
|
87
|
-
return pd.concat(
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
88
|
+
return pd.concat(
|
|
89
|
+
[
|
|
90
|
+
data,
|
|
91
|
+
pd.DataFrame({
|
|
92
|
+
'last_login': login_dates,
|
|
93
|
+
'email_optin': email_optin,
|
|
94
|
+
'credit_card': credit_card,
|
|
95
|
+
'age': age,
|
|
96
|
+
'dollars_spent': dollars_spent,
|
|
97
|
+
}),
|
|
98
|
+
],
|
|
99
|
+
ignore_index=True,
|
|
100
|
+
)
|
|
97
101
|
|
|
98
102
|
|
|
99
103
|
def _get_addon_target(addon_path_name):
|
|
@@ -159,7 +163,7 @@ def _find_addons():
|
|
|
159
163
|
try:
|
|
160
164
|
addon = entry_point.load()
|
|
161
165
|
except Exception: # pylint: disable=broad-exception-caught
|
|
162
|
-
msg = f'Failed to load "{entry_point.name}" from "{entry_point.
|
|
166
|
+
msg = f'Failed to load "{entry_point.name}" from "{entry_point.value}".'
|
|
163
167
|
warnings.warn(msg)
|
|
164
168
|
continue
|
|
165
169
|
|
|
@@ -9,11 +9,20 @@ from copy import deepcopy
|
|
|
9
9
|
import pandas as pd
|
|
10
10
|
|
|
11
11
|
from rdt.errors import (
|
|
12
|
-
ConfigNotSetError,
|
|
13
|
-
|
|
12
|
+
ConfigNotSetError,
|
|
13
|
+
InvalidConfigError,
|
|
14
|
+
InvalidDataError,
|
|
15
|
+
NotFittedError,
|
|
16
|
+
TransformerInputError,
|
|
17
|
+
TransformerProcessingError,
|
|
18
|
+
)
|
|
14
19
|
from rdt.transformers import (
|
|
15
|
-
BaseMultiColumnTransformer,
|
|
16
|
-
|
|
20
|
+
BaseMultiColumnTransformer,
|
|
21
|
+
BaseTransformer,
|
|
22
|
+
get_class_by_transformer_name,
|
|
23
|
+
get_default_transformer,
|
|
24
|
+
get_transformers_by_type,
|
|
25
|
+
)
|
|
17
26
|
from rdt.transformers.utils import flatten_column_list
|
|
18
27
|
|
|
19
28
|
LOGGER = logging.getLogger(__name__)
|
|
@@ -31,7 +40,7 @@ class Config(dict):
|
|
|
31
40
|
|
|
32
41
|
config = {
|
|
33
42
|
'sdtypes': self['sdtypes'],
|
|
34
|
-
'transformers': {str(k): repr(v) for k, v in self['transformers'].items()}
|
|
43
|
+
'transformers': {str(k): repr(v) for k, v in self['transformers'].items()},
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
printed = json.dumps(config, indent=4)
|
|
@@ -59,11 +68,7 @@ class HyperTransformer:
|
|
|
59
68
|
'b': 'boolean',
|
|
60
69
|
'M': 'datetime',
|
|
61
70
|
}
|
|
62
|
-
_DEFAULT_OUTPUT_SDTYPES = [
|
|
63
|
-
'numerical',
|
|
64
|
-
'float',
|
|
65
|
-
'integer'
|
|
66
|
-
]
|
|
71
|
+
_DEFAULT_OUTPUT_SDTYPES = ['numerical', 'float', 'integer']
|
|
67
72
|
_REFIT_MESSAGE = (
|
|
68
73
|
"For this change to take effect, please refit your data using 'fit' or 'fit_transform'."
|
|
69
74
|
)
|
|
@@ -92,11 +97,7 @@ class HyperTransformer:
|
|
|
92
97
|
|
|
93
98
|
@staticmethod
|
|
94
99
|
def _subset(input_list, other_list, not_in=False):
|
|
95
|
-
return [
|
|
96
|
-
element
|
|
97
|
-
for element in input_list
|
|
98
|
-
if (element in other_list) ^ not_in
|
|
99
|
-
]
|
|
100
|
+
return [element for element in input_list if (element in other_list) ^ not_in]
|
|
100
101
|
|
|
101
102
|
def _create_multi_column_fields(self):
|
|
102
103
|
multi_column_fields = {}
|
|
@@ -109,9 +110,11 @@ class HyperTransformer:
|
|
|
109
110
|
def _validate_field_transformers(self):
|
|
110
111
|
for field in self.field_transformers:
|
|
111
112
|
if self._field_in_set(field, self._specified_fields):
|
|
112
|
-
raise ValueError(
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f'Multiple transformers specified for the field {field}. '
|
|
115
|
+
'Each field can have at most one transformer defined in '
|
|
116
|
+
'field_transformers.'
|
|
117
|
+
)
|
|
115
118
|
|
|
116
119
|
self._add_field_to_set(field, self._specified_fields)
|
|
117
120
|
|
|
@@ -150,7 +153,7 @@ class HyperTransformer:
|
|
|
150
153
|
"""
|
|
151
154
|
return Config({
|
|
152
155
|
'sdtypes': self.field_sdtypes,
|
|
153
|
-
'transformers': self.field_transformers
|
|
156
|
+
'transformers': self.field_transformers,
|
|
154
157
|
})
|
|
155
158
|
|
|
156
159
|
@staticmethod
|
|
@@ -250,7 +253,9 @@ class HyperTransformer:
|
|
|
250
253
|
|
|
251
254
|
def _validate_update_columns(self, update_columns):
|
|
252
255
|
unknown_columns = self._subset(
|
|
253
|
-
flatten_column_list(update_columns),
|
|
256
|
+
flatten_column_list(update_columns),
|
|
257
|
+
self.field_sdtypes.keys(),
|
|
258
|
+
not_in=True,
|
|
254
259
|
)
|
|
255
260
|
if unknown_columns:
|
|
256
261
|
raise InvalidConfigError(
|
|
@@ -279,7 +284,8 @@ class HyperTransformer:
|
|
|
279
284
|
warnings.warn(self._REFIT_MESSAGE)
|
|
280
285
|
|
|
281
286
|
def _validate_update_transformers_by_sdtype(
|
|
282
|
-
|
|
287
|
+
self, sdtype, transformer, transformer_name, transformer_parameters
|
|
288
|
+
):
|
|
283
289
|
if not self.field_sdtypes:
|
|
284
290
|
raise ConfigNotSetError(
|
|
285
291
|
'Nothing to update. Use the `detect_initial_config` method to '
|
|
@@ -301,10 +307,14 @@ class HyperTransformer:
|
|
|
301
307
|
)
|
|
302
308
|
|
|
303
309
|
else:
|
|
304
|
-
if
|
|
305
|
-
|
|
310
|
+
if (
|
|
311
|
+
transformer_name not in get_class_by_transformer_name()
|
|
312
|
+
or sdtype
|
|
313
|
+
not in get_class_by_transformer_name()[transformer_name].get_supported_sdtypes()
|
|
314
|
+
):
|
|
306
315
|
raise InvalidConfigError(
|
|
307
|
-
f"Invalid transformer name '{transformer_name}' for the '{sdtype}' sdtype."
|
|
316
|
+
f"Invalid transformer name '{transformer_name}' for the '{sdtype}' sdtype."
|
|
317
|
+
)
|
|
308
318
|
|
|
309
319
|
if transformer_parameters is not None:
|
|
310
320
|
transformer = get_class_by_transformer_name()[transformer_name]
|
|
@@ -325,14 +335,15 @@ class HyperTransformer:
|
|
|
325
335
|
warnings.warn(
|
|
326
336
|
"The 'transformer' parameter will no longer be supported in future versions "
|
|
327
337
|
"of the RDT. Using the 'transformer_name' parameter instead.",
|
|
328
|
-
FutureWarning
|
|
338
|
+
FutureWarning,
|
|
329
339
|
)
|
|
330
340
|
|
|
331
341
|
else:
|
|
332
342
|
warnings.warn(
|
|
333
343
|
"The 'transformer' parameter will no longer be supported in future versions "
|
|
334
344
|
"of the RDT. Please use the 'transformer_name' and 'transformer_parameters' "
|
|
335
|
-
'parameters instead.',
|
|
345
|
+
'parameters instead.',
|
|
346
|
+
FutureWarning,
|
|
336
347
|
)
|
|
337
348
|
|
|
338
349
|
def _remove_column_in_multi_column_fields(self, column):
|
|
@@ -349,7 +360,7 @@ class HyperTransformer:
|
|
|
349
360
|
new_tuple = tuple(item for item in old_tuple if item != column)
|
|
350
361
|
|
|
351
362
|
if len(new_tuple) == 1:
|
|
352
|
-
new_tuple, = new_tuple
|
|
363
|
+
(new_tuple,) = new_tuple
|
|
353
364
|
self._multi_column_fields.pop(new_tuple, None)
|
|
354
365
|
else:
|
|
355
366
|
for col in new_tuple:
|
|
@@ -369,7 +380,7 @@ class HyperTransformer:
|
|
|
369
380
|
|
|
370
381
|
columns_to_sdtypes = self._get_columns_to_sdtypes(field)
|
|
371
382
|
try:
|
|
372
|
-
transformer._validate_sdtypes(
|
|
383
|
+
transformer._validate_sdtypes( # pylint: disable=protected-access
|
|
373
384
|
columns_to_sdtypes
|
|
374
385
|
)
|
|
375
386
|
except TransformerInputError:
|
|
@@ -384,7 +395,12 @@ class HyperTransformer:
|
|
|
384
395
|
self._multi_column_fields = self._create_multi_column_fields()
|
|
385
396
|
|
|
386
397
|
def update_transformers_by_sdtype(
|
|
387
|
-
|
|
398
|
+
self,
|
|
399
|
+
sdtype,
|
|
400
|
+
transformer=None,
|
|
401
|
+
transformer_name=None,
|
|
402
|
+
transformer_parameters=None,
|
|
403
|
+
):
|
|
388
404
|
"""Update the transformers for the specified ``sdtype``.
|
|
389
405
|
|
|
390
406
|
Given an ``sdtype`` and a ``transformer``, change all the fields of the ``sdtype``
|
|
@@ -403,15 +419,17 @@ class HyperTransformer:
|
|
|
403
419
|
A dict of the kwargs of the transformer.
|
|
404
420
|
"""
|
|
405
421
|
self._validate_update_transformers_by_sdtype(
|
|
406
|
-
sdtype, transformer, transformer_name, transformer_parameters
|
|
422
|
+
sdtype, transformer, transformer_name, transformer_parameters
|
|
423
|
+
)
|
|
407
424
|
self._warn_update_transformers_by_sdtype(transformer, transformer_name)
|
|
408
425
|
|
|
409
426
|
transformer_instance = transformer
|
|
410
427
|
|
|
411
428
|
if transformer_name is not None:
|
|
412
429
|
if transformer_parameters is not None:
|
|
413
|
-
transformer_instance =
|
|
414
|
-
|
|
430
|
+
transformer_instance = get_class_by_transformer_name()[transformer_name](
|
|
431
|
+
**transformer_parameters
|
|
432
|
+
)
|
|
415
433
|
|
|
416
434
|
else:
|
|
417
435
|
transformer_instance = get_class_by_transformer_name()[transformer_name]()
|
|
@@ -620,7 +638,7 @@ class HyperTransformer:
|
|
|
620
638
|
|
|
621
639
|
config = Config({
|
|
622
640
|
'sdtypes': self.field_sdtypes,
|
|
623
|
-
'transformers': self.field_transformers
|
|
641
|
+
'transformers': self.field_transformers,
|
|
624
642
|
})
|
|
625
643
|
|
|
626
644
|
LOGGER.info('Config:')
|
|
@@ -675,7 +693,6 @@ class HyperTransformer:
|
|
|
675
693
|
|
|
676
694
|
next_transformers = transformer.get_next_transformers()
|
|
677
695
|
for column_name, next_transformer in next_transformers.items():
|
|
678
|
-
|
|
679
696
|
# If the column is part of a multi-column field, and at least one column
|
|
680
697
|
# isn't present in the data, then it should not fit the next transformer
|
|
681
698
|
if self._field_in_data(column_name, data):
|
|
@@ -2,7 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
|
|
5
|
-
from rdt.performance.datasets import
|
|
5
|
+
from rdt.performance.datasets import (
|
|
6
|
+
boolean,
|
|
7
|
+
categorical,
|
|
8
|
+
datetime,
|
|
9
|
+
numerical,
|
|
10
|
+
pii,
|
|
11
|
+
text,
|
|
12
|
+
)
|
|
6
13
|
from rdt.performance.datasets.base import BaseDatasetGenerator
|
|
7
14
|
|
|
8
15
|
__all__ = [
|
|
@@ -36,18 +36,12 @@ class RandomBooleanGenerator(BooleanGenerator):
|
|
|
36
36
|
def get_performance_thresholds():
|
|
37
37
|
"""Return the expected threseholds."""
|
|
38
38
|
return {
|
|
39
|
-
'fit': {
|
|
40
|
-
|
|
41
|
-
'memory': 400.0
|
|
42
|
-
},
|
|
43
|
-
'transform': {
|
|
44
|
-
'time': 1e-5,
|
|
45
|
-
'memory': 400.0
|
|
46
|
-
},
|
|
39
|
+
'fit': {'time': 2e-5, 'memory': 400.0},
|
|
40
|
+
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
47
41
|
'reverse_transform': {
|
|
48
42
|
'time': 5e-5,
|
|
49
43
|
'memory': 500.0,
|
|
50
|
-
}
|
|
44
|
+
},
|
|
51
45
|
}
|
|
52
46
|
|
|
53
47
|
|
|
@@ -71,18 +65,12 @@ class RandomBooleanNaNsGenerator(BooleanGenerator):
|
|
|
71
65
|
def get_performance_thresholds():
|
|
72
66
|
"""Return the expected threseholds."""
|
|
73
67
|
return {
|
|
74
|
-
'fit': {
|
|
75
|
-
|
|
76
|
-
'memory': 400.0
|
|
77
|
-
},
|
|
78
|
-
'transform': {
|
|
79
|
-
'time': 1e-5,
|
|
80
|
-
'memory': 1000.0
|
|
81
|
-
},
|
|
68
|
+
'fit': {'time': 2e-5, 'memory': 400.0},
|
|
69
|
+
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
82
70
|
'reverse_transform': {
|
|
83
71
|
'time': 5e-5,
|
|
84
72
|
'memory': 1000.0,
|
|
85
|
-
}
|
|
73
|
+
},
|
|
86
74
|
}
|
|
87
75
|
|
|
88
76
|
|
|
@@ -104,18 +92,12 @@ class RandomSkewedBooleanGenerator(BooleanGenerator):
|
|
|
104
92
|
def get_performance_thresholds():
|
|
105
93
|
"""Return the expected threseholds."""
|
|
106
94
|
return {
|
|
107
|
-
'fit': {
|
|
108
|
-
|
|
109
|
-
'memory': 400.0
|
|
110
|
-
},
|
|
111
|
-
'transform': {
|
|
112
|
-
'time': 1e-5,
|
|
113
|
-
'memory': 400.0
|
|
114
|
-
},
|
|
95
|
+
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
96
|
+
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
115
97
|
'reverse_transform': {
|
|
116
98
|
'time': 5e-5,
|
|
117
99
|
'memory': 500.0,
|
|
118
|
-
}
|
|
100
|
+
},
|
|
119
101
|
}
|
|
120
102
|
|
|
121
103
|
|
|
@@ -139,18 +121,12 @@ class RandomSkewedBooleanNaNsGenerator(BooleanGenerator):
|
|
|
139
121
|
def get_performance_thresholds():
|
|
140
122
|
"""Return the expected threseholds."""
|
|
141
123
|
return {
|
|
142
|
-
'fit': {
|
|
143
|
-
|
|
144
|
-
'memory': 400.0
|
|
145
|
-
},
|
|
146
|
-
'transform': {
|
|
147
|
-
'time': 1e-5,
|
|
148
|
-
'memory': 1000.0
|
|
149
|
-
},
|
|
124
|
+
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
125
|
+
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
150
126
|
'reverse_transform': {
|
|
151
127
|
'time': 5e-5,
|
|
152
128
|
'memory': 1000.0,
|
|
153
|
-
}
|
|
129
|
+
},
|
|
154
130
|
}
|
|
155
131
|
|
|
156
132
|
|
|
@@ -167,18 +143,12 @@ class ConstantBooleanGenerator(BooleanGenerator):
|
|
|
167
143
|
def get_performance_thresholds():
|
|
168
144
|
"""Return the expected threseholds."""
|
|
169
145
|
return {
|
|
170
|
-
'fit': {
|
|
171
|
-
|
|
172
|
-
'memory': 400.0
|
|
173
|
-
},
|
|
174
|
-
'transform': {
|
|
175
|
-
'time': 1e-5,
|
|
176
|
-
'memory': 400.0
|
|
177
|
-
},
|
|
146
|
+
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
147
|
+
'transform': {'time': 1e-5, 'memory': 400.0},
|
|
178
148
|
'reverse_transform': {
|
|
179
149
|
'time': 5e-5,
|
|
180
150
|
'memory': 500.0,
|
|
181
|
-
}
|
|
151
|
+
},
|
|
182
152
|
}
|
|
183
153
|
|
|
184
154
|
|
|
@@ -201,16 +171,10 @@ class ConstantBooleanNaNsGenerator(BooleanGenerator):
|
|
|
201
171
|
def get_performance_thresholds():
|
|
202
172
|
"""Return the expected threseholds."""
|
|
203
173
|
return {
|
|
204
|
-
'fit': {
|
|
205
|
-
|
|
206
|
-
'memory': 400.0
|
|
207
|
-
},
|
|
208
|
-
'transform': {
|
|
209
|
-
'time': 1e-5,
|
|
210
|
-
'memory': 1000.0
|
|
211
|
-
},
|
|
174
|
+
'fit': {'time': 1e-5, 'memory': 400.0},
|
|
175
|
+
'transform': {'time': 1e-5, 'memory': 1000.0},
|
|
212
176
|
'reverse_transform': {
|
|
213
177
|
'time': 5e-5,
|
|
214
178
|
'memory': 1000.0,
|
|
215
|
-
}
|
|
179
|
+
},
|
|
216
180
|
}
|