rdt 1.14.0.dev0__tar.gz → 1.15.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rdt-1.14.0.dev0/rdt.egg-info → rdt-1.15.0.dev0}/PKG-INFO +12 -7
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/pyproject.toml +12 -7
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/__init__.py +1 -1
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/__init__.py +3 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/base.py +11 -11
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/boolean.py +1 -1
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/datetime.py +2 -2
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/numerical.py +224 -3
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/utils.py +56 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0/rdt.egg-info}/PKG-INFO +12 -7
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt.egg-info/requires.txt +8 -2
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/LICENSE +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/README.md +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/errors.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/hyper_transformer.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/__init__.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/__init__.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/base.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/boolean.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/categorical.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/datetime.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/id.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/numerical.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/pii.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/text.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/datasets/utils.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/performance.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/performance/profiling.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/_validators.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/categorical.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/id.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/null.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/pii/__init__.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/pii/anonymization.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/pii/anonymizer.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/pii/utils.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt/transformers/text.py +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt.egg-info/SOURCES.txt +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt.egg-info/dependency_links.txt +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt.egg-info/entry_points.txt +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/rdt.egg-info/top_level.txt +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/setup.cfg +0 -0
- {rdt-1.14.0.dev0 → rdt-1.15.0.dev0}/tests/test_tasks.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: rdt
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.15.0.dev0
|
|
4
4
|
Summary: Reversible Data Transforms
|
|
5
5
|
Author-email: "DataCebo, Inc." <info@sdv.dev>
|
|
6
6
|
License: BSL-1.1
|
|
@@ -20,26 +20,31 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
22
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
24
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
-
Requires-Python: <3.
|
|
25
|
+
Requires-Python: <3.14,>=3.8
|
|
25
26
|
Description-Content-Type: text/markdown
|
|
26
27
|
License-File: LICENSE
|
|
27
28
|
Requires-Dist: numpy>=1.21.0; python_version < "3.10"
|
|
28
29
|
Requires-Dist: numpy>=1.23.3; python_version >= "3.10" and python_version < "3.12"
|
|
29
|
-
Requires-Dist: numpy>=1.26.0; python_version >= "3.12"
|
|
30
|
+
Requires-Dist: numpy>=1.26.0; python_version >= "3.12" and python_version < "3.13"
|
|
31
|
+
Requires-Dist: numpy>=2.1.0; python_version >= "3.13"
|
|
30
32
|
Requires-Dist: pandas>=1.4.0; python_version < "3.11"
|
|
31
33
|
Requires-Dist: pandas>=1.5.0; python_version >= "3.11" and python_version < "3.12"
|
|
32
|
-
Requires-Dist: pandas>=2.1.1; python_version >= "3.12"
|
|
34
|
+
Requires-Dist: pandas>=2.1.1; python_version >= "3.12" and python_version < "3.13"
|
|
35
|
+
Requires-Dist: pandas>=2.2.3; python_version >= "3.13"
|
|
33
36
|
Requires-Dist: scipy>=1.7.3; python_version < "3.10"
|
|
34
37
|
Requires-Dist: scipy>=1.9.2; python_version >= "3.10" and python_version < "3.12"
|
|
35
|
-
Requires-Dist: scipy>=1.12.0; python_version >= "3.12"
|
|
38
|
+
Requires-Dist: scipy>=1.12.0; python_version >= "3.12" and python_version < "3.13"
|
|
39
|
+
Requires-Dist: scipy>=1.14.1; python_version >= "3.13"
|
|
36
40
|
Requires-Dist: scikit-learn>=1.0.2; python_version < "3.10"
|
|
37
41
|
Requires-Dist: scikit-learn>=1.1.0; python_version >= "3.10" and python_version < "3.11"
|
|
38
42
|
Requires-Dist: scikit-learn>=1.1.3; python_version >= "3.11" and python_version < "3.12"
|
|
39
|
-
Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12"
|
|
43
|
+
Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12" and python_version < "3.13"
|
|
44
|
+
Requires-Dist: scikit-learn>=1.5.2; python_version >= "3.13"
|
|
40
45
|
Requires-Dist: Faker>=17
|
|
41
46
|
Provides-Extra: copulas
|
|
42
|
-
Requires-Dist: copulas>=0.12.
|
|
47
|
+
Requires-Dist: copulas>=0.12.1; extra == "copulas"
|
|
43
48
|
Provides-Extra: pyarrow
|
|
44
49
|
Requires-Dist: pyarrow>=17.0.0; extra == "pyarrow"
|
|
45
50
|
Provides-Extra: test
|
|
@@ -13,27 +13,32 @@ classifiers = [
|
|
|
13
13
|
'Programming Language :: Python :: 3.10',
|
|
14
14
|
'Programming Language :: Python :: 3.11',
|
|
15
15
|
'Programming Language :: Python :: 3.12',
|
|
16
|
+
'Programming Language :: Python :: 3.13',
|
|
16
17
|
'Topic :: Scientific/Engineering :: Artificial Intelligence',
|
|
17
18
|
]
|
|
18
19
|
keywords = ['machine learning', 'synthetic data generation', 'benchmark', 'generative models']
|
|
19
20
|
dynamic = ['version']
|
|
20
21
|
license = { text = 'BSL-1.1' }
|
|
21
|
-
requires-python = '>=3.8,<3.
|
|
22
|
+
requires-python = '>=3.8,<3.14'
|
|
22
23
|
readme = 'README.md'
|
|
23
24
|
dependencies = [
|
|
24
25
|
"numpy>=1.21.0;python_version<'3.10'",
|
|
25
26
|
"numpy>=1.23.3;python_version>='3.10' and python_version<'3.12'",
|
|
26
|
-
"numpy>=1.26.0;python_version>='3.12'",
|
|
27
|
+
"numpy>=1.26.0;python_version>='3.12' and python_version<'3.13'",
|
|
28
|
+
"numpy>=2.1.0;python_version>='3.13'",
|
|
27
29
|
"pandas>=1.4.0;python_version<'3.11'",
|
|
28
30
|
"pandas>=1.5.0;python_version>='3.11' and python_version<'3.12'",
|
|
29
|
-
"pandas>=2.1.1;python_version>='3.12'",
|
|
31
|
+
"pandas>=2.1.1;python_version>='3.12' and python_version<'3.13'",
|
|
32
|
+
"pandas>=2.2.3;python_version>='3.13'",
|
|
30
33
|
"scipy>=1.7.3;python_version<'3.10'",
|
|
31
34
|
"scipy>=1.9.2;python_version>='3.10' and python_version<'3.12'",
|
|
32
|
-
"scipy>=1.12.0;python_version>='3.12'",
|
|
35
|
+
"scipy>=1.12.0;python_version>='3.12' and python_version<'3.13'",
|
|
36
|
+
"scipy>=1.14.1;python_version>='3.13'",
|
|
33
37
|
"scikit-learn>=1.0.2;python_version<'3.10'",
|
|
34
38
|
"scikit-learn>=1.1.0;python_version>='3.10' and python_version<'3.11'",
|
|
35
39
|
"scikit-learn>=1.1.3;python_version>='3.11' and python_version<'3.12'",
|
|
36
|
-
"scikit-learn>=1.3.1;python_version>='3.12'",
|
|
40
|
+
"scikit-learn>=1.3.1;python_version>='3.12' and python_version<'3.13'",
|
|
41
|
+
"scikit-learn>=1.5.2;python_version>='3.13'",
|
|
37
42
|
'Faker>=17',
|
|
38
43
|
]
|
|
39
44
|
|
|
@@ -48,7 +53,7 @@ dependencies = [
|
|
|
48
53
|
rdt = { main = 'rdt.cli.__main__:main' }
|
|
49
54
|
|
|
50
55
|
[project.optional-dependencies]
|
|
51
|
-
copulas = ['copulas>=0.12.
|
|
56
|
+
copulas = ['copulas>=0.12.1',]
|
|
52
57
|
pyarrow = ['pyarrow>=17.0.0']
|
|
53
58
|
test = [
|
|
54
59
|
'rdt[pyarrow]',
|
|
@@ -139,7 +144,7 @@ collect_ignore = ['pyproject.toml']
|
|
|
139
144
|
exclude_lines = ['NotImplementedError()']
|
|
140
145
|
|
|
141
146
|
[tool.bumpversion]
|
|
142
|
-
current_version = "1.
|
|
147
|
+
current_version = "1.15.0.dev0"
|
|
143
148
|
parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
|
|
144
149
|
serialize = [
|
|
145
150
|
'{major}.{minor}.{patch}.{release}{candidate}',
|
|
@@ -28,6 +28,8 @@ from rdt.transformers.numerical import (
|
|
|
28
28
|
ClusterBasedNormalizer,
|
|
29
29
|
FloatFormatter,
|
|
30
30
|
GaussianNormalizer,
|
|
31
|
+
LogScaler,
|
|
32
|
+
LogitScaler,
|
|
31
33
|
)
|
|
32
34
|
from rdt.transformers.pii.anonymizer import (
|
|
33
35
|
AnonymizedFaker,
|
|
@@ -46,6 +48,7 @@ __all__ = [
|
|
|
46
48
|
'FrequencyEncoder',
|
|
47
49
|
'GaussianNormalizer',
|
|
48
50
|
'LabelEncoder',
|
|
51
|
+
'LogScaler',
|
|
49
52
|
'NullTransformer',
|
|
50
53
|
'OneHotEncoder',
|
|
51
54
|
'OptimizedTimestampEncoder',
|
|
@@ -142,17 +142,6 @@ class BaseTransformer:
|
|
|
142
142
|
elif model_missing_values is False:
|
|
143
143
|
self._set_missing_value_generation('random')
|
|
144
144
|
|
|
145
|
-
def _set_missing_value_replacement(self, default, missing_value_replacement):
|
|
146
|
-
if missing_value_replacement is None:
|
|
147
|
-
warnings.warn(
|
|
148
|
-
"Setting 'missing_value_replacement' to 'None' is no longer supported. "
|
|
149
|
-
f"Imputing with the '{default}' instead.",
|
|
150
|
-
FutureWarning,
|
|
151
|
-
)
|
|
152
|
-
self.missing_value_replacement = default
|
|
153
|
-
else:
|
|
154
|
-
self.missing_value_replacement = missing_value_replacement
|
|
155
|
-
|
|
156
145
|
@classmethod
|
|
157
146
|
def get_name(cls):
|
|
158
147
|
"""Return transformer name.
|
|
@@ -478,7 +467,18 @@ class BaseTransformer:
|
|
|
478
467
|
|
|
479
468
|
data = data.copy()
|
|
480
469
|
columns_data = self._get_columns_data(data, self.output_columns)
|
|
470
|
+
original_missing_values = self.missing_value_generation
|
|
471
|
+
if self.missing_value_generation is not None and pd.isna(columns_data).any().any():
|
|
472
|
+
warnings.warn(
|
|
473
|
+
"The 'missing_value_generation' parameter is set to '"
|
|
474
|
+
f"{self.missing_value_generation}' but the data already contains missing values."
|
|
475
|
+
' Missing value generation will be skipped.',
|
|
476
|
+
UserWarning,
|
|
477
|
+
)
|
|
478
|
+
self.missing_value_generation = None
|
|
479
|
+
|
|
481
480
|
reversed_data = self._reverse_transform(columns_data)
|
|
481
|
+
self.missing_value_generation = original_missing_values
|
|
482
482
|
data = data.drop(self.output_columns, axis=1)
|
|
483
483
|
data = self._add_columns_to_data(data, reversed_data, self.columns)
|
|
484
484
|
|
|
@@ -46,8 +46,8 @@ class BinaryEncoder(BaseTransformer):
|
|
|
46
46
|
missing_value_generation='random',
|
|
47
47
|
):
|
|
48
48
|
super().__init__()
|
|
49
|
+
self.missing_value_replacement = missing_value_replacement
|
|
49
50
|
self._set_missing_value_generation(missing_value_generation)
|
|
50
|
-
self._set_missing_value_replacement('random', missing_value_replacement)
|
|
51
51
|
if model_missing_values is not None:
|
|
52
52
|
self._set_model_missing_values(model_missing_values)
|
|
53
53
|
|
|
@@ -60,7 +60,7 @@ class UnixTimestampEncoder(BaseTransformer):
|
|
|
60
60
|
enforce_min_max_values=False,
|
|
61
61
|
):
|
|
62
62
|
super().__init__()
|
|
63
|
-
self.
|
|
63
|
+
self.missing_value_replacement = missing_value_replacement
|
|
64
64
|
self._set_missing_value_generation(missing_value_generation)
|
|
65
65
|
self.enforce_min_max_values = enforce_min_max_values
|
|
66
66
|
if model_missing_values is not None:
|
|
@@ -275,7 +275,7 @@ class OptimizedTimestampEncoder(UnixTimestampEncoder):
|
|
|
275
275
|
|
|
276
276
|
def __init__(
|
|
277
277
|
self,
|
|
278
|
-
missing_value_replacement=
|
|
278
|
+
missing_value_replacement='mean',
|
|
279
279
|
model_missing_values=None,
|
|
280
280
|
datetime_format=None,
|
|
281
281
|
missing_value_generation='random',
|
|
@@ -8,10 +8,10 @@ import numpy as np
|
|
|
8
8
|
import pandas as pd
|
|
9
9
|
import scipy
|
|
10
10
|
|
|
11
|
-
from rdt.errors import TransformerInputError
|
|
11
|
+
from rdt.errors import InvalidDataError, TransformerInputError
|
|
12
12
|
from rdt.transformers.base import BaseTransformer
|
|
13
13
|
from rdt.transformers.null import NullTransformer
|
|
14
|
-
from rdt.transformers.utils import learn_rounding_digits
|
|
14
|
+
from rdt.transformers.utils import learn_rounding_digits, logit, sigmoid
|
|
15
15
|
|
|
16
16
|
EPSILON = np.finfo(np.float32).eps
|
|
17
17
|
INTEGER_BOUNDS = {
|
|
@@ -87,7 +87,6 @@ class FloatFormatter(BaseTransformer):
|
|
|
87
87
|
super().__init__()
|
|
88
88
|
self.missing_value_replacement = missing_value_replacement
|
|
89
89
|
self._set_missing_value_generation(missing_value_generation)
|
|
90
|
-
self._set_missing_value_replacement('mean', missing_value_replacement)
|
|
91
90
|
if model_missing_values is not None:
|
|
92
91
|
self._set_model_missing_values(model_missing_values)
|
|
93
92
|
|
|
@@ -626,3 +625,225 @@ class ClusterBasedNormalizer(FloatFormatter):
|
|
|
626
625
|
recovered_data = np.stack([recovered_data, data[:, -1]], axis=1) # noqa: PD013
|
|
627
626
|
|
|
628
627
|
return super()._reverse_transform(recovered_data)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
class LogitScaler(FloatFormatter):
|
|
631
|
+
"""Transformer for numerical data by applying a logit function.
|
|
632
|
+
|
|
633
|
+
This transformer works by replacing the values with a scaled
|
|
634
|
+
version and then applying a logit function. The reverse transform
|
|
635
|
+
applies a sigmoid to the data and then scales it back to the original space.
|
|
636
|
+
|
|
637
|
+
Null values are replaced using a ``NullTransformer``.
|
|
638
|
+
|
|
639
|
+
Args:
|
|
640
|
+
missing_value_replacement (object):
|
|
641
|
+
Indicate what to replace the null values with. If an integer or float is given,
|
|
642
|
+
replace them with the given value. If the strings ``'mean'`` or ``'mode'``
|
|
643
|
+
are given, replace them with the corresponding aggregation and if ``'random'``
|
|
644
|
+
replace each null value with a random value in the data range. Defaults to ``mean``.
|
|
645
|
+
missing_value_generation (str or None):
|
|
646
|
+
The way missing values are being handled. There are three strategies:
|
|
647
|
+
|
|
648
|
+
* ``random``: Randomly generates missing values based on the percentage of
|
|
649
|
+
missing values.
|
|
650
|
+
* ``from_column``: Creates a binary column that describes whether the original
|
|
651
|
+
value was missing. Then use it to recreate missing values.
|
|
652
|
+
* ``None``: Do nothing with the missing values on the reverse transform. Simply
|
|
653
|
+
pass whatever data we get through.
|
|
654
|
+
min_value (float):
|
|
655
|
+
The min value for the logit function. Defaults to 0.
|
|
656
|
+
max_value (float):
|
|
657
|
+
max_value (float): The max value for the logit function. Defaults to 1.0.
|
|
658
|
+
learn_rounding_scheme (bool):
|
|
659
|
+
Whether or not to learn what place to round to based on the data seen during ``fit``.
|
|
660
|
+
If ``True``, the data returned by ``reverse_transform`` will be rounded to that place.
|
|
661
|
+
Defaults to ``False``.
|
|
662
|
+
"""
|
|
663
|
+
|
|
664
|
+
def __init__(
|
|
665
|
+
self,
|
|
666
|
+
missing_value_replacement='mean',
|
|
667
|
+
missing_value_generation='random',
|
|
668
|
+
min_value=0.0,
|
|
669
|
+
max_value=1.0,
|
|
670
|
+
learn_rounding_scheme=False,
|
|
671
|
+
):
|
|
672
|
+
if not (isinstance(min_value, int) or isinstance(min_value, float)) or not (
|
|
673
|
+
isinstance(max_value, int) or isinstance(max_value, float)
|
|
674
|
+
):
|
|
675
|
+
error_msg = 'The min_value and max_value must be of type int or float.'
|
|
676
|
+
raise TransformerInputError(error_msg)
|
|
677
|
+
if min_value == max_value:
|
|
678
|
+
error_msg = 'The min_value and max_value for the logit function cannot be equal.'
|
|
679
|
+
raise TransformerInputError(error_msg)
|
|
680
|
+
|
|
681
|
+
super().__init__(
|
|
682
|
+
missing_value_replacement=missing_value_replacement,
|
|
683
|
+
missing_value_generation=missing_value_generation,
|
|
684
|
+
learn_rounding_scheme=learn_rounding_scheme,
|
|
685
|
+
)
|
|
686
|
+
self.min_value = min_value
|
|
687
|
+
self.max_value = max_value
|
|
688
|
+
|
|
689
|
+
def _validate_logit_inputs(self, data):
|
|
690
|
+
out_of_range_vals = data[(data < self.min_value) | (data > self.max_value)]
|
|
691
|
+
if len(out_of_range_vals) > 0:
|
|
692
|
+
num_vals_to_print = 5
|
|
693
|
+
out_of_range_vals = [str(x) for x in sorted(out_of_range_vals, key=lambda x: str(x))]
|
|
694
|
+
if len(out_of_range_vals) > 5:
|
|
695
|
+
extra_missing_vals = f'+ {len(out_of_range_vals) - num_vals_to_print} more'
|
|
696
|
+
out_of_range_vals = (
|
|
697
|
+
f'[{", ".join(out_of_range_vals[:num_vals_to_print])} {extra_missing_vals}]'
|
|
698
|
+
)
|
|
699
|
+
else:
|
|
700
|
+
out_of_range_vals = f'[{", ".join(out_of_range_vals)}]'
|
|
701
|
+
|
|
702
|
+
raise InvalidDataError(
|
|
703
|
+
f"Unable to apply logit function to column '{self.columns[0]}' due to out of "
|
|
704
|
+
f'range values ({out_of_range_vals}).'
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
def _fit(self, data):
|
|
708
|
+
self._validate_logit_inputs(data)
|
|
709
|
+
return super()._fit(data)
|
|
710
|
+
|
|
711
|
+
def _transform(self, data):
|
|
712
|
+
transformed = super()._transform(data)
|
|
713
|
+
transformed_vals = transformed if transformed.ndim == 1 else transformed[:, 0]
|
|
714
|
+
self._validate_logit_inputs(transformed_vals)
|
|
715
|
+
logit_vals = logit(transformed_vals, self.min_value, self.max_value)
|
|
716
|
+
if transformed.ndim == 1:
|
|
717
|
+
return logit_vals
|
|
718
|
+
|
|
719
|
+
transformed[:, 0] = logit_vals
|
|
720
|
+
return transformed
|
|
721
|
+
|
|
722
|
+
def _reverse_transform(self, data):
|
|
723
|
+
if not isinstance(data, np.ndarray):
|
|
724
|
+
data = data.to_numpy()
|
|
725
|
+
|
|
726
|
+
sampled_vals = data if data.ndim == 1 else data[:, 0]
|
|
727
|
+
reversed_values = sigmoid(sampled_vals, self.min_value, self.max_value)
|
|
728
|
+
if data.ndim == 1:
|
|
729
|
+
return super()._reverse_transform(reversed_values)
|
|
730
|
+
|
|
731
|
+
data[:, 0] = reversed_values
|
|
732
|
+
return super()._reverse_transform(data)
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
class LogScaler(FloatFormatter):
|
|
736
|
+
"""Transformer for numerical data using log.
|
|
737
|
+
|
|
738
|
+
This transformer scales numerical values using log and an optional constant.
|
|
739
|
+
Null values are replaced using a ``NullTransformer``.
|
|
740
|
+
|
|
741
|
+
Args:
|
|
742
|
+
missing_value_replacement (object):
|
|
743
|
+
Indicate what to replace the null values with. If an integer or float is given,
|
|
744
|
+
replace them with the given value. If the strings ``'mean'`` or ``'mode'``
|
|
745
|
+
are given, replace them with the corresponding aggregation and if ``'random'``
|
|
746
|
+
replace each null value with a random value in the data range. Defaults to ``mean``.
|
|
747
|
+
missing_value_generation (str or None):
|
|
748
|
+
The way missing values are being handled. There are three strategies:
|
|
749
|
+
* ``random``: Randomly generates missing values based on the percentage of
|
|
750
|
+
missing values.
|
|
751
|
+
* ``from_column``: Creates a binary column that describes whether the original
|
|
752
|
+
value was missing. Then use it to recreate missing values.
|
|
753
|
+
* ``None``: Do nothing with the missing values on the reverse transform. Simply
|
|
754
|
+
pass whatever data we get through.
|
|
755
|
+
constant (float):
|
|
756
|
+
The constant to set as the 0-value for the log-based transform. Defaults to 0
|
|
757
|
+
(do not modify the 0-value of the data).
|
|
758
|
+
invert (bool):
|
|
759
|
+
Whether to invert the data with respect to the constant value. If False, do not
|
|
760
|
+
invert the data (all values will be greater than the constant value). If True,
|
|
761
|
+
invert the data (all the values will be less than the constant value).
|
|
762
|
+
Defaults to False.
|
|
763
|
+
learn_rounding_scheme (bool):
|
|
764
|
+
Whether or not to learn what place to round to based on the data seen during ``fit``.
|
|
765
|
+
If ``True``, the data returned by ``reverse_transform`` will be rounded to that place.
|
|
766
|
+
Defaults to ``False``.
|
|
767
|
+
"""
|
|
768
|
+
|
|
769
|
+
def __init__(
|
|
770
|
+
self,
|
|
771
|
+
missing_value_replacement='mean',
|
|
772
|
+
missing_value_generation='random',
|
|
773
|
+
constant: float = 0.0,
|
|
774
|
+
invert: bool = False,
|
|
775
|
+
learn_rounding_scheme: bool = False,
|
|
776
|
+
):
|
|
777
|
+
if isinstance(constant, (int, float)):
|
|
778
|
+
self.constant = constant
|
|
779
|
+
else:
|
|
780
|
+
raise ValueError('The constant parameter must be a float or int.')
|
|
781
|
+
if isinstance(invert, bool):
|
|
782
|
+
self.invert = invert
|
|
783
|
+
else:
|
|
784
|
+
raise ValueError('The invert parameter must be a bool.')
|
|
785
|
+
|
|
786
|
+
super().__init__(
|
|
787
|
+
missing_value_replacement=missing_value_replacement,
|
|
788
|
+
missing_value_generation=missing_value_generation,
|
|
789
|
+
learn_rounding_scheme=learn_rounding_scheme,
|
|
790
|
+
)
|
|
791
|
+
|
|
792
|
+
def _validate_data(self, data: pd.Series):
|
|
793
|
+
column_name = self.get_input_column()
|
|
794
|
+
if self.invert:
|
|
795
|
+
if not all(data < self.constant):
|
|
796
|
+
raise InvalidDataError(
|
|
797
|
+
f"Unable to apply a log transform to column '{column_name}' due to constant"
|
|
798
|
+
' being too small.'
|
|
799
|
+
)
|
|
800
|
+
else:
|
|
801
|
+
if not all(data > self.constant):
|
|
802
|
+
raise InvalidDataError(
|
|
803
|
+
f"Unable to apply a log transform to column '{column_name}' due to constant"
|
|
804
|
+
' being too large.'
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
def _fit(self, data):
|
|
808
|
+
super()._fit(data)
|
|
809
|
+
data = super()._transform(data)
|
|
810
|
+
|
|
811
|
+
if data.ndim > 1:
|
|
812
|
+
self._validate_data(data[:, 0])
|
|
813
|
+
else:
|
|
814
|
+
self._validate_data(data)
|
|
815
|
+
|
|
816
|
+
def _log_transform(self, data):
|
|
817
|
+
self._validate_data(data)
|
|
818
|
+
|
|
819
|
+
if self.invert:
|
|
820
|
+
return np.log(self.constant - data)
|
|
821
|
+
else:
|
|
822
|
+
return np.log(data - self.constant)
|
|
823
|
+
|
|
824
|
+
def _transform(self, data):
|
|
825
|
+
data = super()._transform(data)
|
|
826
|
+
|
|
827
|
+
if data.ndim > 1:
|
|
828
|
+
data[:, 0] = self._log_transform(data[:, 0])
|
|
829
|
+
else:
|
|
830
|
+
data = self._log_transform(data)
|
|
831
|
+
|
|
832
|
+
return data
|
|
833
|
+
|
|
834
|
+
def _reverse_log(self, data):
|
|
835
|
+
if self.invert:
|
|
836
|
+
return self.constant - np.exp(data)
|
|
837
|
+
else:
|
|
838
|
+
return np.exp(data) + self.constant
|
|
839
|
+
|
|
840
|
+
def _reverse_transform(self, data):
|
|
841
|
+
if not isinstance(data, np.ndarray):
|
|
842
|
+
data = data.to_numpy()
|
|
843
|
+
|
|
844
|
+
if data.ndim > 1:
|
|
845
|
+
data[:, 0] = self._reverse_log(data[:, 0])
|
|
846
|
+
else:
|
|
847
|
+
data = self._reverse_log(data)
|
|
848
|
+
|
|
849
|
+
return super()._reverse_transform(data)
|
|
@@ -6,6 +6,7 @@ import string
|
|
|
6
6
|
import sys
|
|
7
7
|
import warnings
|
|
8
8
|
from collections import defaultdict
|
|
9
|
+
from decimal import Decimal
|
|
9
10
|
|
|
10
11
|
import numpy as np
|
|
11
12
|
import pandas as pd
|
|
@@ -126,6 +127,17 @@ def _from_generators(generators, max_repeat):
|
|
|
126
127
|
yield ''.join(reversed(generated))
|
|
127
128
|
|
|
128
129
|
|
|
130
|
+
def _cast_to_type(data, dtype):
|
|
131
|
+
if isinstance(data, pd.Series):
|
|
132
|
+
data = data.apply(dtype)
|
|
133
|
+
elif isinstance(data, (np.ndarray, list)):
|
|
134
|
+
data = np.array([dtype(value) for value in data])
|
|
135
|
+
else:
|
|
136
|
+
data = dtype(data)
|
|
137
|
+
|
|
138
|
+
return data
|
|
139
|
+
|
|
140
|
+
|
|
129
141
|
def strings_from_regex(regex, max_repeat=16):
|
|
130
142
|
"""Generate strings that match the given regular expression.
|
|
131
143
|
|
|
@@ -280,6 +292,50 @@ def learn_rounding_digits(data):
|
|
|
280
292
|
return None
|
|
281
293
|
|
|
282
294
|
|
|
295
|
+
def logit(data, low, high):
|
|
296
|
+
"""Apply a logit function to the data using ``low`` and ``high``.
|
|
297
|
+
|
|
298
|
+
Args:
|
|
299
|
+
data (pd.Series, pd.DataFrame, np.array, int, or float):
|
|
300
|
+
Data to apply the logit function to.
|
|
301
|
+
low (pd.Series, np.array, int, or float):
|
|
302
|
+
Low value/s to use when scaling.
|
|
303
|
+
high (pd.Series, np.array, int, or float):
|
|
304
|
+
High value/s to use when scaling.
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
Logit scaled version of the input data.
|
|
308
|
+
"""
|
|
309
|
+
data = (data - low) / (high - low)
|
|
310
|
+
data = _cast_to_type(data, Decimal)
|
|
311
|
+
data = data * Decimal(0.95) + Decimal(0.025)
|
|
312
|
+
data = _cast_to_type(data, float)
|
|
313
|
+
return np.log(data / (1.0 - data))
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def sigmoid(data, low, high):
|
|
317
|
+
"""Apply a sigmoid function to the data using ``low`` and ``high``.
|
|
318
|
+
|
|
319
|
+
Args:
|
|
320
|
+
data (pd.Series, pd.DataFrame, np.array, int, float or datetime):
|
|
321
|
+
Data to apply the logit function to.
|
|
322
|
+
low (pd.Series, np.array, int, float or datetime):
|
|
323
|
+
Low value/s to use when scaling.
|
|
324
|
+
high (pd.Series, np.array, int, float or datetime):
|
|
325
|
+
High value/s to use when scaling.
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
Sigmoid transform of the input data.
|
|
329
|
+
"""
|
|
330
|
+
data = 1 / (1 + np.exp(-data))
|
|
331
|
+
data = _cast_to_type(data, Decimal)
|
|
332
|
+
data = (data - Decimal(0.025)) / Decimal(0.95)
|
|
333
|
+
data = _cast_to_type(data, float)
|
|
334
|
+
data = data * (high - low) + low
|
|
335
|
+
|
|
336
|
+
return data
|
|
337
|
+
|
|
338
|
+
|
|
283
339
|
class WarnDict(dict):
|
|
284
340
|
"""Custom dictionary to raise a deprecation warning."""
|
|
285
341
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: rdt
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.15.0.dev0
|
|
4
4
|
Summary: Reversible Data Transforms
|
|
5
5
|
Author-email: "DataCebo, Inc." <info@sdv.dev>
|
|
6
6
|
License: BSL-1.1
|
|
@@ -20,26 +20,31 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
22
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
24
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
-
Requires-Python: <3.
|
|
25
|
+
Requires-Python: <3.14,>=3.8
|
|
25
26
|
Description-Content-Type: text/markdown
|
|
26
27
|
License-File: LICENSE
|
|
27
28
|
Requires-Dist: numpy>=1.21.0; python_version < "3.10"
|
|
28
29
|
Requires-Dist: numpy>=1.23.3; python_version >= "3.10" and python_version < "3.12"
|
|
29
|
-
Requires-Dist: numpy>=1.26.0; python_version >= "3.12"
|
|
30
|
+
Requires-Dist: numpy>=1.26.0; python_version >= "3.12" and python_version < "3.13"
|
|
31
|
+
Requires-Dist: numpy>=2.1.0; python_version >= "3.13"
|
|
30
32
|
Requires-Dist: pandas>=1.4.0; python_version < "3.11"
|
|
31
33
|
Requires-Dist: pandas>=1.5.0; python_version >= "3.11" and python_version < "3.12"
|
|
32
|
-
Requires-Dist: pandas>=2.1.1; python_version >= "3.12"
|
|
34
|
+
Requires-Dist: pandas>=2.1.1; python_version >= "3.12" and python_version < "3.13"
|
|
35
|
+
Requires-Dist: pandas>=2.2.3; python_version >= "3.13"
|
|
33
36
|
Requires-Dist: scipy>=1.7.3; python_version < "3.10"
|
|
34
37
|
Requires-Dist: scipy>=1.9.2; python_version >= "3.10" and python_version < "3.12"
|
|
35
|
-
Requires-Dist: scipy>=1.12.0; python_version >= "3.12"
|
|
38
|
+
Requires-Dist: scipy>=1.12.0; python_version >= "3.12" and python_version < "3.13"
|
|
39
|
+
Requires-Dist: scipy>=1.14.1; python_version >= "3.13"
|
|
36
40
|
Requires-Dist: scikit-learn>=1.0.2; python_version < "3.10"
|
|
37
41
|
Requires-Dist: scikit-learn>=1.1.0; python_version >= "3.10" and python_version < "3.11"
|
|
38
42
|
Requires-Dist: scikit-learn>=1.1.3; python_version >= "3.11" and python_version < "3.12"
|
|
39
|
-
Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12"
|
|
43
|
+
Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12" and python_version < "3.13"
|
|
44
|
+
Requires-Dist: scikit-learn>=1.5.2; python_version >= "3.13"
|
|
40
45
|
Requires-Dist: Faker>=17
|
|
41
46
|
Provides-Extra: copulas
|
|
42
|
-
Requires-Dist: copulas>=0.12.
|
|
47
|
+
Requires-Dist: copulas>=0.12.1; extra == "copulas"
|
|
43
48
|
Provides-Extra: pyarrow
|
|
44
49
|
Requires-Dist: pyarrow>=17.0.0; extra == "pyarrow"
|
|
45
50
|
Provides-Extra: test
|
|
@@ -19,14 +19,20 @@ scipy>=1.9.2
|
|
|
19
19
|
pandas>=1.5.0
|
|
20
20
|
scikit-learn>=1.1.3
|
|
21
21
|
|
|
22
|
-
[:python_version >= "3.12"]
|
|
22
|
+
[:python_version >= "3.12" and python_version < "3.13"]
|
|
23
23
|
numpy>=1.26.0
|
|
24
24
|
pandas>=2.1.1
|
|
25
25
|
scipy>=1.12.0
|
|
26
26
|
scikit-learn>=1.3.1
|
|
27
27
|
|
|
28
|
+
[:python_version >= "3.13"]
|
|
29
|
+
numpy>=2.1.0
|
|
30
|
+
pandas>=2.2.3
|
|
31
|
+
scipy>=1.14.1
|
|
32
|
+
scikit-learn>=1.5.2
|
|
33
|
+
|
|
28
34
|
[copulas]
|
|
29
|
-
copulas>=0.12.
|
|
35
|
+
copulas>=0.12.1
|
|
30
36
|
|
|
31
37
|
[dev]
|
|
32
38
|
rdt[test]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|