rdt 1.9.1__tar.gz → 1.9.2.dev0__tar.gz

{rdt-1.9.1 → rdt-1.9.2.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.9.1
+Version: 1.9.2.dev0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/__init__.py

@@ -5,7 +5,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.9.1'
+__version__ = '1.9.2.dev0'
 
 
 import sys

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/hyper_transformer.py

@@ -357,6 +357,32 @@ class HyperTransformer:
 
         self.field_transformers[new_tuple] = self.field_transformers.pop(old_tuple)
 
+    def _update_multi_column_transformer(self):
+        """Check that multi-columns mappings are valid and update them otherwise."""
+        all_fields_multi_column = set()
+        for columns, transformer in self.field_transformers.items():
+            if isinstance(transformer, BaseMultiColumnTransformer):
+                all_fields_multi_column.add(columns)
+
+        for field in all_fields_multi_column:
+            transformer = self.field_transformers[field]
+
+            columns_to_sdtypes = self._get_columns_to_sdtypes(field)
+            try:
+                transformer._validate_sdtypes(   # pylint: disable=protected-access
+                    columns_to_sdtypes
+                )
+            except TransformerInputError:
+                warnings.warn(
+                    f"Transformer '{transformer.get_name()}' is incompatible with the "
+                    f"multi-column field '{field}'. Assigning default transformer to the columns."
+                )
+                del self.field_transformers[field]
+                for column, sdtype in columns_to_sdtypes.items():
+                    self.field_transformers[column] = deepcopy(get_default_transformer(sdtype))
+
+        self._multi_column_fields = self._create_multi_column_fields()
+
     def update_transformers_by_sdtype(
             self, sdtype, transformer=None, transformer_name=None, transformer_parameters=None):
         """Update the transformers for the specified ``sdtype``.
@@ -397,6 +423,7 @@ class HyperTransformer:
                     self._remove_column_in_multi_column_fields(field)
 
         self._multi_column_fields = self._create_multi_column_fields()
+        self._update_multi_column_transformer()
         self._modified_config = True
 
     def update_sdtypes(self, column_name_to_sdtype):
@@ -445,6 +472,7 @@ class HyperTransformer:
         )
 
         self._multi_column_fields = self._create_multi_column_fields()
+        self._update_multi_column_transformer()
         self._modified_config = True
         if self._fitted:
             warnings.warn(self._REFIT_MESSAGE)
@@ -485,6 +513,7 @@ class HyperTransformer:
             self.field_transformers[column_name] = transformer
 
         self._multi_column_fields = self._create_multi_column_fields()
+        self._update_multi_column_transformer()
         self._modified_config = True
 
     def remove_transformers(self, column_names):
@@ -514,6 +543,7 @@ class HyperTransformer:
 
             self.field_transformers[column_name] = None
 
+        self._update_multi_column_transformer()
         if self._fitted:
             warnings.warn(self._REFIT_MESSAGE)
 
@@ -540,6 +570,7 @@ class HyperTransformer:
 
                 self.field_transformers[column_name] = None
 
+        self._update_multi_column_transformer()
         if self._fitted:
             warnings.warn(self._REFIT_MESSAGE)
 

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/transformers/base.py

@@ -547,6 +547,10 @@ class BaseMultiColumnTransformer(BaseTransformer):
             missing_to_print = ', '.join(missing)
             raise ValueError(f'Columns ({missing_to_print}) are not present in the data.')
 
+    @classmethod
+    def _validate_sdtypes(cls, columns_to_sdtypes):
+        raise NotImplementedError()
+
     def _fit(self, data):
         """Fit the transformer to the data.
 

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/transformers/categorical.py

@@ -9,7 +9,7 @@ from scipy.stats import norm
 
 from rdt.errors import TransformerInputError
 from rdt.transformers.base import BaseTransformer
-from rdt.transformers.utils import fill_nan_with_none
+from rdt.transformers.utils import check_nan_in_transform, fill_nan_with_none, try_convert_to_dtype
 
 LOGGER = logging.getLogger(__name__)
 
@@ -177,6 +177,7 @@ class UniformEncoder(BaseTransformer):
         Returns:
             pandas.Series
         """
+        check_nan_in_transform(data, self.dtype)
         data = data.clip(0, 1)
         bins = [0]
         labels = []
@@ -192,7 +193,10 @@ class UniformEncoder(BaseTransformer):
                 labels.append(key)
 
         result = pd.cut(data, bins=bins, labels=labels, include_lowest=True)
-        return result.replace(nan_name, np.nan).astype(self.dtype)
+        result = result.replace(nan_name, np.nan)
+        result = try_convert_to_dtype(result, self.dtype)
+
+        return result
 
 
 class OrderedUniformEncoder(UniformEncoder):
@@ -333,6 +337,7 @@ class FrequencyEncoder(BaseTransformer):
         )
         super().__init__()
         self.add_noise = add_noise
+        self._is_integer = None
 
     @staticmethod
     def _get_intervals(data):
@@ -516,6 +521,7 @@ class FrequencyEncoder(BaseTransformer):
         Returns:
             pandas.Series
         """
+        check_nan_in_transform(data, self.dtype)
         data = data.clip(0, 1)
         num_rows = len(data)
         num_categories = len(self.means)
@@ -545,6 +551,7 @@ class OneHotEncoder(BaseTransformer):
     _dummy_encoded = False
     _indexer = None
     _uniques = None
+    dtype = None
 
     @staticmethod
     def _prepare_data(data):
@@ -582,6 +589,7 @@ class OneHotEncoder(BaseTransformer):
             data (pandas.Series or pandas.DataFrame):
                 Data to fit the transformer to.
         """
+        self.dtype = data.dtype
         data = self._prepare_data(data)
 
         null = pd.isna(data).to_numpy()
@@ -591,7 +599,7 @@ class OneHotEncoder(BaseTransformer):
         self._indexer = list(range(self._num_dummies))
         self.dummies = self._uniques.copy()
 
-        if not np.issubdtype(data.dtype, np.number):
+        if not np.issubdtype(data.dtype.type, np.number):
             self._dummy_encoded = True
 
         if self._dummy_na:
@@ -657,6 +665,7 @@ class OneHotEncoder(BaseTransformer):
         Returns:
             pandas.Series
         """
+        check_nan_in_transform(data, self.dtype)
         if not isinstance(data, np.ndarray):
             data = data.to_numpy()
 
@@ -664,8 +673,10 @@ class OneHotEncoder(BaseTransformer):
             data = data.reshape(-1, 1)
 
         indices = np.argmax(data, axis=1)
+        result = pd.Series(indices).map(self.dummies.__getitem__)
+        result = try_convert_to_dtype(result, self.dtype)
 
-        return pd.Series(indices).map(self.dummies.__getitem__)
+        return result
 
 
 class LabelEncoder(BaseTransformer):
@@ -801,13 +812,15 @@ class LabelEncoder(BaseTransformer):
         Returns:
             pandas.Series
         """
+        check_nan_in_transform(data, self.dtype)
         if self.add_noise:
             data = np.floor(data)
 
         data = data.clip(min(self.values_to_categories), max(self.values_to_categories))
         data = data.round().map(self.values_to_categories)
+        data = try_convert_to_dtype(data, self.dtype)
 
-        return data.astype(self.dtype)
+        return data
 
 
 class OrderedLabelEncoder(LabelEncoder):
@@ -865,6 +878,7 @@ class OrderedLabelEncoder(LabelEncoder):
             data (pandas.Series):
                 Data to fit the transformer to.
         """
+        self.dtype = data.dtype
         data = data.fillna(np.nan)
         missing = list(data[~data.isin(self.order)].unique())
         if len(missing) > 0:

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/transformers/numerical.py

@@ -1,7 +1,5 @@
 """Transformers for numerical data."""
 import copy
-import logging
-import sys
 import warnings
 
 import numpy as np
@@ -11,11 +9,9 @@ from sklearn.mixture import BayesianGaussianMixture
 
 from rdt.transformers.base import BaseTransformer
 from rdt.transformers.null import NullTransformer
-
-LOGGER = logging.getLogger(__name__)
+from rdt.transformers.utils import learn_rounding_digits
 
 EPSILON = np.finfo(np.float32).eps
-MAX_DECIMALS = sys.float_info.dig - 1
 INTEGER_BOUNDS = {
     'Int8': (-2**7, 2**7 - 1),
     'Int16': (-2**15, 2**15 - 1),
@@ -91,31 +87,6 @@ class FloatFormatter(BaseTransformer):
         self.enforce_min_max_values = enforce_min_max_values
         self.computer_representation = computer_representation
 
-    @staticmethod
-    def _learn_rounding_digits(data):
-        # check if data has any decimals
-        name = data.name
-        data = np.array(data)
-        roundable_data = data[~(np.isinf(data) | pd.isna(data))]
-
-        # Doesn't contain numbers
-        if len(roundable_data) == 0:
-            return None
-
-        # Doesn't contain decimal digits
-        if ((roundable_data % 1) == 0).all():
-            return 0
-
-        # Try to round to fewer digits
-        if (roundable_data == roundable_data.round(MAX_DECIMALS)).all():
-            for decimal in range(MAX_DECIMALS + 1):
-                if (roundable_data == roundable_data.round(decimal)).all():
-                    return decimal
-
-        # Can't round, not equal after MAX_DECIMALS digits of precision
-        LOGGER.info("No rounding scheme detected for column '%s'. Data will not be rounded.", name)
-        return None
-
     def _raise_out_of_bounds_error(self, value, name, bound_type, min_bound, max_bound):
         raise ValueError(
             f"The {bound_type} value in column '{name}' is {value}."
@@ -158,7 +129,7 @@ class FloatFormatter(BaseTransformer):
             self._max_value = data.max()
 
         if self.learn_rounding_scheme:
-            self._rounding_digits = self._learn_rounding_digits(data)
+            self._rounding_digits = learn_rounding_digits(data)
 
         self.null_transformer = NullTransformer(
             self.missing_value_replacement,

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/transformers/text.py

@@ -1,13 +1,15 @@
 """Transformers for text data."""
+import logging
 import warnings
 
 import numpy as np
 import pandas as pd
 
-from rdt.errors import TransformerProcessingError
 from rdt.transformers.base import BaseTransformer
 from rdt.transformers.utils import strings_from_regex
 
+LOGGER = logging.getLogger(__name__)
+
 
 class IDGenerator(BaseTransformer):
     """Generate an ID column.
@@ -139,6 +141,37 @@ class RegexGenerator(BaseTransformer):
         """Drop the input column by returning ``None``."""
         return None
 
+    def _warn_not_enough_unique_values(self, sample_size):
+        """Warn the user that the regex cannot generate enough unique values.
+
+        Args:
+            sample_size (int):
+                Number of samples to be generated.
+        """
+        warned = False
+        if sample_size > self.generator_size:
+            if self.enforce_uniqueness:
+                warnings.warn(
+                    f"The regex for '{self.get_input_column()}' can only generate "
+                    f'{self.generator_size} unique values. Additional values may not exactly '
+                    'follow the provided regex.'
+                )
+                warned = True
+            else:
+                LOGGER.info(
+                    "The data has %s rows but the regex for '%s' can only create %s unique values."
+                    " Some values in '%s' may be repeated.",
+                    sample_size, self.get_input_column(), self.generator_size,
+                    self.get_input_column()
+                )
+
+        remaining = self.generator_size - self.generated
+        if sample_size > remaining and self.enforce_uniqueness and not warned:
+            warnings.warn(
+                f'The regex generator is not able to generate {sample_size} new unique '
+                f'values (only {max(remaining, 0)} unique values left).'
+            )
+
     def _reverse_transform(self, data):
         """Generate new data using the provided ``regex_format``.
 
@@ -154,47 +187,39 @@ class RegexGenerator(BaseTransformer):
         else:
             sample_size = self.data_length
 
-        if sample_size > self.generator_size:
-            if self.enforce_uniqueness:
-                raise TransformerProcessingError(
-                    f'The regex is not able to generate {sample_size} unique values. '
-                    f"Please use a different regex for column ('{self.get_input_column()}')."
-                )
-
-            warnings.warn(
-                f"The data has {sample_size} rows but the regex for '{self.get_input_column()}' "
-                f'can only create {self.generator_size} unique values. Some values in '
-                f"'{self.get_input_column()}' may be repeated."
-            )
+        self._warn_not_enough_unique_values(sample_size)
 
         remaining = self.generator_size - self.generated
-        if sample_size > self.generator_size - self.generated:
-            if self.enforce_uniqueness:
-                raise TransformerProcessingError(
-                    f'The regex generator is not able to generate {sample_size} new unique '
-                    f'values (only {remaining} unique value left). Please use '
-                    "'reset_randomization' in order to restart the generator."
-                )
-
+        if sample_size > remaining:
             self.reset_randomization()
             remaining = self.generator_size
 
         if remaining >= sample_size:
-            reverse_transformed = np.array([
-                next(self.generator)
-                for _ in range(sample_size)
-            ], dtype=object)
-
+            reverse_transformed = [next(self.generator) for _ in range(sample_size)]
             self.generated += sample_size
 
         else:
-            self.generated = self.generator_size
             generated_values = list(self.generator)
-            reverse_transformed = []
-            while len(reverse_transformed) < sample_size:
-                remaining_samples = sample_size - len(reverse_transformed)
-                reverse_transformed.extend(generated_values[:remaining_samples])
-
-            reverse_transformed = np.array(reverse_transformed, dtype=object)
-
-        return reverse_transformed
+            reverse_transformed = generated_values[:]
+            self.generated = self.generator_size
+            if self.enforce_uniqueness:
+                try:
+                    remaining_samples = sample_size - len(reverse_transformed)
+                    start = int(generated_values[-1]) + 1
+                    reverse_transformed.extend(
+                        [str(i) for i in range(start, start + remaining_samples)])
+
+                except ValueError:
+                    counter = 0
+                    while len(reverse_transformed) < sample_size:
+                        remaining_samples = sample_size - len(reverse_transformed)
+                        reverse_transformed.extend(
+                            [f'{i}({counter})' for i in generated_values[:remaining_samples]])
+                        counter += 1
+
+            else:
+                while len(reverse_transformed) < sample_size:
+                    remaining_samples = sample_size - len(reverse_transformed)
+                    reverse_transformed.extend(generated_values[:remaining_samples])
+
+        return np.array(reverse_transformed, dtype=object)

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt/transformers/utils.py

@@ -1,12 +1,20 @@
 """Tools to generate strings from regular expressions."""
 
+import logging
 import re
 import string
+import sys
+import warnings
 
 import numpy as np
+import pandas as pd
 
 import sre_parse  # isort:skip
 
+LOGGER = logging.getLogger(__name__)
+
+MAX_DECIMALS = sys.float_info.dig - 1
+
 
 def _literal(character, max_repeat):
     del max_repeat
@@ -149,7 +157,7 @@ def strings_from_regex(regex, max_repeat=16):
             generators.append((generator, option, args))
             sizes.append(size)
 
-    return _from_generators(generators, max_repeat), np.prod(sizes, dtype=np.complex128)
+    return _from_generators(generators, max_repeat), np.prod(sizes, dtype=np.complex128).real
 
 
 def fill_nan_with_none(data):
@@ -184,3 +192,86 @@ def flatten_column_list(column_list):
             flattened.append(column)
 
     return flattened
+
+
+def check_nan_in_transform(data, dtype):
+    """Check if there are null values in the transformed data.
+
+    Args:
+        data (pd.Series or numpy.ndarray):
+            Data that has been transformed.
+        dtype (str):
+            Data type of the transformed data.
+    """
+    if pd.isna(data).any().any():
+        message = (
+            'There are null values in the transformed data. The reversed '
+            'transformed data will contain null values'
+        )
+        is_integer = pd.api.types.is_integer_dtype(dtype)
+        if is_integer:
+            message += " of type 'float'."
+        else:
+            message += '.'
+
+        warnings.warn(message)
+
+
+def try_convert_to_dtype(data, dtype):
+    """Try to convert data to a given dtype.
+
+    Args:
+        data (pd.Series or numpy.ndarray):
+            Data to convert.
+        dtype (str):
+            Data type to convert to.
+
+    Returns:
+        data:
+            Data converted to the given dtype.
+    """
+    try:
+        data = data.astype(dtype)
+    except ValueError as error:
+        is_integer = pd.api.types.is_integer_dtype(dtype)
+        if is_integer:
+            data = data.astype(float)
+        else:
+            raise error
+
+    return data
+
+
+def learn_rounding_digits(data):
+    """Learn the number of digits to round data to.
+
+    Args:
+        data (pd.Series):
+            Data to learn the number of digits to round to.
+
+    Returns:
+        int or None:
+            Number of digits to round to.
+    """
+    # check if data has any decimals
+    name = data.name
+    data = np.array(data)
+    roundable_data = data[~(np.isinf(data) | pd.isna(data))]
+
+    # Doesn't contain numbers
+    if len(roundable_data) == 0:
+        return None
+
+    # Doesn't contain decimal digits
+    if ((roundable_data % 1) == 0).all():
+        return 0
+
+    # Try to round to fewer digits
+    if (roundable_data == roundable_data.round(MAX_DECIMALS)).all():
+        for decimal in range(MAX_DECIMALS + 1):
+            if (roundable_data == roundable_data.round(decimal)).all():
+                return decimal
+
+    # Can't round, not equal after MAX_DECIMALS digits of precision
+    LOGGER.info("No rounding scheme detected for column '%s'. Data will not be rounded.", name)
+    return None

{rdt-1.9.1 → rdt-1.9.2.dev0}/rdt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.9.1
+Version: 1.9.2.dev0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.

{rdt-1.9.1 → rdt-1.9.2.dev0}/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.1
+current_version = 1.9.2.dev0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?

{rdt-1.9.1 → rdt-1.9.2.dev0}/setup.py

@@ -138,6 +138,6 @@ setup(
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/sdv-dev/RDT',
-    version='1.9.1',
+    version='1.9.2.dev0',
     zip_safe=False,
 )

{rdt-1.9.1 → rdt-1.9.2.dev0}/tests/integration/test_hyper_transformer.py

@@ -7,7 +7,8 @@ import pandas as pd
 import pytest
 
 from rdt import get_demo
-from rdt.errors import ConfigNotSetError, InvalidConfigError, InvalidDataError, NotFittedError
+from rdt.errors import (
+    ConfigNotSetError, InvalidConfigError, InvalidDataError, NotFittedError, TransformerInputError)
 from rdt.hyper_transformer import Config, HyperTransformer
 from rdt.transformers import (
     AnonymizedFaker, BaseMultiColumnTransformer, BaseTransformer, BinaryEncoder,
@@ -67,6 +68,10 @@ class DummyMultiColumnTransformerNumerical(BaseMultiColumnTransformer):
             } for column in self.columns
         }
 
+    @classmethod
+    def _validate_sdtypes(cls, columns_to_sdtype):
+        return None
+
     def _get_prefix(self):
         return None
 
@@ -1846,3 +1851,100 @@ class TestHyperTransformer:
             ]
         })
         pd.testing.assert_frame_equal(result, expected_results)
+
+    expected_sdtype = {
+        'sdtypes': {
+            'A': 'categorical',
+            'B': 'categorical',
+            'D': 'categorical',
+            'E': 'categorical',
+            'C': 'boolean'
+        }
+    }
+    expected_transformer_update = {
+        'transformers': {
+            'A': UniformEncoder(),
+            'E': UniformEncoder(),
+            'C': UniformEncoder(),
+            'B': UniformEncoder(),
+            'D': UniformEncoder()
+        }
+    }
+    expected_transformer_remove = {
+        'transformers': {
+            'A': UniformEncoder(),
+            'E': UniformEncoder(),
+            'C': None,
+            'B': UniformEncoder(),
+            'D': UniformEncoder()
+        }
+    }
+    expected_update = {
+        **expected_sdtype,
+        **expected_transformer_update
+    }
+    expected_remove = {
+        **expected_sdtype,
+        **expected_transformer_remove
+    }
+
+    parametrization = [
+        (
+            'update_transformers', {'column_name_to_transformer': {'C': UniformEncoder()}},
+            expected_update
+        ),
+        (
+            'update_transformers_by_sdtype',
+            {'sdtype': 'boolean', 'transformer': UniformEncoder()}, expected_update
+        ),
+        ('remove_transformers', {'column_names': 'C'}, expected_remove),
+        ('remove_transformers_by_sdtype', {'sdtype': 'boolean'}, expected_remove),
+    ]
+
+    @pytest.mark.parametrize(('method_name', 'method_input', 'expected_result'), parametrization)
+    def test_invalid_multi_column(self, method_name, method_input, expected_result):
+        """Test the ``update`` and ``remove`` methods with invalid multi column transformer.
+
+        When a multi column is no longer valid, all these methods should raise a warning
+        and assign the default transformer to the columns.
+        """
+        # Setup
+        class BadDummyMultiColumnTransformer(DummyMultiColumnTransformerNumerical):
+
+            @classmethod
+            def _validate_sdtypes(cls, columns_to_sdtype):
+                raise TransformerInputError('Invalid sdtype')
+
+        dict_config = {
+            'sdtypes': {
+                'A': 'categorical',
+                'B': 'categorical',
+                'D': 'categorical',
+                'E': 'categorical',
+                'C': 'boolean',
+            },
+            'transformers': {
+                'A': UniformEncoder(),
+                ('B', 'D', 'C'): BadDummyMultiColumnTransformer(),
+                'E': UniformEncoder()
+            }
+        }
+
+        config = Config(dict_config)
+        ht = HyperTransformer()
+        ht.set_config(config)
+
+        # Run
+        expected_warning = re.escape(
+            "Transformer 'BadDummyMultiColumnTransformer' is incompatible with the "
+            "multi-column field '('B', 'D')'. Assigning default transformer to the columns."
+        )
+        with pytest.warns(UserWarning, match=expected_warning):
+            ht.__getattribute__(method_name)(**method_input)
+
+        # Assert
+        new_config = ht.get_config()
+        expected_config = Config(expected_result)
+        expected_multi_columns = {}
+        assert ht._multi_column_fields == expected_multi_columns
+        assert repr(new_config) == repr(expected_config)

{rdt-1.9.1 → rdt-1.9.2.dev0}/tests/integration/transformers/test_base.py

@@ -136,6 +136,7 @@ def test_multi_column_transformer_same_number_of_columns_input_output():
     # Setup
     class AdditionTransformer(BaseMultiColumnTransformer):
         """This transformer takes 3 columns and return the cumulative sum of each row."""
+
         def _fit(self, columns_data):
             self.output_properties = {
                 f'{self.columns[0]}': {'sdtype': 'numerical'},
@@ -188,6 +189,7 @@ def test_multi_column_transformer_less_output_than_input_columns():
         """This transformer takes 4 columns and concatenate them into 2 columns.
         The two first and last columns are concatenated together.
         """
+
         def _fit(self, columns_data):
             self.name_1 = self.columns[0] + '#' + self.columns[1]
             self.name_2 = self.columns[2] + '#' + self.columns[3]