rdt 1.6.0__tar.gz → 1.6.1.dev0__tar.gz

{rdt-1.6.0 → rdt-1.6.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.6.0
+Version: 1.6.1.dev0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.

{rdt-1.6.0 → rdt-1.6.1.dev0}/rdt/__init__.py

@@ -5,7 +5,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.6.0'
+__version__ = '1.6.1.dev0'
 
 
 import sys

{rdt-1.6.0 → rdt-1.6.1.dev0}/rdt/hyper_transformer.py

@@ -110,10 +110,8 @@ class HyperTransformer:
             self._add_field_to_set(field, self._specified_fields)
 
     def __init__(self):
-        self._default_sdtype_transformers = {}
         self.field_sdtypes = {}
         self.field_transformers = {}
-
         self._specified_fields = set()
         self._validate_field_transformers()
         self._valid_output_sdtypes = self._DEFAULT_OUTPUT_SDTYPES
@@ -489,11 +487,7 @@ class HyperTransformer:
                 self._set_field_sdtype(data, field)
             if field not in self.field_transformers:
                 sdtype = self.field_sdtypes[field]
-                if sdtype in self._default_sdtype_transformers:
-                    self.field_transformers[field] = deepcopy(
-                        self._default_sdtype_transformers[sdtype])
-                else:
-                    self.field_transformers[field] = deepcopy(get_default_transformer(sdtype))
+                self.field_transformers[field] = deepcopy(get_default_transformer(sdtype))
 
     def detect_initial_config(self, data):
         """Print the configuration of the data.
@@ -508,7 +502,6 @@ class HyperTransformer:
                 Data which will have its configuration detected.
         """
         # Reset the state of the HyperTransformer
-        self._default_sdtype_transformers = {}
         self.field_sdtypes = {}
         self.field_transformers = {}
 

{rdt-1.6.0 → rdt-1.6.1.dev0}/rdt/transformers/__init__.py

@@ -88,9 +88,11 @@ TRANSFORMERS = {
 
 DEFAULT_TRANSFORMERS = {
     'numerical': FloatFormatter(),
-    'categorical': FrequencyEncoder(),
-    'boolean': BinaryEncoder(),
+    'categorical': LabelEncoder(add_noise=True),
+    'boolean': LabelEncoder(add_noise=True),
     'datetime': UnixTimestampEncoder(),
+    'text': RegexGenerator(),
+    'pii': AnonymizedFaker(),
 }
 
 

{rdt-1.6.0 → rdt-1.6.1.dev0}/rdt/transformers/categorical.py

@@ -439,6 +439,7 @@ class LabelEncoder(BaseTransformer):
     SUPPORTED_SDTYPES = ['categorical', 'boolean']
     values_to_categories = None
     categories_to_values = None
+    dtype = 'O'
 
     def __init__(self, add_noise=False, order_by=None):
         super().__init__()
@@ -483,6 +484,7 @@ class LabelEncoder(BaseTransformer):
             data (pandas.Series):
                 Data to fit the transformer to.
         """
+        self.dtype = data.dtype
         unique_data = pd.unique(data.fillna(np.nan))
         unique_data = self._order_categories(unique_data)
         self.values_to_categories = dict(enumerate(unique_data))
@@ -524,6 +526,7 @@ class LabelEncoder(BaseTransformer):
         )
 
         if self.add_noise:
+            mapped = mapped.astype(float)
             mapped = np.random.uniform(mapped, mapped + 1)
 
         return mapped
@@ -542,7 +545,9 @@ class LabelEncoder(BaseTransformer):
             data = np.floor(data)
 
         data = data.clip(min(self.values_to_categories), max(self.values_to_categories))
-        return data.round().map(self.values_to_categories)
+        data = data.round().map(self.values_to_categories)
+
+        return data.astype(self.dtype)
 
 
 class OrderedLabelEncoder(LabelEncoder):

{rdt-1.6.0 → rdt-1.6.1.dev0}/rdt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.6.0
+Version: 1.6.1.dev0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.

{rdt-1.6.0 → rdt-1.6.1.dev0}/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.0
+current_version = 1.6.1.dev0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?

{rdt-1.6.0 → rdt-1.6.1.dev0}/setup.py

@@ -139,6 +139,6 @@ setup(
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/sdv-dev/RDT',
-    version='1.6.0',
+    version='1.6.1.dev0',
     zip_safe=False,
 )

{rdt-1.6.0 → rdt-1.6.1.dev0}/tests/integration/test_hyper_transformer.py

@@ -92,16 +92,25 @@ def get_transformed_data():
     return pd.DataFrame({
         'integer': [1., 2., 1., 3., 1., 4., 2., 3.],
         'float': [0.1, 0.2, 0.1, 0.2, 0.1, 0.4, 0.2, 0.3],
-        'categorical': [0.3125, 0.3125, .8125, 0.8125, 0.3125, 0.8125, 0.3125, 0.3125],
-        'bool': [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0],
+        'categorical': [
+            0.9690758764963199, 0.8816575994729887, 1.1326495454234662, 1.7988488918189502,
+            0.9265972159030215, 1.885454600378942, 0.9280858691537548, 0.5093227924068265
+        ],
+        'bool': [
+            0.26161253184788935, 0.5735484647493089, 0.026673806296574787, 1.197229599974477,
+            0.8860641570557322, 0.33432787358513416, 1.1089412122841389, 0.6182653878449814
+        ],
         'datetime': datetimes,
-        'names': [0.3125, 0.75, 0.75, 0.3125, 0.3125, 0.9375, 0.3125, 0.3125]
+        'names': [
+            0.24180193241041126, 1.9297787196579723, 1.5617500744772101, 0.6811042561384157,
+            0.48017218468846856, 2.2867787591284823, 0.25476586891248476, 0.620052082101593
+        ]
     }, index=TEST_DATA_INDEX)
 
 
 def get_reversed_data():
     data = get_input_data()
-    data['bool'] = data['bool'].astype('object')
+
     return data
 
 
@@ -161,10 +170,19 @@ def test_hypertransformer_default_inputs():
     expected_transformed = pd.DataFrame({
         'integer': [1., 2., 1., 3., 1., 4., 2., 3.],
         'float': [0.1, 0.2, 0.1, 0.2, 0.1, 0.4, 0.2, 0.3],
-        'categorical': [0.3125, 0.3125, 0.9375, 0.75, 0.3125, 0.75, 0.3125, 0.3125],
-        'bool': [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0],
+        'categorical': [
+            0.9690758764963199, 0.8816575994729887, 1.1326495454234662, 2.7988488918189502,
+            0.9265972159030215, 2.8854546003789423, 0.9280858691537548, 0.5093227924068265
+        ],
+        'bool': [
+            0.26161253184788935, 1.573548464749309, 0.026673806296574787, 2.1972295999744773,
+            0.8860641570557322, 1.334327873585134, 2.108941212284139, 0.6182653878449814
+        ],
         'datetime': expected_datetimes,
-        'names': [0.3125, 0.75, 0.75, 0.3125, 0.3125, 0.9375, 0.3125, 0.3125]
+        'names': [
+            0.24180193241041126, 1.9297787196579723, 1.5617500744772101, 0.6811042561384157,
+            0.48017218468846856, 2.2867787591284823, 0.25476586891248476, 0.620052082101593
+        ]
     }, index=TEST_DATA_INDEX)
     pd.testing.assert_frame_equal(transformed, expected_transformed)
 
@@ -194,10 +212,10 @@ def test_hypertransformer_default_inputs():
 
     assert isinstance(ht.field_transformers['integer'], FloatFormatter)
     assert isinstance(ht.field_transformers['float'], FloatFormatter)
-    assert isinstance(ht.field_transformers['categorical'], FrequencyEncoder)
-    assert isinstance(ht.field_transformers['bool'], BinaryEncoder)
+    assert isinstance(ht.field_transformers['categorical'], LabelEncoder)
+    assert isinstance(ht.field_transformers['bool'], LabelEncoder)
     assert isinstance(ht.field_transformers['datetime'], UnixTimestampEncoder)
-    assert isinstance(ht.field_transformers['names'], FrequencyEncoder)
+    assert isinstance(ht.field_transformers['names'], LabelEncoder)
 
     get_default_transformers.cache_clear()
     get_default_transformer.cache_clear()
@@ -236,10 +254,10 @@ def test_hypertransformer_field_transformers():
         'transformers': {
             'integer': FloatFormatter(missing_value_replacement='mean'),
             'float': FloatFormatter(missing_value_replacement='mean'),
-            'categorical': FrequencyEncoder(),
-            'bool': BinaryEncoder(missing_value_replacement='mode'),
+            'categorical': LabelEncoder(add_noise=True),
+            'bool': LabelEncoder(add_noise=True),
             'datetime': DummyTransformerNotMLReady(),
-            'names': FrequencyEncoder()
+            'names': LabelEncoder(add_noise=True)
         }
     }
 

{rdt-1.6.0 → rdt-1.6.1.dev0}/tests/unit/test_hyper_transformer.py

@@ -108,7 +108,6 @@ class TestHyperTransformer(TestCase):
 
         # Asserts
         assert ht.field_sdtypes == {}
-        assert ht._default_sdtype_transformers == {}
         assert ht.field_transformers == {}
         assert ht._specified_fields == set()
         assert ht._valid_output_sdtypes == ht._DEFAULT_OUTPUT_SDTYPES
@@ -217,19 +216,26 @@ class TestHyperTransformer(TestCase):
         """
         # Setup
         data = self.get_data()
+        data['pii'] = ['a', 'b', 'c', 'd']
+        data['text'] = ['e', 'f', 'g', 'h']
         field_transformers = {
             'integer': FloatFormatter(),
             'float': ClusterBasedNormalizer(),
         }
-        default_sdtype_transformers = {
-            'boolean': BinaryEncoder(),
-            'categorical': FrequencyEncoder()
-        }
-        get_default_transformer_mock.return_value = UnixTimestampEncoder()
+        get_default_transformer_mock.side_effect = [
+            LabelEncoder(),
+            LabelEncoder(),
+            UnixTimestampEncoder(),
+            AnonymizedFaker(),
+            RegexGenerator(),
+        ]
         ht = HyperTransformer()
         ht.field_transformers = field_transformers
-        ht.field_sdtypes = {'datetime': 'datetime'}
-        ht._default_sdtype_transformers = default_sdtype_transformers
+        ht.field_sdtypes = {
+            'datetime': 'datetime',
+            'pii': 'pii',
+            'text': 'text'
+        }
         ht._unfit = Mock()
 
         # Run
@@ -241,14 +247,18 @@ class TestHyperTransformer(TestCase):
             'float': 'numerical',
             'bool': 'boolean',
             'categorical': 'categorical',
-            'datetime': 'datetime'
+            'datetime': 'datetime',
+            'pii': 'pii',
+            'text': 'text',
         }
 
         assert isinstance(ht.field_transformers['integer'], FloatFormatter)
         assert isinstance(ht.field_transformers['float'], ClusterBasedNormalizer)
-        assert isinstance(ht.field_transformers['categorical'], FrequencyEncoder)
-        assert isinstance(ht.field_transformers['bool'], BinaryEncoder)
+        assert isinstance(ht.field_transformers['categorical'], LabelEncoder)
+        assert isinstance(ht.field_transformers['bool'], LabelEncoder)
         assert isinstance(ht.field_transformers['datetime'], UnixTimestampEncoder)
+        assert isinstance(ht.field_transformers['pii'], AnonymizedFaker)
+        assert isinstance(ht.field_transformers['text'], RegexGenerator)
         ht._unfit.assert_called_once()
 
     @patch('rdt.hyper_transformer.LOGGER')
@@ -286,8 +296,8 @@ class TestHyperTransformer(TestCase):
         field_transformers = {k: repr(v) for (k, v) in ht.field_transformers.items()}
         assert field_transformers == {
             'col1': 'FloatFormatter()',
-            'col2': 'FrequencyEncoder()',
-            'col3': 'BinaryEncoder()',
+            'col2': 'LabelEncoder(add_noise=True)',
+            'col3': 'LabelEncoder(add_noise=True)',
             'col4': 'UnixTimestampEncoder()',
             'col5': 'FloatFormatter()'
         }
@@ -303,8 +313,8 @@ class TestHyperTransformer(TestCase):
             '    },',
             '    "transformers": {',
             '        "col1": FloatFormatter(),',
-            '        "col2": FrequencyEncoder(),',
-            '        "col3": BinaryEncoder(),',
+            '        "col2": LabelEncoder(add_noise=True),',
+            '        "col3": LabelEncoder(add_noise=True),',
             '        "col4": UnixTimestampEncoder(),',
             '        "col5": FloatFormatter()',
             '    }',

{rdt-1.6.0 → rdt-1.6.1.dev0}/tests/unit/transformers/test___init__.py

@@ -1,6 +1,8 @@
 import pytest
 
-from rdt.transformers import BinaryEncoder, get_transformer_class, get_transformer_name
+from rdt.transformers import (
+    AnonymizedFaker, BinaryEncoder, FloatFormatter, LabelEncoder, RegexGenerator,
+    UnixTimestampEncoder, get_default_transformers, get_transformer_class, get_transformer_name)
 from rdt.transformers.addons.identity.identity import IdentityTransformer
 
 
@@ -94,3 +96,25 @@ def test_get_transformer_class_transformer_path_addon():
 
     # Assert
     assert returned == IdentityTransformer
+
+
+def test_get_default_transformers():
+    """Test the ``get_default_transformers`` method.
+
+    Check that the right default transformer is returned for each type.
+    """
+    # Run
+    default_transformer_dict = get_default_transformers()
+
+    # Assert
+    expected_dict = {
+        'numerical': FloatFormatter,
+        'categorical': LabelEncoder,
+        'boolean': LabelEncoder,
+        'datetime': UnixTimestampEncoder,
+        'text': RegexGenerator,
+        'pii': AnonymizedFaker,
+    }
+
+    for sdtype, transformer in expected_dict.items():
+        assert isinstance(default_transformer_dict[sdtype], transformer)