rdt 1.5.1.dev1__tar.gz → 1.6.0__tar.gz

{rdt-1.5.1.dev1 → rdt-1.6.0}/HISTORY.md

@@ -1,5 +1,29 @@
 # History
 
+## 1.6.0 -2023-07-12
+
+This release adds the ability to generate missing values to the `AnonymizedFaker`. Users can now provide the `missing_value_generation` parameter during initialization. They can set it to `None` to not generate any missing values, or `'random'` to generate random missing values in the same proportion as the fitted data.
+
+Additionally, this release improves the `NullTransformer` by allowing nulls to be replaced on the forward transform even if `missing_value_generation` is set to None. It also fixes a bug that was causing the `UnixTimestampEncoder` to return a different dtype than the input on `reverse_transform`. This was particularly problematic when datetime columns are represented as ints.
+
+### New Features
+
+* AnonymizedFaker should be able to model and generate missing values - Issue [#660](https://github.com/sdv-dev/RDT/issues/660) by @R-Palazzo
+
+### Bugs
+
+* The datetime transformers don't give me back the same dtype sometimes - Issue [#657](https://github.com/sdv-dev/RDT/issues/657) by @frances-h
+* RDT NullTransformer doesn't replace nulls if missing_value_generation is None - Issue [#658](https://github.com/sdv-dev/RDT/issues/658) by @amontanez24
+
+### Maintenance
+
+* Remove python 3.7 builds - Issue [#663](https://github.com/sdv-dev/RDT/issues/663) by @amontanez24
+* Drop support for Python 3.7 - Issue [#666](https://github.com/sdv-dev/RDT/issues/666) by @amontanez24
+
+### Internal
+
+* Add add-on modules to sys.modules - Issue [#653](https://github.com/sdv-dev/RDT/issues/653) by @amontanez24
+
 ## 1.5.0 - 2023-06-01
 
 This release adds a new parameter called `missing_value_generation` to the initialization of certain transformers to specify how missing values should be created. The parameter can be used in the `FloatFormatter`, `BinaryEncoder`, `UnixTimestampEncoder`, `OptimizedTimestampEncoder`, `GaussianNormalizer` and `ClusterBasedNormalizer`. Additionally, it fixes a bug that was causing every column that had nulls to generate them in the same place.

{rdt-1.5.1.dev1 → rdt-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.5.1.dev1
+Version: 1.6.0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.
@@ -225,6 +225,30 @@ for specific needs.
 
 # History
 
+## 1.6.0 -2023-07-12
+
+This release adds the ability to generate missing values to the `AnonymizedFaker`. Users can now provide the `missing_value_generation` parameter during initialization. They can set it to `None` to not generate any missing values, or `'random'` to generate random missing values in the same proportion as the fitted data.
+
+Additionally, this release improves the `NullTransformer` by allowing nulls to be replaced on the forward transform even if `missing_value_generation` is set to None. It also fixes a bug that was causing the `UnixTimestampEncoder` to return a different dtype than the input on `reverse_transform`. This was particularly problematic when datetime columns are represented as ints.
+
+### New Features
+
+* AnonymizedFaker should be able to model and generate missing values - Issue [#660](https://github.com/sdv-dev/RDT/issues/660) by @R-Palazzo
+
+### Bugs
+
+* The datetime transformers don't give me back the same dtype sometimes - Issue [#657](https://github.com/sdv-dev/RDT/issues/657) by @frances-h
+* RDT NullTransformer doesn't replace nulls if missing_value_generation is None - Issue [#658](https://github.com/sdv-dev/RDT/issues/658) by @amontanez24
+
+### Maintenance
+
+* Remove python 3.7 builds - Issue [#663](https://github.com/sdv-dev/RDT/issues/663) by @amontanez24
+* Drop support for Python 3.7 - Issue [#666](https://github.com/sdv-dev/RDT/issues/666) by @amontanez24
+
+### Internal
+
+* Add add-on modules to sys.modules - Issue [#653](https://github.com/sdv-dev/RDT/issues/653) by @amontanez24
+
 ## 1.5.0 - 2023-06-01
 
 This release adds a new parameter called `missing_value_generation` to the initialization of certain transformers to specify how missing values should be created. The parameter can be used in the `FloatFormatter`, `BinaryEncoder`, `UnixTimestampEncoder`, `OptimizedTimestampEncoder`, `GaussianNormalizer` and `ClusterBasedNormalizer`. Additionally, it fixes a bug that was causing every column that had nulls to generate them in the same place.

{rdt-1.5.1.dev1 → rdt-1.6.0}/rdt/__init__.py

@@ -5,7 +5,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.5.1.dev1'
+__version__ = '1.6.0'
 
 
 import sys

{rdt-1.5.1.dev1 → rdt-1.6.0}/rdt/transformers/pii/anonymizer.py

@@ -36,6 +36,14 @@ class AnonymizedFaker(BaseTransformer):
             Whether or not to ensure that the new anonymized data is all unique. If it isn't
             possible to create the requested number of rows, then an error will be raised.
             Defaults to ``False``.
+        missing_value_generation (str or None):
+            The way missing values are being handled. There are two strategies:
+
+                * ``random``: Randomly generates missing values based on the percentage of
+                  missing values.
+                * ``None``: Don't learn anything during fit. Then during reverse transform,
+                  don't create any missing values.
+
     """
 
     IS_GENERATOR = True
@@ -85,7 +93,7 @@ class AnonymizedFaker(BaseTransformer):
             )
 
     def __init__(self, provider_name=None, function_name=None, function_kwargs=None,
-                 locales=None, enforce_uniqueness=False):
+                 locales=None, enforce_uniqueness=False, missing_value_generation='random'):
         super().__init__()
         self.data_length = None
         self.enforce_uniqueness = enforce_uniqueness
@@ -107,6 +115,15 @@ class AnonymizedFaker(BaseTransformer):
         if self.locales:
             self._check_locales()
 
+        if missing_value_generation not in ['random', None]:
+            raise TransformerInputError(
+                f"Missing value generation '{missing_value_generation}' is not supported "
+                "for AnonymizedFaker. Please use either 'random' or None."
+            )
+
+        self.missing_value_generation = missing_value_generation
+        self._nan_frequency = 0.0
+
     def reset_randomization(self):
         """Create a new ``Faker`` instance."""
         super().reset_randomization()
@@ -138,6 +155,8 @@ class AnonymizedFaker(BaseTransformer):
         """
         self._set_faker_seed(data)
         self.data_length = len(data)
+        if self.missing_value_generation == 'random':
+            self._nan_frequency = data.isna().sum() / len(data)
 
     def _transform(self, _data):
         """Drop the input column by returning ``None``."""
@@ -170,6 +189,11 @@ class AnonymizedFaker(BaseTransformer):
                 f"('{self.get_input_column()}')."
             ) from exception
 
+        if self.missing_value_generation == 'random':
+            num_nans = int(self._nan_frequency * sample_size)
+            nan_indices = np.random.choice(sample_size, num_nans, replace=False)
+            reverse_transformed[nan_indices] = np.nan
+
         return reverse_transformed
 
     def __repr__(self):

{rdt-1.5.1.dev1 → rdt-1.6.0}/rdt.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdt
-Version: 1.5.1.dev1
+Version: 1.6.0
 Summary: Reversible Data Transforms
 Home-page: https://github.com/sdv-dev/RDT
 Author: DataCebo, Inc.
@@ -225,6 +225,30 @@ for specific needs.
 
 # History
 
+## 1.6.0 -2023-07-12
+
+This release adds the ability to generate missing values to the `AnonymizedFaker`. Users can now provide the `missing_value_generation` parameter during initialization. They can set it to `None` to not generate any missing values, or `'random'` to generate random missing values in the same proportion as the fitted data.
+
+Additionally, this release improves the `NullTransformer` by allowing nulls to be replaced on the forward transform even if `missing_value_generation` is set to None. It also fixes a bug that was causing the `UnixTimestampEncoder` to return a different dtype than the input on `reverse_transform`. This was particularly problematic when datetime columns are represented as ints.
+
+### New Features
+
+* AnonymizedFaker should be able to model and generate missing values - Issue [#660](https://github.com/sdv-dev/RDT/issues/660) by @R-Palazzo
+
+### Bugs
+
+* The datetime transformers don't give me back the same dtype sometimes - Issue [#657](https://github.com/sdv-dev/RDT/issues/657) by @frances-h
+* RDT NullTransformer doesn't replace nulls if missing_value_generation is None - Issue [#658](https://github.com/sdv-dev/RDT/issues/658) by @amontanez24
+
+### Maintenance
+
+* Remove python 3.7 builds - Issue [#663](https://github.com/sdv-dev/RDT/issues/663) by @amontanez24
+* Drop support for Python 3.7 - Issue [#666](https://github.com/sdv-dev/RDT/issues/666) by @amontanez24
+
+### Internal
+
+* Add add-on modules to sys.modules - Issue [#653](https://github.com/sdv-dev/RDT/issues/653) by @amontanez24
+
 ## 1.5.0 - 2023-06-01
 
 This release adds a new parameter called `missing_value_generation` to the initialization of certain transformers to specify how missing values should be created. The parameter can be used in the `FloatFormatter`, `BinaryEncoder`, `UnixTimestampEncoder`, `OptimizedTimestampEncoder`, `GaussianNormalizer` and `ClusterBasedNormalizer`. Additionally, it fixes a bug that was causing every column that had nulls to generate them in the same place.

{rdt-1.5.1.dev1 → rdt-1.6.0}/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.1.dev1
+current_version = 1.6.0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?

{rdt-1.5.1.dev1 → rdt-1.6.0}/setup.py

@@ -139,6 +139,6 @@ setup(
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/sdv-dev/RDT',
-    version='1.5.1.dev1',
+    version='1.6.0',
     zip_safe=False,
 )

{rdt-1.5.1.dev1 → rdt-1.6.0}/tests/integration/transformers/pii/test_anonymizer.py

@@ -69,6 +69,26 @@ def test_anonymizedfaker_with_nans():
         'id': [1, 2, 3, 4, 5],
     })
 
+    pd.testing.assert_frame_equal(transformed, expected_transformed)
+    assert len(reverse_transform['username']) == 5
+    assert reverse_transform['username'].isna().sum() == 1
+
+
+def test_anonymizedfaker_with_nans_missing_value_generation_none():
+    """End to end test settings missing_value_generation=None."""
+    data = pd.DataFrame({
+        'id': [1, 2, 3, 4, 5],
+        'username': ['a', np.nan, 'c', 'd', 'e']
+    })
+
+    instance = AnonymizedFaker(missing_value_generation=None)
+    transformed = instance.fit_transform(data, 'username')
+    reverse_transform = instance.reverse_transform(transformed)
+
+    expected_transformed = pd.DataFrame({
+        'id': [1, 2, 3, 4, 5],
+    })
+
     pd.testing.assert_frame_equal(transformed, expected_transformed)
     assert len(reverse_transform['username']) == 5
     assert reverse_transform['username'].isna().sum() == 0
@@ -102,7 +122,7 @@ def test_anonymizedfaker_custom_provider_with_nans():
 
     pd.testing.assert_frame_equal(transformed, expected_transformed)
     assert len(reverse_transform['cc']) == 5
-    assert reverse_transform['cc'].isna().sum() == 0
+    assert reverse_transform['cc'].isna().sum() == 1
 
 
 def test_anonymizedfaker_enforce_uniqueness():

{rdt-1.5.1.dev1 → rdt-1.6.0}/tests/unit/transformers/pii/test_anonymizer.py

@@ -220,6 +220,23 @@ class TestAnonymizedFaker:
         assert instance.locales is None
         assert mock_faker.Faker.called_once_with(None)
         assert instance.enforce_uniqueness is False
+        assert instance.missing_value_generation == 'random'
+
+    def test___init__error_missing_value_generation(self):
+        """Test that the ``__init__`` raises an error if the missing value generation is invalid.
+
+        Test that the ``__init__`` raises an error if the missing value generation is invalid.
+
+        Side effects:
+            - ``ValueError`` is raised.
+        """
+        expected_message = (
+            "Missing value generation 'invalid' is not supported "
+            "for AnonymizedFaker. Please use either 'random' or None."
+        )
+        # Run and Assert
+        with pytest.raises(TransformerInputError, match=expected_message):
+            AnonymizedFaker(missing_value_generation='invalid')
 
     @patch('rdt.transformers.pii.anonymizer.faker')
     @patch('rdt.transformers.pii.anonymizer.AnonymizedFaker.check_provider_function')
@@ -312,15 +329,16 @@ class TestAnonymizedFaker:
         """
         # Setup
         transformer = AnonymizedFaker()
-        columns_data = pd.Series(['1', '2', '3'])
+        columns_data = pd.Series(['1', '2', '3', None, np.nan])
         transformer.columns = ['col']
 
         # Run
         transformer._fit(columns_data)
 
         # Assert
-        assert transformer.data_length == 3
+        assert transformer.data_length == 5
         assert transformer.output_properties == {None: {'next_transformer': None}}
+        assert transformer._nan_frequency == 0.4
 
     def test__transform(self):
         """Test the ``_transform`` method.
@@ -377,6 +395,26 @@ class TestAnonymizedFaker:
         assert function.call_args_list == [call(), call(), call()]
         np.testing.assert_array_equal(result, np.array(['a', 'b', 'c']))
 
+    def test__reverse_transform_with_nans(self):
+        """Test that ``_reverse_transform`` generates NaNs."""
+        # Setup
+        instance = AnonymizedFaker()
+        instance.data_length = 4
+        instance._nan_frequency = 0.25
+        function = Mock()
+        function.side_effect = ['a', 'b', 'c', 'd']
+
+        instance._function = function
+
+        # Run
+        result = instance._reverse_transform(None)
+        result = pd.Series(result)
+
+        # Assert
+        assert function.call_args_list == [call(), call(), call(), call()]
+        assert instance.missing_value_generation == 'random'
+        assert result.isna().sum() == 1
+
     def test__reverse_transform_not_enough_unique_values(self):
         """Test the ``_reverse_transform`` method.