rdt 1.21.1.dev0__tar.gz → 1.21.2.dev0__tar.gz

{rdt-1.21.1.dev0/rdt.egg-info → rdt-1.21.2.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdt
-Version: 1.21.1.dev0
+Version: 1.21.2.dev0
 Summary: Reversible Data Transforms
 Author-email: "DataCebo, Inc." <info@sdv.dev>
 License-Expression: BUSL-1.1

{rdt-1.21.1.dev0 → rdt-1.21.2.dev0}/pyproject.toml

@@ -150,7 +150,7 @@ collect_ignore = ['pyproject.toml']
 exclude_lines = ['NotImplementedError()']
 
 [tool.bumpversion]
-current_version = "1.21.1.dev0"
+current_version = "1.21.2.dev0"
 parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
 serialize = [
     '{major}.{minor}.{patch}.{release}{candidate}',

{rdt-1.21.1.dev0 → rdt-1.21.2.dev0}/rdt/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.21.1.dev0'
+__version__ = '1.21.2.dev0'
 
 
 import sys

{rdt-1.21.1.dev0 → rdt-1.21.2.dev0}/rdt/transformers/id.py

@@ -2,6 +2,7 @@
 
 import logging
 import warnings
+from itertools import count
 
 import numpy as np
 import pandas as pd
@@ -132,6 +133,25 @@ class RegexGenerator(BaseTransformer):
         state.pop('generator')
         return state
 
+    def _create_numerical_fallback_generator(self):
+        """Create fallback generator."""
+        regex_generator, _ = strings_from_regex(self.regex_format)
+        last_template = ''
+        for _ in range(self.generated):
+            last_template = next(regex_generator)
+
+        try:
+            # count is like range, but generates infinite values
+            for value in count(int(last_template) + 1):
+                yield str(value)
+
+        except ValueError:
+            # Generate values like A(0), B(0), A(1), B(1), etc
+            for counter in count():
+                regex_generator, _ = strings_from_regex(self.regex_format)
+                for value in regex_generator:
+                    yield f'{value}({counter})'
+
     def __setstate__(self, state):
         """Set the generator when pickling."""
         generator_size = state.get('generator_size')
@@ -141,13 +161,18 @@ class RegexGenerator(BaseTransformer):
             state['generator_size'] = size
         if generated is None:
             state['generated'] = 0
-
-        if generated:
+        if '_num_fallback_samples_generated' not in state:
+            state['_num_fallback_samples_generated'] = 0
+        if generated and not state['_num_fallback_samples_generated']:
             for _ in range(generated):
                 next(generator)
 
         state['generator'] = generator
         self.__dict__ = state
+        if self._num_fallback_samples_generated:
+            self.generator = self._create_numerical_fallback_generator()
+            for _ in range(self._num_fallback_samples_generated):
+                next(self.generator)
 
     def __init__(
         self,
@@ -180,12 +205,14 @@ class RegexGenerator(BaseTransformer):
         # Used otherwise
         self.generator_size = None
         self.generated = None
+        self._num_fallback_samples_generated = 0
 
     def reset_randomization(self):
         """Create a new generator and reset the generated values counter."""
         super().reset_randomization()
         self.generator, self.generator_size = strings_from_regex(self.regex_format)
         self.generated = 0
+        self._num_fallback_samples_generated = 0
 
         if hasattr(self, 'cardinality_rule') and self.cardinality_rule == 'scale':
             self._remaining_samples['repetitions'] = 0
@@ -193,6 +220,17 @@ class RegexGenerator(BaseTransformer):
 
     def _sample_fallback(self, num_samples, template_samples):
         """Sample num_samples values such that they are all unique, disregarding the regex."""
+        unique_condition = (
+            self.cardinality_rule == 'unique'
+            if hasattr(self, 'cardinality_rule')
+            else self.enforce_uniqueness
+        )
+        if unique_condition:
+            if not self._num_fallback_samples_generated:
+                self.generator = self._create_numerical_fallback_generator()
+
+            return [next(self.generator) for _ in range(num_samples)]
+
         try:
             # Integer-based fallback: attempt to convert the last template sample to an integer
             # and then generate values in a sequential manner.
@@ -247,6 +285,9 @@ class RegexGenerator(BaseTransformer):
             match_cardinality (bool):
                 Whether or not to match the cardinality of the data.
         """
+        if unique_condition and self._num_fallback_samples_generated:
+            return
+
         warned = False
         warn_msg = (
             f"The regex for '{self.get_input_column()}' can only generate "
@@ -389,16 +430,27 @@ class RegexGenerator(BaseTransformer):
             if self.cardinality_rule == 'scale':
                 return self._sample_scale(num_samples)
 
-        # If there aren't enough values left in the generator, reset it
-        if num_samples > self.generator_size - self.generated:
+        if unique_condition and self._num_fallback_samples_generated:
+            samples = self._sample_fallback(num_samples, [])
+            self._num_fallback_samples_generated += len(samples)
+            return samples
+
+        # If there aren't enough values left in the generator, reset it if cardinality_rule!=unique
+        remaining = max(self.generator_size - self.generated, 0)
+        if num_samples > remaining and not unique_condition:
             self.reset_randomization()
 
         samples = self._sample_from_generator(num_samples)
+
+        # Need more samples than the generator can produce
         if num_samples > len(samples):
             if unique_condition:
-                new_samples = self._sample_fallback(num_samples - len(samples), samples)
+                fallback_size = num_samples - len(samples)
+                new_samples = self._sample_fallback(fallback_size, samples)
+                self._num_fallback_samples_generated += len(new_samples)
             else:
                 new_samples = self._sample_from_template(num_samples - len(samples), samples)
+
             samples.extend(new_samples)
 
         return samples

{rdt-1.21.1.dev0 → rdt-1.21.2.dev0/rdt.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdt
-Version: 1.21.1.dev0
+Version: 1.21.2.dev0
 Summary: Reversible Data Transforms
 Author-email: "DataCebo, Inc." <info@sdv.dev>
 License-Expression: BUSL-1.1