rdt 1.16.0.dev0__tar.gz → 1.17.0.dev0__tar.gz

{rdt-1.16.0.dev0/rdt.egg-info → rdt-1.17.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdt
-Version: 1.16.0.dev0
+Version: 1.17.0.dev0
 Summary: Reversible Data Transforms
 Author-email: "DataCebo, Inc." <info@sdv.dev>
 License: BSL-1.1
@@ -43,6 +43,7 @@ Requires-Dist: scikit-learn>=1.1.3; python_version >= "3.11" and python_version
 Requires-Dist: scikit-learn>=1.3.1; python_version >= "3.12" and python_version < "3.13"
 Requires-Dist: scikit-learn>=1.5.2; python_version >= "3.13"
 Requires-Dist: Faker>=17
+Requires-Dist: python-dateutil>=2.9
 Provides-Extra: copulas
 Requires-Dist: copulas>=0.12.1; extra == "copulas"
 Provides-Extra: pyarrow

{rdt-1.16.0.dev0 → rdt-1.17.0.dev0}/pyproject.toml

@@ -40,6 +40,7 @@ dependencies = [
     "scikit-learn>=1.3.1;python_version>='3.12' and python_version<'3.13'",
     "scikit-learn>=1.5.2;python_version>='3.13'",
     'Faker>=17',
+    'python-dateutil>=2.9',
 ]
 
 [project.urls]
@@ -144,7 +145,7 @@ collect_ignore = ['pyproject.toml']
 exclude_lines = ['NotImplementedError()']
 
 [tool.bumpversion]
-current_version = "1.16.0.dev0"
+current_version = "1.17.0.dev0"
 parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
 serialize = [
     '{major}.{minor}.{patch}.{release}{candidate}',

{rdt-1.16.0.dev0 → rdt-1.17.0.dev0}/rdt/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.16.0.dev0'
+__version__ = '1.17.0.dev0'
 
 
 import sys

rdt-1.17.0.dev0/rdt/_utils.py

@@ -0,0 +1,47 @@
+from collections import defaultdict
+
+from rdt.errors import InvalidConfigError
+
+
+def _validate_unique_transformer_instances(column_name_to_transformer):
+    """Validate that the transformer instance for each field is unique.
+
+    Args:
+        column_name_to_transformer (dict):
+            A dictionary mapping column names to their current transformer.
+
+    Raises:
+        - ``InvalidConfigError`` if transformers in ``column_name_to_transformer`` are repeated.
+    """
+    seen_transformers = defaultdict(set)
+    for column_name, transformer in column_name_to_transformer.items():
+        if transformer is not None:
+            seen_transformers[transformer].add(column_name)
+
+    duplicated_transformers = {
+        transformer: columns
+        for transformer, columns in seen_transformers.items()
+        if len(columns) > 1
+    }
+    if duplicated_transformers:
+        duplicated_column_messages = []
+        for duplicated_columns in duplicated_transformers.values():
+            columns = ', '.join(
+                sorted([
+                    str(columns) if not isinstance(columns, str) else f"'{columns}'"
+                    for columns in duplicated_columns
+                ])
+            )
+            duplicated_column_messages.append(f'columns ({columns})')
+
+        if len(duplicated_column_messages) > 1:
+            plurality = 'instances are'
+        else:
+            plurality = 'instance is'
+
+        error_message = (
+            f'The same transformer {plurality} being assigned to '
+            f'{", ".join(duplicated_column_messages)}. Please create different transformer objects '
+            'for each assignment.'
+        )
+        raise InvalidConfigError(error_message)

{rdt-1.16.0.dev0 → rdt-1.17.0.dev0}/rdt/hyper_transformer.py

@@ -8,6 +8,7 @@ from copy import deepcopy
 
 import pandas as pd
 
+from rdt._utils import _validate_unique_transformer_instances
 from rdt.errors import (
     ConfigNotSetError,
     InvalidConfigError,
@@ -181,6 +182,8 @@ class HyperTransformer:
                 'Please assign an rdt transformer instance to each column name.'
             )
 
+        _validate_unique_transformer_instances(column_name_to_transformer)
+
     @staticmethod
     def _validate_sdtypes(sdtypes):
         """Validate the given sdtypes are valid.
@@ -496,6 +499,33 @@ class HyperTransformer:
         if self._fitted:
             warnings.warn(self._REFIT_MESSAGE)
 
+    def _validate_updated_transformer_unique(self, column_name, transformer):
+        """Validate the transformer being updated is not reused in the config."""
+        if (
+            transformer in self.field_transformers.values()
+            and self.field_transformers.get(column_name) != transformer
+        ):
+            existing_column = [
+                column
+                for column, existing_transformer in self.field_transformers.items()
+                if transformer == existing_transformer
+            ][0]
+            column_name = (
+                f"column ('{column_name}')"
+                if isinstance(column_name, str)
+                else f'columns {str(column_name)}'
+            )
+            existing_column = (
+                f"column ('{existing_column}')"
+                if isinstance(existing_column, str)
+                else f'columns {str(existing_column)}'
+            )
+            raise InvalidConfigError(
+                f'The transformer for {column_name} is already assigned to '
+                f'{existing_column}. Please create different transformer objects for '
+                'each assignment.'
+            )
+
     def update_transformers(self, column_name_to_transformer):
         """Update any of the transformers assigned to each of the column names.
 
@@ -514,6 +544,9 @@ class HyperTransformer:
         self._validate_transformers(column_name_to_transformer)
 
         for column_name, transformer in column_name_to_transformer.items():
+            if transformer is not None:
+                self._validate_updated_transformer_unique(column_name, transformer)
+
             columns = column_name if isinstance(column_name, tuple) else (column_name,)
             for column in columns:
                 if transformer is not None:

{rdt-1.16.0.dev0 → rdt-1.17.0.dev0}/rdt/transformers/datetime.py

@@ -1,5 +1,7 @@
 """Transformer for datetime data."""
 
+import warnings
+
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_datetime64_dtype, is_numeric_dtype
@@ -8,6 +10,7 @@ from pandas.core.tools.datetimes import _guess_datetime_format_for_array
 from rdt.errors import TransformerInputError
 from rdt.transformers.base import BaseTransformer
 from rdt.transformers.null import NullTransformer
+from rdt.transformers.utils import _safe_parse_datetime, data_has_multiple_timezones
 
 
 class UnixTimestampEncoder(BaseTransformer):
@@ -68,41 +71,75 @@ class UnixTimestampEncoder(BaseTransformer):
 
         self.datetime_format = datetime_format
         self._dtype = None
+        self._has_multiple_timezones = False
+        self._timezone_offset = None
+
+    def _warn_if_mixed_timezones(self):
+        if self._has_multiple_timezones:
+            warnings.warn(
+                'Mixed timezones are not supported in SDV Community. Data will be converted to UTC.'
+            )
+
+    def _raise_appropriate_conversion_error(self, error):
+        message = str(error)
+        if 'Unknown string' in message or 'Unknown datetime string' in message:
+            raise TypeError('Data must be of dtype datetime, or castable to datetime.') from None
+
+        raise ValueError('Data does not match specified datetime format.') from None
+
+    def _learn_timezone_offest(self, data):
+        """Extracts and stores the timezone offset from the first valid datetime in the data."""
+        has_timezone = self.datetime_format and '%z' in self.datetime_format.lower()
+        if has_timezone and not self._has_multiple_timezones:
+            for val in data:
+                if pd.notna(val):
+                    try:
+                        dt = _safe_parse_datetime(
+                            str(val), warn=True, datetime_format=self.datetime_format
+                        )
+                        self._timezone_offset = dt.tzinfo
+                        break
+
+                    except (ValueError, TypeError, AttributeError):
+                        self._timezone_offset = None
+
+    def _learn_has_multiple_timezones(self, data):
+        """Determines if the data contains multiple timezones and stores the result."""
+        if self.datetime_format and '%z' not in self.datetime_format.lower():
+            return
+
+        if not isinstance(data, pd.Series):
+            data = data.to_series()
+
+        self._has_multiple_timezones = data_has_multiple_timezones(
+            data, datetime_format=self.datetime_format
+        )
 
-    def _convert_to_datetime(self, data):
-        """Convert datetime column into datetime dtype.
+    def _needs_datetime_conversion(self, data):
+        """Determines if the data requires datetime conversion."""
+        return self.datetime_format is not None or not is_numeric_dtype(data)
 
-        Convert the datetime column to datetime dtype using the ``datetime_format``.
-        All non-numeric columns will automatically be cast to datetimes. Numeric columns
-        with a ``datetime_format`` will be treated as strings and cast to datetime. Numeric
-        columns without a ``datetime_format`` will be treated as already converted datetimes.
+    def _get_pandas_datetime_format(self):
+        """Converts the instance's datetime format to a pandas-compatible format."""
+        if self.datetime_format:
+            return self.datetime_format.replace('%#', '%').replace('%-', '%')
 
-        Args:
-            data (pandas.Series):
-                The datetime column.
+        return None
 
-        Raises:
-            - ``TypeError`` if data cannot be converted to datetime.
-            - ``ValueError`` if data does not match the specified datetime format
+    def _to_datetime(self, data):
+        """Converts data to datetime using the instance's datetime format."""
+        return pd.to_datetime(
+            data,
+            format=self._get_pandas_datetime_format(),
+            utc=getattr(self, '_has_multiple_timezones', False),  # Backward compatibility
+        )
 
-        Returns:
-            pandas.Series:
-                The datetime column converted to the datetime dtype.
-        """
-        if self.datetime_format or not is_numeric_dtype(data):
+    def _convert_to_datetime(self, data):
+        if self._needs_datetime_conversion(data):
             try:
-                pandas_datetime_format = None
-                if self.datetime_format:
-                    pandas_datetime_format = self.datetime_format.replace('%-', '%')
-
-                data = pd.to_datetime(data, format=pandas_datetime_format)
-
+                return self._to_datetime(data)
             except ValueError as error:
-                if 'Unknown string' in str(error) or 'Unknown datetime string' in str(error):
-                    message = 'Data must be of dtype datetime, or castable to datetime.'
-                    raise TypeError(message) from None
-
-                raise ValueError('Data does not match specified datetime format.') from None
+                self._raise_appropriate_conversion_error(error)
 
         return data
 
@@ -116,15 +153,6 @@ class UnixTimestampEncoder(BaseTransformer):
 
         return transformed
 
-    def _reverse_transform_helper(self, data):
-        """Transform integer values back into datetimes."""
-        if not isinstance(data, np.ndarray):
-            data = data.to_numpy()
-
-        data = self.null_transformer.reverse_transform(data)
-        data = np.round(data.astype(np.float64))
-        return data
-
     def _fit(self, data):
         """Fit the transformer to the data.
 
@@ -137,7 +165,11 @@ class UnixTimestampEncoder(BaseTransformer):
             datetime_array = data[data.notna()].astype(str).to_numpy()
             self.datetime_format = _guess_datetime_format_for_array(datetime_array)
 
+        self._learn_has_multiple_timezones(data)
         transformed = self._transform_helper(data)
+        self._learn_timezone_offest(data)
+        self._warn_if_mixed_timezones()
+
         if self.enforce_min_max_values:
             self._min_value = transformed.min()
             self._max_value = transformed.max()
@@ -200,6 +232,46 @@ class UnixTimestampEncoder(BaseTransformer):
         data = self._transform_helper(data)
         return self.null_transformer.transform(data)
 
+    def _convert_data_to_pandas_datetime(self, data):
+        """Converts data to pandas datetime, applying UTC or specific timezone offset if set."""
+        if hasattr(self, '_has_multiple_timezones') and self._has_multiple_timezones:
+            return pd.to_datetime(data, utc=True)
+
+        elif hasattr(self, '_timezone_offset') and self._timezone_offset is not None:
+            datime_data = pd.to_datetime(data, utc=True)
+            return datime_data.dt.tz_convert(self._timezone_offset)
+
+        return pd.to_datetime(data)
+
+    def _handle_datetime_formatting(self, datetime_data):
+        """Formats datetime data based on datetime format or converts to numeric if needed."""
+        if self.datetime_format:
+            return self._format_datetime(datetime_data)
+
+        elif is_numeric_dtype(self._dtype):
+            datetime_data = pd.to_numeric(datetime_data.astype('object'), errors='coerce')
+            return datetime_data.astype(self._dtype)
+
+        return datetime_data
+
+    def _format_datetime(self, datetime_data):
+        """Formats datetime data to string or datetime using learned datatime format and dtype."""
+        if is_datetime64_dtype(self._dtype) and '.%f' not in self.datetime_format:
+            return pd.to_datetime(
+                datetime_data.dt.strftime(self.datetime_format), format=self.datetime_format
+            )
+
+        return datetime_data.dt.strftime(self.datetime_format).astype(self._dtype)
+
+    def _reverse_transform_helper(self, data):
+        """Transform integer values back into datetimes."""
+        if not isinstance(data, np.ndarray):
+            data = data.to_numpy()
+
+        data = self.null_transformer.reverse_transform(data)
+        data = np.round(data.astype(np.float64))
+        return data
+
     def _reverse_transform(self, data):
         """Convert float values back to datetimes.
 
@@ -214,19 +286,8 @@ class UnixTimestampEncoder(BaseTransformer):
         if self.enforce_min_max_values:
             data = data.clip(self._min_value, self._max_value)
 
-        datetime_data = pd.to_datetime(data)
-        if self.datetime_format:
-            if is_datetime64_dtype(self._dtype) and '.%f' not in self.datetime_format:
-                datetime_data = pd.to_datetime(
-                    datetime_data.dt.strftime(self.datetime_format),
-                    format=self.datetime_format,
-                )
-            else:
-                datetime_data = datetime_data.dt.strftime(self.datetime_format).astype(self._dtype)
-        elif is_numeric_dtype(self._dtype):
-            datetime_data = pd.to_numeric(datetime_data.astype('object'), errors='coerce')
-            datetime_data = datetime_data.astype(self._dtype)
-
+        datetime_data = self._convert_data_to_pandas_datetime(data)
+        datetime_data = self._handle_datetime_formatting(datetime_data)
         return datetime_data