sdmetrics 0.28.3.dev0__tar.gz → 0.29.1.dev0__tar.gz

{sdmetrics-0.28.3.dev0/sdmetrics.egg-info → sdmetrics-0.29.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdmetrics
-Version: 0.28.3.dev0
+Version: 0.29.1.dev0
 Summary: Metrics for Synthetic Data Generation Projects
 Author-email: MIT Data To AI Lab <dailabmit@gmail.com>
 License-Expression: MIT

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/pyproject.toml

@@ -148,7 +148,7 @@ convention = 'google'
 add-ignore = ['D107', 'D407', 'D417']
 
 [tool.bumpversion]
-current_version = "0.28.3.dev0"
+current_version = "0.29.1.dev0"
 parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
 serialize = [
     '{major}.{minor}.{patch}.{release}{candidate}',

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = 'MIT Data To AI Lab'
 __email__ = 'dailabmit@gmail.com'
-__version__ = '0.28.3.dev0'
+__version__ = '0.29.1.dev0'
 
 import sys
 import warnings as python_warnings

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/_utils_metadata.py

@@ -100,6 +100,15 @@ def handle_single_and_multi_table(single_table_func):
     return wrapper
 
 
+def _convert_column_to_string(column_data, column_metadata):
+    if is_datetime(column_data):
+        datetime_format = column_metadata.get('datetime_format')
+        if datetime_format is not None:
+            return column_data.dt.strftime(datetime_format)
+
+    return column_data.astype(str)
+
+
 def _convert_datetime_column(column_name, column_data, column_metadata):
     if is_datetime(column_data):
         return column_data

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/demos/multi_table/metadata.json

@@ -5,7 +5,7 @@
             "columns": {
                 "user_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "country": {
                     "sdtype": "categorical"
@@ -24,11 +24,11 @@
             "columns": {
                 "session_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "user_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "device": {
                     "sdtype": "categorical"
@@ -43,11 +43,11 @@
             "columns": {
                 "transaction_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "session_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "timestamp": {
                     "sdtype": "datetime",

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/demos/single_table/metadata.json

@@ -21,7 +21,7 @@
                 },
                 "student_id": {
                     "sdtype": "id",
-                    "regex_format": "\\d{30}"
+                    "regex_format": "\\d{1,30}"
                 },
                 "high_perc": {
                     "sdtype": "numerical",

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/reports/single_table/_properties/data_validity.py

@@ -2,9 +2,16 @@ import numpy as np
 import pandas as pd
 import plotly.express as px
 
+from sdmetrics._utils_metadata import _convert_column_to_string
 from sdmetrics.reports.single_table._properties import BaseSingleTableProperty
 from sdmetrics.reports.utils import PlotConfig
-from sdmetrics.single_column import BoundaryAdherence, CategoryAdherence, KeyUniqueness
+from sdmetrics.single_column import (
+    BoundaryAdherence,
+    CategoryAdherence,
+    DatetimeFormatAdherence,
+    KeyUniqueness,
+    RegexFormatAdherence,
+)
 from sdmetrics.utils import (
     get_alternate_keys,
     get_columns_from_metadata,
@@ -21,16 +28,53 @@ class DataValidity(BaseSingleTableProperty):
     The BoundaryAdherence metric is used for numerical and datetime columns, the CategoryAdherence
     is used for categorical and boolean columns and the KeyUniqueness for primary
     and alternate keys. The other column types are ignored by this property.
+
+    If the metadata defines the valid range of a column, it is passed down to the metric,
+    which uses it instead of computing it from the real data.
     """
 
     _num_iteration_case = 'column'
+    _metric_to_arguments = {
+        DatetimeFormatAdherence: ('datetime_format',),
+        RegexFormatAdherence: ('regex_format',),
+        BoundaryAdherence: ('range_min', 'range_max', 'range_is_nullable'),
+        CategoryAdherence: ('range_values', 'range_is_nullable'),
+    }
     _sdtype_to_metric = {
-        'numerical': BoundaryAdherence,
-        'datetime': BoundaryAdherence,
-        'categorical': CategoryAdherence,
-        'boolean': CategoryAdherence,
+        'numerical': [BoundaryAdherence],
+        'datetime': [BoundaryAdherence, DatetimeFormatAdherence],
+        'categorical': [CategoryAdherence],
+        'boolean': [CategoryAdherence],
+        'id': [KeyUniqueness, RegexFormatAdherence],
     }
 
+    @classmethod
+    def _get_metric_arguments(cls, metric, column_name, columns_meta):
+        """Get the information of a column defined in the metadata.
+
+        Args:
+            metric (SingleColumnMetric):
+                The metric to compute the column score with.
+            column_name (str):
+                The name of the column.
+            columns_meta (dict):
+                The metadata of every column of the table.
+
+        Returns:
+            dict:
+                The arguments to pass down to the metric. Any information that is not
+                defined in the metadata is omitted.
+        """
+        arguments = cls._metric_to_arguments.get(metric)
+        if not arguments:
+            return {}
+
+        column_meta = columns_meta[column_name]
+        column_arguments = {
+            argument: column_meta[argument] for argument in arguments if argument in column_meta
+        }
+        return column_arguments
+
     def _generate_details(self, real_data, synthetic_data, metadata, progress_bar=None):
         """Generate the _details dataframe for the data validity property.
 
@@ -66,28 +110,45 @@ class DataValidity(BaseSingleTableProperty):
             is_unique = primary_key_match or alternate_key_match
             is_sequence_index = column_name == sequence_index
 
-            try:
-                if sdtype not in self._sdtype_to_metric and not is_unique:
-                    continue
-
-                if is_sequence_index and self._sdtype_to_metric.get(sdtype) == BoundaryAdherence:
-                    continue
-
-                metric = self._sdtype_to_metric.get(sdtype, KeyUniqueness)
-                column_score = metric.compute(real_data[column_name], synthetic_data[column_name])
-                error_message = None
-
-            except Exception as e:
-                column_score = np.nan
-                error_message = f'{type(e).__name__}: {e}'
-            finally:
-                if progress_bar:
-                    progress_bar.update()
-
-            column_names.append(column_name)
-            metric_names.append(metric.__name__)
-            scores.append(column_score)
-            error_messages.append(error_message)
+            if (sdtype == 'id' or sdtype not in self._sdtype_to_metric) and not is_unique:
+                continue
+
+            if is_sequence_index and BoundaryAdherence in self._sdtype_to_metric.get(sdtype):
+                continue
+
+            metrics = self._sdtype_to_metric.get(sdtype, [KeyUniqueness])
+            for metric in metrics:
+                try:
+                    metric_arguments = self._get_metric_arguments(metric, column_name, columns_meta)
+                    if metric in [DatetimeFormatAdherence, RegexFormatAdherence]:
+                        column_meta = columns_meta[column_name]
+                        real_column = _convert_column_to_string(real_data[column_name], column_meta)
+                        synthetic_column = _convert_column_to_string(
+                            synthetic_data[column_name], column_meta
+                        )
+                        column_score = metric.compute(
+                            real_column, synthetic_column, **metric_arguments
+                        )
+                    else:
+                        column_score = metric.compute(
+                            real_data[column_name],
+                            synthetic_data[column_name],
+                            **metric_arguments,
+                        )
+
+                    error_message = None
+
+                except Exception as e:
+                    column_score = np.nan
+                    error_message = f'{type(e).__name__}: {e}'
+                finally:
+                    if progress_bar:
+                        progress_bar.update()
+
+                column_names.append(column_name)
+                metric_names.append(metric.__name__)
+                scores.append(column_score)
+                error_messages.append(error_message)
 
         result = pd.DataFrame({
             'Column': column_names,

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/single_column/base.py

@@ -41,7 +41,7 @@ class SingleColumnMetric(BaseMetric):
         raise NotImplementedError()
 
     @classmethod
-    def compute_breakdown(cls, real_data, synthetic_data):
+    def compute_breakdown(cls, real_data, synthetic_data, **kwargs):
         """Compute this metric breakdown.
 
         Args:
@@ -51,9 +51,11 @@ class SingleColumnMetric(BaseMetric):
             synthetic_data (Union[numpy.ndarray, pandas.Series]):
                 The values from the synthetic dataset, passed as a 1d numpy
                 array or as a pandas.Series.
+            **kwargs:
+                Additional keyword arguments to pass to the ``compute`` method.
 
         Returns:
             dict
                 Mapping of the metric output. Must include the key 'score'.
         """
-        return {'score': cls.compute(real_data, synthetic_data)}
+        return {'score': cls.compute(real_data, synthetic_data, **kwargs)}

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/single_column/statistical/boundary_adherence.py

@@ -11,7 +11,8 @@ class BoundaryAdherence(SingleColumnMetric):
     """Boundary adherence metric.
 
     Compute the fraction of rows in the synthetic data that are within the min and max
-    bounds of the real data
+    bounds of the real data. If any of ``range_min``, ``range_max`` or ``range_is_nullable``
+    are provided, they are used instead of the values computed from the real data.
 
     Attributes:
         name (str):
@@ -30,7 +31,9 @@ class BoundaryAdherence(SingleColumnMetric):
     max_value = 1.0
 
     @classmethod
-    def compute(cls, real_data, synthetic_data):
+    def compute(
+        cls, real_data, synthetic_data, range_min=None, range_max=None, range_is_nullable=None
+    ):
         """Compute the boundary adherence of two continuous columns.
 
         Args:
@@ -38,6 +41,15 @@ class BoundaryAdherence(SingleColumnMetric):
                 The values from the real dataset.
             synthetic_data (Union[numpy.ndarray, pandas.Series]):
                 The values from the synthetic dataset.
+            range_min (float or datetime, optional):
+                The minimum value the column is allowed to take. If ``None``, the minimum
+                value of the real data is used instead. Defaults to ``None``.
+            range_max (float or datetime, optional):
+                The maximum value the column is allowed to take. If ``None``, the maximum
+                value of the real data is used instead. Defaults to ``None``.
+            range_is_nullable (bool, optional):
+                Whether the column is allowed to contain missing values.
+                Defaults to ``None``.
 
         Returns:
             float:
@@ -45,15 +57,24 @@ class BoundaryAdherence(SingleColumnMetric):
         """
         real_data = pd.Series(real_data)
         synthetic_data = pd.Series(synthetic_data)
-        if any(pd.isna(real_data)):
-            real_data = real_data.dropna()
+
+        range_min = real_data.min() if range_min is None else range_min
+        range_max = real_data.max() if range_max is None else range_max
+
+        if range_is_nullable is None:
+            range_is_nullable = any(pd.isna(real_data))
+
+        real_data = real_data.dropna()
+        if range_is_nullable:
             synthetic_data = synthetic_data.dropna()
 
         if is_datetime(real_data):
+            bounds = pd.to_datetime([range_min, range_max]).astype(real_data.dtype)
             real_data = pd.to_numeric(real_data)
             synthetic_data = pd.to_numeric(synthetic_data)
+            range_min, range_max = pd.to_numeric(bounds)
 
-        valid = synthetic_data.between(real_data.min(), real_data.max())
+        valid = synthetic_data.between(range_min, range_max)
 
         return valid.sum() / len(synthetic_data)
 

sdmetrics-0.29.1.dev0/sdmetrics/single_column/statistical/category_adherence.py

@@ -0,0 +1,96 @@
+"""Category Adherence Metric."""
+
+import numpy as np
+import pandas as pd
+
+from sdmetrics.goal import Goal
+from sdmetrics.single_column.base import SingleColumnMetric
+
+
+class CategoryAdherence(SingleColumnMetric):
+    """Category adherence metric.
+
+    The proportion of synthetic data points that match an existing category from the real data.
+    If any of ``range_values`` or ``range_is_nullable`` are provided, they are used instead of
+    the categories computed from the real data.
+
+    Attributes:
+        name (str):
+            Name to use when reports about this metric are printed.
+        goal (sdmetrics.goal.Goal):
+            The goal of this metric.
+        min_value (Union[float, tuple[float]]):
+            Minimum value or values that this metric can take.
+        max_value (Union[float, tuple[float]]):
+            Maximum value or values that this metric can take.
+    """
+
+    name = 'CategoryAdherence'
+    goal = Goal.MAXIMIZE
+    min_value = 0.0
+    max_value = 1.0
+
+    @classmethod
+    def compute_breakdown(
+        cls, real_data, synthetic_data, range_values=None, range_is_nullable=None
+    ):
+        """Compute the score breakdown of the category adherence metric.
+
+        Args:
+            real_data (pandas.Series):
+                The real data.
+            synthetic_data (pandas.Series):
+                The synthetic data.
+            range_values (List[str], optional):
+                The list of categories the column is allowed to take. If ``None``, the
+                categories of the real data are used instead. Defaults to ``None``.
+            range_is_nullable (bool, optional):
+                Whether the column is allowed to contain missing values.
+                Defaults to ``None``.
+
+        Returns:
+            dict:
+                The score breakdown of the category adherence metric.
+        """
+        real_data = real_data.fillna(np.nan)
+        synthetic_data = synthetic_data.fillna(np.nan)
+        if range_values is None:
+            valid_values = pd.Series(real_data.unique())
+        else:
+            valid_values = pd.Series(list(range_values))
+
+        if range_is_nullable is None:
+            range_is_nullable = any(pd.isna(real_data)) or any(pd.isna(valid_values))
+
+        valid = synthetic_data.isin(valid_values.dropna())
+        if range_is_nullable:
+            valid = valid | pd.isna(synthetic_data)
+
+        return {'score': valid.mean()}
+
+    @classmethod
+    def compute(cls, real_data, synthetic_data, range_values=None, range_is_nullable=None):
+        """Compute the category adherence of two columns.
+
+        Args:
+            real_data (pandas.Series):
+                The real data.
+            synthetic_data (pandas.Series):
+                The synthetic data.
+            range_values (List[str], optional):
+                The list of categories the column is allowed to take. If ``None``, the
+                categories of the real data are used instead. Defaults to ``None``.
+            range_is_nullable (bool, optional):
+                Whether the column is allowed to contain missing values.
+                Defaults to ``None``.
+
+        Returns:
+            float:
+                The category adherence metric score.
+        """
+        return cls.compute_breakdown(
+            real_data,
+            synthetic_data,
+            range_values=range_values,
+            range_is_nullable=range_is_nullable,
+        )['score']

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/single_column/statistical/datetime_format_adherence.py

@@ -6,10 +6,10 @@ from datetime import datetime
 
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_string_dtype
 
 from sdmetrics.goal import Goal
 from sdmetrics.single_column.base import SingleColumnMetric
+from sdmetrics.utils import is_datetime
 
 
 class DatetimeFormatAdherence(SingleColumnMetric):
@@ -92,13 +92,11 @@ class DatetimeFormatAdherence(SingleColumnMetric):
             float:
                 The proportion of data points in the synthetic data that match the datetime format.
         """
-        real_data_nan = pd.isna(real_data)
-        synthetic_data_nan = pd.isna(synthetic_data)
-        if not is_string_dtype(synthetic_data[~synthetic_data_nan]):
+        if is_datetime(synthetic_data):
             return np.nan
 
         cls._validate_datetime_format(datetime_format)
-        if is_string_dtype(real_data[~real_data_nan]):
+        if not is_datetime(real_data):
             real_valid_rows = cls._filter_valid_datetime_rows(real_data, datetime_format)
             if len(real_valid_rows) != len(real_data):
                 invalid = real_data[~real_data.index.isin(real_valid_rows.index)]

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0}/sdmetrics/visualization.py

@@ -5,9 +5,9 @@ from functools import wraps
 
 import pandas as pd
 import plotly.express as px
-import plotly.figure_factory as ff
 import plotly.graph_objects as go
 import plotly.io as pio
+import scipy
 from pandas.api.types import is_datetime64_dtype
 
 from sdmetrics.reports.utils import PlotConfig
@@ -252,22 +252,30 @@ def _generate_column_distplot(real_data, synthetic_data, plot_kwargs={}):
         col_names.append('Synthetic')
         colors.append(PlotConfig.DATACEBO_GREEN)
 
-    default_distplot_kwargs = {
-        'show_hist': False,
-        'show_rug': False,
-        'colors': colors,
-    }
-
     has_data = any(len(data) > 0 for data in hist_data)
 
-    if has_data:
-        return ff.create_distplot(
-            hist_data,
-            col_names,
-            **{**default_distplot_kwargs, **plot_kwargs},
+    if not has_data:
+        return go.Figure()
+
+    traces = []
+    for index, data in enumerate(hist_data):
+        start = min(data)
+        end = max(data)
+
+        curve_x = [start + x * (end - start) / 500 for x in range(500)]
+        curve_y = scipy.stats.gaussian_kde(data)(curve_x)
+
+        traces.append(
+            go.Scatter(
+                x=curve_x,
+                y=curve_y,
+                mode='lines',
+                name=col_names[index],
+                line={'color': colors[index]},
+            )
         )
 
-    return go.Figure()
+    return go.Figure(data=traces, **plot_kwargs)
 
 
 def _generate_column_plot(

{sdmetrics-0.28.3.dev0 → sdmetrics-0.29.1.dev0/sdmetrics.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdmetrics
-Version: 0.28.3.dev0
+Version: 0.29.1.dev0
 Summary: Metrics for Synthetic Data Generation Projects
 Author-email: MIT Data To AI Lab <dailabmit@gmail.com>
 License-Expression: MIT

sdmetrics-0.28.3.dev0/sdmetrics/single_column/statistical/category_adherence.py

@@ -1,64 +0,0 @@
-"""Category Adherence Metric."""
-
-import numpy as np
-
-from sdmetrics.goal import Goal
-from sdmetrics.single_column.base import SingleColumnMetric
-
-
-class CategoryAdherence(SingleColumnMetric):
-    """Category adherence metric.
-
-    The proportion of synthetic data points that match an existing category from the real data.
-
-    Attributes:
-        name (str):
-            Name to use when reports about this metric are printed.
-        goal (sdmetrics.goal.Goal):
-            The goal of this metric.
-        min_value (Union[float, tuple[float]]):
-            Minimum value or values that this metric can take.
-        max_value (Union[float, tuple[float]]):
-            Maximum value or values that this metric can take.
-    """
-
-    name = 'CategoryAdherence'
-    goal = Goal.MAXIMIZE
-    min_value = 0.0
-    max_value = 1.0
-
-    @classmethod
-    def compute_breakdown(cls, real_data, synthetic_data):
-        """Compute the score breakdown of the category adherence metric.
-
-        Args:
-            real_data (pandas.Series):
-                The real data.
-            synthetic_data (pandas.Series):
-                The synthetic data.
-
-        Returns:
-            dict:
-                The score breakdown of the category adherence metric.
-        """
-        real_data = real_data.fillna(np.nan)
-        synthetic_data = synthetic_data.fillna(np.nan)
-        score = synthetic_data.isin(real_data).mean()
-
-        return {'score': score}
-
-    @classmethod
-    def compute(cls, real_data, synthetic_data):
-        """Compute the category adherence of two columns.
-
-        Args:
-            real_data (pandas.Series):
-                The real data.
-            synthetic_data (pandas.Series):
-                The synthetic data.
-
-        Returns:
-            float:
-                The category adherence metric score.
-        """
-        return cls.compute_breakdown(real_data, synthetic_data)['score']