sdv 1.35.2.dev0__tar.gz → 1.36.1.dev0__tar.gz

{sdv-1.35.2.dev0/sdv.egg-info → sdv-1.36.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdv
-Version: 1.35.2.dev0
+Version: 1.36.1.dev0
 Summary: Generate synthetic data for single table, multi table and sequential data
 Author-email: "DataCebo, Inc." <info@sdv.dev>
 License-Expression: BUSL-1.1

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/pyproject.toml

@@ -149,7 +149,7 @@ namespaces = false
 version = {attr = 'sdv.__version__'}
 
 [tool.bumpversion]
-current_version = "1.35.2.dev0"
+current_version = "1.36.1.dev0"
 parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
 serialize = [
     '{major}.{minor}.{patch}.{release}{candidate}',

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/sdv/__init__.py

@@ -6,7 +6,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.35.2.dev0'
+__version__ = '1.36.1.dev0'
 
 
 import sys

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/sdv/_utils.py

@@ -536,3 +536,8 @@ def _get_unreferenced_keys(parent_columns, child_columns):
     merged = merged[merged[indicator] == 'left_only'][list(child_columns.columns)]
     merged = merged.dropna(how='all')
     return merged.dropna(how='all')
+
+
+def _validate_boolean_parameter(parameter, parameter_name):
+    if not isinstance(parameter, bool):
+        raise ValueError(f"'{parameter_name}' must be a boolean value.")

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/sdv/metadata/metadata.py

@@ -4,6 +4,7 @@ import warnings
 
 import pandas as pd
 
+from sdv._utils import _validate_boolean_parameter
 from sdv.metadata.errors import InvalidMetadataError
 from sdv.metadata.multi_table import MultiTableMetadata
 from sdv.metadata.single_table import SingleTableMetadata
@@ -61,11 +62,6 @@ class Metadata(MultiTableMetadata):
         instance._set_metadata_dict(metadata_dict, single_table_name)
         return instance
 
-    @staticmethod
-    def _validate_infer_sdtypes(infer_sdtypes):
-        if not isinstance(infer_sdtypes, bool):
-            raise ValueError("'infer_sdtypes' must be a boolean value.")
-
     @staticmethod
     def _validate_foreign_key_inference_algorithm(foreign_key_inference_algorithm):
         if foreign_key_inference_algorithm != 'column_name_match':
@@ -78,6 +74,7 @@ class Metadata(MultiTableMetadata):
         infer_sdtypes=True,
         infer_keys='primary_and_foreign',
         foreign_key_inference_algorithm='column_name_match',
+        verbose=False,
     ):
         if not data or not all(isinstance(df, pd.DataFrame) for df in data.values()):
             raise ValueError('The provided dictionary must contain only pandas DataFrame objects.')
@@ -86,16 +83,20 @@ class Metadata(MultiTableMetadata):
                 "'infer_keys' must be one of: 'primary_and_foreign', 'primary_only', None."
             )
         cls._validate_foreign_key_inference_algorithm(foreign_key_inference_algorithm)
-        cls._validate_infer_sdtypes(infer_sdtypes)
+        _validate_boolean_parameter(infer_sdtypes, 'infer_sdtypes')
 
         metadata = Metadata()
         for table_name, dataframe in data.items():
             metadata.detect_table_from_dataframe(
-                table_name, dataframe, infer_sdtypes, None if infer_keys is None else 'primary_only'
+                table_name,
+                dataframe,
+                infer_sdtypes,
+                None if infer_keys is None else 'primary_only',
+                verbose,
             )
 
         if infer_keys == 'primary_and_foreign':
-            metadata._detect_relationships(data, foreign_key_inference_algorithm)
+            metadata._detect_relationships(data, foreign_key_inference_algorithm, verbose)
 
         return metadata
 
@@ -106,6 +107,7 @@ class Metadata(MultiTableMetadata):
         infer_sdtypes=True,
         infer_keys='primary_and_foreign',
         foreign_key_inference_algorithm='column_name_match',
+        verbose=False,
     ):
         """Detect the metadata for all tables in a dictionary of dataframes.
 
@@ -130,6 +132,11 @@ class Metadata(MultiTableMetadata):
             foreign_key_inference_algorithm (str):
                 Which algorithm to use for detecting foreign keys. Currently only one option,
                 'column_name_match'. Defaults to 'column_name_match'.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
 
         Returns:
             Metadata:
@@ -140,8 +147,25 @@ class Metadata(MultiTableMetadata):
             infer_sdtypes=infer_sdtypes,
             infer_keys=infer_keys,
             foreign_key_inference_algorithm=foreign_key_inference_algorithm,
+            verbose=verbose,
         )
 
+    @classmethod
+    def _detect_from_dataframe(
+        cls, data, table_name=None, infer_sdtypes=True, infer_keys='primary_only', verbose=False
+    ):
+        """Detect the metadata for a DataFrame."""
+        table_name = table_name or cls.DEFAULT_SINGLE_TABLE_NAME
+        if not isinstance(data, pd.DataFrame):
+            raise ValueError('The provided data must be a pandas DataFrame object.')
+        if infer_keys not in ['primary_only', None]:
+            raise ValueError("'infer_keys' must be one of: 'primary_only', None.")
+
+        _validate_boolean_parameter(infer_sdtypes, 'infer_sdtypes')
+        metadata = Metadata()
+        metadata.detect_table_from_dataframe(table_name, data, infer_sdtypes, infer_keys, verbose)
+        return metadata
+
     @classmethod
     def detect_from_dataframe(
         cls,
@@ -149,6 +173,7 @@ class Metadata(MultiTableMetadata):
         table_name=DEFAULT_SINGLE_TABLE_NAME,
         infer_sdtypes=True,
         infer_keys='primary_only',
+        verbose=False,
     ):
         """Detect the metadata for a DataFrame.
 
@@ -157,7 +182,10 @@ class Metadata(MultiTableMetadata):
 
         Args:
             data (pandas.DataFrame):
-                Dictionary of table names to dataframes.
+                The data to detect metadata from.
+            table_name (str):
+                The name of the table to detect. If None, a default name will be used.
+                Defaults to None.
             infer_sdtypes (bool):
                 A boolean describing whether to infer the sdtypes of each column.
                 If True it infers the sdtypes based on the data.
@@ -168,20 +196,23 @@ class Metadata(MultiTableMetadata):
                     - 'primary_only': Infer only the primary keys of each table
                     - None: Do not infer any keys
                 Defaults to 'primary_only'.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
 
         Returns:
             Metadata:
                 A new metadata object with the sdtypes detected from the data.
         """
-        if not isinstance(data, pd.DataFrame):
-            raise ValueError('The provided data must be a pandas DataFrame object.')
-        if infer_keys not in ['primary_only', None]:
-            raise ValueError("'infer_keys' must be one of: 'primary_only', None.")
-        cls._validate_infer_sdtypes(infer_sdtypes)
-
-        metadata = Metadata()
-        metadata.detect_table_from_dataframe(table_name, data, infer_sdtypes, infer_keys)
-        return metadata
+        return cls._detect_from_dataframe(
+            data=data,
+            table_name=table_name,
+            infer_sdtypes=infer_sdtypes,
+            infer_keys=infer_keys,
+            verbose=verbose,
+        )
 
     def _set_metadata_dict(self, metadata, single_table_name=None):
         """Set a ``metadata`` dictionary to the current instance.

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/sdv/metadata/multi_table.py

@@ -3,6 +3,7 @@
 import datetime
 import json
 import logging
+import sys
 import warnings
 from collections import defaultdict
 from copy import deepcopy
@@ -547,7 +548,7 @@ class MultiTableMetadata:
                 f'The relationships in the dataset are disjointed. {table_msg}'
             )
 
-    def _detect_foreign_keys_by_column_name(self, data):
+    def _detect_foreign_keys_by_column_name(self, data, verbose=False):
         """Detect the foreign keys based on if a column name matches a primary key.
 
         If a column name (a child table) is a primary key, it will also be considered
@@ -557,7 +558,15 @@ class MultiTableMetadata:
             data (dict):
                 Dictionary of table names to dataframes.
                 NOTE: this is only used in SDV-Enterprise.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
+        is_foreign_keys_found = False
+        if verbose:
+            sys.stdout.write('\nDetecting foreign keys:\n')
         for parent_candidate in self.tables.keys():
             primary_key = self.tables[parent_candidate].primary_key
             if primary_key is None:
@@ -573,15 +582,25 @@ class MultiTableMetadata:
                         continue
 
                     try:
+                        sdtype_updated = False
                         if pk_sdtype == 'id' and original_fk_sdtype != 'id':
                             self.update_column(
                                 table_name=child_candidate,
                                 column_name=primary_key,
                                 sdtype='id',
                             )
+                            sdtype_updated = True
                         self.add_relationship(
                             parent_candidate, child_candidate, primary_key, primary_key
                         )
+                        is_foreign_keys_found = True
+                        if verbose:
+                            child_col = f"'{child_candidate}.{primary_key}'"
+                            parent_col = f"'{parent_candidate}.{primary_key}'"
+                            suffix = " (updating sdtype to 'id')" if sdtype_updated else ''
+                            sys.stdout.write(
+                                f'- Column {child_col} refers to column {parent_col}{suffix}\n'
+                            )
 
                     except InvalidMetadataError:
                         # circular relationship
@@ -592,8 +611,12 @@ class MultiTableMetadata:
                                 **original_fk_meta,
                             )
                         continue
+        if verbose and not is_foreign_keys_found:
+            sys.stdout.write('- No foreign keys found\n')
 
-    def _detect_relationships(self, data=None, foreign_key_inference_algorithm='column_name_match'):
+    def _detect_relationships(
+        self, data=None, foreign_key_inference_algorithm='column_name_match', verbose=False
+    ):
         """Automatically detect relationships between tables.
 
         Args:
@@ -603,12 +626,22 @@ class MultiTableMetadata:
             foreign_key_inference_algorithm (str):
                 Which algorithm to use for detecting foreign keys. Currently only one option,
                 'column_name_match'.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
         if foreign_key_inference_algorithm == 'column_name_match':
-            self._detect_foreign_keys_by_column_name(data)
+            self._detect_foreign_keys_by_column_name(data, verbose)
 
     def detect_table_from_dataframe(
-        self, table_name, data, infer_sdtypes=True, infer_keys='primary_only'
+        self,
+        table_name,
+        data,
+        infer_sdtypes=True,
+        infer_keys='primary_only',
+        verbose=False,
     ):
         """Detect the metadata for a table from a dataframe.
 
@@ -630,14 +663,19 @@ class MultiTableMetadata:
                     - 'primary_only': Infer only the primary keys of each table
                     - None: Do not infer any keys
                 Defaults to 'primary_only'.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
         self._validate_table_not_detected(table_name)
         table = SingleTableMetadata()
-        table._detect_columns(data, table_name, infer_sdtypes, infer_keys)
+        table._detect_columns(data, table_name, infer_sdtypes, infer_keys, verbose)
         self.tables[table_name] = table
         self._log_detected_table(table)
 
-    def detect_from_dataframes(self, data):
+    def detect_from_dataframes(self, data, verbose=False):
         """Detect the metadata for all tables in a dictionary of dataframes.
 
         This method automatically detects the ``sdtypes`` for the given ``pandas.DataFrame``.
@@ -646,14 +684,19 @@ class MultiTableMetadata:
         Args:
             data (dict):
                 Dictionary of table names to dataframes.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
         if not data or not all(isinstance(df, pd.DataFrame) for df in data.values()):
             raise ValueError('The provided dictionary must contain only pandas DataFrame objects.')
 
         for table_name, dataframe in data.items():
-            self.detect_table_from_dataframe(table_name, dataframe)
+            self.detect_table_from_dataframe(table_name, dataframe, verbose=verbose)
 
-        self._detect_relationships(data)
+        self._detect_relationships(data, verbose=verbose)
 
     def detect_from_csvs(self, folder_name, read_csv_parameters=None):
         """Detect the metadata for all tables in a folder of csv files.

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0}/sdv/metadata/single_table.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import re
+import sys
 import warnings
 from collections import Counter, defaultdict
 from copy import deepcopy
@@ -28,7 +29,13 @@ from sdv.errors import InvalidDataError
 from sdv.logging import get_sdv_logger
 from sdv.metadata.errors import InvalidMetadataError
 from sdv.metadata.metadata_upgrader import convert_metadata
-from sdv.metadata.utils import _validate_file_mode, read_json, validate_file_does_not_exist
+from sdv.metadata.utils import (
+    _format_column_metadata,
+    _print_primary_key_detection,
+    _validate_file_mode,
+    read_json,
+    validate_file_does_not_exist,
+)
 from sdv.metadata.visualization import (
     create_columns_node,
     create_summarized_columns_node,
@@ -651,7 +658,9 @@ class SingleTableMetadata:
         except Exception as e:
             self._handle_detection_error(e, column_name, table_name)
 
-    def _select_primary_key(self, infer_sdtypes, pk_candidates, pii_pk_candidates):
+    def _select_primary_key(
+        self, infer_sdtypes, pk_candidates, pii_pk_candidates, table_name=None, verbose=False
+    ):
         """Select the primary key from a list of candidates.
 
         If there are any non-pii candidates, we select the first one. Otherwise, we select the
@@ -666,22 +675,48 @@ class SingleTableMetadata:
                 A list of primary key candidates that aren't pii.
             pii_pk_candidates (list):
                 A list of primary key candidates that are pii.
+            table_name (str):
+                The name of the table to be analyzed. Defaults to ``None``.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
+        if verbose:
+            table_str = f" for table '{table_name}'" if table_name else ''
+            sys.stdout.write(f'\nDetecting primary key{table_str}:\n')
+        chosen_pk = None
+        sdtype_updated = False
+        pii_removed = False
+
         if pk_candidates:
             selected_pk = pk_candidates[0]
             self.primary_key = selected_pk
+            original_sdtype = self.columns.get(self.primary_key, {}).get('sdtype')
             self.columns[self.primary_key]['sdtype'] = 'id'
+            chosen_pk = self.primary_key
+            sdtype_updated = original_sdtype != 'id'
 
         elif pii_pk_candidates:
             self.primary_key = pii_pk_candidates[0]
+            chosen_pk = self.primary_key
             if not infer_sdtypes:
+                original_sdtype = self.columns.get(self.primary_key, {}).get('sdtype')
                 self.columns[self.primary_key]['sdtype'] = 'id'
+                sdtype_updated = original_sdtype != 'id'
 
         if self.primary_key and self.columns[self.primary_key].get('sdtype') == 'id':
             if self.columns[self.primary_key].get('pii') is not None:
                 del self.columns[self.primary_key]['pii']
+                pii_removed = True
+
+        if verbose:
+            _print_primary_key_detection(chosen_pk, sdtype_updated, pii_removed)
 
-    def _detect_columns(self, data, table_name=None, infer_sdtypes=True, infer_keys='primary_only'):
+    def _detect_columns(
+        self, data, table_name=None, infer_sdtypes=True, infer_keys='primary_only', verbose=False
+    ):
         """Detect metadata information for each column in the data.
 
         Args:
@@ -699,7 +734,16 @@ class SingleTableMetadata:
                     - 'primary_only': Infer the primary keys.
                     - None: Do not infer any keys.
                 Defaults to 'primary_only'.
+            verbose (bool):
+                A boolean that determines if information should be printed regarding detection.
+                If True, it prints out information about what is detected.
+                If False, it does not print out any information about what is detected.
+                Defaults to False.
         """
+        if verbose and infer_sdtypes:
+            table_str = f"table '{table_name}'" if table_name else 'table'
+            sys.stdout.write(f'\nDetecting {table_str}:\n')
+
         old_columns = data.columns
         data.columns = data.columns.astype(str)
         pk_candidates = []
@@ -728,13 +772,19 @@ class SingleTableMetadata:
                 column_dict['pii'] = True
 
             column_dict['sdtype'] = sdtype
-            self.columns[field] = deepcopy(column_dict)
 
+            if verbose and infer_sdtypes:
+                column_metadata = _format_column_metadata(column_dict)
+                sys.stdout.write(f"- Column '{field}': {column_metadata}\n")
+
+            self.columns[field] = deepcopy(column_dict)
         if infer_keys == 'primary_only':
             self._select_primary_key(
                 infer_sdtypes=infer_sdtypes,
                 pk_candidates=pk_candidates,
                 pii_pk_candidates=pii_pk_candidates,
+                table_name=table_name,
+                verbose=verbose,
             )
 
         self._updated = True

sdv-1.36.1.dev0/sdv/metadata/utils.py

@@ -0,0 +1,82 @@
+"""Tools to generate strings from regular expressions."""
+
+import json
+import sys
+from pathlib import Path
+
+
+def read_json(filepath):
+    """Validate and open a file path."""
+    filepath = Path(filepath)
+    if not filepath.exists():
+        raise ValueError(
+            f"A file named '{filepath.name}' does not exist. Please specify a different filename."
+        )
+
+    with open(filepath, 'r', encoding='utf-8') as metadata_file:
+        return json.load(metadata_file)
+
+
+def validate_file_does_not_exist(filepath):
+    """Validate a file path doesn't exist."""
+    filepath = Path(filepath)
+    if filepath.exists():
+        raise ValueError(
+            f"A file named '{filepath.name}' already exists in this folder. Please specify "
+            'a different filename.'
+        )
+
+
+def _validate_file_mode(mode):
+    possible_modes = ['write', 'overwrite']
+    if mode not in possible_modes:
+        raise ValueError(f"Mode '{mode}' must be in {possible_modes}.")
+
+
+def _format_metadata_value(value):
+    """Format a value for display, quoting only strings.
+
+    Args:
+        value:
+            The value to format. Boolean and None are returned as their
+            string representation; all other values are wrapped in single quotes.
+
+    Returns:
+        str:
+            The formatted value as a string.
+    """
+    if isinstance(value, bool) or value is None:
+        return str(value)
+    return f"'{value}'"
+
+
+def _format_column_metadata(sdtype_info):
+    """Format a column's metadata dictionary as a display string, with sdtype first.
+
+    Args:
+        sdtype_info (dict):
+            A dictionary of column metadata (`{'sdtype': 'ssn', 'pii': False}`).
+
+    Returns:
+        str:
+            A comma-separated `key=value` string with 'sdtype' first.
+            (`sdtype='numerical', computer_representation='Float'`)
+    """
+    parts = [f'{k}={_format_metadata_value(v)}' for k, v in sdtype_info.items()]
+    parts.sort(key=lambda p: not p.startswith('sdtype='))
+    return ', '.join(parts)
+
+
+def _print_primary_key_detection(chosen_pk, sdtype_updated, pii_removed):
+    if not chosen_pk:
+        sys.stdout.write('- No primary key found\n')
+        return
+
+    notes = []
+    if sdtype_updated:
+        notes.append("updating sdtype to 'id'")
+    if pii_removed:
+        notes.append("removing 'pii' field")
+
+    suffix = f' ({", ".join(notes)})' if notes else ''
+    sys.stdout.write(f"- primary_key='{chosen_pk}'{suffix}\n")

{sdv-1.35.2.dev0 → sdv-1.36.1.dev0/sdv.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdv
-Version: 1.35.2.dev0
+Version: 1.36.1.dev0
 Summary: Generate synthetic data for single table, multi table and sequential data
 Author-email: "DataCebo, Inc." <info@sdv.dev>
 License-Expression: BUSL-1.1

sdv-1.35.2.dev0/sdv/metadata/utils.py

@@ -1,32 +0,0 @@
-"""Tools to generate strings from regular expressions."""
-
-import json
-from pathlib import Path
-
-
-def read_json(filepath):
-    """Validate and open a file path."""
-    filepath = Path(filepath)
-    if not filepath.exists():
-        raise ValueError(
-            f"A file named '{filepath.name}' does not exist. Please specify a different filename."
-        )
-
-    with open(filepath, 'r', encoding='utf-8') as metadata_file:
-        return json.load(metadata_file)
-
-
-def validate_file_does_not_exist(filepath):
-    """Validate a file path doesn't exist."""
-    filepath = Path(filepath)
-    if filepath.exists():
-        raise ValueError(
-            f"A file named '{filepath.name}' already exists in this folder. Please specify "
-            'a different filename.'
-        )
-
-
-def _validate_file_mode(mode):
-    possible_modes = ['write', 'overwrite']
-    if mode not in possible_modes:
-        raise ValueError(f"Mode '{mode}' must be in {possible_modes}.")