sdg-core-lib 0.1.7.dev2__tar.gz → 0.1.7.dev4__tar.gz

{sdg_core_lib-0.1.7.dev2 → sdg_core_lib-0.1.7.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdg-core-lib
-Version: 0.1.7.dev2
+Version: 0.1.7.dev4
 Summary: Add your description here
 Author: emiliocimino
 Author-email: emiliocimino <emilio.cimino@outlook.it>

{sdg_core_lib-0.1.7.dev2 → sdg_core_lib-0.1.7.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdg-core-lib"
-version = "0.1.7.dev2"
+version = "0.1.7.dev4"
 description = "Add your description here"
 readme = "README.md"
 authors = [

sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/data_generator/models/GANs/CTGANComponents.py

@@ -0,0 +1,315 @@
+import tensorflow as tf
+from keras import ops
+from keras.api import layers
+import numpy as np
+import keras
+
+
+class CTGANCritic(keras.Model):
+    def __init__(self, pac_size: int = 10, hidden: int = 256, dropout: float = 0.2, **kwargs):
+        super(CTGANCritic, self).__init__(**kwargs)
+        self.pac_size = pac_size
+        self.fc1 = layers.Dense(hidden)
+        self.fc2 = layers.Dense(hidden)
+        self.out = layers.Dense(1)
+        self.leaky = layers.LeakyReLU(negative_slope=0.2)
+        self.drop = layers.Dropout(dropout)
+    
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'pac_size': self.pac_size,
+            'hidden': self.fc1.units,
+            'dropout': self.drop.rate,
+        })
+        return config
+    
+    @classmethod
+    def from_config(cls, config):
+        # Filter out only the parameters our constructor expects
+        constructor_params = {
+            'pac_size': config.get('pac_size', 10),
+            'hidden': config.get('hidden', 256),
+            'dropout': config.get('dropout', 0.2),
+        }
+        return cls(**constructor_params)
+
+    def call(self, x, training=False):
+        batch_size = tf.shape(x)[0]
+        feature_dim = tf.shape(x)[1]
+        remainder = batch_size % self.pac_size
+        
+        def pad_batch():
+            padding_size = self.pac_size - remainder
+            last_sample = tf.expand_dims(x[-1], axis=0)
+            padding = tf.tile(last_sample, [padding_size, 1])
+            return tf.concat([x, padding], axis=0), padding_size
+        
+        def no_padding():
+            return x, 0
+        
+        x_padded, pad_size = tf.cond(
+            remainder > 0,
+            pad_batch,
+            no_padding
+        )
+        
+        x_reshaped = tf.reshape(x_padded, [-1, self.pac_size * feature_dim])
+
+        h = self.fc1(x_reshaped)
+        h = self.leaky(h)
+        h = self.drop(h, training=training)
+        h = self.fc2(h)
+        h = self.leaky(h)
+        h = self.drop(h, training=training)
+        score = tf.squeeze(self.out(h), axis=1)
+
+        def remove_padding():
+            valid_groups = (batch_size + self.pac_size - 1) // self.pac_size
+            return score[:valid_groups]
+        
+        def keep_all():
+            return score
+        
+        final_score = tf.cond(
+            remainder > 0,
+            remove_padding,
+            keep_all
+        )
+
+        return tf.cast(final_score, tf.float64)
+
+
+def gumbel_softmax(logits, tau=0.2, hard=True):
+    u = tf.random.uniform(tf.shape(logits), minval=0, maxval=1)
+    gumbel = -tf.math.log(-tf.math.log(u + 1e-20) + 1e-20)
+    y = tf.nn.softmax((logits + gumbel) / tau)
+    if hard:
+        y_hard = tf.cast(tf.equal(y, tf.reduce_max(y, axis=-1, keepdims=True)), y.dtype)
+        y = tf.stop_gradient(y_hard - y) + y
+    return y
+
+
+class CTGANGenerator(keras.Model):
+    def __init__(self, skeleton, modes_per_continuous_column, categories_per_discrete_column, hidden=256):
+        super().__init__()
+        self.skeleton = skeleton
+        self.tau = 0.2
+        self.modes_cont = modes_per_continuous_column
+        self.cats_disc = categories_per_discrete_column
+        self.fc1 = layers.Dense(hidden)
+        self.bn1 = layers.BatchNormalization()
+        self.fc2 = layers.Dense(hidden)
+        self.bn2 = layers.BatchNormalization()
+        self.alpha_heads = [layers.Dense(1) for _ in self.modes_cont]
+        self.beta_heads = [layers.Dense(m) for m in self.modes_cont]
+        self.d_heads = [layers.Dense(d) for d in self.cats_disc]
+    
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            'skeleton': self.skeleton,
+            'modes_per_continuous_column': self.modes_cont,
+            'categories_per_discrete_column': self.cats_disc,
+            'hidden': self.fc1.units,
+        })
+        return config
+    
+    @classmethod
+    def from_config(cls, config):
+        # Filter out only the parameters our constructor expects
+        constructor_params = {
+            'skeleton': config.get('skeleton'),
+            'modes_per_continuous_column': config.get('modes_per_continuous_column'),
+            'categories_per_discrete_column': config.get('categories_per_discrete_column'),
+            'hidden': config.get('hidden', 256),
+        }
+        return cls(**constructor_params)
+
+    def call(self, inputs, training=False):
+        z, cond = inputs
+        h = tf.concat([z, cond], axis=1)
+        h = tf.nn.relu(self.bn1(self.fc1(h), training=training))
+        h = tf.nn.relu(self.bn2(self.fc2(h), training=training))
+
+        alphas, betas, ds = [], [], []
+        for i in range(len(self.alpha_heads)):
+            alphas.append(tf.nn.tanh(self.alpha_heads[i](h)))
+            betas.append(gumbel_softmax(self.beta_heads[i](h), self.tau))
+        for j in range(len(self.d_heads)):
+            ds.append(gumbel_softmax(self.d_heads[j](h), self.tau))
+
+        parts = []
+        c_idx, d_idx = 0, 0
+        for col in self.skeleton:
+            if col["feature_type"] == "continuous":
+                parts.append(alphas[c_idx])
+                parts.append(betas[c_idx])
+                c_idx += 1
+            else:
+                parts.append(ds[d_idx])
+                d_idx += 1
+
+        full_row = tf.concat(parts, axis=1)
+        return full_row, alphas, betas, ds
+
+
+class CTGANModel(keras.Model):
+    def __init__(self, generator: CTGANGenerator, critic: CTGANCritic, onehot_discrete_indexes: list[int] | None = None):
+        super().__init__()
+        self.generator = generator
+        self.critic = critic
+        self.onehot_discrete_indexes = onehot_discrete_indexes
+        self.gen_loss_tracker = keras.metrics.Mean(name="generator_loss")
+        self.critic_loss_tracker = keras.metrics.Mean(name="discriminator_loss")
+        self._train_data = None
+        self.probability_mass_function_list = None
+        self.row_dim = sum(generator.modes_cont) + sum(generator.cats_disc) + len(generator.modes_cont)
+
+    @property
+    def metrics(self):
+        return [self.gen_loss_tracker, self.critic_loss_tracker]
+
+    @tf.function
+    def generate_batch_cond(self, batch_size):
+        num_cats = len(self.generator.cats_disc)
+        total_cond_dim = sum(self.generator.cats_disc)
+        cats_disc = tf.convert_to_tensor(self.generator.cats_disc, dtype=tf.int32)
+
+        col_indices = tf.random.uniform(
+            shape=[batch_size], minval=0, maxval=num_cats, dtype=tf.int32
+        )
+
+        relevant_pmfs = tf.gather(self.probability_mass_function_list, col_indices)
+
+        cat_indices = tf.random.categorical(tf.math.log(relevant_pmfs), num_samples=1)
+        cat_indices = tf.cast(tf.squeeze(cat_indices, axis=1), tf.int32)
+
+        offsets_table = tf.concat([[0], tf.cumsum(cats_disc)[:-1]], axis=0)
+        batch_offsets = tf.gather(offsets_table, col_indices)
+
+        global_hot_indices = batch_offsets + cat_indices
+        row_indices = tf.range(batch_size)
+        scatter_indices = tf.stack([row_indices, global_hot_indices], axis=1)
+
+        cond_batch = tf.scatter_nd(
+            indices=scatter_indices,
+            updates=tf.ones([batch_size], dtype=tf.float32),
+            shape=[batch_size, total_cond_dim],
+        )
+
+        return cond_batch
+
+    @staticmethod
+    @tf.function
+    def sample_real_data(train_tensor, cond, discrete_onehot_indexes):
+        if tf.rank(cond) == 1:
+            cond = tf.expand_dims(cond, axis=0)
+
+        discrete_indices = tf.constant(discrete_onehot_indexes, dtype=tf.int32)
+        cond_indices = tf.cast(tf.argmax(cond, axis=1), tf.int32)
+        target_columns = tf.gather(discrete_indices, cond_indices)
+
+        def sample_single_row(col):
+            mask = tf.equal(train_tensor[:, col], 1.0)
+            elems = tf.boolean_mask(train_tensor, mask)
+            num_elems = tf.shape(elems)[0]
+            tf.Assert(num_elems > 0, ["No row found for condition!"])
+            logits = tf.zeros([1, num_elems])
+            random_idx = tf.random.categorical(logits, 1)
+            random_idx = tf.cast(tf.reshape(random_idx, []), tf.int32)
+            return tf.gather(elems, random_idx)
+
+        return tf.map_fn(
+            sample_single_row, target_columns, fn_output_signature=train_tensor.dtype
+        )
+
+    @tf.function
+    def train_critic(self, real_data, z, cond):
+        with tf.GradientTape() as tape:
+            fake_data, _, _, _ = self.generator([z, cond], training=True)
+
+            alpha = tf.random.uniform([ops.shape(real_data)[0], 1], 0.0, 1.0)
+            alpha = tf.cast(alpha, tf.float64)
+            real_data = tf.cast(real_data, tf.float64)
+            fake_data = tf.cast(fake_data, tf.float64)
+            interpolated = alpha * real_data + (1 - alpha) * fake_data
+            
+            with tf.GradientTape() as gp_tape:
+                gp_tape.watch(interpolated)
+                pred = self.critic(interpolated, training=True)
+            grads = gp_tape.gradient(pred, [interpolated])[0]
+            norm = tf.sqrt(tf.reduce_sum(tf.square(grads), axis=1) + 1e-12)
+            gp = tf.cast(tf.reduce_mean((norm - 1.0) ** 2) * 10.0, tf.float64)
+            
+            real_score = self.critic(real_data, training=True)
+            fake_score = self.critic(fake_data, training=True)
+            c_loss = tf.reduce_mean(fake_score) - tf.reduce_mean(real_score) + gp
+
+        grads_c = tape.gradient(c_loss, self.critic.trainable_variables)
+        self.critic.optimizer.apply_gradients(
+            zip(grads_c, self.critic.trainable_variables)
+        )
+        return c_loss
+
+    @tf.function
+    def train_gen(self, z, cond):
+        with tf.GradientTape() as tape:
+            fake_data, _, _, d_list = self.generator([z, cond], training=True)
+            fake_score = self.critic(fake_data, training=True)
+
+            adv_loss = -tf.reduce_mean(fake_score)
+            d_logits = tf.concat(d_list, axis=1)
+            cond_loss = -tf.reduce_mean(
+                tf.reduce_sum(cond * tf.math.log(d_logits + 1e-8), axis=1)
+            )
+            g_loss = adv_loss + tf.cast(cond_loss, tf.float64)
+
+        grads_g = tape.gradient(g_loss, self.generator.trainable_variables)
+        self.generator.optimizer.apply_gradients(
+            zip(grads_g, self.generator.trainable_variables)
+        )
+        return g_loss
+
+    def get_pmfs(self, train_data):
+        onehot_all = tf.gather(train_data, self.onehot_discrete_indexes, axis=1)
+        pmfs = []
+        curr = 0
+        for sz in self.generator.cats_disc:
+            chunk = onehot_all[:, curr:curr + sz]
+            log_freqs = tf.math.log(tf.reduce_sum(chunk, axis=0) + 1.0)
+            pmfs.append(log_freqs / tf.reduce_sum(log_freqs))
+            curr += sz
+        return pmfs
+
+    def train_step(self, data):
+        batch = ops.shape(data)[0]
+        self.row_dim = ops.shape(data)[1]
+        z = tf.random.normal([batch, self.row_dim - sum(self.generator.cats_disc)])
+        cond = self.generate_batch_cond(batch)
+        real_batch = CTGANModel.sample_real_data(
+            self._train_data, cond, self.onehot_discrete_indexes
+        )
+        c_loss = self.train_critic(real_batch, z, cond)
+        g_loss = self.train_gen(z, cond)
+        
+        self.gen_loss_tracker.update_state(g_loss)
+        self.critic_loss_tracker.update_state(c_loss)
+        return {
+            "g_loss": self.gen_loss_tracker.result(),
+            "d_loss": self.critic_loss_tracker.result(),
+        }
+
+    def compile(self, g_optimizer, d_optimizer):
+        super().compile()
+        self.generator.compile(g_optimizer)
+        self.critic.compile(d_optimizer)
+
+    def generate(self, batch_size: int = 100) -> np.ndarray:
+        if self.generator is None or self.probability_mass_function_list is None:
+            raise RuntimeError("In order to generate some data you need to fit a dataset first!")
+
+        z = keras.random.normal(shape=(batch_size, self.row_dim - sum(self.generator.cats_disc)), seed=42)
+        cond = self.generate_batch_cond(batch_size)
+        gen_x, _, _, _ = self.generator([z, cond], training=False)
+        return ops.convert_to_numpy(gen_x)

sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/data_generator/models/GANs/implementation/CTGAN.py

@@ -0,0 +1,193 @@
+import os
+
+from sdg_core_lib.data_generator.models.UnspecializedModel import UnspecializedModel
+from sdg_core_lib.data_generator.models.ModelInfo import ModelInfo
+from sdg_core_lib.commons import AllowedData, DataType
+
+os.environ["KERAS_BACKEND"] = "tensorflow"
+from sdg_core_lib.data_generator.models.GANs.CTGANComponents import (
+    CTGANGenerator,
+    CTGANCritic,
+    CTGANModel,
+)
+import keras
+from sdg_core_lib.data_generator.models.TrainingInfo import TrainingInfo
+import numpy as np
+
+
+class CTGAN(UnspecializedModel):
+    def __init__(
+        self,
+        metadata: dict,
+        model_name: str,
+        input_shape: str = None,
+        load_path: str = None,
+        gen_hidden=256,
+        critic_hidden=256,
+        pac_size=10,
+        learning_rate=1e-3,
+        batch_size=100,
+        epochs=10,
+        gen_steps=4,
+        critic_dropout=0.2,
+    ):
+        super().__init__(metadata, model_name, input_shape, load_path)
+        self._batch_size = batch_size
+        self._epochs = epochs
+        self._gen_steps = gen_steps
+        self._pac_size = pac_size
+        self._gen_hidden = gen_hidden
+        self._critic_hidden = critic_hidden
+        self._learning_rate = learning_rate
+        self._critic_dropout = critic_dropout
+        self._instantiate()
+
+    @staticmethod
+    def infer_data_structure(skeleton):
+        cats, modes, idxs = [], [], []
+        true_index = 0
+        for col in skeleton:
+            try:
+                f_size = int(col["feature_size"])
+                if col["feature_type"] == "categorical":
+                    cats.append(f_size)
+                    # These are the actual global column indices in the train_tensor
+                    idxs.extend(range(true_index, true_index + f_size))
+                elif f_size <= 1:
+                    raise AttributeError(
+                        "Continous column after normalization must have at least size 2 (1 column "
+                        "for the norm values and another for indicating the onehot of"
+                        "a single mode"
+                    )
+                else:
+                    modes.append(f_size - 1)
+            except KeyError as e:
+                raise AttributeError(
+                    f"The CTGAN needs a valid data schema for each column, "
+                    f"key {e.args[0]} is missing"
+                )
+            true_index += f_size
+        if not cats:
+            raise AttributeError("At least a categorical column must be passed!")
+        return cats, modes, idxs
+
+    def _build(self, input_shape: tuple[int, ...]):
+        """
+            This method is called during init if there is no load path,
+            otherwise the method _load will be called
+        :param input_shape:
+        :return:
+        """
+
+        if (
+            not isinstance(self._metadata, list)
+            or not self._metadata
+            or any(not isinstance(item, dict) or not item for item in self._metadata)
+        ):
+            raise AttributeError("CTGAN needs a data schema in order to work!")
+        # Infer dimensions and indices
+        (
+            categories_per_discrete_column,
+            modes_per_continuous_column,
+            onehot_discrete_indexes,
+        ) = CTGAN.infer_data_structure(self._metadata)
+        self.generator = CTGANGenerator(
+            self._metadata,
+            modes_per_continuous_column,
+            categories_per_discrete_column,
+            self._gen_hidden,
+        )
+        self.critic = CTGANCritic(
+            self._pac_size, self._critic_hidden, self._critic_dropout
+        )
+        return CTGANModel(self.generator, self.critic, onehot_discrete_indexes)
+
+    def _load(self, folder_path: str):
+        # Should set the _model variable CTGAN Model complete with Generator and Critic
+        # Does NOT return the model
+        # self._metadata is available
+        _, _, onehot_discrete_indexes = CTGAN.infer_data_structure(self._metadata)
+        critic = keras.saving.load_model(os.path.join(folder_path, "critic.keras"))
+        generator = keras.saving.load_model(
+            os.path.join(folder_path, "generator.keras")
+        )
+        self._model = CTGANModel(generator, critic, onehot_discrete_indexes)
+        
+        # Load probability_mass_function_list if it exists
+        pmf_path = os.path.join(folder_path, "probability_mass_function_list.npy")
+        if os.path.exists(pmf_path):
+            self._model.probability_mass_function_list = np.load(pmf_path, allow_pickle=True)
+
+    def save(self, folder_path: str):
+        keras.saving.save_model(
+            self._model.generator, os.path.join(folder_path, "generator.keras")
+        )
+        keras.saving.save_model(
+            self._model.critic, os.path.join(folder_path, "critic.keras")
+        )
+        
+        if hasattr(self._model, 'probability_mass_function_list') and self._model.probability_mass_function_list is not None:
+            np.save(
+                os.path.join(folder_path, "probability_mass_function_list.npy"),
+                self._model.probability_mass_function_list
+            )
+
+    def train(self, data: np.ndarray):
+        """
+        The idea is to condense training hyperparams here and call
+        Since learning_rate and other hyperparams comes from user, it should be better defining
+        generator optimizer and critic optimizer here and pass them through the model.fit method.
+        self._model.fit(data, gen_opt, crit_opt, ....)
+        :param data:
+        :return: Nothing
+        IMPORTANT: Here TrainingInfo should be defined. See KerasBaseVAE train method
+        """
+        self._model.compile(
+            g_optimizer=keras.optimizers.Adam(self._learning_rate, beta_1=0.5, beta_2=0.9),
+            d_optimizer=keras.optimizers.Adam(self._learning_rate, beta_1=0.5, beta_2=0.9),
+        )
+        self._model._train_data = data
+        probability_mass_function_list = self._model.get_pmfs(data)
+        self._model.probability_mass_function_list = keras.ops.convert_to_numpy(probability_mass_function_list)
+        history = self._model.fit(data, batch_size=self._batch_size, epochs=self._epochs, verbose=1)
+        self.training_info = TrainingInfo(
+            loss_fn="Generator Adversary Loss + Log-frequency weighted cross-entropy",
+            train_loss= history.history["g_loss"][-1].numpy().item(),
+            train_samples=data.shape[0],
+            validation_loss=-1,
+            validation_samples=0,
+        )
+
+    def fine_tune(self, data: np.ndarray, **kwargs):
+        raise NotImplementedError
+
+    def infer(self, n_rows: int, **kwargs):
+        return self._model.generate(n_rows)
+
+    def set_hyperparameters(self, **kwargs):
+        """
+        Define some hyperarams that can be defined outside using kwargs
+        :param kwargs:
+        :return:
+        """
+        self._batch_size = int(kwargs.get("batch_size", self._batch_size))
+        self._epochs = int(kwargs.get("epochs", self._epochs))
+        self._pac_size = kwargs.get("pac_size", self._pac_size)
+        self._gen_hidden = kwargs.get("gen_hidden", self._gen_hidden)
+        self._critic_hidden = kwargs.get("critic_hidden", self._critic_hidden)
+        self._learning_rate = float(kwargs.get("learning_rate", self._learning_rate))
+        self._critic_dropout = kwargs.get("critic_dropout", self._critic_dropout)
+
+    @classmethod
+    def self_describe(cls):
+        return ModelInfo(
+            name=f"{cls.__module__}.{cls.__qualname__}",
+            default_loss_function="Mean",
+            description="A Conditional Tabular Generative Adversarial Network for data generation",
+            allowed_data=[
+                AllowedData(DataType.float32, False),
+                AllowedData(DataType.int32, False),
+                AllowedData(DataType.int32, True),
+                AllowedData(DataType.string, True),
+            ],
+        ).get_model_info()

{sdg_core_lib-0.1.7.dev2 → sdg_core_lib-0.1.7.dev4}/src/sdg_core_lib/data_generator/models/ModelInfo.py

@@ -14,7 +14,7 @@ class ModelInfo:
         self.description = description
         self.allowed_data = allowed_data
 
-    def get_model_info(self):
+    def get_model_info(self) -> dict:
         """
         Returns a dictionary containing the model information.
 

{sdg_core_lib-0.1.7.dev2 → sdg_core_lib-0.1.7.dev4}/src/sdg_core_lib/data_generator/models/UnspecializedModel.py

@@ -8,18 +8,17 @@ class UnspecializedModel(ABC):
     by all subclasses.
 
     Attributes:
-        _metadata (dict): A dictionary containing miscellaneous information about the model.
+        _metadata (dict): A dictionary containing miscellaneous information about the data structure used by a model.
         model_name (str): The model name, used to identify the model itself.
         input_shape (tuple): A tuple containing the input shape of the model.
         _load_path (str): A string containing the path where to load the model from.
         _model (keras.Model): The model instance.
-        _scaler (Scaler): The scaler instance.
         training_info (TrainingInfo): The training info instance.
     """
 
     def __init__(
         self,
-        metadata: dict,
+        metadata: list[dict],
         model_name: str,
         input_shape: str = None,
         load_path: str = None,
@@ -30,7 +29,6 @@ class UnspecializedModel(ABC):
         self._load_path = load_path
         self._model = None  # Placeholder for the model instance
         self.training_info = None  # Placeholder for training info
-        self._model_misc = None  # Placeholder for model miscellaneous info
 
     @abstractmethod
     def _build(self, input_shape: tuple[int, ...]):

{sdg_core_lib-0.1.7.dev2/src/sdg_core_lib/data_generator/models/keras → sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/data_generator/models/VAEs}/implementation/AutoTabularVAE.py

@@ -7,8 +7,8 @@ from keras_tuner import HyperParameters
 from sdg_core_lib.data_generator.models.TrainingInfo import TrainingInfo
 from sdg_core_lib.data_generator.models.ModelInfo import ModelInfo
 from sdg_core_lib.commons import AllowedData, DataType
-from sdg_core_lib.data_generator.models.keras.VAE import Sampling, VAE
-from sdg_core_lib.data_generator.models.keras.implementation.TabularVAE import (
+from sdg_core_lib.data_generator.models.VAEs.VAE import Sampling, VAE
+from sdg_core_lib.data_generator.models.VAEs.implementation.TabularVAE import (
     TabularVAE,
 )
 

{sdg_core_lib-0.1.7.dev2/src/sdg_core_lib/data_generator/models/keras → sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/data_generator/models/VAEs}/implementation/TabularVAE.py

@@ -3,8 +3,8 @@ from keras import layers
 
 from sdg_core_lib.data_generator.models.ModelInfo import ModelInfo
 from sdg_core_lib.commons import AllowedData, DataType
-from sdg_core_lib.data_generator.models.keras.KerasBaseVAE import KerasBaseVAE
-from sdg_core_lib.data_generator.models.keras.VAE import Sampling, VAE
+from sdg_core_lib.data_generator.models.VAEs.KerasBaseVAE import KerasBaseVAE
+from sdg_core_lib.data_generator.models.VAEs.VAE import Sampling, VAE
 
 
 class TabularVAE(KerasBaseVAE):

{sdg_core_lib-0.1.7.dev2/src/sdg_core_lib/data_generator/models/keras → sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/data_generator/models/VAEs}/implementation/TimeSeriesVAE.py

@@ -3,10 +3,10 @@ import keras
 
 from sdg_core_lib.data_generator.models.ModelInfo import ModelInfo
 from sdg_core_lib.commons import AllowedData, DataType
-from sdg_core_lib.data_generator.models.keras.KerasBaseVAE import KerasBaseVAE
+from sdg_core_lib.data_generator.models.VAEs.KerasBaseVAE import KerasBaseVAE
 from keras import layers
 
-from sdg_core_lib.data_generator.models.keras.VAE import Sampling, VAE
+from sdg_core_lib.data_generator.models.VAEs.VAE import Sampling, VAE
 
 
 class TimeSeriesVAE(KerasBaseVAE):

{sdg_core_lib-0.1.7.dev2 → sdg_core_lib-0.1.7.dev4}/src/sdg_core_lib/job.py

@@ -1,10 +1,11 @@
 from typing import Optional
 
+from sdg_core_lib.data_generator.models.GANs.implementation.CTGAN import CTGAN
 from sdg_core_lib.data_generator.models.UnspecializedModel import UnspecializedModel
-from sdg_core_lib.data_generator.models.keras.implementation.TabularVAE import (
+from sdg_core_lib.data_generator.models.VAEs.implementation.TabularVAE import (
     TabularVAE,
 )
-from sdg_core_lib.data_generator.models.keras.implementation.TimeSeriesVAE import (
+from sdg_core_lib.data_generator.models.VAEs.implementation.TimeSeriesVAE import (
     TimeSeriesVAE,
 )
 from sdg_core_lib.dataset.datasets import Dataset, Table, TimeSeries
@@ -16,6 +17,7 @@ from sdg_core_lib.preprocess.strategies.vae_strategy import (
     TabularVAEPreprocessingStrategy,
     TimeSeriesVAEPreprocessingStrategy,
 )
+from sdg_core_lib.preprocess.strategies.ctgan_strategy import CTGANPreprocessingStrategy
 from sdg_core_lib.evaluate.tables import TabularComparisonEvaluator
 from sdg_core_lib.evaluate.time_series import TimeSeriesComparisonEvaluator
 import importlib
@@ -23,13 +25,13 @@ import os
 
 
 def get_hyperparameters() -> dict:
+    hyperparams_name = ["EPOCHS", "LEARNING_RATE", "BATCH_SIZE"]
     return {
-        "epochs": os.environ.get("EPOCHS"),
-        "learning_rate": os.environ.get("LEARNING_RATE"),
-        "batch_size": os.environ.get("BATCH_SIZE"),
+        hp.lower(): os.environ.get(hp)
+        for hp in hyperparams_name
+        if os.environ.get(hp) is not None
     }
 
-
 class Job:
     dataset_mapping = {
         "table": {
@@ -47,6 +49,7 @@ class Job:
     model_strategy_mapping: dict[type, BasePreprocessingStrategy] = {
         TabularVAE: TabularVAEPreprocessingStrategy,
         TimeSeriesVAE: TimeSeriesVAEPreprocessingStrategy,
+        CTGAN: CTGANPreprocessingStrategy,
     }
 
     def __init__(

sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/preprocess/strategies/ctgan_strategy.py

@@ -0,0 +1,23 @@
+from sdg_core_lib.dataset.columns import Column, Numeric, Categorical
+from sdg_core_lib.preprocess.strategies.base_strategy import BasePreprocessingStrategy
+from sdg_core_lib.preprocess.strategies.steps import (
+    Step,
+    NoneStep,
+    PerModeNormalization,
+    OneHotEncoderWrapper,
+)
+
+
+class CTGANPreprocessingStrategy(BasePreprocessingStrategy):
+    @staticmethod
+    def get_steps_per_feature(feature: Column) -> list[Step]:
+        step_list = []
+        if isinstance(feature, Numeric):
+            step_list.append(PerModeNormalization(feature.position, feature.name))
+        elif isinstance(feature, Categorical):
+            step_list.append(OneHotEncoderWrapper(feature.position, feature.name))
+        elif type(feature) is Column:
+            step_list.append(NoneStep(feature.position))
+        else:
+            raise NotImplementedError()
+        return step_list

sdg_core_lib-0.1.7.dev4/src/sdg_core_lib/preprocess/strategies/steps.py

@@ -0,0 +1,259 @@
+from abc import ABC, abstractmethod
+from typing import Literal
+
+import numpy as np
+from sklearn.preprocessing import (
+    MinMaxScaler,
+    StandardScaler,
+    OneHotEncoder,
+    OrdinalEncoder,
+)
+import os
+import skops.io as sio
+from sklearn.mixture import BayesianGaussianMixture
+
+
+class Step(ABC):
+    def __init__(self, type_name: str, position: int, col_name: str, mode: str):
+        self.type_name = type_name
+        self.mode = mode
+        self.position = position
+        self.col_name = col_name
+        self.operator = None
+        self.filename = (
+            f"{self.position}_{self.col_name}_{self.mode}_{self.type_name}.skops"
+        )
+
+    @abstractmethod
+    def _set_operator(self):
+        raise NotImplementedError
+
+    def save_if_not_exist(self, directory_path: str):
+        if self.operator is None:
+            raise ValueError("Operator is not created")
+        os.makedirs(directory_path, exist_ok=True)
+        filename = os.path.join(directory_path, self.filename)
+        if not os.path.exists(filename):
+            sio.dump(self.operator, filename)
+
+    def load(self, directory_path: str):
+        filename = os.path.join(directory_path, self.filename)
+        if not os.path.isfile(filename):
+            raise FileNotFoundError(f"Operator file not found: {filename}")
+        self.operator = sio.load(filename)
+
+    def fit_transform(self, data: np.ndarray) -> np.ndarray:
+        self.operator = self._set_operator()
+        return self.operator.fit_transform(data)
+
+    def transform(self, data: np.ndarray) -> np.ndarray:
+        if self.operator is None:
+            raise ValueError("Operator not initialized")
+        return self.operator.transform(data)
+
+    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
+        if self.operator is None:
+            raise ValueError("Operator not initialized")
+        return self.operator.inverse_transform(data)
+
+
+class NoneStep(Step):
+    def __init__(self, position: int, mode="", type_name="none"):
+        super().__init__(type_name=type_name, position=position, col_name="", mode=mode)
+
+    def save_if_not_exist(self, directory_path: str):
+        pass
+
+    def load(self, directory_path: str):
+        pass
+
+    def _set_operator(self):
+        pass
+
+    def fit_transform(self, data: np.ndarray) -> np.ndarray:
+        return data
+
+    def transform(self, data: np.ndarray) -> np.ndarray:
+        return data
+
+    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
+        return data
+
+
+class ScalerWrapper(Step):
+    def __init__(
+        self,
+        position: int,
+        col_name: str,
+        mode: Literal["minmax", "standard"] = "standard",
+        type_name="scaler",
+    ):
+        super().__init__(
+            type_name=type_name, position=position, col_name=col_name, mode=mode
+        )
+
+    def _set_operator(self):
+        if self.mode == "minmax":
+            return MinMaxScaler()
+        elif self.mode == "standard":
+            return StandardScaler()
+        else:
+            raise ValueError("Invalid mode while setting the scaler")
+
+
+class OrdinalEncoderWrapper(Step):
+    def __init__(
+        self, position: int, col_name: str, mode="ordinal", type_name="encoder"
+    ):
+        super().__init__(
+            type_name=type_name, position=position, col_name=col_name, mode=mode
+        )
+
+    def _set_operator(self):
+        return OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan)
+
+
+class OneHotEncoderWrapper(Step):
+    def __init__(
+        self, position: int, col_name: str, mode="one_hot", type_name="encoder"
+    ):
+        super().__init__(
+            type_name=type_name, position=position, col_name=col_name, mode=mode
+        )
+
+    def _set_operator(self):
+        return OneHotEncoder(handle_unknown="error")
+
+    def fit_transform(self, data: np.ndarray) -> np.ndarray:
+        return super().fit_transform(data).toarray()
+
+    def transform(self, data: np.ndarray) -> np.ndarray:
+        return super().transform(data).toarray()
+
+    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
+        # Numerical stability for all zeros
+        data = data + np.ones(data.shape) * 1e-9
+        return super().inverse_transform(data)
+
+
+class PerModeNormalization(Step):
+    """
+    This step estimates using variational gaussian mixtures models
+    the number of modes the data may come from and performs mode specific
+    normalization that will be later used by a CTGAN. This step also
+    saves this information in order to perform inverse transformations
+    """
+
+    def __init__(
+        self,
+        position: int,
+        col_name: str,
+        mode: str = "",
+        type_name="per_mode_normalization",
+        n_components=10,
+        max_iter=1000,
+        random_state=42,
+    ):
+        super().__init__(
+            type_name=type_name, position=position, col_name=col_name, mode=mode
+        )
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.random_state = random_state
+
+    def _set_operator(self):
+        vbgmm = BayesianGaussianMixture(
+            n_components=self.n_components,
+            weight_concentration_prior_type="dirichlet_process",
+            covariance_type="full",
+            max_iter=1000,
+            random_state=self.random_state,
+        )
+        return vbgmm
+
+    def fit_transform(self, data: np.ndarray) -> np.ndarray:
+        self.operator = self._set_operator()
+        self.operator.fit(data)
+        return self.transform(data)
+
+    def transform(self, data: np.ndarray) -> np.ndarray:
+        if self.operator is None:
+            raise ValueError("Operator not initialized")
+        column = data.reshape(-1, 1)
+        active_weights_indx = np.where(self.operator.weights_ > 0.01)
+        weights = self.operator.weights_[active_weights_indx]
+        means = self.operator.means_[active_weights_indx].flatten()
+        stds = np.sqrt(self.operator.covariances_[active_weights_indx].flatten())
+        mixture_probability_density = []
+        for w, m, s in zip(weights, means, stds):
+            mixture_probability_density.append(
+                w
+                * PerModeNormalization._gaussian_probability_density_function(
+                    column, m, s
+                )
+            )
+        marginal_mixture_probability_density = np.hstack(mixture_probability_density)
+        responsibilities = PerModeNormalization._compute_responsibilities(
+            marginal_mixture_probability_density
+        )
+        rng = np.random.default_rng(self.random_state)
+        n, K = responsibilities.shape
+        sampled_mode = np.array(
+            [rng.choice(K, p=responsibilities[i]) for i in range(n)]
+        )
+        f = np.zeros((n, K), dtype=int)
+        f[np.arange(n), sampled_mode] = 1
+        mu_sel = means[sampled_mode]
+        std_sel = stds[sampled_mode]
+        normalized_value = (column.reshape(-1) - mu_sel) / (4.0 * std_sel)
+        to_return = np.concatenate([normalized_value.reshape(-1, 1), f], axis=1)
+        return to_return
+
+    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
+        if self.operator is None:
+            raise ValueError("Operator not initialized")
+        active_weights_indx = np.where(self.operator.weights_ > 0.01)
+        means = self.operator.means_[active_weights_indx].flatten()
+        stds = np.sqrt(self.operator.covariances_[active_weights_indx].flatten())
+        
+        # Handle both 1D and 2D input data
+        if data.ndim == 1:
+            # Data is 1D, reshape to 2D for processing
+            data = data.reshape(1, -1)
+            was_1d = True
+        else:
+            was_1d = False
+            
+        rows = data.shape[0]
+        
+        # Find the active mode for each row (argmax of one-hot encoded modes)
+        active_modes = np.argmax(data[:, 1:], axis=1)
+        
+        # Get the means and stds for the active modes
+        selected_mus = means[active_modes]
+        selected_devs = stds[active_modes]
+        
+        # Get the normalized values (first column)
+        normalized_values = data[:, 0]
+        
+        # Denormalize the values
+        values = (normalized_values * 4 * selected_devs) + selected_mus
+        
+        # Always return 2D array with shape (n_samples, 1) for consistency
+        return values.reshape(-1, 1)
+
+    @staticmethod
+    def _gaussian_probability_density_function(
+        x: np.ndarray, mean: np.ndarray, std: np.ndarray
+    ):
+        """
+        This function computes the probability density function of the gaussian mixture
+        given the mean and standard deviation
+        """
+        return (1 / (np.sqrt(2 * np.pi) * std)) * np.exp(
+            -0.5 * (x - mean) ** 2 / (std**2)
+        )
+
+    @staticmethod
+    def _compute_responsibilities(pdf_per_mode: np.ndarray) -> np.ndarray:
+        return pdf_per_mode / pdf_per_mode.sum(axis=1, keepdims=True)

sdg_core_lib-0.1.7.dev2/src/sdg_core_lib/preprocess/strategies/steps.py

@@ -1,137 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Literal
-
-import numpy as np
-from sklearn.preprocessing import (
-    MinMaxScaler,
-    StandardScaler,
-    OneHotEncoder,
-    OrdinalEncoder,
-)
-import os
-import skops.io as sio
-
-# TODO: What if steps change data types? Should I implement some "old_type / new_type" mechanics?
-
-
-class Step(ABC):
-    def __init__(self, type_name: str, position: int, col_name: str, mode: str):
-        self.type_name = type_name
-        self.mode = mode
-        self.position = position
-        self.col_name = col_name
-        self.operator = None
-        self.filename = (
-            f"{self.position}_{self.col_name}_{self.mode}_{self.type_name}.skops"
-        )
-
-    @abstractmethod
-    def _set_operator(self):
-        raise NotImplementedError
-
-    def save_if_not_exist(self, directory_path: str):
-        if self.operator is None:
-            raise ValueError("Operator is not created")
-        os.makedirs(directory_path, exist_ok=True)
-        filename = os.path.join(directory_path, self.filename)
-        if not os.path.exists(filename):
-            sio.dump(self.operator, filename)
-
-    def load(self, directory_path: str):
-        filename = os.path.join(directory_path, self.filename)
-        if not os.path.isfile(filename):
-            raise FileNotFoundError(f"Operator file not found: {filename}")
-        self.operator = sio.load(filename)
-
-    def fit_transform(self, data: np.ndarray) -> np.ndarray:
-        self.operator = self._set_operator()
-        return self.operator.fit_transform(data)
-
-    def transform(self, data: np.ndarray) -> np.ndarray:
-        if self.operator is None:
-            raise ValueError("Operator not initialized")
-        return self.operator.transform(data)
-
-    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
-        if self.operator is None:
-            raise ValueError("Operator not initialized")
-        return self.operator.inverse_transform(data)
-
-
-class NoneStep(Step):
-    def __init__(self, position: int, mode="", type_name="none"):
-        super().__init__(type_name=type_name, position=position, col_name="", mode=mode)
-
-    def save_if_not_exist(self, directory_path: str):
-        pass
-
-    def load(self, directory_path: str):
-        pass
-
-    def _set_operator(self):
-        pass
-
-    def fit_transform(self, data: np.ndarray) -> np.ndarray:
-        return data
-
-    def transform(self, data: np.ndarray) -> np.ndarray:
-        return data
-
-    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
-        return data
-
-
-class ScalerWrapper(Step):
-    def __init__(
-        self,
-        position: int,
-        col_name: str,
-        mode: Literal["minmax", "standard"] = "standard",
-        type_name="scaler",
-    ):
-        super().__init__(
-            type_name=type_name, position=position, col_name=col_name, mode=mode
-        )
-
-    def _set_operator(self):
-        if self.mode == "minmax":
-            return MinMaxScaler()
-        elif self.mode == "standard":
-            return StandardScaler()
-        else:
-            raise ValueError("Invalid mode while setting the scaler")
-
-
-class OrdinalEncoderWrapper(Step):
-    def __init__(
-        self, position: int, col_name: str, mode="ordinal", type_name="encoder"
-    ):
-        super().__init__(
-            type_name=type_name, position=position, col_name=col_name, mode=mode
-        )
-
-    def _set_operator(self):
-        return OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan)
-
-
-class OneHotEncoderWrapper(Step):
-    def __init__(
-        self, position: int, col_name: str, mode="one_hot", type_name="encoder"
-    ):
-        super().__init__(
-            type_name=type_name, position=position, col_name=col_name, mode=mode
-        )
-
-    def _set_operator(self):
-        return OneHotEncoder(handle_unknown="error")
-
-    def fit_transform(self, data: np.ndarray) -> np.ndarray:
-        return super().fit_transform(data).toarray()
-
-    def transform(self, data: np.ndarray) -> np.ndarray:
-        return super().transform(data).toarray()
-
-    def inverse_transform(self, data: np.ndarray) -> np.ndarray:
-        # Numerical stability for all zeros
-        data = data + np.ones(data.shape) * 1e-9
-        return super().inverse_transform(data)