calm-data-generator 2.2.0__tar.gz → 2.2.1__tar.gz

{calm_data_generator-2.2.0/calm_data_generator.egg-info → calm_data_generator-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calm-data-generator
-Version: 2.2.0
+Version: 2.2.1
 Summary: CALM-Data-Generator: A Python library for synthetic data generation with support for drift injection and clinical data.
 Author-email: Alejandro Belda Fernandez <alejandrobeldafernandez@gmail.com>
 License: MIT

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/calm_data_generator/__init__.py

@@ -1,11 +1,11 @@
 # CALM-Data-Generator - Synthetic Data Generation Library
 
-from calm_data_generator.generators.tabular import RealGenerator, QualityReporter
+from calm_data_generator import presets
 from calm_data_generator.generators.clinical import ClinicalDataGenerator
-from calm_data_generator.generators.drift import DriftInjector
-from calm_data_generator.generators.dynamics import ScenarioInjector, CausalEngine
 from calm_data_generator.generators.complex import ComplexGenerator
-from calm_data_generator import presets
+from calm_data_generator.generators.drift import DriftInjector
+from calm_data_generator.generators.dynamics import CausalEngine, ScenarioInjector
+from calm_data_generator.generators.tabular import QualityReporter, RealGenerator
 
 # Optional imports that may fail
 try:
@@ -13,7 +13,7 @@ try:
 except ImportError:
     StreamGenerator = None
 
-__version__ = "2.1.0"
+__version__ = "2.2.1"
 
 __all__ = [
     # Generators

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/calm_data_generator/generators/tabular/RealBlockGenerator.py

@@ -13,11 +13,11 @@ Key Features:
 - **Timestamp Alignment**: Can inject timestamps that are aligned with the block structure.
 """
 
-import os
 import logging
+import os
 import warnings
 from pathlib import Path
-from typing import Optional, Dict, Any, List, Union
+from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -27,9 +27,10 @@ warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 
-from .RealGenerator import RealGenerator
-from calm_data_generator.reports.QualityReporter import QualityReporter
-from calm_data_generator.generators.configs import DriftConfig, ReportConfig
+from calm_data_generator.generators.configs import DriftConfig, ReportConfig  # noqa: E402
+from calm_data_generator.reports.QualityReporter import QualityReporter  # noqa: E402
+
+from .RealGenerator import RealGenerator  # noqa: E402
 
 
 class RealBlockGenerator(RealGenerator):
@@ -59,7 +60,7 @@ class RealBlockGenerator(RealGenerator):
         self.logger = logging.getLogger(self.__class__.__name__)
         self.logger.setLevel(logging.INFO)
 
-       
+
         self.logger.info("RealBlockGenerator initialized.")
 
     # --------------------------------------------------------------------------- #
@@ -162,10 +163,10 @@ class RealBlockGenerator(RealGenerator):
             data=block_data_no_block,
             method=method,
             target_col=target_col,
-            model_params=model_params,
             n_samples=n_samples,
             output_dir=output_dir,
             custom_distributions=custom_distributions,
+            **(model_params or {}),
         )
 
         if synthetic_block is None:

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/calm_data_generator/generators/tabular/RealGenerator.py

@@ -642,6 +642,250 @@ class RealGenerator(BaseGenerator):
 
         return None
 
+    def encode_to_latent(
+        self,
+        data,
+        target_col: Optional[str] = None,
+    ) -> "torch.Tensor":
+        """
+        Encodes ``data`` into the model's latent space, handling the full
+        preprocessing pipeline for each supported method.
+
+        - **tvae / rtvae**: runs TabularEncoder → VAE encoder → returns ``mu``
+          (mean of the approximate posterior).  The TabularEncoder was fitted
+          on the *complete* training DataFrame (including ``target_col``), so
+          ``data`` must contain the same columns, in the same order, as the
+          training data — ``target_col`` included.
+        - **scvi / scanvi**: calls ``model.get_latent_representation()`` and
+          returns the result as a float32 tensor.
+
+        Unlike :meth:`get_encoder`, which exposes the raw PyTorch module, this
+        method handles the method-specific preprocessing automatically. Use it
+        when implementing external drift analyses that need latent
+        representations for both TVAE and SCVI without rewriting the encoding
+        pipeline each time.
+
+        Parameters
+        ----------
+        data:
+            - TVAE/RTVAE: a :class:`pandas.DataFrame` with **all** columns
+              used during training (including ``target_col``).
+            - SCVI/SCANVI: a :class:`pandas.DataFrame` *or* an
+              ``AnnData`` object whose ``var_names`` match the training genes.
+        target_col:
+            Label column used to build the optional conditioning tensor for
+            TVAE models trained with a conditional dimension.  For SCVI this
+            argument is ignored.
+
+        Returns
+        -------
+        torch.Tensor
+            Float32 tensor of shape ``(n_samples, n_latent)``.
+
+        Raises
+        ------
+        RuntimeError
+            If no model has been trained yet, or the method is not supported.
+        """
+        if not self.synthesizer:
+            raise RuntimeError("No trained model found. Call generate() first.")
+
+        if self.method in ("tvae", "rtvae"):
+            tabular_model = self.synthesizer.model
+            pytorch_model = tabular_model.model
+            pytorch_model.eval()
+
+            # TVAE trains the TabularEncoder on the FULL DataFrame (including
+            # target_col).  We must pass the same columns here; do not drop
+            # target_col before encoding.
+            data_encoded = tabular_model.encode(data)
+            data_tensor = torch.tensor(
+                data_encoded.values, dtype=torch.float32
+            ).to(pytorch_model.device)
+
+            # Append conditional dimensions expected by the encoder.
+            cond_dim = getattr(pytorch_model, "n_units_conditional", 0)
+            if cond_dim > 0 and target_col and isinstance(data, pd.DataFrame) and target_col in data.columns:
+                unique_classes = data[target_col].unique()
+                label_to_idx = {str(c): i for i, c in enumerate(unique_classes)}
+                label_indices = torch.tensor(
+                    [label_to_idx.get(str(l), 0) for l in data[target_col].values],
+                    dtype=torch.long,
+                    device=pytorch_model.device,
+                )
+                cond_tensor = torch.zeros(
+                    len(data), cond_dim, device=pytorch_model.device
+                )
+                cond_tensor.scatter_(
+                    1, label_indices.unsqueeze(1).clamp(max=cond_dim - 1), 1.0
+                )
+                data_tensor = torch.cat([data_tensor, cond_tensor], dim=1)
+
+            with torch.no_grad():
+                encoder_out = pytorch_model.encoder(data_tensor)
+                mu = encoder_out[0] if isinstance(encoder_out, (tuple, list)) else encoder_out
+            return mu
+
+        if self.method in ("scvi", "scanvi"):
+            import torch as _torch
+            model = self.synthesizer
+            z = model.get_latent_representation()
+            return _torch.tensor(z, dtype=_torch.float32)
+
+        raise RuntimeError(
+            f"encode_to_latent() is not supported for method '{self.method}'. "
+            "Supported: tvae, rtvae, scvi, scanvi."
+        )
+
+    def decode_from_latent(
+        self,
+        z: "torch.Tensor",
+        data=None,
+        target_col: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """
+        Decodes latent vectors ``z`` back to the original feature space.
+
+        - **tvae / rtvae**: runs VAE decoder → TabularEncoder inverse_transform.
+        - **scvi / scanvi**: calls the model's generative pass and returns
+          a DataFrame with the original gene columns.
+
+        Parameters
+        ----------
+        z:
+            Float32 tensor of shape ``(n_samples, n_latent)`` produced by
+            :meth:`encode_to_latent` (or a manually perturbed version of it).
+        data:
+            Reference data used to infer library size (SCVI only) and column
+            names.  Can be a :class:`pandas.DataFrame` or ``AnnData``.
+        target_col:
+            Label column to attach to the output DataFrame.  When provided
+            together with ``data``, labels are copied from the reference.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Decoded samples in the original feature space.
+
+        Raises
+        ------
+        RuntimeError
+            If no model has been trained yet, or the method is not supported.
+        """
+        if not self.synthesizer:
+            raise RuntimeError("No trained model found. Call generate() first.")
+
+        n_samples = z.shape[0]
+
+        if self.method in ("tvae", "rtvae"):
+            tabular_model = self.synthesizer.model
+            pytorch_model = tabular_model.model
+            pytorch_model.eval()
+
+            cond_dim = getattr(pytorch_model, "n_units_conditional", 0)
+            cond_tensor = None
+            if cond_dim > 0 and target_col is not None and data is not None and isinstance(data, pd.DataFrame):
+                unique_classes = data[target_col].unique()
+                label_to_idx = {str(c): i for i, c in enumerate(unique_classes)}
+                label_indices = torch.tensor(
+                    [label_to_idx.get(str(l), 0) for l in data[target_col].values],
+                    dtype=torch.long,
+                    device=pytorch_model.device,
+                )
+                cond_tensor = torch.zeros(
+                    n_samples, cond_dim, device=pytorch_model.device
+                )
+                cond_tensor.scatter_(
+                    1, label_indices.unsqueeze(1).clamp(max=cond_dim - 1), 1.0
+                )
+
+            z = z.to(pytorch_model.device)
+            with torch.no_grad():
+                reconstructed = pytorch_model.decoder(z, cond_tensor)
+
+            # Get encoded column names: encode the full data (TabularEncoder
+            # was fitted on the complete DataFrame including target_col).
+            enc_cols = tabular_model.encode(data).columns if data is not None else None
+            reconstructed_df = pd.DataFrame(
+                reconstructed.cpu().numpy(),
+                columns=enc_cols,
+            )
+            synth_df = tabular_model.decode(reconstructed_df)
+            return synth_df
+
+        if self.method in ("scvi", "scanvi"):
+            model = self.synthesizer
+            device = model.device if hasattr(model, "device") else next(model.module.parameters()).device
+
+            if data is not None and hasattr(data, "X"):
+                raw = data.X.toarray() if hasattr(data.X, "toarray") else np.array(data.X)
+                orig_log_lib = np.log(raw.sum(axis=1) + 1e-8)
+            elif data is not None and isinstance(data, pd.DataFrame):
+                num_cols = data.select_dtypes(include=[np.number]).columns
+                if target_col:
+                    num_cols = [c for c in num_cols if c != target_col]
+                orig_log_lib = np.log(data[num_cols].values.sum(axis=1) + 1e-8)
+            else:
+                orig_log_lib = np.zeros(n_samples)
+
+            z = z.to(device)
+            library_tensor = torch.tensor(
+                orig_log_lib, dtype=torch.float32
+            ).unsqueeze(1).to(device)
+            batch_index = torch.zeros(n_samples, 1, dtype=torch.long).to(device)
+
+            y_tensor = None
+            if getattr(model.module, "dispersion", "gene") == "gene-label" and data is not None:
+                try:
+                    label_registry = model.adata_manager.get_state_registry("labels")
+                    cat_mapping = label_registry.categorical_mapping
+                    label_map = {str(cat): i for i, cat in enumerate(cat_mapping)}
+                    if hasattr(data, "obs") and target_col in data.obs.columns:
+                        labels = data.obs[target_col].astype(str).values
+                    elif isinstance(data, pd.DataFrame) and target_col in data.columns:
+                        labels = data[target_col].astype(str).values
+                    else:
+                        labels = None
+                    if labels is not None:
+                        y_tensor = torch.tensor(
+                            [label_map.get(l, 0) for l in labels],
+                            dtype=torch.long,
+                        ).unsqueeze(1).to(device)
+                except Exception:
+                    pass
+
+            with torch.no_grad():
+                gen_out = model.module.generative(
+                    z=z,
+                    library=library_tensor,
+                    batch_index=batch_index,
+                    y=y_tensor,
+                )
+                px_dist = gen_out["px"]
+                vals = (
+                    px_dist.sample().cpu().numpy()
+                    if hasattr(px_dist, "sample")
+                    else px_dist.mean.cpu().numpy()
+                )
+
+            col_names = (
+                data.var_names.tolist()
+                if hasattr(data, "var_names")
+                else [c for c in (data.columns if isinstance(data, pd.DataFrame) else []) if c != target_col]
+            )
+            synth_df = pd.DataFrame(vals, columns=col_names)
+            if target_col is not None and data is not None:
+                if hasattr(data, "obs") and target_col in data.obs.columns:
+                    synth_df[target_col] = data.obs[target_col].values[:n_samples]
+                elif isinstance(data, pd.DataFrame) and target_col in data.columns:
+                    synth_df[target_col] = data[target_col].values[:n_samples]
+            return synth_df
+
+        raise RuntimeError(
+            f"decode_from_latent() is not supported for method '{self.method}'. "
+            "Supported: tvae, rtvae, scvi, scanvi."
+        )
+
     @staticmethod
     def to_anndata(
         df: pd.DataFrame,
@@ -1087,7 +1331,7 @@ class RealGenerator(BaseGenerator):
 
         self.synthesizer = syn
         self.method = "tvae"
-        self.metadata = {"columns": data.columns.tolist()}
+        self.metadata = {"columns": data.columns.tolist(), "target_col": target_col}
 
         if differentiation_factor > 0.0 and target_col and target_col in data.columns:
               synth_df = self.apply_latent_differentiation(
@@ -1189,17 +1433,22 @@ class RealGenerator(BaseGenerator):
         result = result.drop(columns=[stage_col])
         if custom_distributions:
             col = next(iter(custom_distributions), None)
-            if col and col in data.columns:
+            if col and col in result.columns:
                 dist = custom_distributions[col]
                 self.logger.info(
-                    f" Generating conditionally per class on '{col}' — {dist}"
+                    f" Applying custom_distributions on '{col}' — {dist} (resampling from staged result)"
                 )
                 frames = []
                 for cls, proportion in dist.items():
                     n_cls = max(1, round(n_samples * proportion))
-                    cls_df = syn.generate(count=n_cls, random_state=self.random_state).dataframe()
-                    cls_df[col] = cls
-                    frames.append(cls_df)
+                    cls_pool = result[result[col] == cls]
+                    if len(cls_pool) == 0:
+                        cls_pool = result
+                    frames.append(
+                        cls_pool.sample(n=n_cls, replace=len(cls_pool) < n_cls,
+                                        random_state=self.random_state)
+                        .assign(**{col: cls})
+                    )
                 result = (
                     pd.concat(frames, ignore_index=True)
                     .sample(frac=1, random_state=self.random_state)
@@ -1249,7 +1498,7 @@ class RealGenerator(BaseGenerator):
 
         self.synthesizer = syn
         self.method = "rtvae"
-        self.metadata = {"columns": data.columns.tolist()}
+        self.metadata = {"columns": data.columns.tolist(), "target_col": target_col}
 
         if differentiation_factor > 0.0 and target_col and target_col in data.columns:
             synth_df = self.apply_latent_differentiation(

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/calm_data_generator/reports/ExternalReporter.py

@@ -3,10 +3,11 @@ External Reporter Module
 Wraps YData Profiling to generate advanced HTML reports.
 """
 
+import logging
 import os
-import pandas as pd
 from typing import Optional
-import logging
+
+import pandas as pd
 
 # Initializing logger
 logger = logging.getLogger("ExternalReporter")
@@ -83,6 +84,7 @@ class ExternalReporter:
                 sortby=sortby,
                 html={"style": {"full_width": True}},
                 progress_bar=False,
+                minimal=minimal,
             )
 
             profile.to_file(output_path)

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/calm_data_generator/reports/QualityReporter.py

@@ -6,22 +6,24 @@ static report comparing a real dataset with a synthetic one.
 Uses YData Profiling for analysis and Plotly for interactive visualizations.
 """
 
-import pandas as pd
-import numpy as np
-from typing import Optional, Dict, Any, List, Union
-import warnings
+import contextlib
+import io
+import json
 import logging
-from datetime import datetime
 import os
-import json
-import io
-import contextlib
+import warnings
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
 sc = None
 ad = None
 try:
-    from scgft_evaluator import ScGFT_Evaluator
-    import scanpy as sc
     import anndata as ad
+    import scanpy as sc
+    from scgft_evaluator import ScGFT_Evaluator
     SCGFT_AVAILABLE = True
 except ImportError:
     SCGFT_AVAILABLE = False
@@ -33,12 +35,12 @@ try:
 except ImportError:
     SKLEARN_AVAILABLE = False
 
-from calm_data_generator.reports.ExternalReporter import ExternalReporter
-from calm_data_generator.reports.Visualizer import Visualizer
-from calm_data_generator.reports.LocalIndexGenerator import LocalIndexGenerator
-from calm_data_generator.reports.base import BaseReporter
-from calm_data_generator.reports.DiscriminatorReporter import DiscriminatorReporter
-from calm_data_generator.generators.configs import ReportConfig
+from calm_data_generator.generators.configs import ReportConfig  # noqa: E402
+from calm_data_generator.reports.base import BaseReporter  # noqa: E402
+from calm_data_generator.reports.DiscriminatorReporter import DiscriminatorReporter  # noqa: E402
+from calm_data_generator.reports.ExternalReporter import ExternalReporter  # noqa: E402
+from calm_data_generator.reports.LocalIndexGenerator import LocalIndexGenerator  # noqa: E402
+from calm_data_generator.reports.Visualizer import Visualizer  # noqa: E402
 
 # Direct usage of sdmetrics
 try:
@@ -113,9 +115,9 @@ class QualityReporter(BaseReporter):
         return self._assess_quality_scores(real_df, synthetic_df)
 
     def calculate_ari(
-        self, 
-        real_df: pd.DataFrame, 
-        synthetic_df: pd.DataFrame, 
+        self,
+        real_df: pd.DataFrame,
+        synthetic_df: pd.DataFrame,
         target_col: str
     ) -> Dict[str, float]:
         """
@@ -589,10 +591,15 @@ class QualityReporter(BaseReporter):
 
         # Cross duplicates
         try:
-            real_unique = real_df.drop_duplicates()
-            merged = synthetic_df.merge(
-                real_unique, on=list(synthetic_df.columns), how="left", indicator=True
-            )
+            shared_cols = [c for c in synthetic_df.columns if c in real_df.columns]
+            real_unique = real_df[shared_cols].drop_duplicates()
+            synth_cast = synthetic_df[shared_cols].copy()
+            for col in shared_cols:
+                try:
+                    synth_cast[col] = synth_cast[col].astype(real_unique[col].dtype)
+                except (ValueError, TypeError):
+                    pass
+            merged = synth_cast.merge(real_unique, on=shared_cols, how="left", indicator=True)
             cross_dup_count = (merged["_merge"] == "both").sum()
         except Exception as e:
             self.logger.warning(f"Could not compute cross-duplication count; defaulting to 0. Reason: {e}")
@@ -755,9 +762,9 @@ class QualityReporter(BaseReporter):
             return None
 
     def _calculate_ari_metrics(
-        self, 
-        real_df: pd.DataFrame, 
-        synthetic_df: pd.DataFrame, 
+        self,
+        real_df: pd.DataFrame,
+        synthetic_df: pd.DataFrame,
         target_col: Optional[str]
     ) -> Optional[Dict[str, float]]:
         """
@@ -775,21 +782,21 @@ class QualityReporter(BaseReporter):
                 features = df.select_dtypes(include=[np.number]).drop(columns=[t_col], errors='ignore')
                 if features.empty:
                     return None
-                
+
                 # Fill NaNs for KMeans
                 X = features.fillna(0).values
-                
+
                 # Handle cases where we have fewer samples than clusters
                 k = min(2, len(X))
                 if k < 2:
                     return 0.0
-                
+
                 kmeans = KMeans(n_clusters=k, n_init=10, random_state=42)
                 cluster_labels = kmeans.fit_predict(X)
-                
+
                 # Get true labels (convert to categorical codes if necessary)
                 true_labels = pd.Categorical(df[t_col]).codes
-                
+
                 return float(adjusted_rand_score(true_labels, cluster_labels))
 
             ari_real = get_ari(real_df, target_col)
@@ -805,9 +812,9 @@ class QualityReporter(BaseReporter):
             return None
 
     def _run_scgft_evaluation(
-        self, 
-        real_df: pd.DataFrame, 
-        synthetic_df: pd.DataFrame, 
+        self,
+        real_df: pd.DataFrame,
+        synthetic_df: pd.DataFrame,
         output_dir: str,
         target_col: Optional[str] = None
     ) -> None:
@@ -830,10 +837,10 @@ class QualityReporter(BaseReporter):
             numeric_cols = real_df.select_dtypes(include=[np.number]).columns.tolist()
             if target_col and target_col in numeric_cols:
                 numeric_cols.remove(target_col)
-            
+
             adata_real = ad.AnnData(real_df[numeric_cols])
             adata_synth = ad.AnnData(synthetic_df[numeric_cols])
-            
+
             if target_col and target_col in real_df.columns:
                 adata_real.obs["cell_type"] = real_df[target_col].astype(str).values
                 adata_synth.obs["cell_type"] = synthetic_df[target_col].astype(str).values
@@ -845,7 +852,7 @@ class QualityReporter(BaseReporter):
             # 2. Basic Preprocessing for scvi metrics (PCA is required)
             if self.verbose:
                 print("   -> Preprocessing AnnData (PCA)...")
-            
+
             sc.pp.pca(adata_real)
             sc.pp.pca(adata_synth)
 
@@ -878,7 +885,7 @@ class QualityReporter(BaseReporter):
 
             # 4. Save to HTML Report
             scgft_report_path = os.path.join(output_dir, "scgft_report.html")
-            
+
             html_content = f"""
             <html>
             <head>
@@ -911,7 +918,7 @@ class QualityReporter(BaseReporter):
             </body>
             </html>
             """
-            
+
             with open(scgft_report_path, "w") as html_file:
                 html_file.write(html_content)
 

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1/calm_data_generator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calm-data-generator
-Version: 2.2.0
+Version: 2.2.1
 Summary: CALM-Data-Generator: A Python library for synthetic data generation with support for drift injection and clinical data.
 Author-email: Alejandro Belda Fernandez <alejandrobeldafernandez@gmail.com>
 License: MIT

{calm_data_generator-2.2.0 → calm_data_generator-2.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "calm-data-generator"
-version = "2.2.0"
+version = "2.2.1"
 description = "CALM-Data-Generator: A Python library for synthetic data generation with support for drift injection and clinical data."
 readme = "README.md"
 requires-python = ">=3.10"