misata 0.1.0b0__py3-none-any.whl

misata/codegen.py

@@ -0,0 +1,153 @@
+"""
+Code generation module for creating standalone Python scripts.
+
+Generates executable Python scripts from SchemaConfig that can be
+run independently to produce synthetic data.
+"""
+
+import json
+from pathlib import Path
+
+from misata.schema import SchemaConfig
+
+
+class ScriptGenerator:
+    """
+    Generates standalone Python scripts for data generation.
+
+    The generated scripts are self-contained and can be executed
+    without the Misata library installed.
+    """
+
+    def __init__(self, config: SchemaConfig):
+        """
+        Initialize the script generator.
+
+        Args:
+            config: Schema configuration to generate script from
+        """
+        self.config = config
+
+    def _generate_imports(self) -> str:
+        """Generate import statements."""
+        return """import pandas as pd
+import numpy as np
+import warnings
+from collections import defaultdict, deque
+import os
+
+# Pure Python text generator (no external dependencies)
+from misata.generators import TextGenerator
+"""
+
+    def _generate_config_dict(self) -> str:
+        """Generate the configuration as a Python dictionary."""
+        config_dict = self.config.model_dump()
+        return f"CONFIG = {json.dumps(config_dict, indent=2)}"
+
+    def _generate_simulator_class(self) -> str:
+        """Generate the DataSimulator class code."""
+        # Read the simulator.py file and extract the class
+        simulator_path = Path(__file__).parent / "simulator.py"
+        with open(simulator_path, "r") as f:
+            content = f.read()
+
+        # Extract just the class definition (simplified - in production use AST)
+        # For now, include the entire simulator module
+        return content.replace("from misata.schema import", "# from misata.schema import")
+
+    def generate(self, output_path: str, include_export: bool = True) -> None:
+        """
+        Generate a standalone Python script.
+
+        Args:
+            output_path: Path where the script should be saved
+            include_export: Whether to include CSV export code at the end
+        """
+        script_parts = []
+
+        # Header comment
+        script_parts.append('"""')
+        script_parts.append("Auto-generated synthetic data script by Misata")
+        script_parts.append(f"Dataset: {self.config.name}")
+        if self.config.description:
+            script_parts.append(f"Description: {self.config.description}")
+        script_parts.append('"""')
+        script_parts.append("")
+
+        # Imports
+        script_parts.append(self._generate_imports())
+        script_parts.append("")
+
+        # Configuration
+        script_parts.append("# " + "=" * 70)
+        script_parts.append("# CONFIGURATION")
+        script_parts.append("# " + "=" * 70)
+        script_parts.append(self._generate_config_dict())
+        script_parts.append("")
+
+        # Main execution
+        script_parts.append("# " + "=" * 70)
+        script_parts.append("# MAIN EXECUTION")
+        script_parts.append("# " + "=" * 70)
+        script_parts.append("")
+        script_parts.append("def main():")
+        script_parts.append("    \"\"\"Generate synthetic data and export to CSV.\"\"\"")
+        script_parts.append("    ")
+        script_parts.append("    # Note: This is a simplified version.")
+        script_parts.append("    # For full functionality, use the Misata library directly.")
+        script_parts.append("    ")
+        script_parts.append("    print('Generating synthetic data...')")
+        script_parts.append("    print(f'Dataset: {CONFIG[\"name\"]}')")
+        script_parts.append("    print(f'Tables: {len(CONFIG[\"tables\"])}')")
+        script_parts.append("    ")
+        script_parts.append("    # Initialize random seed")
+        script_parts.append("    seed = CONFIG.get('seed', 42)")
+        script_parts.append("    np.random.seed(seed)")
+        script_parts.append("    rng = np.random.default_rng(seed)")
+        script_parts.append("    text_gen = TextGenerator(seed=seed)")
+        script_parts.append("    ")
+        script_parts.append("    # TODO: Import and use DataSimulator from Misata")
+        script_parts.append("    # For now, this is a placeholder")
+        script_parts.append("    print('Please install misata library:')")
+        script_parts.append("    print('  pip install misata')")
+        script_parts.append("    print('Then use:')")
+        script_parts.append("    print('  from misata import DataSimulator, SchemaConfig')")
+        script_parts.append("    ")
+
+        if include_export:
+            script_parts.append("    # Export placeholder")
+            script_parts.append("    output_dir = 'generated_data'")
+            script_parts.append("    os.makedirs(output_dir, exist_ok=True)")
+            script_parts.append("    print(f'Output directory: {output_dir}')")
+
+        script_parts.append("")
+        script_parts.append("")
+        script_parts.append("if __name__ == '__main__':")
+        script_parts.append("    main()")
+
+        # Write to file
+        output_path_obj = Path(output_path)
+        output_path_obj.parent.mkdir(parents=True, exist_ok=True)
+        output_path_obj.write_text("\n".join(script_parts))
+
+        print(f"Generated script saved to: {output_path}")
+
+    def generate_yaml_config(self, output_path: str) -> None:
+        """
+        Generate a YAML configuration file.
+
+        Args:
+            output_path: Path where the YAML should be saved
+        """
+        import yaml
+
+        config_dict = self.config.model_dump()
+
+        output_path_obj = Path(output_path)
+        output_path_obj.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(output_path_obj, "w") as f:
+            yaml.dump(config_dict, f, default_flow_style=False, sort_keys=False)
+
+        print(f"Generated YAML config saved to: {output_path}")

misata/curve_fitting.py

@@ -0,0 +1,106 @@
+
+"""
+Mathematical Graph Solver for Misata.
+
+This module uses scipy.optimize to find the best distribution parameters
+that match a set of control points provided by the LLM or user.
+This allows users/LLMs to "draw" a distribution shape rather than
+guessing abstract parameters like alpha/beta/gamma.
+"""
+
+from typing import Dict, List
+
+import numpy as np
+from scipy.optimize import minimize
+from scipy.stats import norm, lognorm, expon, beta, gamma, uniform
+
+class CurveFitter:
+    """
+    Fits statistical distributions to control points using optimization.
+    """
+
+    def __init__(self):
+        """Initialize the curve fitter."""
+        self.distributions = {
+            "normal": norm,
+            "lognormal": lognorm,
+            "exponential": expon,
+            "beta": beta,
+            "gamma": gamma,
+            "uniform": uniform
+        }
+
+    def fit_distribution(
+        self,
+        targets: List[Dict[str, float]],
+        distribution_type: str = "normal"
+    ) -> Dict[str, float]:
+        """
+        Find best parameters for a distribution to match target points.
+
+        Args:
+            targets: List of points dicts [{"x": 10, "y": 0.1}, ...]
+                     where x is the value and y is the desired PDF probability density.
+            distribution_type: Name of distribution to fit.
+
+        Returns:
+            Dictionary of best-fit parameters (e.g., {"mean": 10, "std": 5})
+        """
+        if distribution_type not in self.distributions:
+            raise ValueError(f"Unsupported distribution: {distribution_type}")
+
+        dist_func = self.distributions[distribution_type]
+        points = np.array([(p["x"], p["y"]) for p in targets])
+        x_vals = points[:, 0]
+        y_targets = points[:, 1]
+
+        # Define objective function (MSE)
+        def objective(params):
+            try:
+                # Scipy stats distributions take args/scale/loc differently
+                # We need to map generic params array to specific distribution args
+                # This is tricky. Simplified approach:
+                if distribution_type == "normal":
+                    # params[0] = mean (loc), params[1] = std (scale)
+                    y_pred = dist_func.pdf(x_vals, loc=params[0], scale=abs(params[1]))
+                elif distribution_type == "exponential":
+                     # params[0] = scale (1/lambda)
+                     y_pred = dist_func.pdf(x_vals, scale=abs(params[0]))
+                elif distribution_type == "uniform":
+                    # params[0] = min (loc), params[1] = range (scale)
+                    y_pred = dist_func.pdf(x_vals, loc=params[0], scale=abs(params[1]))
+                elif distribution_type == "lognormal":
+                     # s=shape, scale=exp(mean), loc=0 usually
+                     y_pred = dist_func.pdf(x_vals, s=abs(params[0]), scale=abs(params[1]))
+                else:
+                     # General fallback?
+                     return 1e9
+
+                # Mean Squared Error
+                mse = np.mean((y_pred - y_targets) ** 2)
+                return mse
+            except Exception:
+                return 1e9
+
+        # Initial guesses
+        initial_guess = [np.mean(x_vals), np.std(x_vals)]
+        if distribution_type == "exponential":
+            initial_guess = [np.mean(x_vals)]
+        elif distribution_type == "lognormal":
+            initial_guess = [1.0, np.mean(x_vals)]
+
+        # Optimize
+        result = minimize(objective, initial_guess, method='Nelder-Mead')
+        best_params = result.x
+
+        # Map back to named parameters
+        if distribution_type == "normal":
+            return {"mean": float(best_params[0]), "std": float(abs(best_params[1]))}
+        elif distribution_type == "exponential":
+            return {"scale": float(abs(best_params[0]))}
+        elif distribution_type == "uniform":
+             return {"min": float(best_params[0]), "max": float(best_params[0] + abs(best_params[1]))}
+        elif distribution_type == "lognormal":
+            return {"shape": float(abs(best_params[0])), "scale": float(abs(best_params[1]))}
+
+        return {}

misata/customization.py

@@ -0,0 +1,256 @@
+"""
+Attribute customization module for fine-grained control over data generation.
+
+Allows users to:
+- Override column values with custom generators
+- Apply conditional logic to values
+- Define custom value pools per column
+- Apply transformations post-generation
+"""
+
+from typing import Any, Callable, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+
+class ColumnOverride:
+    """
+    Define custom generation logic for a specific column.
+
+    Usage:
+        override = ColumnOverride(
+            name="price",
+            generator=lambda n: np.random.uniform(10, 100, n),
+            post_process=lambda x: round(x, 2)
+        )
+    """
+
+    def __init__(
+        self,
+        name: str,
+        generator: Optional[Callable[[int], np.ndarray]] = None,
+        value_pool: Optional[List[Any]] = None,
+        conditional: Optional[Dict[str, Any]] = None,
+        post_process: Optional[Callable[[Any], Any]] = None,
+        null_rate: float = 0.0,
+    ):
+        """
+        Initialize a column override.
+
+        Args:
+            name: Column name to override
+            generator: Function that takes size N and returns N values
+            value_pool: List of values to sample from
+            conditional: Dict with {condition_column: {value: override_value}}
+            post_process: Function to apply to each value after generation
+            null_rate: Rate of null values to inject
+        """
+        self.name = name
+        self.generator = generator
+        self.value_pool = value_pool
+        self.conditional = conditional
+        self.post_process = post_process
+        self.null_rate = null_rate
+
+    def apply(self, df: pd.DataFrame, rng: np.random.Generator) -> pd.DataFrame:
+        """Apply this override to a DataFrame."""
+        result = df.copy()
+
+        if self.generator is not None:
+            result[self.name] = self.generator(len(df))
+
+        elif self.value_pool is not None:
+            result[self.name] = rng.choice(self.value_pool, size=len(df))
+
+        # Apply conditional overrides
+        if self.conditional is not None:
+            for cond_col, value_map in self.conditional.items():
+                if cond_col not in result.columns:
+                    continue
+                for cond_value, override_value in value_map.items():
+                    mask = result[cond_col] == cond_value
+                    result.loc[mask, self.name] = override_value
+
+        # Apply post-processing
+        if self.post_process is not None:
+            result[self.name] = result[self.name].apply(self.post_process)
+
+        # Inject nulls
+        if self.null_rate > 0:
+            mask = rng.random(len(result)) < self.null_rate
+            result.loc[mask, self.name] = np.nan
+
+        return result
+
+
+class Customizer:
+    """
+    Central customization engine for attribute-level control.
+
+    Usage:
+        customizer = Customizer()
+        customizer.add_override("users", ColumnOverride(
+            name="age",
+            generator=lambda n: np.random.normal(35, 10, n).clip(18, 80).astype(int)
+        ))
+        customizer.add_conditional("orders", "shipping_cost", {
+            "country": {"US": 5.99, "UK": 9.99, "CA": 7.99}
+        })
+
+        df = customizer.apply(df, "users")
+    """
+
+    def __init__(self, seed: Optional[int] = None):
+        """Initialize the customizer."""
+        self.overrides: Dict[str, List[ColumnOverride]] = {}
+        self.rng = np.random.default_rng(seed)
+
+    def add_override(self, table: str, override: ColumnOverride) -> "Customizer":
+        """Add a column override for a table."""
+        if table not in self.overrides:
+            self.overrides[table] = []
+        self.overrides[table].append(override)
+        return self
+
+    def add_conditional(
+        self,
+        table: str,
+        column: str,
+        conditions: Dict[str, Dict[Any, Any]],
+    ) -> "Customizer":
+        """
+        Add a conditional value override.
+
+        Args:
+            table: Table name
+            column: Column to override
+            conditions: {condition_column: {condition_value: new_value}}
+
+        Example:
+            customizer.add_conditional("products", "tax_rate", {
+                "category": {"Electronics": 0.08, "Food": 0.0, "Clothing": 0.05}
+            })
+        """
+        override = ColumnOverride(name=column, conditional=conditions)
+        return self.add_override(table, override)
+
+    def add_value_pool(
+        self,
+        table: str,
+        column: str,
+        values: List[Any],
+        probabilities: Optional[List[float]] = None,
+    ) -> "Customizer":
+        """
+        Add a custom value pool for a column.
+
+        Args:
+            table: Table name
+            column: Column to override
+            values: List of possible values
+            probabilities: Optional weights (must sum to 1)
+        """
+        if probabilities:
+            def gen(n):
+                return self.rng.choice(values, size=n, p=probabilities)
+        else:
+            def gen(n):
+                return self.rng.choice(values, size=n)
+
+        override = ColumnOverride(name=column, generator=gen)
+        return self.add_override(table, override)
+
+    def add_formula(
+        self,
+        table: str,
+        column: str,
+        formula: Callable[[pd.DataFrame], pd.Series],
+    ) -> "Customizer":
+        """
+        Add a formula-based column using other columns.
+
+        Args:
+            table: Table name
+            column: Column to create/override
+            formula: Function that takes DataFrame and returns Series
+
+        Example:
+            customizer.add_formula("orders", "total",
+                lambda df: df["quantity"] * df["unit_price"] * (1 + df["tax_rate"]))
+        """
+        # Store as a special override that uses post-processing
+        class FormulaOverride(ColumnOverride):
+            def __init__(self, name, formula_fn):
+                super().__init__(name=name)
+                self.formula_fn = formula_fn
+
+            def apply(self, df: pd.DataFrame, rng: np.random.Generator) -> pd.DataFrame:
+                result = df.copy()
+                result[self.name] = self.formula_fn(result)
+                return result
+
+        override = FormulaOverride(column, formula)
+        return self.add_override(table, override)
+
+    def apply(self, df: pd.DataFrame, table: str) -> pd.DataFrame:
+        """Apply all overrides for a table to a DataFrame."""
+        result = df.copy()
+
+        if table not in self.overrides:
+            return result
+
+        for override in self.overrides[table]:
+            result = override.apply(result, self.rng)
+
+        return result
+
+
+# Convenience functions for common patterns
+
+def price_generator(min_val: float = 1.0, max_val: float = 1000.0, decimals: int = 2):
+    """Create a price generator with realistic distribution."""
+    def gen(n):
+        # Log-normal distribution for prices (more small items than expensive ones)
+        log_min = np.log(max(min_val, 0.01))
+        log_max = np.log(max_val)
+        log_prices = np.random.uniform(log_min, log_max, n)
+        prices = np.exp(log_prices)
+        return np.round(prices, decimals)
+    return gen
+
+
+def age_generator(mean: int = 35, std: int = 12, min_age: int = 18, max_age: int = 80):
+    """Create an age generator with realistic distribution."""
+    def gen(n):
+        ages = np.random.normal(mean, std, n)
+        return np.clip(ages, min_age, max_age).astype(int)
+    return gen
+
+
+def rating_generator(min_rating: float = 1.0, max_rating: float = 5.0, skew: str = "positive"):
+    """Create a rating generator with configurable skew."""
+    def gen(n):
+        if skew == "positive":
+            # Most ratings are 4-5 stars (beta distribution)
+            ratings = np.random.beta(5, 2, n) * (max_rating - min_rating) + min_rating
+        elif skew == "negative":
+            ratings = np.random.beta(2, 5, n) * (max_rating - min_rating) + min_rating
+        else:
+            ratings = np.random.uniform(min_rating, max_rating, n)
+        return np.round(ratings, 1)
+    return gen
+
+
+def percentage_generator(realistic: bool = True):
+    """Create a percentage generator."""
+    def gen(n):
+        if realistic:
+            # Most percentages cluster around common values
+            common = [0, 5, 10, 15, 20, 25, 30, 50, 75, 100]
+            base = np.random.choice(common, n)
+            noise = np.random.uniform(-2, 2, n)
+            return np.clip(base + noise, 0, 100)
+        else:
+            return np.random.uniform(0, 100, n)
+    return gen