misata 0.2.0b0__tar.gz → 0.3.0b0__tar.gz

{misata-0.2.0b0 → misata-0.3.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.2.0b0
+Version: 0.3.0b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT

misata-0.3.0b0/misata/__init__.py

@@ -0,0 +1,134 @@
+"""
+Misata - AI-Powered Synthetic Data Engine
+
+Generate realistic multi-table datasets from natural language descriptions.
+Supports OpenAI, Groq, Gemini, and Ollama for intelligent schema generation.
+
+Usage:
+    from misata import DataSimulator, SchemaConfig
+
+    # Or use the CLI:
+    #   misata generate --story "A SaaS with 50k users..."
+    
+    # Or use pre-built templates:
+    from misata.templates.library import load_template
+    config = load_template("ecommerce")
+"""
+
+__version__ = "0.3.0b0"
+__author__ = "Muhammed Rasin"
+
+from misata.schema import (
+    Column,
+    Constraint,
+    Relationship,
+    ScenarioEvent,
+    SchemaConfig,
+    Table,
+)
+from misata.simulator import DataSimulator
+from misata.generators import TextGenerator
+from misata.generators.base import (
+    BaseGenerator,
+    IntegerGenerator,
+    FloatGenerator,
+    BooleanGenerator,
+    CategoricalGenerator,
+    DateGenerator,
+    ForeignKeyGenerator,
+    GeneratorFactory,
+)
+from misata.constraints import (
+    BaseConstraint,
+    SumConstraint,
+    RangeConstraint,
+    UniqueConstraint,
+    NotNullConstraint,
+    RatioConstraint,
+    ConstraintEngine,
+)
+from misata.context import GenerationContext
+from misata.exceptions import (
+    MisataError,
+    SchemaValidationError,
+    ColumnGenerationError,
+    LLMError,
+    ConfigurationError,
+    ExportError,
+)
+from misata.smart_values import SmartValueGenerator
+from misata.noise import NoiseInjector, add_noise
+from misata.customization import Customizer, ColumnOverride
+from misata.quality import DataQualityChecker, check_quality
+from misata.templates.library import load_template, list_templates
+from misata.profiles import (
+    DistributionProfile,
+    get_profile,
+    list_profiles,
+    generate_with_profile,
+)
+from misata.generators.base import (
+    ConditionalCategoricalGenerator,
+    CONDITIONAL_LOOKUPS,
+    create_conditional_generator,
+)
+
+__all__ = [
+    # Core
+    "Column",
+    "Constraint",
+    "Relationship",
+    "ScenarioEvent",
+    "SchemaConfig",
+    "Table",
+    "DataSimulator",
+    # Generators
+    "TextGenerator",
+    "BaseGenerator",
+    "IntegerGenerator",
+    "FloatGenerator",
+    "BooleanGenerator",
+    "CategoricalGenerator",
+    "DateGenerator",
+    "ForeignKeyGenerator",
+    "GeneratorFactory",
+    "ConditionalCategoricalGenerator",
+    "CONDITIONAL_LOOKUPS",
+    "create_conditional_generator",
+    # Constraints
+    "BaseConstraint",
+    "SumConstraint",
+    "RangeConstraint",
+    "UniqueConstraint",
+    "NotNullConstraint",
+    "RatioConstraint",
+    "ConstraintEngine",
+    # Context
+    "GenerationContext",
+    # Exceptions
+    "MisataError",
+    "SchemaValidationError",
+    "ColumnGenerationError",
+    "LLMError",
+    "ConfigurationError",
+    "ExportError",
+    # Smart Values
+    "SmartValueGenerator",
+    # Distribution Profiles
+    "DistributionProfile",
+    "get_profile",
+    "list_profiles",
+    "generate_with_profile",
+    # ML-ready features
+    "NoiseInjector",
+    "add_noise",
+    "Customizer",
+    "ColumnOverride",
+    # Quality
+    "DataQualityChecker",
+    "check_quality",
+    # Templates
+    "load_template",
+    "list_templates",
+]
+

misata-0.3.0b0/misata/cache.py

@@ -0,0 +1,258 @@
+"""
+Caching utilities for Misata.
+
+Provides LLM response caching using diskcache for performance optimization.
+"""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+try:
+    import diskcache
+    HAS_DISKCACHE = True
+except ImportError:
+    HAS_DISKCACHE = False
+
+
+class LLMCache:
+    """Cache for LLM responses to avoid repeated API calls.
+    
+    Uses diskcache for persistent storage with automatic expiration.
+    Falls back to in-memory dict if diskcache is not installed.
+    
+    Example:
+        cache = LLMCache()
+        
+        # Check cache first
+        key = cache.make_key("groq", "llama-3.3", prompt)
+        cached = cache.get(key)
+        if cached:
+            return cached
+        
+        # Make LLM call
+        response = llm.generate(prompt)
+        
+        # Cache the result
+        cache.set(key, response)
+    """
+    
+    def __init__(
+        self,
+        cache_dir: Optional[str] = None,
+        max_size_mb: int = 100,
+        expire_days: int = 7,
+    ):
+        """Initialize the cache.
+        
+        Args:
+            cache_dir: Directory for cache storage (default: ~/.misata/cache)
+            max_size_mb: Maximum cache size in MB
+            expire_days: Days before cache entries expire
+        """
+        self.expire_seconds = expire_days * 24 * 60 * 60
+        
+        if cache_dir is None:
+            cache_dir = os.path.expanduser("~/.misata/cache/llm")
+        
+        self.cache_dir = Path(cache_dir)
+        
+        if HAS_DISKCACHE:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            self._cache = diskcache.Cache(
+                str(self.cache_dir),
+                size_limit=max_size_mb * 1024 * 1024,
+            )
+        else:
+            # Fallback to in-memory cache
+            self._cache: Dict[str, Any] = {}
+            self._memory_mode = True
+    
+    def make_key(
+        self,
+        provider: str,
+        model: str,
+        prompt: str,
+        temperature: float = 0.0,
+        **kwargs: Any
+    ) -> str:
+        """Create a cache key from request parameters.
+        
+        Args:
+            provider: LLM provider (groq, openai, etc.)
+            model: Model name
+            prompt: The prompt text
+            temperature: Temperature setting
+            **kwargs: Additional parameters to include in key
+            
+        Returns:
+            Hash-based cache key
+        """
+        key_data = {
+            "provider": provider,
+            "model": model,
+            "prompt": prompt,
+            "temperature": temperature,
+            **kwargs
+        }
+        key_str = json.dumps(key_data, sort_keys=True)
+        return hashlib.sha256(key_str.encode()).hexdigest()[:32]
+    
+    def get(self, key: str) -> Optional[Any]:
+        """Get a cached value.
+        
+        Args:
+            key: Cache key
+            
+        Returns:
+            Cached value or None if not found/expired
+        """
+        if HAS_DISKCACHE:
+            return self._cache.get(key)
+        else:
+            return self._cache.get(key)
+    
+    def set(self, key: str, value: Any) -> None:
+        """Store a value in cache.
+        
+        Args:
+            key: Cache key
+            value: Value to cache (must be JSON-serializable for persistence)
+        """
+        if HAS_DISKCACHE:
+            self._cache.set(key, value, expire=self.expire_seconds)
+        else:
+            self._cache[key] = value
+    
+    def delete(self, key: str) -> bool:
+        """Delete a cached value.
+        
+        Args:
+            key: Cache key
+            
+        Returns:
+            True if deleted, False if not found
+        """
+        if HAS_DISKCACHE:
+            return self._cache.delete(key)
+        else:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+    
+    def clear(self) -> None:
+        """Clear all cached values."""
+        if HAS_DISKCACHE:
+            self._cache.clear()
+        else:
+            self._cache.clear()
+    
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        if HAS_DISKCACHE:
+            return {
+                "type": "diskcache",
+                "directory": str(self.cache_dir),
+                "size_bytes": self._cache.volume(),
+                "count": len(self._cache),
+            }
+        else:
+            return {
+                "type": "memory",
+                "count": len(self._cache),
+            }
+    
+    def __contains__(self, key: str) -> bool:
+        return self.get(key) is not None
+    
+    def close(self) -> None:
+        """Close the cache (required for diskcache)."""
+        if HAS_DISKCACHE:
+            self._cache.close()
+
+
+class SmartValueCache:
+    """Cache for SmartValueGenerator pools.
+    
+    Caches generated value pools by domain/context to avoid
+    repeated LLM calls for the same domain.
+    """
+    
+    def __init__(self, cache: Optional[LLMCache] = None):
+        self._cache = cache or LLMCache(
+            cache_dir=os.path.expanduser("~/.misata/cache/smart_values")
+        )
+    
+    def get_pool(
+        self,
+        domain: str,
+        context: Optional[str] = None,
+        provider: str = "groq"
+    ) -> Optional[list]:
+        """Get cached value pool for a domain.
+        
+        Args:
+            domain: Domain type (disease, prescription, etc.)
+            context: Additional context
+            provider: LLM provider used
+            
+        Returns:
+            List of values or None if not cached
+        """
+        key = self._make_pool_key(domain, context, provider)
+        return self._cache.get(key)
+    
+    def set_pool(
+        self,
+        domain: str,
+        values: list,
+        context: Optional[str] = None,
+        provider: str = "groq"
+    ) -> None:
+        """Cache a value pool.
+        
+        Args:
+            domain: Domain type
+            values: List of generated values
+            context: Additional context
+            provider: LLM provider used
+        """
+        key = self._make_pool_key(domain, context, provider)
+        self._cache.set(key, values)
+    
+    def _make_pool_key(
+        self,
+        domain: str,
+        context: Optional[str],
+        provider: str
+    ) -> str:
+        key_data = f"{provider}:{domain}:{context or ''}"
+        return hashlib.sha256(key_data.encode()).hexdigest()[:24]
+    
+    def clear(self) -> None:
+        """Clear all cached pools."""
+        self._cache.clear()
+
+
+# Global cache instances
+_llm_cache: Optional[LLMCache] = None
+_smart_value_cache: Optional[SmartValueCache] = None
+
+
+def get_llm_cache() -> LLMCache:
+    """Get the global LLM cache instance."""
+    global _llm_cache
+    if _llm_cache is None:
+        _llm_cache = LLMCache()
+    return _llm_cache
+
+
+def get_smart_value_cache() -> SmartValueCache:
+    """Get the global smart value cache instance."""
+    global _smart_value_cache
+    if _smart_value_cache is None:
+        _smart_value_cache = SmartValueCache()
+    return _smart_value_cache

misata-0.3.0b0/misata/constraints.py

@@ -0,0 +1,307 @@
+"""
+Constraint handling for Misata data generation.
+
+Provides constraint classes for applying business rules to generated data.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from misata.exceptions import ConstraintError
+
+
+class BaseConstraint(ABC):
+    """Abstract base class for all constraints."""
+    
+    @abstractmethod
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Apply the constraint to a DataFrame.
+        
+        Args:
+            df: DataFrame to constrain
+            
+        Returns:
+            Constrained DataFrame
+        """
+        pass
+    
+    @abstractmethod
+    def validate(self, df: pd.DataFrame) -> bool:
+        """Check if the constraint is satisfied.
+        
+        Args:
+            df: DataFrame to check
+            
+        Returns:
+            True if constraint is satisfied
+        """
+        pass
+
+
+class SumConstraint(BaseConstraint):
+    """Ensures sum of a column (optionally grouped) doesn't exceed a value."""
+    
+    def __init__(
+        self,
+        column: str,
+        max_sum: float,
+        group_by: Optional[List[str]] = None,
+        action: str = "cap"
+    ):
+        self.column = column
+        self.max_sum = max_sum
+        self.group_by = group_by or []
+        self.action = action  # 'cap', 'redistribute', 'drop'
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        if self.column not in df.columns:
+            return df
+        
+        df = df.copy()
+        
+        if not self.group_by:
+            # Global sum constraint
+            current_sum = df[self.column].sum()
+            if current_sum > self.max_sum:
+                if self.action == "cap":
+                    scale = self.max_sum / current_sum
+                    df[self.column] = df[self.column] * scale
+                elif self.action == "drop":
+                    # Keep first N rows that fit
+                    cumsum = df[self.column].cumsum()
+                    df = df[cumsum <= self.max_sum]
+        else:
+            # Grouped sum constraint
+            def cap_group(group):
+                current_sum = group[self.column].sum()
+                if current_sum > self.max_sum:
+                    if self.action == "cap":
+                        scale = self.max_sum / current_sum
+                        group = group.copy()
+                        group[self.column] = group[self.column] * scale
+                return group
+            
+            df = df.groupby(self.group_by, group_keys=False).apply(cap_group)
+        
+        return df
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        if self.column not in df.columns:
+            return True
+        
+        if not self.group_by:
+            return df[self.column].sum() <= self.max_sum
+        
+        group_sums = df.groupby(self.group_by)[self.column].sum()
+        return (group_sums <= self.max_sum).all()
+
+
+class RangeConstraint(BaseConstraint):
+    """Ensures values in a column stay within a range."""
+    
+    def __init__(self, column: str, min_val: Optional[float] = None, max_val: Optional[float] = None):
+        self.column = column
+        self.min_val = min_val
+        self.max_val = max_val
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        if self.column not in df.columns:
+            return df
+        
+        df = df.copy()
+        
+        if self.min_val is not None:
+            df[self.column] = df[self.column].clip(lower=self.min_val)
+        
+        if self.max_val is not None:
+            df[self.column] = df[self.column].clip(upper=self.max_val)
+        
+        return df
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        if self.column not in df.columns:
+            return True
+        
+        values = df[self.column]
+        
+        if self.min_val is not None and (values < self.min_val).any():
+            return False
+        
+        if self.max_val is not None and (values > self.max_val).any():
+            return False
+        
+        return True
+
+
+class UniqueConstraint(BaseConstraint):
+    """Ensures values in column(s) are unique."""
+    
+    def __init__(self, columns: Union[str, List[str]]):
+        self.columns = [columns] if isinstance(columns, str) else columns
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        # Drop duplicates
+        return df.drop_duplicates(subset=self.columns)
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        return not df.duplicated(subset=self.columns).any()
+
+
+class NotNullConstraint(BaseConstraint):
+    """Ensures a column has no null values."""
+    
+    def __init__(self, column: str, fill_value: Any = None):
+        self.column = column
+        self.fill_value = fill_value
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        if self.column not in df.columns:
+            return df
+        
+        df = df.copy()
+        
+        if self.fill_value is not None:
+            df[self.column] = df[self.column].fillna(self.fill_value)
+        else:
+            df = df.dropna(subset=[self.column])
+        
+        return df
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        if self.column not in df.columns:
+            return True
+        return not df[self.column].isnull().any()
+
+
+class RatioConstraint(BaseConstraint):
+    """Ensures ratio between categories matches target distribution."""
+    
+    def __init__(self, column: str, target_ratios: Dict[Any, float]):
+        self.column = column
+        self.target_ratios = target_ratios
+        # Normalize ratios
+        total = sum(target_ratios.values())
+        self.target_ratios = {k: v / total for k, v in target_ratios.items()}
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        if self.column not in df.columns:
+            return df
+        
+        df = df.copy()
+        n = len(df)
+        
+        # Calculate target counts
+        target_counts = {k: int(n * v) for k, v in self.target_ratios.items()}
+        
+        # Randomly assign categories
+        categories = []
+        for cat, count in target_counts.items():
+            categories.extend([cat] * count)
+        
+        # Fill remaining
+        remaining = n - len(categories)
+        if remaining > 0:
+            most_common = max(self.target_ratios, key=self.target_ratios.get)
+            categories.extend([most_common] * remaining)
+        
+        np.random.shuffle(categories)
+        df[self.column] = categories[:n]
+        
+        return df
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        if self.column not in df.columns:
+            return True
+        
+        actual = df[self.column].value_counts(normalize=True)
+        
+        for cat, target in self.target_ratios.items():
+            actual_ratio = actual.get(cat, 0)
+            if abs(actual_ratio - target) > 0.05:  # 5% tolerance
+                return False
+        
+        return True
+
+
+class TemporalConstraint(BaseConstraint):
+    """Ensures temporal ordering between columns."""
+    
+    def __init__(self, before_column: str, after_column: str, min_gap_days: int = 0):
+        self.before_column = before_column
+        self.after_column = after_column
+        self.min_gap_days = min_gap_days
+    
+    def apply(self, df: pd.DataFrame) -> pd.DataFrame:
+        if self.before_column not in df.columns or self.after_column not in df.columns:
+            return df
+        
+        df = df.copy()
+        
+        before = pd.to_datetime(df[self.before_column])
+        after = pd.to_datetime(df[self.after_column])
+        
+        # Fix violations
+        mask = after < before
+        if mask.any():
+            # Swap dates where violated
+            df.loc[mask, self.after_column], df.loc[mask, self.before_column] = \
+                df.loc[mask, self.before_column], df.loc[mask, self.after_column]
+        
+        # Apply minimum gap
+        if self.min_gap_days > 0:
+            before = pd.to_datetime(df[self.before_column])
+            after = pd.to_datetime(df[self.after_column])
+            gap = (after - before).dt.days
+            
+            mask = gap < self.min_gap_days
+            if mask.any():
+                df.loc[mask, self.after_column] = (
+                    before[mask] + pd.Timedelta(days=self.min_gap_days)
+                ).dt.strftime('%Y-%m-%d')
+        
+        return df
+    
+    def validate(self, df: pd.DataFrame) -> bool:
+        if self.before_column not in df.columns or self.after_column not in df.columns:
+            return True
+        
+        before = pd.to_datetime(df[self.before_column])
+        after = pd.to_datetime(df[self.after_column])
+        
+        gap = (after - before).dt.days
+        return (gap >= self.min_gap_days).all()
+
+
+class ConstraintEngine:
+    """Engine for applying multiple constraints to a DataFrame."""
+    
+    def __init__(self, constraints: Optional[List[BaseConstraint]] = None):
+        self.constraints = constraints or []
+    
+    def add(self, constraint: BaseConstraint) -> "ConstraintEngine":
+        """Add a constraint."""
+        self.constraints.append(constraint)
+        return self
+    
+    def apply_all(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Apply all constraints in order."""
+        for constraint in self.constraints:
+            try:
+                df = constraint.apply(df)
+            except Exception as e:
+                raise ConstraintError(
+                    f"Failed to apply constraint: {e}",
+                    constraint_type=type(constraint).__name__
+                )
+        return df
+    
+    def validate_all(self, df: pd.DataFrame) -> Dict[str, bool]:
+        """Check all constraints and return results."""
+        results = {}
+        for constraint in self.constraints:
+            name = type(constraint).__name__
+            results[name] = constraint.validate(df)
+        return results