misata 0.1.0b0__tar.gz → 0.3.0b0__tar.gz

misata-0.3.0b0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Muhammed Rasin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{misata-0.1.0b0 → misata-0.3.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.1.0b0
+Version: 0.3.0b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT
@@ -23,6 +23,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Database
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pydantic>=2.0.0
@@ -41,6 +42,7 @@ Requires-Dist: pytest-benchmark>=4.0.0; extra == "dev"
 Requires-Dist: black>=23.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: mypy>=1.5.0; extra == "dev"
+Dynamic: license-file
 
 # 🧠 Misata
 
@@ -48,7 +50,7 @@ Requires-Dist: mypy>=1.5.0; extra == "dev"
 
 No schema writing. No training data. Just describe what you need.
 
-[![Version](https://img.shields.io/badge/version-0.1.0--beta-purple.svg)]()
+[![Version](https://img.shields.io/badge/version-0.2.0--beta-purple.svg)]()
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)]()
 [![Python](https://img.shields.io/badge/python-3.10+-green.svg)]()
 

{misata-0.1.0b0 → misata-0.3.0b0}/README.md

@@ -4,7 +4,7 @@
 
 No schema writing. No training data. Just describe what you need.
 
-[![Version](https://img.shields.io/badge/version-0.1.0--beta-purple.svg)]()
+[![Version](https://img.shields.io/badge/version-0.2.0--beta-purple.svg)]()
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)]()
 [![Python](https://img.shields.io/badge/python-3.10+-green.svg)]()
 

misata-0.3.0b0/misata/__init__.py

@@ -0,0 +1,134 @@
+"""
+Misata - AI-Powered Synthetic Data Engine
+
+Generate realistic multi-table datasets from natural language descriptions.
+Supports OpenAI, Groq, Gemini, and Ollama for intelligent schema generation.
+
+Usage:
+    from misata import DataSimulator, SchemaConfig
+
+    # Or use the CLI:
+    #   misata generate --story "A SaaS with 50k users..."
+    
+    # Or use pre-built templates:
+    from misata.templates.library import load_template
+    config = load_template("ecommerce")
+"""
+
+__version__ = "0.3.0b0"
+__author__ = "Muhammed Rasin"
+
+from misata.schema import (
+    Column,
+    Constraint,
+    Relationship,
+    ScenarioEvent,
+    SchemaConfig,
+    Table,
+)
+from misata.simulator import DataSimulator
+from misata.generators import TextGenerator
+from misata.generators.base import (
+    BaseGenerator,
+    IntegerGenerator,
+    FloatGenerator,
+    BooleanGenerator,
+    CategoricalGenerator,
+    DateGenerator,
+    ForeignKeyGenerator,
+    GeneratorFactory,
+)
+from misata.constraints import (
+    BaseConstraint,
+    SumConstraint,
+    RangeConstraint,
+    UniqueConstraint,
+    NotNullConstraint,
+    RatioConstraint,
+    ConstraintEngine,
+)
+from misata.context import GenerationContext
+from misata.exceptions import (
+    MisataError,
+    SchemaValidationError,
+    ColumnGenerationError,
+    LLMError,
+    ConfigurationError,
+    ExportError,
+)
+from misata.smart_values import SmartValueGenerator
+from misata.noise import NoiseInjector, add_noise
+from misata.customization import Customizer, ColumnOverride
+from misata.quality import DataQualityChecker, check_quality
+from misata.templates.library import load_template, list_templates
+from misata.profiles import (
+    DistributionProfile,
+    get_profile,
+    list_profiles,
+    generate_with_profile,
+)
+from misata.generators.base import (
+    ConditionalCategoricalGenerator,
+    CONDITIONAL_LOOKUPS,
+    create_conditional_generator,
+)
+
+__all__ = [
+    # Core
+    "Column",
+    "Constraint",
+    "Relationship",
+    "ScenarioEvent",
+    "SchemaConfig",
+    "Table",
+    "DataSimulator",
+    # Generators
+    "TextGenerator",
+    "BaseGenerator",
+    "IntegerGenerator",
+    "FloatGenerator",
+    "BooleanGenerator",
+    "CategoricalGenerator",
+    "DateGenerator",
+    "ForeignKeyGenerator",
+    "GeneratorFactory",
+    "ConditionalCategoricalGenerator",
+    "CONDITIONAL_LOOKUPS",
+    "create_conditional_generator",
+    # Constraints
+    "BaseConstraint",
+    "SumConstraint",
+    "RangeConstraint",
+    "UniqueConstraint",
+    "NotNullConstraint",
+    "RatioConstraint",
+    "ConstraintEngine",
+    # Context
+    "GenerationContext",
+    # Exceptions
+    "MisataError",
+    "SchemaValidationError",
+    "ColumnGenerationError",
+    "LLMError",
+    "ConfigurationError",
+    "ExportError",
+    # Smart Values
+    "SmartValueGenerator",
+    # Distribution Profiles
+    "DistributionProfile",
+    "get_profile",
+    "list_profiles",
+    "generate_with_profile",
+    # ML-ready features
+    "NoiseInjector",
+    "add_noise",
+    "Customizer",
+    "ColumnOverride",
+    # Quality
+    "DataQualityChecker",
+    "check_quality",
+    # Templates
+    "load_template",
+    "list_templates",
+]
+

misata-0.3.0b0/misata/cache.py

@@ -0,0 +1,258 @@
+"""
+Caching utilities for Misata.
+
+Provides LLM response caching using diskcache for performance optimization.
+"""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+try:
+    import diskcache
+    HAS_DISKCACHE = True
+except ImportError:
+    HAS_DISKCACHE = False
+
+
+class LLMCache:
+    """Cache for LLM responses to avoid repeated API calls.
+    
+    Uses diskcache for persistent storage with automatic expiration.
+    Falls back to in-memory dict if diskcache is not installed.
+    
+    Example:
+        cache = LLMCache()
+        
+        # Check cache first
+        key = cache.make_key("groq", "llama-3.3", prompt)
+        cached = cache.get(key)
+        if cached:
+            return cached
+        
+        # Make LLM call
+        response = llm.generate(prompt)
+        
+        # Cache the result
+        cache.set(key, response)
+    """
+    
+    def __init__(
+        self,
+        cache_dir: Optional[str] = None,
+        max_size_mb: int = 100,
+        expire_days: int = 7,
+    ):
+        """Initialize the cache.
+        
+        Args:
+            cache_dir: Directory for cache storage (default: ~/.misata/cache)
+            max_size_mb: Maximum cache size in MB
+            expire_days: Days before cache entries expire
+        """
+        self.expire_seconds = expire_days * 24 * 60 * 60
+        
+        if cache_dir is None:
+            cache_dir = os.path.expanduser("~/.misata/cache/llm")
+        
+        self.cache_dir = Path(cache_dir)
+        
+        if HAS_DISKCACHE:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            self._cache = diskcache.Cache(
+                str(self.cache_dir),
+                size_limit=max_size_mb * 1024 * 1024,
+            )
+        else:
+            # Fallback to in-memory cache
+            self._cache: Dict[str, Any] = {}
+            self._memory_mode = True
+    
+    def make_key(
+        self,
+        provider: str,
+        model: str,
+        prompt: str,
+        temperature: float = 0.0,
+        **kwargs: Any
+    ) -> str:
+        """Create a cache key from request parameters.
+        
+        Args:
+            provider: LLM provider (groq, openai, etc.)
+            model: Model name
+            prompt: The prompt text
+            temperature: Temperature setting
+            **kwargs: Additional parameters to include in key
+            
+        Returns:
+            Hash-based cache key
+        """
+        key_data = {
+            "provider": provider,
+            "model": model,
+            "prompt": prompt,
+            "temperature": temperature,
+            **kwargs
+        }
+        key_str = json.dumps(key_data, sort_keys=True)
+        return hashlib.sha256(key_str.encode()).hexdigest()[:32]
+    
+    def get(self, key: str) -> Optional[Any]:
+        """Get a cached value.
+        
+        Args:
+            key: Cache key
+            
+        Returns:
+            Cached value or None if not found/expired
+        """
+        if HAS_DISKCACHE:
+            return self._cache.get(key)
+        else:
+            return self._cache.get(key)
+    
+    def set(self, key: str, value: Any) -> None:
+        """Store a value in cache.
+        
+        Args:
+            key: Cache key
+            value: Value to cache (must be JSON-serializable for persistence)
+        """
+        if HAS_DISKCACHE:
+            self._cache.set(key, value, expire=self.expire_seconds)
+        else:
+            self._cache[key] = value
+    
+    def delete(self, key: str) -> bool:
+        """Delete a cached value.
+        
+        Args:
+            key: Cache key
+            
+        Returns:
+            True if deleted, False if not found
+        """
+        if HAS_DISKCACHE:
+            return self._cache.delete(key)
+        else:
+            if key in self._cache:
+                del self._cache[key]
+                return True
+            return False
+    
+    def clear(self) -> None:
+        """Clear all cached values."""
+        if HAS_DISKCACHE:
+            self._cache.clear()
+        else:
+            self._cache.clear()
+    
+    def stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        if HAS_DISKCACHE:
+            return {
+                "type": "diskcache",
+                "directory": str(self.cache_dir),
+                "size_bytes": self._cache.volume(),
+                "count": len(self._cache),
+            }
+        else:
+            return {
+                "type": "memory",
+                "count": len(self._cache),
+            }
+    
+    def __contains__(self, key: str) -> bool:
+        return self.get(key) is not None
+    
+    def close(self) -> None:
+        """Close the cache (required for diskcache)."""
+        if HAS_DISKCACHE:
+            self._cache.close()
+
+
+class SmartValueCache:
+    """Cache for SmartValueGenerator pools.
+    
+    Caches generated value pools by domain/context to avoid
+    repeated LLM calls for the same domain.
+    """
+    
+    def __init__(self, cache: Optional[LLMCache] = None):
+        self._cache = cache or LLMCache(
+            cache_dir=os.path.expanduser("~/.misata/cache/smart_values")
+        )
+    
+    def get_pool(
+        self,
+        domain: str,
+        context: Optional[str] = None,
+        provider: str = "groq"
+    ) -> Optional[list]:
+        """Get cached value pool for a domain.
+        
+        Args:
+            domain: Domain type (disease, prescription, etc.)
+            context: Additional context
+            provider: LLM provider used
+            
+        Returns:
+            List of values or None if not cached
+        """
+        key = self._make_pool_key(domain, context, provider)
+        return self._cache.get(key)
+    
+    def set_pool(
+        self,
+        domain: str,
+        values: list,
+        context: Optional[str] = None,
+        provider: str = "groq"
+    ) -> None:
+        """Cache a value pool.
+        
+        Args:
+            domain: Domain type
+            values: List of generated values
+            context: Additional context
+            provider: LLM provider used
+        """
+        key = self._make_pool_key(domain, context, provider)
+        self._cache.set(key, values)
+    
+    def _make_pool_key(
+        self,
+        domain: str,
+        context: Optional[str],
+        provider: str
+    ) -> str:
+        key_data = f"{provider}:{domain}:{context or ''}"
+        return hashlib.sha256(key_data.encode()).hexdigest()[:24]
+    
+    def clear(self) -> None:
+        """Clear all cached pools."""
+        self._cache.clear()
+
+
+# Global cache instances
+_llm_cache: Optional[LLMCache] = None
+_smart_value_cache: Optional[SmartValueCache] = None
+
+
+def get_llm_cache() -> LLMCache:
+    """Get the global LLM cache instance."""
+    global _llm_cache
+    if _llm_cache is None:
+        _llm_cache = LLMCache()
+    return _llm_cache
+
+
+def get_smart_value_cache() -> SmartValueCache:
+    """Get the global smart value cache instance."""
+    global _smart_value_cache
+    if _smart_value_cache is None:
+        _smart_value_cache = SmartValueCache()
+    return _smart_value_cache