misata 0.1.0b0__tar.gz → 0.2.0b0__tar.gz

misata-0.2.0b0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Muhammed Rasin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{misata-0.1.0b0 → misata-0.2.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.1.0b0
+Version: 0.2.0b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT
@@ -23,6 +23,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Database
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pydantic>=2.0.0
@@ -41,6 +42,7 @@ Requires-Dist: pytest-benchmark>=4.0.0; extra == "dev"
 Requires-Dist: black>=23.0.0; extra == "dev"
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: mypy>=1.5.0; extra == "dev"
+Dynamic: license-file
 
 # 🧠 Misata
 
@@ -48,7 +50,7 @@ Requires-Dist: mypy>=1.5.0; extra == "dev"
 
 No schema writing. No training data. Just describe what you need.
 
-[![Version](https://img.shields.io/badge/version-0.1.0--beta-purple.svg)]()
+[![Version](https://img.shields.io/badge/version-0.2.0--beta-purple.svg)]()
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)]()
 [![Python](https://img.shields.io/badge/python-3.10+-green.svg)]()
 

{misata-0.1.0b0 → misata-0.2.0b0}/README.md

@@ -4,7 +4,7 @@
 
 No schema writing. No training data. Just describe what you need.
 
-[![Version](https://img.shields.io/badge/version-0.1.0--beta-purple.svg)]()
+[![Version](https://img.shields.io/badge/version-0.2.0--beta-purple.svg)]()
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)]()
 [![Python](https://img.shields.io/badge/python-3.10+-green.svg)]()
 

{misata-0.1.0b0 → misata-0.2.0b0}/misata/__init__.py

@@ -9,9 +9,13 @@ Usage:
 
     # Or use the CLI:
     #   misata generate --story "A SaaS with 50k users..."
+    
+    # Or use pre-built templates:
+    from misata.templates.library import load_template
+    config = load_template("ecommerce")
 """
 
-__version__ = "0.1.0-beta"
+__version__ = "0.2.0-beta"
 __author__ = "Muhammed Rasin"
 
 from misata.schema import (
@@ -26,6 +30,8 @@ from misata.simulator import DataSimulator
 from misata.generators import TextGenerator
 from misata.noise import NoiseInjector, add_noise
 from misata.customization import Customizer, ColumnOverride
+from misata.quality import DataQualityChecker, check_quality
+from misata.templates.library import load_template, list_templates
 
 __all__ = [
     # Core
@@ -43,6 +49,11 @@ __all__ = [
     "add_noise",
     "Customizer",
     "ColumnOverride",
+    # Quality
+    "DataQualityChecker",
+    "check_quality",
+    # Templates
+    "load_template",
+    "list_templates",
 ]
 
-

{misata-0.1.0b0 → misata-0.2.0b0}/misata/llm_parser.py

@@ -24,7 +24,11 @@ def _load_env():
     """Load environment variables from .env file."""
     env_paths = [
         Path.cwd() / ".env",
-        Path(__file__).parent.parent / ".env",
+        Path.cwd().parent / ".env",  # apps/.env or api parent
+        Path.cwd().parent.parent / ".env",  # Misata root from apps/api
+        Path(__file__).parent.parent / ".env",  # packages/core/.env
+        Path(__file__).parent.parent.parent / ".env",  # packages/.env
+        Path(__file__).parent.parent.parent.parent / ".env",  # Misata root from packages/core/misata
         Path.home() / ".misata" / ".env",
     ]
 
@@ -35,7 +39,9 @@ def _load_env():
                     line = line.strip()
                     if line and not line.startswith("#") and "=" in line:
                         key, _, value = line.partition("=")
-                        os.environ.setdefault(key.strip(), value.strip())
+                        # Remove quotes if present
+                        value = value.strip().strip("'\"")
+                        os.environ.setdefault(key.strip(), value)
             break
 
 _load_env()
@@ -82,6 +88,39 @@ Instead of guessing parameters, you can provide "control_points" to draw the sha
 Format: {"distribution": "normal", "control_points": [{"x": 10, "y": 0.1}, {"x": 50, "y": 0.9}]}
 Misata will mathematically solve for the best parameters.
 
+### SMART DEFAULTS (Use These for Realistic Data):
+
+**Age columns:**
+- type: "int", distribution: "normal", mean: 35, std: 12, min: 18, max: 80
+
+**Price/Amount columns:**
+- type: "float", distribution: "exponential", scale: 50, min: 0.01, decimals: 2
+- OR for products: uniform min: 9.99, max: 499.99
+
+**Rating columns (1-5 stars):**
+- type: "int", distribution: "categorical", choices: [1,2,3,4,5], probabilities: [0.05, 0.08, 0.15, 0.32, 0.40]
+
+**Quantity/Count columns:**
+- type: "int", distribution: "poisson", lambda: 3, min: 1
+
+**Duration (minutes):**
+- type: "int", distribution: "normal", mean: 45, std: 20, min: 5, max: 180
+
+**Percentage columns:**
+- type: "float", distribution: "uniform", min: 0.0, max: 100.0, decimals: 1
+
+**Status columns:**
+- type: "categorical", choices: ["active", "inactive", "pending"], probabilities: [0.70, 0.20, 0.10]
+
+**Boolean probabilities:**
+- is_verified: probability: 0.85
+- is_premium: probability: 0.25
+- is_active: probability: 0.80
+
+**Date columns:**
+- For recent data: bias last 30% of range with 70% of values
+- Always use realistic date ranges (not 1970-2100)
+
 ## OUTPUT FORMAT
 
 {

misata-0.2.0b0/misata/quality.py

@@ -0,0 +1,329 @@
+"""
+Data Quality Checker for Synthetic Data Validation.
+
+This module validates generated synthetic data for:
+- Distribution plausibility
+- Referential integrity
+- Temporal consistency
+- Domain-specific rules
+"""
+
+from typing import Dict, List, Any, Optional, Tuple
+from dataclasses import dataclass, field
+import warnings
+
+
+@dataclass
+class QualityIssue:
+    """Represents a single data quality issue."""
+    severity: str  # "error", "warning", "info"
+    category: str  # "distribution", "integrity", "temporal", "domain"
+    table: str
+    column: Optional[str]
+    message: str
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class QualityReport:
+    """Complete quality report for generated data."""
+    score: float  # 0-100
+    issues: List[QualityIssue]
+    stats: Dict[str, Any]
+    
+    @property
+    def passed(self) -> bool:
+        """Returns True if no errors (warnings OK)."""
+        return not any(i.severity == "error" for i in self.issues)
+    
+    def summary(self) -> str:
+        """Human-readable summary."""
+        errors = sum(1 for i in self.issues if i.severity == "error")
+        warnings = sum(1 for i in self.issues if i.severity == "warning")
+        return f"Quality Score: {self.score:.1f}/100 | Errors: {errors} | Warnings: {warnings}"
+
+
+class DataQualityChecker:
+    """
+    Validate generated synthetic data for realism and correctness.
+    
+    Usage:
+        checker = DataQualityChecker()
+        report = checker.check_all(tables, relationships, schema)
+        
+        if not report.passed:
+            print("Issues found:", report.issues)
+    """
+    
+    # Domain-specific plausibility rules
+    PLAUSIBILITY_RULES = {
+        # Column name patterns -> (min, max, description)
+        "age": (0, 120, "Human age"),
+        "price": (0, 1_000_000, "Price"),
+        "quantity": (0, 10_000, "Quantity"),
+        "rating": (1, 5, "Rating"),
+        "percentage": (0, 100, "Percentage"),
+        "year": (1900, 2100, "Year"),
+        "month": (1, 12, "Month"),
+        "day": (1, 31, "Day"),
+        "hour": (0, 23, "Hour"),
+        "minute": (0, 59, "Minute"),
+        "score": (0, 100, "Score"),
+        "count": (0, 1_000_000, "Count"),
+        "duration": (0, 10_000, "Duration"),
+    }
+    
+    def __init__(self, strict: bool = False):
+        """
+        Initialize the quality checker.
+        
+        Args:
+            strict: If True, warnings become errors
+        """
+        self.strict = strict
+        self.issues: List[QualityIssue] = []
+    
+    def _add_issue(
+        self,
+        severity: str,
+        category: str,
+        table: str,
+        column: Optional[str],
+        message: str,
+        details: Optional[Dict] = None,
+    ):
+        """Add an issue to the list."""
+        if self.strict and severity == "warning":
+            severity = "error"
+        
+        self.issues.append(QualityIssue(
+            severity=severity,
+            category=category,
+            table=table,
+            column=column,
+            message=message,
+            details=details or {},
+        ))
+    
+    def check_distribution_plausibility(
+        self,
+        df: "pd.DataFrame",
+        table_name: str,
+    ) -> None:
+        """
+        Check if numeric distributions are plausible for their domains.
+        
+        Args:
+            df: DataFrame to check
+            table_name: Name of the table
+        """
+        import pandas as pd
+        import numpy as np
+        
+        for col in df.columns:
+            col_lower = col.lower()
+            
+            # Check against plausibility rules
+            for pattern, (min_val, max_val, description) in self.PLAUSIBILITY_RULES.items():
+                if pattern in col_lower:
+                    if pd.api.types.is_numeric_dtype(df[col]):
+                        actual_min = df[col].min()
+                        actual_max = df[col].max()
+                        
+                        if actual_min < min_val:
+                            self._add_issue(
+                                "warning", "distribution", table_name, col,
+                                f"{description} column '{col}' has min {actual_min} < expected {min_val}",
+                                {"actual_min": actual_min, "expected_min": min_val}
+                            )
+                        
+                        if actual_max > max_val:
+                            self._add_issue(
+                                "warning", "distribution", table_name, col,
+                                f"{description} column '{col}' has max {actual_max} > expected {max_val}",
+                                {"actual_max": actual_max, "expected_max": max_val}
+                            )
+                    break
+            
+            # Check for all-null columns
+            if df[col].isna().all():
+                self._add_issue(
+                    "error", "distribution", table_name, col,
+                    f"Column '{col}' is entirely NULL",
+                )
+            
+            # Check for zero variance (all same value)
+            if pd.api.types.is_numeric_dtype(df[col]) and df[col].std() == 0:
+                self._add_issue(
+                    "warning", "distribution", table_name, col,
+                    f"Column '{col}' has zero variance (all values identical)",
+                    {"value": df[col].iloc[0]}
+                )
+    
+    def check_referential_integrity(
+        self,
+        tables: Dict[str, "pd.DataFrame"],
+        relationships: List[Any],
+    ) -> None:
+        """
+        Verify all foreign key references are valid.
+        
+        Args:
+            tables: Dict of table_name -> DataFrame
+            relationships: List of Relationship objects
+        """
+        for rel in relationships:
+            parent_table = rel.parent_table
+            child_table = rel.child_table
+            parent_key = rel.parent_key
+            child_key = rel.child_key
+            
+            if parent_table not in tables:
+                self._add_issue(
+                    "error", "integrity", child_table, child_key,
+                    f"Parent table '{parent_table}' not found for FK '{child_key}'",
+                )
+                continue
+            
+            if child_table not in tables:
+                continue  # Child table might not exist yet
+            
+            parent_df = tables[parent_table]
+            child_df = tables[child_table]
+            
+            if parent_key not in parent_df.columns:
+                self._add_issue(
+                    "error", "integrity", parent_table, parent_key,
+                    f"Parent key '{parent_key}' not found in table '{parent_table}'",
+                )
+                continue
+            
+            if child_key not in child_df.columns:
+                self._add_issue(
+                    "error", "integrity", child_table, child_key,
+                    f"Child key '{child_key}' not found in table '{child_table}'",
+                )
+                continue
+            
+            # Check for orphaned records
+            parent_ids = set(parent_df[parent_key].dropna().unique())
+            child_ids = set(child_df[child_key].dropna().unique())
+            orphans = child_ids - parent_ids
+            
+            if orphans:
+                orphan_pct = len(orphans) / len(child_ids) * 100
+                self._add_issue(
+                    "error" if orphan_pct > 1 else "warning",
+                    "integrity", child_table, child_key,
+                    f"{len(orphans)} orphaned FK values ({orphan_pct:.1f}%) in '{child_key}' -> '{parent_table}.{parent_key}'",
+                    {"orphan_count": len(orphans), "orphan_pct": orphan_pct}
+                )
+    
+    def check_temporal_consistency(
+        self,
+        df: "pd.DataFrame",
+        table_name: str,
+    ) -> None:
+        """
+        Ensure temporal columns are consistent.
+        
+        Checks:
+        - created_at < updated_at
+        - start_date < end_date
+        - birth_date in past
+        """
+        import pandas as pd
+        
+        date_cols = [c for c in df.columns if pd.api.types.is_datetime64_any_dtype(df[c])]
+        
+        # Check created < updated
+        if "created_at" in date_cols and "updated_at" in date_cols:
+            violations = (df["created_at"] > df["updated_at"]).sum()
+            if violations > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, "created_at",
+                    f"{violations} rows have created_at > updated_at",
+                    {"violation_count": violations}
+                )
+        
+        # Check start < end
+        if "start_date" in date_cols and "end_date" in date_cols:
+            violations = (df["start_date"] > df["end_date"]).sum()
+            if violations > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, "start_date",
+                    f"{violations} rows have start_date > end_date",
+                    {"violation_count": violations}
+                )
+        
+        # Check birth_date is in past
+        if "birth_date" in date_cols or "date_of_birth" in date_cols:
+            col = "birth_date" if "birth_date" in date_cols else "date_of_birth"
+            future_births = (df[col] > pd.Timestamp.now()).sum()
+            if future_births > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, col,
+                    f"{future_births} rows have birth_date in the future",
+                    {"violation_count": future_births}
+                )
+    
+    def check_all(
+        self,
+        tables: Dict[str, "pd.DataFrame"],
+        relationships: Optional[List[Any]] = None,
+        schema: Optional[Any] = None,
+    ) -> QualityReport:
+        """
+        Run all quality checks and generate a report.
+        
+        Args:
+            tables: Dict of table_name -> DataFrame
+            relationships: Optional list of Relationship objects
+            schema: Optional SchemaConfig for additional checks
+            
+        Returns:
+            QualityReport with score and issues
+        """
+        self.issues = []  # Reset
+        
+        # Check each table
+        for table_name, df in tables.items():
+            self.check_distribution_plausibility(df, table_name)
+            self.check_temporal_consistency(df, table_name)
+        
+        # Check referential integrity
+        if relationships:
+            self.check_referential_integrity(tables, relationships)
+        
+        # Calculate score
+        base_score = 100
+        for issue in self.issues:
+            if issue.severity == "error":
+                base_score -= 10
+            elif issue.severity == "warning":
+                base_score -= 3
+            else:
+                base_score -= 1
+        
+        score = max(0, min(100, base_score))
+        
+        # Gather stats
+        stats = {
+            "tables_checked": len(tables),
+            "total_rows": sum(len(df) for df in tables.values()),
+            "total_columns": sum(len(df.columns) for df in tables.values()),
+            "error_count": sum(1 for i in self.issues if i.severity == "error"),
+            "warning_count": sum(1 for i in self.issues if i.severity == "warning"),
+        }
+        
+        return QualityReport(
+            score=score,
+            issues=self.issues.copy(),
+            stats=stats,
+        )
+
+
+def check_quality(tables: Dict[str, "pd.DataFrame"], **kwargs) -> QualityReport:
+    """Convenience function for quick quality checks."""
+    checker = DataQualityChecker()
+    return checker.check_all(tables, **kwargs)

{misata-0.1.0b0 → misata-0.2.0b0}/misata/schema.py

@@ -23,7 +23,7 @@ class Column(BaseModel):
     """
 
     name: str
-    type: Literal["int", "float", "date", "categorical", "foreign_key", "text", "boolean"]
+    type: Literal["int", "float", "date", "time", "datetime", "categorical", "foreign_key", "text", "boolean"]
     distribution_params: Dict[str, Any] = Field(default_factory=dict)
     nullable: bool = False
     unique: bool = False
@@ -39,8 +39,13 @@ class Column(BaseModel):
 
         if col_type == "date":
             if "relative_to" not in v:
-                if "start" not in v or "end" not in v:
-                    raise ValueError("Date columns must have 'start' and 'end' OR 'relative_to' in distribution_params")
+                # Provide sensible defaults if start/end not specified
+                if "start" not in v:
+                    from datetime import datetime, timedelta
+                    v["start"] = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
+                if "end" not in v:
+                    from datetime import datetime
+                    v["end"] = datetime.now().strftime("%Y-%m-%d")
 
         if col_type in ["int", "float"]:
             if "distribution" not in v:

{misata-0.1.0b0 → misata-0.2.0b0}/misata/simulator.py

@@ -35,7 +35,8 @@ class DataSimulator:
     """
 
     def __init__(self, config: SchemaConfig,
-                 apply_semantic_fixes: bool = True, batch_size: int = 10_000):
+                 apply_semantic_fixes: bool = True, batch_size: int = 10_000,
+                 smart_mode: bool = False, use_llm: bool = True):
         """
         Initialize the simulator.
 
@@ -43,13 +44,19 @@ class DataSimulator:
             config: Schema configuration defining tables, columns, and relationships
             apply_semantic_fixes: Auto-fix column types based on semantic patterns
             batch_size: Number of rows to generate per batch
+            smart_mode: Enable LLM-powered context-aware value generation
+            use_llm: If smart_mode is True, whether to use LLM (vs curated fallbacks)
         """
         self.config = config
         self.context: Dict[str, pd.DataFrame] = {}  # Lightweight context (IDs only)
         self.text_gen = TextGenerator(seed=config.seed)
         self.batch_size = batch_size
+        self.smart_mode = smart_mode
+        self.use_llm = use_llm
+        self._smart_gen = None  # Lazy init
         self._unique_pools: Dict[str, np.ndarray] = {}  # Store pre-generated unique values
         self._unique_counters: Dict[str, int] = {}      # Track usage of unique pools
+        self._smart_pools: Dict[str, np.ndarray] = {}   # Cache smart value pools
 
         # Apply semantic inference to fix column types
         if apply_semantic_fixes:
@@ -60,6 +67,16 @@ class DataSimulator:
         seed = config.seed if config.seed is not None else np.random.randint(0, 2**32 - 1)
         self.rng = np.random.default_rng(seed)
         np.random.seed(seed)  # For legacy numpy.random calls
+    
+    def _get_smart_gen(self):
+        """Lazy initialize SmartValueGenerator."""
+        if self._smart_gen is None:
+            try:
+                from misata.smart_values import SmartValueGenerator
+                self._smart_gen = SmartValueGenerator()
+            except Exception:
+                self._smart_gen = None
+        return self._smart_gen
 
     def topological_sort(self) -> List[str]:
         """
@@ -210,13 +227,21 @@ class DataSimulator:
 
         # CATEGORICAL
         if column.type == "categorical":
-            choices = params["choices"]
+            choices = params.get("choices", ["A", "B", "C"])
             probabilities = params.get("probabilities", None)
 
+            # Ensure choices is a list
+            if not isinstance(choices, list):
+                choices = list(choices)
+
             if probabilities is not None:
-                # Normalize probabilities
-                probabilities = np.array(probabilities)
-                probabilities = probabilities / probabilities.sum()
+                # Convert to float array and normalize
+                probabilities = np.array(probabilities, dtype=float)
+                prob_sum = probabilities.sum()
+                if prob_sum > 0:
+                    probabilities = probabilities / prob_sum
+                else:
+                    probabilities = None
 
             values = self.rng.choice(choices, size=size, p=probabilities)
             return values
@@ -413,6 +438,35 @@ class DataSimulator:
         # TEXT
         elif column.type == "text":
             text_type = params.get("text_type", "sentence")
+            
+            # Smart value generation - check for domain-specific content
+            smart_generate = params.get("smart_generate", False) or self.smart_mode
+            if smart_generate:
+                smart_gen = self._get_smart_gen()
+                if smart_gen:
+                    # Check for explicit domain hint or auto-detect
+                    domain_hint = params.get("domain_hint")
+                    context = params.get("context", "")
+                    
+                    # Create cache key for this column's pool
+                    pool_key = f"{table_name}.{column.name}"
+                    
+                    if pool_key not in self._smart_pools:
+                        pool = smart_gen.get_pool(
+                            column_name=column.name,
+                            table_name=table_name,
+                            domain_hint=domain_hint,
+                            context=context,
+                            size=100,
+                            use_llm=self.use_llm,
+                        )
+                        if pool:
+                            self._smart_pools[pool_key] = np.array(pool)
+                    
+                    if pool_key in self._smart_pools:
+                        pool = self._smart_pools[pool_key]
+                        values = self.rng.choice(pool, size=size)
+                        return values
 
             if text_type == "name":
                 values = np.array([self.text_gen.name() for _ in range(size)])
@@ -441,6 +495,28 @@ class DataSimulator:
             values = self.rng.random(size) < probability
             return values
 
+        # TIME
+        elif column.type == "time":
+            # Generate random times as HH:MM:SS strings
+            start_hour = params.get("start_hour", 0)
+            end_hour = params.get("end_hour", 24)
+            hours = self.rng.integers(start_hour, end_hour, size=size)
+            minutes = self.rng.integers(0, 60, size=size)
+            seconds = self.rng.integers(0, 60, size=size)
+            values = np.array([f"{h:02d}:{m:02d}:{s:02d}" for h, m, s in zip(hours, minutes, seconds)])
+            return values
+
+        # DATETIME
+        elif column.type == "datetime":
+            # Generate random datetimes within a range
+            start = pd.to_datetime(params.get("start", "2020-01-01"))
+            end = pd.to_datetime(params.get("end", "2024-12-31"))
+            start_int = start.value
+            end_int = end.value
+            random_ints = self.rng.integers(start_int, end_int, size=size)
+            values = pd.to_datetime(random_ints)
+            return values
+
         else:
             raise ValueError(f"Unknown column type: {column.type}")