misata 0.1.0b0__py3-none-any.whl

misata/validation.py

@@ -0,0 +1,313 @@
+"""
+Data validation layer for post-generation quality checks.
+
+This module validates generated data to ensure:
+- No negative values where inappropriate
+- Valid date ranges
+- Referential integrity (FK -> PK exists)
+- Business logic rules
+- Statistical distribution accuracy
+"""
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+
+class Severity(Enum):
+    """Validation issue severity levels."""
+    INFO = "info"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+@dataclass
+class ValidationIssue:
+    """A single validation issue found in the data."""
+    severity: Severity
+    table: str
+    column: Optional[str]
+    message: str
+    affected_rows: int = 0
+    sample_values: List[Any] = field(default_factory=list)
+
+    def __str__(self):
+        severity_icon = {"info": "ℹ️", "warning": "⚠️", "error": "❌"}[self.severity.value]
+        col = f".{self.column}" if self.column else ""
+        return f"{severity_icon} [{self.table}{col}] {self.message} ({self.affected_rows} rows)"
+
+
+@dataclass
+class ValidationReport:
+    """Complete validation report for generated data."""
+    issues: List[ValidationIssue] = field(default_factory=list)
+    tables_checked: int = 0
+    columns_checked: int = 0
+    total_rows: int = 0
+
+    @property
+    def has_errors(self) -> bool:
+        return any(i.severity == Severity.ERROR for i in self.issues)
+
+    @property
+    def has_warnings(self) -> bool:
+        return any(i.severity == Severity.WARNING for i in self.issues)
+
+    @property
+    def is_clean(self) -> bool:
+        return len(self.issues) == 0
+
+    def summary(self) -> str:
+        """Get a summary of the validation report."""
+        errors = sum(1 for i in self.issues if i.severity == Severity.ERROR)
+        warnings = sum(1 for i in self.issues if i.severity == Severity.WARNING)
+        info = sum(1 for i in self.issues if i.severity == Severity.INFO)
+
+        lines = [
+            "=" * 50,
+            "DATA VALIDATION REPORT",
+            "=" * 50,
+            f"Tables checked: {self.tables_checked}",
+            f"Columns checked: {self.columns_checked}",
+            f"Total rows: {self.total_rows:,}",
+            "-" * 50,
+            f"❌ Errors: {errors}",
+            f"⚠️ Warnings: {warnings}",
+            f"ℹ️ Info: {info}",
+            "-" * 50,
+        ]
+
+        if self.is_clean:
+            lines.append("✅ All validations passed!")
+        else:
+            lines.append("Issues found:")
+            for issue in self.issues:
+                lines.append(f"  {issue}")
+
+        lines.append("=" * 50)
+        return "\n".join(lines)
+
+
+class DataValidator:
+    """
+    Validates generated data for quality and accuracy.
+    """
+
+    def __init__(
+        self,
+        tables: Dict[str, pd.DataFrame],
+        schema_config: Optional[Any] = None,
+    ):
+        """
+        Initialize validator with generated tables.
+
+        Args:
+            tables: Dict mapping table name to DataFrame
+            schema_config: Optional schema config for relationship checking
+        """
+        self.tables = tables
+        self.schema_config = schema_config
+        self.issues: List[ValidationIssue] = []
+
+    def validate_all(self) -> ValidationReport:
+        """
+        Run all validation checks.
+
+        Returns:
+            Complete validation report
+        """
+        self.issues = []
+
+        for table_name, df in self.tables.items():
+            self._validate_table(table_name, df)
+
+        # Validate referential integrity
+        self._validate_referential_integrity()
+
+        return ValidationReport(
+            issues=self.issues,
+            tables_checked=len(self.tables),
+            columns_checked=sum(len(df.columns) for df in self.tables.values()),
+            total_rows=sum(len(df) for df in self.tables.values()),
+        )
+
+    def _validate_table(self, table_name: str, df: pd.DataFrame) -> None:
+        """Validate a single table."""
+        for col in df.columns:
+            self._validate_column(table_name, df, col)
+
+    def _validate_column(self, table_name: str, df: pd.DataFrame, col: str) -> None:
+        """Validate a single column."""
+        col.lower()
+        values = df[col]
+
+        # Check for nulls
+        null_count = values.isna().sum()
+        if null_count > 0:
+            self.issues.append(ValidationIssue(
+                severity=Severity.INFO,
+                table=table_name,
+                column=col,
+                message=f"Contains {null_count} null values",
+                affected_rows=null_count,
+            ))
+
+        # Numeric column checks
+        if pd.api.types.is_numeric_dtype(values):
+            self._validate_numeric_column(table_name, col, values)
+
+        # Date column checks
+        if pd.api.types.is_datetime64_any_dtype(values):
+            self._validate_date_column(table_name, col, values)
+
+        # String column checks
+        if pd.api.types.is_string_dtype(values) or pd.api.types.is_object_dtype(values):
+            self._validate_string_column(table_name, col, values)
+
+    def _validate_numeric_column(self, table_name: str, col: str, values: pd.Series) -> None:
+        """Validate numeric columns."""
+        col_lower = col.lower()
+
+        # Check for negative values in columns that should be positive
+        positive_patterns = ['price', 'cost', 'amount', 'age', 'quantity', 'count',
+                             'duration', 'weight', 'height', 'salary', 'revenue']
+
+        if any(p in col_lower for p in positive_patterns):
+            negative_count = (values < 0).sum()
+            if negative_count > 0:
+                self.issues.append(ValidationIssue(
+                    severity=Severity.ERROR,
+                    table=table_name,
+                    column=col,
+                    message=f"Contains {negative_count} negative values (should be positive)",
+                    affected_rows=negative_count,
+                    sample_values=values[values < 0].head(5).tolist(),
+                ))
+
+        # Check for unreasonable ages
+        if 'age' in col_lower:
+            invalid_ages = ((values < 0) | (values > 150)).sum()
+            if invalid_ages > 0:
+                self.issues.append(ValidationIssue(
+                    severity=Severity.WARNING,
+                    table=table_name,
+                    column=col,
+                    message=f"Contains {invalid_ages} unrealistic age values (< 0 or > 150)",
+                    affected_rows=invalid_ages,
+                ))
+
+        # Check for unreasonable prices
+        if 'price' in col_lower or 'cost' in col_lower:
+            very_high = (values > 1000000).sum()
+            if very_high > 0:
+                self.issues.append(ValidationIssue(
+                    severity=Severity.INFO,
+                    table=table_name,
+                    column=col,
+                    message=f"Contains {very_high} values over $1M",
+                    affected_rows=very_high,
+                ))
+
+    def _validate_date_column(self, table_name: str, col: str, values: pd.Series) -> None:
+        """Validate date columns."""
+        # Check for dates too far in the future
+        future_cutoff = pd.Timestamp.now() + pd.Timedelta(days=365*5)
+        far_future = (values > future_cutoff).sum()
+        if far_future > 0:
+            self.issues.append(ValidationIssue(
+                severity=Severity.WARNING,
+                table=table_name,
+                column=col,
+                message=f"Contains {far_future} dates more than 5 years in the future",
+                affected_rows=far_future,
+            ))
+
+        # Check for dates too far in the past
+        past_cutoff = pd.Timestamp('1900-01-01')
+        far_past = (values < past_cutoff).sum()
+        if far_past > 0:
+            self.issues.append(ValidationIssue(
+                severity=Severity.ERROR,
+                table=table_name,
+                column=col,
+                message=f"Contains {far_past} dates before 1900",
+                affected_rows=far_past,
+            ))
+
+    def _validate_string_column(self, table_name: str, col: str, values: pd.Series) -> None:
+        """Validate string columns."""
+        col_lower = col.lower()
+
+        # Check for email format
+        if 'email' in col_lower:
+            # Simple email check - contains @
+            invalid_emails = (~values.astype(str).str.contains('@', na=False)).sum()
+            if invalid_emails > 0:
+                self.issues.append(ValidationIssue(
+                    severity=Severity.ERROR,
+                    table=table_name,
+                    column=col,
+                    message=f"Contains {invalid_emails} invalid email addresses",
+                    affected_rows=invalid_emails,
+                ))
+
+        # Check for empty strings
+        empty_count = (values.astype(str).str.strip() == '').sum()
+        if empty_count > 0:
+            self.issues.append(ValidationIssue(
+                severity=Severity.WARNING,
+                table=table_name,
+                column=col,
+                message=f"Contains {empty_count} empty strings",
+                affected_rows=empty_count,
+            ))
+
+    def _validate_referential_integrity(self) -> None:
+        """Validate foreign key relationships."""
+        if not self.schema_config:
+            return
+
+        for rel in self.schema_config.relationships:
+            if rel.parent_table not in self.tables or rel.child_table not in self.tables:
+                continue
+
+            parent_df = self.tables[rel.parent_table]
+            child_df = self.tables[rel.child_table]
+
+            if rel.parent_key not in parent_df.columns or rel.child_key not in child_df.columns:
+                continue
+
+            parent_ids = set(parent_df[rel.parent_key].dropna())
+            child_fks = child_df[rel.child_key].dropna()
+
+            orphans = ~child_fks.isin(parent_ids)
+            orphan_count = orphans.sum()
+
+            if orphan_count > 0:
+                self.issues.append(ValidationIssue(
+                    severity=Severity.ERROR,
+                    table=rel.child_table,
+                    column=rel.child_key,
+                    message=f"Contains {orphan_count} orphan references (FK not found in {rel.parent_table})",
+                    affected_rows=orphan_count,
+                ))
+
+
+def validate_data(
+    tables: Dict[str, pd.DataFrame],
+    schema_config: Optional[Any] = None,
+) -> ValidationReport:
+    """
+    Quick validation of generated data.
+
+    Args:
+        tables: Generated tables
+        schema_config: Optional schema for FK validation
+
+    Returns:
+        Validation report
+    """
+    validator = DataValidator(tables, schema_config)
+    return validator.validate_all()

misata-0.1.0b0.dist-info/METADATA

@@ -0,0 +1,291 @@
+Metadata-Version: 2.4
+Name: misata
+Version: 0.1.0b0
+Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
+Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/rasinmuhammed/misata
+Project-URL: Documentation, https://github.com/rasinmuhammed/misata#readme
+Project-URL: Repository, https://github.com/rasinmuhammed/misata
+Project-URL: Issues, https://github.com/rasinmuhammed/misata/issues
+Keywords: synthetic-data,data-generation,fake-data,machine-learning,testing,llm,ai,database
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Database
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: click>=8.1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: groq>=0.4.0
+Requires-Dist: fastapi>=0.109.0
+Requires-Dist: uvicorn>=0.27.0
+Requires-Dist: python-multipart>=0.0.6
+Requires-Dist: simpleeval>=0.9.0
+Requires-Dist: scipy>=1.10.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-benchmark>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.5.0; extra == "dev"
+
+# 🧠 Misata
+
+**Generate realistic multi-table datasets from natural language.**
+
+No schema writing. No training data. Just describe what you need.
+
+[![Version](https://img.shields.io/badge/version-0.1.0--beta-purple.svg)]()
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)]()
+[![Python](https://img.shields.io/badge/python-3.10+-green.svg)]()
+
+## ✨ What Makes Misata Different
+
+| Feature | Faker | SDV | **Misata** |
+|---------|-------|-----|------------|
+| Natural language input | ❌ | ❌ | ✅ |
+| Auto schema generation | ❌ | ❌ | ✅ |
+| Relational integrity | ❌ | ✅ | ✅ |
+| Business constraints | ❌ | ❌ | ✅ |
+| No training data needed | ✅ | ❌ | ✅ |
+| Streaming (10M+ rows) | ❌ | ❌ | ✅ |
+
+## 🚀 Quick Start
+
+```bash
+pip install misata
+```
+
+### With Groq (Free, Fast)
+
+```bash
+export GROQ_API_KEY=your_key  # Get free: https://console.groq.com
+misata generate --story "A SaaS with 50K users, subscriptions, and payments" --use-llm
+```
+
+### With OpenAI
+
+```bash
+export OPENAI_API_KEY=your_key
+misata generate --story "E-commerce with products and orders" --use-llm --provider openai
+```
+
+### With Ollama (Local, Free, Private)
+
+```bash
+ollama run llama3  # Start Ollama first
+misata generate --story "Fitness app with workouts" --use-llm --provider ollama
+```
+
+## 📊 Example Output
+
+```
+$ misata generate --story "A fitness app with 50K users" --use-llm
+
+🧠 Using Groq (llama-3.3-70b-versatile) for intelligent parsing...
+✅ LLM schema generated successfully!
+
+📋 Schema: FitnessApp
+   Tables: 5
+   Relationships: 4
+
+🔧 Generating 5 table(s)...
+
+   ✓ exercises     (10 rows)
+   ✓ plans         (5 rows)
+   ✓ users         (50,000 rows)
+   ✓ subscriptions (45,000 rows)
+   ✓ workouts      (500,000 rows)
+
+⏱️  Generation time: 2.34 seconds
+🚀 Performance: 213,675 rows/second
+💾 Data saved to: ./generated_data
+```
+
+## 💻 Python API
+
+```python
+from misata import DataSimulator, SchemaConfig
+from misata.llm_parser import LLMSchemaGenerator
+
+# Generate schema from story
+llm = LLMSchemaGenerator(provider="groq")  # or "openai", "ollama"
+config = llm.generate_from_story(
+    "A mobile fitness app with 50K users, workout tracking, "
+    "premium subscriptions, and January signup spikes"
+)
+
+# Generate data
+for table_name, batch in DataSimulator(config).generate_all():
+    print(f"Generated {len(batch)} rows for {table_name}")
+```
+
+## 🔧 CLI Reference
+
+```bash
+# Basic generation (rule-based, no API key needed)
+misata generate --story "SaaS company with users and subscriptions"
+
+# LLM-powered generation
+misata generate --story "..." --use-llm
+
+# Specify provider and model
+misata generate --story "..." --use-llm --provider ollama --model llama3
+
+# Custom output directory
+misata generate --story "..." --use-llm --output-dir ./my_data
+
+# Set row count
+misata generate --story "..." --use-llm --rows 100000
+
+# Reproducible with seed
+misata generate --story "..." --use-llm --seed 42
+```
+
+## 🎯 Business Rule Constraints
+
+Define rules like "employees can't log >8 hours/day":
+
+```python
+from misata import Constraint, Table
+
+timesheets = Table(
+    name="timesheets",
+    row_count=10000,
+    constraints=[
+        Constraint(
+            name="max_daily_hours",
+            type="sum_limit",
+            group_by=["employee_id", "date"],
+            column="hours",
+            value=8.0,
+            action="redistribute"
+        )
+    ]
+)
+```
+
+## 🔑 LLM Providers
+
+| Provider | Env Variable | Free Tier | Notes |
+|----------|--------------|-----------|-------|
+| **Groq** | `GROQ_API_KEY` | ✅ 30 req/min | Fastest, recommended |
+| **OpenAI** | `OPENAI_API_KEY` | ❌ | Best quality |
+| **Ollama** | None | ✅ Local | Private, no internet |
+
+## 📈 Extending Data Pools
+
+```python
+from misata import TextGenerator
+
+# Add custom names
+TextGenerator.extend_pool("first_names", ["Arjun", "Priya", "Rahul"])
+
+# Load from file
+TextGenerator.load_pools_from_file("custom_pools.json")
+
+# Save for reuse
+TextGenerator.save_pools_to_file("expanded_pools.json")
+```
+
+## 🤖 ML Training Data
+
+Make your synthetic data **indistinguishable from real-world data** with noise injection:
+
+```python
+from misata import add_noise, NoiseInjector
+
+# Quick noise injection
+noisy_df = add_noise(df,
+    null_rate=0.05,      # 5% missing values
+    outlier_rate=0.02,   # 2% statistical outliers
+    typo_rate=0.01,      # 1% typos in text
+    duplicate_rate=0.03, # 3% duplicate rows
+    seed=42
+)
+
+# Advanced: Temporal distribution drift
+injector = NoiseInjector(seed=42)
+df = injector.apply_temporal_drift(df, 
+    date_column="created_at",
+    value_column="revenue", 
+    drift_rate=0.15,      # 15% increase over time
+    drift_direction="up"
+)
+```
+
+### Attribute Customization
+
+```python
+from misata import Customizer, ColumnOverride
+import numpy as np
+
+customizer = Customizer(seed=42)
+
+# Custom age distribution (realistic, not uniform)
+customizer.add_override("users", ColumnOverride(
+    name="age",
+    generator=lambda n: np.random.normal(35, 12, n).clip(18, 80).astype(int)
+))
+
+# Conditional values based on other columns
+customizer.add_conditional("orders", "shipping_cost", {
+    "country": {"US": 5.99, "UK": 9.99, "DE": 7.99}
+})
+
+# Apply to generated data
+df = customizer.apply(df, "users")
+```
+
+## ⚡ Performance
+
+| Rows | Time | Speed |
+|------|------|-------|
+| 10K | 0.03s | 333K rows/sec |
+| 100K | 0.26s | 385K rows/sec |
+| 1M | 2.6s | 390K rows/sec |
+| 10M | 26s | 390K rows/sec (streaming) |
+
+## � Try It Now
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/rasinmuhammed/misata/blob/main/examples/getting_started.ipynb)
+
+Try Misata in your browser without installing anything!
+
+## 💼 Enterprise & Consulting
+
+**Need help with complex scenarios?**
+
+- 🏢 Custom enterprise data schemas (10M+ rows)
+- 🔧 Integration with your existing pipelines
+- 📊 Industry-specific realistic data generation
+- 🎓 Training and onboarding for your team
+
+📧 **Contact: rasinbinabdulla@gmail.com**
+
+## �📄 License
+
+MIT License
+
+## 👤 Author
+
+Built by **Muhammed Rasin**
+
+---
+
+**Misata** - From story to synthetic database in one command.
+
+

misata-0.1.0b0.dist-info/RECORD

@@ -0,0 +1,25 @@
+misata/__init__.py,sha256=QebW3oeV4C6fLQ8B3O3YK_N7A4tInyoi-lbuFohALIY,1023
+misata/api.py,sha256=Wq2H3iJzocNTsCzb9vhYJxDyag3Yiucvb-GVF0tdKhI,14999
+misata/audit.py,sha256=4eUCHT2STptemfakWeNODbVuBRhyD8Q32LlB2eufvuw,12291
+misata/benchmark.py,sha256=Y1-tuKegJyAlTneROQpPo276qnfmMmupGDbVDs9k5J8,12358
+misata/cli.py,sha256=a7YijZCUYrkCYGVYJ2nZSL9J3JfFqbXQQOad6bhy7zM,22642
+misata/codegen.py,sha256=m7ykTtLgITvaqzVB1cVhs1b9Puo2X4uyzngZ85wi6J0,5791
+misata/curve_fitting.py,sha256=gLj4BkIxNWKkfo3QKZFI_aq60bsXlI53K5yZX4hc9EU,4126
+misata/customization.py,sha256=pw-BEsPKN091hyOrQWWQoRhTrlmQ9_PXXopm2FZSEvs,8551
+misata/feedback.py,sha256=HBEsoKi_vdRqwRzMoVFVj_cjfzQ5SUAaGz40s1HMD50,13313
+misata/formulas.py,sha256=KOTq5YN_19vv1ERd92bdzKot9yo9rrrwjOuWO13nFCg,11210
+misata/generators.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
+misata/hybrid.py,sha256=5oopAdfOLWUYzdRWlc0plVeVEVg7Nu1CVGNNCDSjQt8,13104
+misata/llm_parser.py,sha256=1BrX5jUkRqewpHDypfJdglgVXtsLILByAJmI7zrSQ3o,19118
+misata/noise.py,sha256=UO7MokzQ5Y5Vj7JaayDUG0JwCLnpHtnpQTcJ4UHWibo,10460
+misata/schema.py,sha256=vHy-KesQIaHJIH6CxRed_uQ6oUxjkXYhME0mAacAPd8,8696
+misata/semantic.py,sha256=0fauGWJ75wlbHVqT0hohYTN4m_nscdaMaVAIfkhTZXk,7087
+misata/simulator.py,sha256=XF8jLCOak_-9NsSU2pGH_AqWIu5zGpJF86LNG6tAcHE,30550
+misata/story_parser.py,sha256=7N7so3KWisl2UxkOtENQwP-4hN2cs9vTKsPHVRZB2Mc,15964
+misata/validation.py,sha256=5yJSN7jecVNLJ8ss6y7l2U4hF1Ljn27Q6Xs9N1iDPcw,10791
+misata/templates/__init__.py,sha256=0RcZz9d4bmCqLAr77h0gpMfHncqAPeZCguqsuGCz7rE,25245
+misata-0.1.0b0.dist-info/METADATA,sha256=6MaVv1gWkZ-2g5hW_qDnGAkka95ZrgG3R3XaHx2VXnA,8070
+misata-0.1.0b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+misata-0.1.0b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
+misata-0.1.0b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
+misata-0.1.0b0.dist-info/RECORD,,

misata-0.1.0b0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

misata-0.1.0b0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+misata = misata.cli:main

misata-0.1.0b0.dist-info/top_level.txt

@@ -0,0 +1 @@
+misata