misata 0.1.0b0__py3-none-any.whl → 0.3.0b0__py3-none-any.whl

misata/profiles.py

@@ -0,0 +1,332 @@
+"""
+Distribution Profiles for Realistic Data Generation.
+
+Pre-configured distribution parameters that match real-world patterns
+for common data types like age, salary, prices, etc.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+import numpy as np
+
+
+class DistributionProfile:
+    """A named distribution configuration for realistic generation.
+    
+    Example:
+        profile = DistributionProfile(
+            name="age",
+            distribution="mixture",
+            params={
+                "components": [
+                    {"mean": 35, "std": 12, "weight": 0.6},  # Working age
+                    {"mean": 70, "std": 8, "weight": 0.2},   # Retirees
+                    {"mean": 12, "std": 4, "weight": 0.2},   # Children
+                ]
+            }
+        )
+        values = profile.generate(1000)
+    """
+    
+    def __init__(
+        self,
+        name: str,
+        distribution: str,
+        params: Dict[str, Any],
+        min_value: Optional[float] = None,
+        max_value: Optional[float] = None,
+        decimals: Optional[int] = None,
+    ):
+        self.name = name
+        self.distribution = distribution
+        self.params = params
+        self.min_value = min_value
+        self.max_value = max_value
+        self.decimals = decimals
+    
+    def generate(
+        self, 
+        size: int, 
+        rng: Optional[np.random.Generator] = None
+    ) -> np.ndarray:
+        """Generate values according to this profile."""
+        if rng is None:
+            rng = np.random.default_rng()
+        
+        if self.distribution == "normal":
+            mean = self.params.get("mean", 50)
+            std = self.params.get("std", 10)
+            values = rng.normal(mean, std, size)
+        
+        elif self.distribution == "lognormal":
+            mean = self.params.get("mean", 0)
+            sigma = self.params.get("sigma", 1)
+            values = rng.lognormal(mean, sigma, size)
+        
+        elif self.distribution == "exponential":
+            scale = self.params.get("scale", 1.0)
+            values = rng.exponential(scale, size)
+        
+        elif self.distribution == "pareto":
+            alpha = self.params.get("alpha", 2.0)
+            min_val = self.params.get("min", 1.0)
+            values = (rng.pareto(alpha, size) + 1) * min_val
+        
+        elif self.distribution == "beta":
+            a = self.params.get("a", 2)
+            b = self.params.get("b", 5)
+            scale = self.params.get("scale", 1.0)
+            values = rng.beta(a, b, size) * scale
+        
+        elif self.distribution == "mixture":
+            # Gaussian mixture model
+            components = self.params.get("components", [])
+            if not components:
+                values = rng.normal(0, 1, size)
+            else:
+                weights = np.array([c.get("weight", 1) for c in components])
+                weights = weights / weights.sum()
+                
+                # Sample component indices
+                component_indices = rng.choice(
+                    len(components), size=size, p=weights
+                )
+                
+                values = np.zeros(size)
+                for i, comp in enumerate(components):
+                    mask = component_indices == i
+                    n = mask.sum()
+                    if n > 0:
+                        values[mask] = rng.normal(
+                            comp.get("mean", 0),
+                            comp.get("std", 1),
+                            n
+                        )
+        
+        elif self.distribution == "zipf":
+            # Zipf distribution for long-tail data
+            a = self.params.get("alpha", 2.0)
+            values = rng.zipf(a, size).astype(float)
+        
+        elif self.distribution == "uniform":
+            low = self.params.get("min", 0)
+            high = self.params.get("max", 100)
+            values = rng.uniform(low, high, size)
+        
+        else:
+            # Default to uniform
+            values = rng.uniform(0, 100, size)
+        
+        # Apply constraints
+        if self.min_value is not None:
+            values = np.maximum(values, self.min_value)
+        if self.max_value is not None:
+            values = np.minimum(values, self.max_value)
+        if self.decimals is not None:
+            values = np.round(values, self.decimals)
+        
+        return values
+
+
+# ============ Pre-built Profiles ============
+
+PROFILES: Dict[str, DistributionProfile] = {}
+
+
+def _register_profile(profile: DistributionProfile) -> None:
+    """Register a profile by name."""
+    PROFILES[profile.name] = profile
+
+
+# Age distributions
+_register_profile(DistributionProfile(
+    name="age_adult",
+    distribution="mixture",
+    params={
+        "components": [
+            {"mean": 28, "std": 6, "weight": 0.3},   # Young adults
+            {"mean": 42, "std": 10, "weight": 0.45}, # Middle age
+            {"mean": 62, "std": 8, "weight": 0.25},  # Older adults
+        ]
+    },
+    min_value=18,
+    max_value=100,
+    decimals=0,
+))
+
+_register_profile(DistributionProfile(
+    name="age_population",
+    distribution="mixture",
+    params={
+        "components": [
+            {"mean": 8, "std": 4, "weight": 0.15},   # Children
+            {"mean": 25, "std": 8, "weight": 0.25},  # Young adults
+            {"mean": 42, "std": 12, "weight": 0.35}, # Middle age
+            {"mean": 68, "std": 10, "weight": 0.25}, # Seniors
+        ]
+    },
+    min_value=0,
+    max_value=105,
+    decimals=0,
+))
+
+# Salary distributions
+_register_profile(DistributionProfile(
+    name="salary_usd",
+    distribution="lognormal",
+    params={"mean": 11.0, "sigma": 0.5},  # Log of ~$60k median
+    min_value=25000,
+    max_value=500000,
+    decimals=0,
+))
+
+_register_profile(DistributionProfile(
+    name="salary_tech",
+    distribution="mixture",
+    params={
+        "components": [
+            {"mean": 75000, "std": 15000, "weight": 0.2},   # Junior
+            {"mean": 120000, "std": 25000, "weight": 0.4},  # Mid
+            {"mean": 180000, "std": 40000, "weight": 0.3},  # Senior
+            {"mean": 280000, "std": 60000, "weight": 0.1},  # Staff+
+        ]
+    },
+    min_value=50000,
+    max_value=600000,
+    decimals=0,
+))
+
+# Price distributions
+_register_profile(DistributionProfile(
+    name="price_retail",
+    distribution="lognormal",
+    params={"mean": 3.5, "sigma": 1.2},  # ~$30 median
+    min_value=0.99,
+    max_value=10000,
+    decimals=2,
+))
+
+_register_profile(DistributionProfile(
+    name="price_saas",
+    distribution="mixture",
+    params={
+        "components": [
+            {"mean": 15, "std": 5, "weight": 0.3},     # Basic tier
+            {"mean": 49, "std": 15, "weight": 0.4},    # Pro tier
+            {"mean": 199, "std": 50, "weight": 0.25},  # Enterprise
+            {"mean": 999, "std": 200, "weight": 0.05}, # Custom
+        ]
+    },
+    min_value=0,
+    max_value=5000,
+    decimals=0,
+))
+
+# Transaction amounts
+_register_profile(DistributionProfile(
+    name="transaction_amount",
+    distribution="pareto",
+    params={"alpha": 2.5, "min": 10},
+    min_value=1,
+    max_value=100000,
+    decimals=2,
+))
+
+# Counts / quantities
+_register_profile(DistributionProfile(
+    name="order_quantity",
+    distribution="zipf",
+    params={"alpha": 2.0},
+    min_value=1,
+    max_value=100,
+    decimals=0,
+))
+
+# Time-related
+_register_profile(DistributionProfile(
+    name="session_duration_seconds",
+    distribution="lognormal",
+    params={"mean": 5.5, "sigma": 1.5},  # ~4 min median
+    min_value=1,
+    max_value=7200,  # 2 hours max
+    decimals=0,
+))
+
+# Ratings and scores
+_register_profile(DistributionProfile(
+    name="rating_5star",
+    distribution="beta",
+    params={"a": 5, "b": 2, "scale": 5},  # Skewed towards higher ratings
+    min_value=1,
+    max_value=5,
+    decimals=1,
+))
+
+_register_profile(DistributionProfile(
+    name="nps_score",
+    distribution="mixture",
+    params={
+        "components": [
+            {"mean": 3, "std": 2, "weight": 0.15},   # Detractors
+            {"mean": 7, "std": 1, "weight": 0.25},   # Passives
+            {"mean": 9, "std": 0.8, "weight": 0.6},  # Promoters
+        ]
+    },
+    min_value=0,
+    max_value=10,
+    decimals=0,
+))
+
+# Percentages
+_register_profile(DistributionProfile(
+    name="conversion_rate",
+    distribution="beta",
+    params={"a": 2, "b": 50, "scale": 100},  # Low conversion (1-5%)
+    min_value=0,
+    max_value=100,
+    decimals=2,
+))
+
+_register_profile(DistributionProfile(
+    name="churn_rate",
+    distribution="beta",
+    params={"a": 1.5, "b": 30, "scale": 100},  # ~5% typical
+    min_value=0,
+    max_value=100,
+    decimals=2,
+))
+
+
+def get_profile(name: str) -> Optional[DistributionProfile]:
+    """Get a profile by name."""
+    return PROFILES.get(name)
+
+
+def list_profiles() -> List[str]:
+    """List all available profile names."""
+    return list(PROFILES.keys())
+
+
+def generate_with_profile(
+    profile_name: str,
+    size: int,
+    rng: Optional[np.random.Generator] = None
+) -> np.ndarray:
+    """Generate values using a named profile.
+    
+    Args:
+        profile_name: Name of the profile (e.g., "salary_tech")
+        size: Number of values to generate
+        rng: Random number generator
+        
+    Returns:
+        Array of generated values
+        
+    Raises:
+        ValueError: If profile not found
+    """
+    profile = get_profile(profile_name)
+    if profile is None:
+        available = ", ".join(list_profiles())
+        raise ValueError(f"Unknown profile: {profile_name}. Available: {available}")
+    
+    return profile.generate(size, rng)

misata/quality.py

@@ -0,0 +1,329 @@
+"""
+Data Quality Checker for Synthetic Data Validation.
+
+This module validates generated synthetic data for:
+- Distribution plausibility
+- Referential integrity
+- Temporal consistency
+- Domain-specific rules
+"""
+
+from typing import Dict, List, Any, Optional, Tuple
+from dataclasses import dataclass, field
+import warnings
+
+
+@dataclass
+class QualityIssue:
+    """Represents a single data quality issue."""
+    severity: str  # "error", "warning", "info"
+    category: str  # "distribution", "integrity", "temporal", "domain"
+    table: str
+    column: Optional[str]
+    message: str
+    details: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class QualityReport:
+    """Complete quality report for generated data."""
+    score: float  # 0-100
+    issues: List[QualityIssue]
+    stats: Dict[str, Any]
+    
+    @property
+    def passed(self) -> bool:
+        """Returns True if no errors (warnings OK)."""
+        return not any(i.severity == "error" for i in self.issues)
+    
+    def summary(self) -> str:
+        """Human-readable summary."""
+        errors = sum(1 for i in self.issues if i.severity == "error")
+        warnings = sum(1 for i in self.issues if i.severity == "warning")
+        return f"Quality Score: {self.score:.1f}/100 | Errors: {errors} | Warnings: {warnings}"
+
+
+class DataQualityChecker:
+    """
+    Validate generated synthetic data for realism and correctness.
+    
+    Usage:
+        checker = DataQualityChecker()
+        report = checker.check_all(tables, relationships, schema)
+        
+        if not report.passed:
+            print("Issues found:", report.issues)
+    """
+    
+    # Domain-specific plausibility rules
+    PLAUSIBILITY_RULES = {
+        # Column name patterns -> (min, max, description)
+        "age": (0, 120, "Human age"),
+        "price": (0, 1_000_000, "Price"),
+        "quantity": (0, 10_000, "Quantity"),
+        "rating": (1, 5, "Rating"),
+        "percentage": (0, 100, "Percentage"),
+        "year": (1900, 2100, "Year"),
+        "month": (1, 12, "Month"),
+        "day": (1, 31, "Day"),
+        "hour": (0, 23, "Hour"),
+        "minute": (0, 59, "Minute"),
+        "score": (0, 100, "Score"),
+        "count": (0, 1_000_000, "Count"),
+        "duration": (0, 10_000, "Duration"),
+    }
+    
+    def __init__(self, strict: bool = False):
+        """
+        Initialize the quality checker.
+        
+        Args:
+            strict: If True, warnings become errors
+        """
+        self.strict = strict
+        self.issues: List[QualityIssue] = []
+    
+    def _add_issue(
+        self,
+        severity: str,
+        category: str,
+        table: str,
+        column: Optional[str],
+        message: str,
+        details: Optional[Dict] = None,
+    ):
+        """Add an issue to the list."""
+        if self.strict and severity == "warning":
+            severity = "error"
+        
+        self.issues.append(QualityIssue(
+            severity=severity,
+            category=category,
+            table=table,
+            column=column,
+            message=message,
+            details=details or {},
+        ))
+    
+    def check_distribution_plausibility(
+        self,
+        df: "pd.DataFrame",
+        table_name: str,
+    ) -> None:
+        """
+        Check if numeric distributions are plausible for their domains.
+        
+        Args:
+            df: DataFrame to check
+            table_name: Name of the table
+        """
+        import pandas as pd
+        import numpy as np
+        
+        for col in df.columns:
+            col_lower = col.lower()
+            
+            # Check against plausibility rules
+            for pattern, (min_val, max_val, description) in self.PLAUSIBILITY_RULES.items():
+                if pattern in col_lower:
+                    if pd.api.types.is_numeric_dtype(df[col]):
+                        actual_min = df[col].min()
+                        actual_max = df[col].max()
+                        
+                        if actual_min < min_val:
+                            self._add_issue(
+                                "warning", "distribution", table_name, col,
+                                f"{description} column '{col}' has min {actual_min} < expected {min_val}",
+                                {"actual_min": actual_min, "expected_min": min_val}
+                            )
+                        
+                        if actual_max > max_val:
+                            self._add_issue(
+                                "warning", "distribution", table_name, col,
+                                f"{description} column '{col}' has max {actual_max} > expected {max_val}",
+                                {"actual_max": actual_max, "expected_max": max_val}
+                            )
+                    break
+            
+            # Check for all-null columns
+            if df[col].isna().all():
+                self._add_issue(
+                    "error", "distribution", table_name, col,
+                    f"Column '{col}' is entirely NULL",
+                )
+            
+            # Check for zero variance (all same value)
+            if pd.api.types.is_numeric_dtype(df[col]) and df[col].std() == 0:
+                self._add_issue(
+                    "warning", "distribution", table_name, col,
+                    f"Column '{col}' has zero variance (all values identical)",
+                    {"value": df[col].iloc[0]}
+                )
+    
+    def check_referential_integrity(
+        self,
+        tables: Dict[str, "pd.DataFrame"],
+        relationships: List[Any],
+    ) -> None:
+        """
+        Verify all foreign key references are valid.
+        
+        Args:
+            tables: Dict of table_name -> DataFrame
+            relationships: List of Relationship objects
+        """
+        for rel in relationships:
+            parent_table = rel.parent_table
+            child_table = rel.child_table
+            parent_key = rel.parent_key
+            child_key = rel.child_key
+            
+            if parent_table not in tables:
+                self._add_issue(
+                    "error", "integrity", child_table, child_key,
+                    f"Parent table '{parent_table}' not found for FK '{child_key}'",
+                )
+                continue
+            
+            if child_table not in tables:
+                continue  # Child table might not exist yet
+            
+            parent_df = tables[parent_table]
+            child_df = tables[child_table]
+            
+            if parent_key not in parent_df.columns:
+                self._add_issue(
+                    "error", "integrity", parent_table, parent_key,
+                    f"Parent key '{parent_key}' not found in table '{parent_table}'",
+                )
+                continue
+            
+            if child_key not in child_df.columns:
+                self._add_issue(
+                    "error", "integrity", child_table, child_key,
+                    f"Child key '{child_key}' not found in table '{child_table}'",
+                )
+                continue
+            
+            # Check for orphaned records
+            parent_ids = set(parent_df[parent_key].dropna().unique())
+            child_ids = set(child_df[child_key].dropna().unique())
+            orphans = child_ids - parent_ids
+            
+            if orphans:
+                orphan_pct = len(orphans) / len(child_ids) * 100
+                self._add_issue(
+                    "error" if orphan_pct > 1 else "warning",
+                    "integrity", child_table, child_key,
+                    f"{len(orphans)} orphaned FK values ({orphan_pct:.1f}%) in '{child_key}' -> '{parent_table}.{parent_key}'",
+                    {"orphan_count": len(orphans), "orphan_pct": orphan_pct}
+                )
+    
+    def check_temporal_consistency(
+        self,
+        df: "pd.DataFrame",
+        table_name: str,
+    ) -> None:
+        """
+        Ensure temporal columns are consistent.
+        
+        Checks:
+        - created_at < updated_at
+        - start_date < end_date
+        - birth_date in past
+        """
+        import pandas as pd
+        
+        date_cols = [c for c in df.columns if pd.api.types.is_datetime64_any_dtype(df[c])]
+        
+        # Check created < updated
+        if "created_at" in date_cols and "updated_at" in date_cols:
+            violations = (df["created_at"] > df["updated_at"]).sum()
+            if violations > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, "created_at",
+                    f"{violations} rows have created_at > updated_at",
+                    {"violation_count": violations}
+                )
+        
+        # Check start < end
+        if "start_date" in date_cols and "end_date" in date_cols:
+            violations = (df["start_date"] > df["end_date"]).sum()
+            if violations > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, "start_date",
+                    f"{violations} rows have start_date > end_date",
+                    {"violation_count": violations}
+                )
+        
+        # Check birth_date is in past
+        if "birth_date" in date_cols or "date_of_birth" in date_cols:
+            col = "birth_date" if "birth_date" in date_cols else "date_of_birth"
+            future_births = (df[col] > pd.Timestamp.now()).sum()
+            if future_births > 0:
+                self._add_issue(
+                    "error", "temporal", table_name, col,
+                    f"{future_births} rows have birth_date in the future",
+                    {"violation_count": future_births}
+                )
+    
+    def check_all(
+        self,
+        tables: Dict[str, "pd.DataFrame"],
+        relationships: Optional[List[Any]] = None,
+        schema: Optional[Any] = None,
+    ) -> QualityReport:
+        """
+        Run all quality checks and generate a report.
+        
+        Args:
+            tables: Dict of table_name -> DataFrame
+            relationships: Optional list of Relationship objects
+            schema: Optional SchemaConfig for additional checks
+            
+        Returns:
+            QualityReport with score and issues
+        """
+        self.issues = []  # Reset
+        
+        # Check each table
+        for table_name, df in tables.items():
+            self.check_distribution_plausibility(df, table_name)
+            self.check_temporal_consistency(df, table_name)
+        
+        # Check referential integrity
+        if relationships:
+            self.check_referential_integrity(tables, relationships)
+        
+        # Calculate score
+        base_score = 100
+        for issue in self.issues:
+            if issue.severity == "error":
+                base_score -= 10
+            elif issue.severity == "warning":
+                base_score -= 3
+            else:
+                base_score -= 1
+        
+        score = max(0, min(100, base_score))
+        
+        # Gather stats
+        stats = {
+            "tables_checked": len(tables),
+            "total_rows": sum(len(df) for df in tables.values()),
+            "total_columns": sum(len(df.columns) for df in tables.values()),
+            "error_count": sum(1 for i in self.issues if i.severity == "error"),
+            "warning_count": sum(1 for i in self.issues if i.severity == "warning"),
+        }
+        
+        return QualityReport(
+            score=score,
+            issues=self.issues.copy(),
+            stats=stats,
+        )
+
+
+def check_quality(tables: Dict[str, "pd.DataFrame"], **kwargs) -> QualityReport:
+    """Convenience function for quick quality checks."""
+    checker = DataQualityChecker()
+    return checker.check_all(tables, **kwargs)

misata/schema.py

@@ -23,7 +23,7 @@ class Column(BaseModel):
     """
 
     name: str
-    type: Literal["int", "float", "date", "categorical", "foreign_key", "text", "boolean"]
+    type: Literal["int", "float", "date", "time", "datetime", "categorical", "foreign_key", "text", "boolean"]
     distribution_params: Dict[str, Any] = Field(default_factory=dict)
     nullable: bool = False
     unique: bool = False
@@ -39,8 +39,13 @@ class Column(BaseModel):
 
         if col_type == "date":
             if "relative_to" not in v:
-                if "start" not in v or "end" not in v:
-                    raise ValueError("Date columns must have 'start' and 'end' OR 'relative_to' in distribution_params")
+                # Provide sensible defaults if start/end not specified
+                if "start" not in v:
+                    from datetime import datetime, timedelta
+                    v["start"] = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
+                if "end" not in v:
+                    from datetime import datetime
+                    v["end"] = datetime.now().strftime("%Y-%m-%d")
 
         if col_type in ["int", "float"]:
             if "distribution" not in v: