misata 0.1.0b0__py3-none-any.whl

misata/formulas.py

@@ -0,0 +1,362 @@
+"""
+Formula engine for derived columns.
+
+This module enables columns that are computed from other columns,
+supporting expressions like:
+- calories_burned = duration_minutes * @exercises.calories_per_minute
+- total_price = quantity * @products.price
+- discount_amount = total_price * 0.1
+"""
+
+import re
+from typing import Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+import ast
+import operator
+from simpleeval import simple_eval, NameNotDefined
+
+# Whitelist of safe functions
+SAFE_FUNCTIONS = {
+    'where': np.where,
+    'abs': np.abs,
+    'round': np.round,
+    'ceil': np.ceil,
+    'floor': np.floor,
+    'min': np.minimum,
+    'max': np.maximum,
+    'sin': np.sin,
+    'cos': np.cos,
+    'tan': np.tan,
+    'log': np.log,
+    'exp': np.exp,
+    'sqrt': np.sqrt,
+    'random': np.random.random,
+    'randint': np.random.randint,
+}
+
+# Standard operators to bypass simpleeval's string length checks which fail on numpy arrays
+SAFE_OPERATORS = {
+    ast.Add: operator.add,
+    ast.Sub: operator.sub,
+    ast.Mult: operator.mul,
+    ast.Div: operator.truediv,
+    ast.FloorDiv: operator.floordiv,
+    ast.Pow: operator.pow,
+    ast.Mod: operator.mod,
+    ast.Eq: operator.eq,
+    ast.NotEq: operator.ne,
+    ast.Gt: operator.gt,
+    ast.Lt: operator.lt,
+    ast.GtE: operator.ge,
+    ast.LtE: operator.le,
+    ast.BitAnd: operator.and_,
+    ast.BitOr: operator.or_,
+    ast.BitXor: operator.xor,
+    ast.USub: operator.neg,
+    ast.UAdd: operator.pos,
+    ast.Not: operator.not_,
+    ast.In: lambda a, b: a in b,
+}
+
+class SafeNumpy:
+    """Proxy for numpy to allow safe access to whitelisted functions."""
+    def __getattr__(self, name):
+        if name in SAFE_FUNCTIONS:
+            return SAFE_FUNCTIONS[name]
+        raise NameNotDefined(name, f"Function 'np.{name}' is not allowed in formulas.")
+
+class FormulaEngine:
+    """
+    Evaluate column formulas using safe expressions.
+
+    Supports:
+    - Simple arithmetic: duration * 10
+    - Column references: quantity * unit_price
+    - Cross-table references: @exercises.calories_per_minute
+    - Conditional expressions: np.where(status == 'active', 1, 0)
+    """
+
+    def __init__(self, tables: Dict[str, pd.DataFrame]):
+        """
+        Initialize with generated tables for cross-table lookups.
+
+        Args:
+            tables: Dict mapping table names to DataFrames
+        """
+        self.tables = tables
+
+    def evaluate(
+        self,
+        df: pd.DataFrame,
+        formula: str,
+        fk_column: Optional[str] = None,
+    ) -> np.ndarray:
+        """
+        Evaluate a formula on a DataFrame.
+
+        Args:
+            df: DataFrame to evaluate on
+            formula: Expression string
+            fk_column: Foreign key column name for cross-table lookups
+
+        Returns:
+            Array of computed values
+        """
+        # Replace cross-table references with actual values
+        processed_formula = self._resolve_cross_table_refs(df, formula, fk_column)
+
+        # Create evaluation context
+        names = {
+            'np': SafeNumpy(),
+            'pd': pd, # Needed for some checks, but ideally we restrict this too
+            # simpleeval defaults allow basic math
+        }
+
+        # Add columns to context
+        for col in df.columns:
+            names[col] = df[col].values
+
+        # Evaluate the expression safely
+        try:
+            result = simple_eval(
+                processed_formula,
+                names=names,
+                functions=SAFE_FUNCTIONS, # Allow top-level functions too
+                operators=SAFE_OPERATORS
+            )
+            return np.array(result)
+        except Exception as e:
+            raise ValueError(f"Failed to evaluate formula '{formula}': {e}")
+
+    def _resolve_cross_table_refs(
+        self,
+        df: pd.DataFrame,
+        formula: str,
+        fk_column: Optional[str] = None,
+    ) -> str:
+        """
+        Replace @table.column references with actual looked-up values.
+
+        Pattern: @tablename.columnname
+
+        Args:
+            df: Current DataFrame
+            formula: Formula with potential cross-table refs
+            fk_column: FK column to use for lookups
+
+        Returns:
+            Formula with refs replaced by _lookup_N variables
+        """
+        # Pattern to match @table.column
+        pattern = r'@(\w+)\.(\w+)'
+        matches = re.findall(pattern, formula)
+
+        if not matches:
+            return formula
+
+        result = formula
+
+        for i, (table_name, col_name) in enumerate(matches):
+            if table_name not in self.tables:
+                raise ValueError(f"Table '{table_name}' not found for formula lookup")
+
+            ref_table = self.tables[table_name]
+
+            if col_name not in ref_table.columns:
+                raise ValueError(f"Column '{col_name}' not found in table '{table_name}'")
+
+            # Determine the FK column to use for lookup
+            actual_fk = fk_column or f"{table_name}_id"
+
+            if actual_fk not in df.columns:
+                raise ValueError(
+                    f"Cannot lookup @{table_name}.{col_name}: "
+                    f"no foreign key column '{actual_fk}' in current table"
+                )
+
+            # Create lookup mapping and apply
+            # Handle duplicates in index by grouping or taking first (safety)
+            if not ref_table['id'].is_unique:
+                 # If IDs are not unique in ref table, we have a problem.
+                 # Assuming IDs are unique for lookups.
+                 pass
+
+            lookup_map = ref_table.set_index('id')[col_name].to_dict()
+            df[actual_fk].map(lookup_map).values
+
+            var_name = f'_lookup_{i}'
+            # Store in a temporary dict doesn't work well with clean state
+            # Instead we'll inject into the names dict in evaluate
+            # But specific method needs to handle this exchange.
+            # Refactoring to return both formula and context updates would be better
+            # For now, sticking to string replacement and hoping evaluate calls context filler
+
+            # CRITICAL: This method only returns string.
+            # The previous implementation put it in _temp_lookups but implementation was messy.
+            # We will use evaluate_with_lookups which is the main entry point
+
+            result = result.replace(f'@{table_name}.{col_name}', var_name)
+
+        return result
+
+    def evaluate_with_lookups(
+        self,
+        df: pd.DataFrame,
+        formula: str,
+        fk_mappings: Optional[Dict[str, str]] = None,
+    ) -> np.ndarray:
+        """
+        Evaluate formula with automatic cross-table lookups.
+
+        Args:
+            df: DataFrame to evaluate on
+            formula: Expression with @table.column references
+            fk_mappings: Optional dict mapping table name to FK column name
+                         e.g., {"exercises": "exercise_id", "products": "product_id"}
+
+        Returns:
+            Array of computed values
+        """
+        fk_mappings = fk_mappings or {}
+
+        # Pattern to match @table.column
+        pattern = r'@(\w+)\.(\w+)'
+        matches = re.findall(pattern, formula)
+
+        result = formula
+        names = {
+            'np': SafeNumpy(),
+            'pd': pd,
+        }
+
+        # Add columns to context
+        for col in df.columns:
+            names[col] = df[col].values
+
+        # Resolve each cross-table reference
+        for i, (table_name, col_name) in enumerate(matches):
+            if table_name not in self.tables:
+                raise ValueError(f"Table '{table_name}' not found")
+
+            ref_table = self.tables[table_name]
+
+            # Determine FK column
+            fk_col = fk_mappings.get(table_name, f"{table_name}_id")
+            if fk_col.endswith("s_id"):
+                # Try without trailing 's' (exercises -> exercise_id)
+                alt_fk = fk_col.replace("s_id", "_id")
+                if alt_fk in df.columns:
+                    fk_col = alt_fk
+
+            if fk_col not in df.columns:
+                # Try common patterns
+                for pattern_fk in [f"{table_name}_id", f"{table_name[:-1]}_id", "id"]:
+                    if pattern_fk in df.columns:
+                        fk_col = pattern_fk
+                        break
+
+            if fk_col not in df.columns:
+                raise ValueError(f"No FK column found for table '{table_name}'")
+
+            # Create lookup and add to context
+            if 'id' not in ref_table.columns:
+                raise ValueError(f"Reference table '{table_name}' has no 'id' column")
+
+            lookup_map = ref_table.set_index('id')[col_name].to_dict()
+            looked_up = df[fk_col].map(lookup_map).fillna(0).values
+
+            var_name = f'_ref_{i}'
+            names[var_name] = looked_up
+            result = result.replace(f'@{table_name}.{col_name}', var_name)
+
+        # Evaluate safely
+        try:
+            return np.array(simple_eval(
+                result,
+                names=names,
+                functions=SAFE_FUNCTIONS,
+                operators=SAFE_OPERATORS
+            ))
+        except Exception as e:
+            raise ValueError(f"Failed to evaluate formula '{formula}': {e}")
+
+
+class FormulaColumn:
+    """
+    Definition of a formula-based column.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        formula: str,
+        result_type: str = "float",
+        fk_mappings: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Define a formula column.
+
+        Args:
+            name: Column name
+            formula: Expression (can include @table.column refs)
+            result_type: Type of result (int, float, boolean)
+            fk_mappings: Map table names to FK column names
+        """
+        self.name = name
+        self.formula = formula
+        self.result_type = result_type
+        self.fk_mappings = fk_mappings or {}
+
+    def evaluate(
+        self,
+        df: pd.DataFrame,
+        tables: Dict[str, pd.DataFrame],
+    ) -> np.ndarray:
+        """
+        Evaluate this formula column.
+
+        Args:
+            df: Current table DataFrame
+            tables: All generated tables for cross-table lookups
+
+        Returns:
+            Array of computed values
+        """
+        engine = FormulaEngine(tables)
+        result = engine.evaluate_with_lookups(df, self.formula, self.fk_mappings)
+
+        # Cast to result type
+        if self.result_type == "int":
+            return result.astype(int)
+        elif self.result_type == "float":
+            return result.astype(float)
+        elif self.result_type == "boolean":
+            return result.astype(bool)
+
+        return result
+
+
+def apply_formula_columns(
+    df: pd.DataFrame,
+    formulas: List[FormulaColumn],
+    tables: Dict[str, pd.DataFrame],
+) -> pd.DataFrame:
+    """
+    Apply formula columns to a DataFrame.
+
+    Args:
+        df: DataFrame to add columns to
+        formulas: List of formula column definitions
+        tables: All tables for cross-table lookups
+
+    Returns:
+        DataFrame with formula columns added
+    """
+    result = df.copy()
+
+    for formula_col in formulas:
+        result[formula_col.name] = formula_col.evaluate(result, tables)
+
+    return result

misata/generators.py

@@ -0,0 +1,247 @@
+"""
+Pure Python text generators for Misata.
+
+Replaces Mimesis with lightweight, built-in generators using curated data pools.
+No external dependencies required.
+"""
+
+import random
+import string
+from typing import Dict, List, Optional
+
+# ============================================
+# DATA POOLS
+# ============================================
+
+FIRST_NAMES = [
+    "James", "Mary", "Robert", "Patricia", "John", "Jennifer", "Michael", "Linda",
+    "David", "Elizabeth", "William", "Barbara", "Richard", "Susan", "Joseph", "Jessica",
+    "Thomas", "Sarah", "Christopher", "Karen", "Charles", "Lisa", "Daniel", "Nancy",
+    "Matthew", "Betty", "Anthony", "Margaret", "Mark", "Sandra", "Donald", "Ashley",
+    "Steven", "Kimberly", "Paul", "Emily", "Andrew", "Donna", "Joshua", "Michelle",
+    "Kenneth", "Dorothy", "Kevin", "Carol", "Brian", "Amanda", "George", "Melissa",
+    "Timothy", "Deborah", "Ronald", "Stephanie", "Edward", "Rebecca", "Jason", "Sharon",
+]
+
+LAST_NAMES = [
+    "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis",
+    "Rodriguez", "Martinez", "Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson",
+    "Thomas", "Taylor", "Moore", "Jackson", "Martin", "Lee", "Perez", "Thompson",
+    "White", "Harris", "Sanchez", "Clark", "Ramirez", "Lewis", "Robinson", "Walker",
+    "Young", "Allen", "King", "Wright", "Scott", "Torres", "Nguyen", "Hill",
+    "Flores", "Green", "Adams", "Nelson", "Baker", "Hall", "Rivera", "Campbell",
+]
+
+EMAIL_DOMAINS = [
+    "gmail.com", "yahoo.com", "outlook.com", "hotmail.com", "icloud.com",
+    "protonmail.com", "mail.com", "aol.com", "zoho.com", "fastmail.com",
+]
+
+COMPANY_NAMES = [
+    "Acme Corp", "Globex", "Initech", "Umbrella Corp", "Stark Industries",
+    "Wayne Enterprises", "Cyberdyne Systems", "Soylent Corp", "Massive Dynamic",
+    "Aperture Science", "InGen", "Tyrell Corporation", "Weyland-Yutani", "OsCorp",
+    "LexCorp", "Oscorp Industries", "Dharma Initiative", "Dunder Mifflin",
+    "Sterling Cooper", "Wonka Industries", "Prestige Worldwide", "Vandelay Industries",
+]
+
+STREET_NAMES = [
+    "Main", "Oak", "Maple", "Cedar", "Elm", "Pine", "Washington", "Lake",
+    "Hill", "Park", "River", "Sunset", "Highland", "Valley", "Forest", "Spring",
+]
+
+STREET_SUFFIXES = ["St", "Ave", "Blvd", "Dr", "Ln", "Rd", "Way", "Ct", "Pl", "Cir"]
+
+CITIES = [
+    "New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia",
+    "San Antonio", "San Diego", "Dallas", "San Jose", "Austin", "Jacksonville",
+    "Fort Worth", "Columbus", "Charlotte", "Seattle", "Denver", "Boston",
+]
+
+STATES = [
+    "NY", "CA", "TX", "FL", "IL", "PA", "OH", "GA", "NC", "MI",
+    "NJ", "VA", "WA", "AZ", "MA", "CO", "TN", "IN", "MO", "MD",
+]
+
+LOREM_WORDS = [
+    "lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit",
+    "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore",
+    "magna", "aliqua", "enim", "ad", "minim", "veniam", "quis", "nostrud",
+    "exercitation", "ullamco", "laboris", "nisi", "aliquip", "ex", "ea", "commodo",
+]
+
+URLS = [
+    "https://example.com", "https://test.org", "https://demo.io", "https://sample.net",
+]
+
+
+# ============================================
+# GENERATOR CLASS
+# ============================================
+
+class TextGenerator:
+    """
+    Pure Python text generator for synthetic data.
+
+    Drop-in replacement for Mimesis functionality.
+    Supports expandable data pools that can grow over time.
+    """
+
+    # Class-level pools (shared across instances, can be extended)
+    _pools = {
+        "first_names": list(FIRST_NAMES),
+        "last_names": list(LAST_NAMES),
+        "email_domains": list(EMAIL_DOMAINS),
+        "company_names": list(COMPANY_NAMES),
+        "street_names": list(STREET_NAMES),
+        "cities": list(CITIES),
+        "states": list(STATES),
+        "lorem_words": list(LOREM_WORDS),
+    }
+
+    def __init__(self, seed: Optional[int] = None):
+        """Initialize with optional random seed."""
+        self.rng = random.Random(seed)
+
+    @classmethod
+    def extend_pool(cls, pool_name: str, values: List[str]) -> int:
+        """
+        Extend a data pool with new values.
+
+        Args:
+            pool_name: Name of pool (first_names, last_names, etc.)
+            values: List of new values to add
+
+        Returns:
+            New pool size
+        """
+        if pool_name not in cls._pools:
+            cls._pools[pool_name] = []
+
+        # Add only unique values
+        existing = set(cls._pools[pool_name])
+        new_values = [v for v in values if v not in existing]
+        cls._pools[pool_name].extend(new_values)
+
+        return len(cls._pools[pool_name])
+
+    @classmethod
+    def load_pools_from_file(cls, filepath: str) -> Dict[str, int]:
+        """
+        Load and extend pools from a JSON file.
+
+        Args:
+            filepath: Path to JSON file with pool data
+
+        Returns:
+            Dict of pool names to their new sizes
+        """
+        import json
+        with open(filepath, 'r') as f:
+            data = json.load(f)
+
+        sizes = {}
+        for pool_name, values in data.items():
+            if isinstance(values, list):
+                sizes[pool_name] = cls.extend_pool(pool_name, values)
+
+        return sizes
+
+    @classmethod
+    def save_pools_to_file(cls, filepath: str) -> None:
+        """
+        Save current pools to a JSON file.
+
+        Args:
+            filepath: Path to save JSON file
+        """
+        import json
+        with open(filepath, 'w') as f:
+            json.dump(cls._pools, f, indent=2)
+
+    @classmethod
+    def get_pool_sizes(cls) -> Dict[str, int]:
+        """Get sizes of all pools."""
+        return {name: len(values) for name, values in cls._pools.items()}
+
+    def name(self) -> str:
+        """Generate a full name."""
+        first = self.rng.choice(self._pools["first_names"])
+        last = self.rng.choice(self._pools["last_names"])
+        return f"{first} {last}"
+
+    def first_name(self) -> str:
+        """Generate a first name."""
+        return self.rng.choice(self._pools["first_names"])
+
+    def last_name(self) -> str:
+        """Generate a last name."""
+        return self.rng.choice(self._pools["last_names"])
+
+    def email(self) -> str:
+        """Generate an email address."""
+        first = self.rng.choice(self._pools["first_names"]).lower()
+        last = self.rng.choice(self._pools["last_names"]).lower()
+        domain = self.rng.choice(self._pools["email_domains"])
+        separator = self.rng.choice([".", "_", ""])
+        num = self.rng.randint(1, 99) if self.rng.random() > 0.5 else ""
+        return f"{first}{separator}{last}{num}@{domain}"
+
+    def company(self) -> str:
+        """Generate a company name."""
+        return self.rng.choice(self._pools["company_names"])
+
+    def address(self) -> str:
+        """Generate a street address."""
+        number = self.rng.randint(1, 9999)
+        street = self.rng.choice(self._pools["street_names"])
+        suffix = self.rng.choice(STREET_SUFFIXES)
+        return f"{number} {street} {suffix}"
+
+    def full_address(self) -> str:
+        """Generate a full address with city, state, zip."""
+        addr = self.address()
+        city = self.rng.choice(self._pools["cities"])
+        state = self.rng.choice(self._pools["states"])
+        zipcode = self.rng.randint(10000, 99999)
+        return f"{addr}, {city}, {state} {zipcode}"
+
+    def phone_number(self) -> str:
+        """Generate a phone number."""
+        area = self.rng.randint(200, 999)
+        prefix = self.rng.randint(200, 999)
+        line = self.rng.randint(1000, 9999)
+        return f"({area}) {prefix}-{line}"
+
+    def url(self) -> str:
+        """Generate a URL."""
+        base = self.rng.choice(URLS)
+        path = ''.join(self.rng.choices(string.ascii_lowercase, k=8))
+        return f"{base}/{path}"
+
+    def sentence(self, words: int = 8) -> str:
+        """Generate a lorem ipsum sentence."""
+        selected = [self.rng.choice(self._pools["lorem_words"]) for _ in range(words)]
+        selected[0] = selected[0].capitalize()
+        return ' '.join(selected) + '.'
+
+    def word(self) -> str:
+        """Generate a single word."""
+        return self.rng.choice(self._pools["lorem_words"])
+
+    def text(self, sentences: int = 3) -> str:
+        """Generate a paragraph of text."""
+        return ' '.join(self.sentence() for _ in range(sentences))
+
+    def uuid(self) -> str:
+        """Generate a UUID-like string."""
+        hex_chars = '0123456789abcdef'
+        parts = [
+            ''.join(self.rng.choices(hex_chars, k=8)),
+            ''.join(self.rng.choices(hex_chars, k=4)),
+            ''.join(self.rng.choices(hex_chars, k=4)),
+            ''.join(self.rng.choices(hex_chars, k=4)),
+            ''.join(self.rng.choices(hex_chars, k=12)),
+        ]
+        return '-'.join(parts)
+