misata 0.2.0b0__py3-none-any.whl → 0.3.1b0__py3-none-any.whl

misata/smart_values.py

@@ -580,14 +580,183 @@ Return ONLY a JSON array of strings, no explanation. Example:
     def get_fallback_pool(self, domain: str) -> List[str]:
         """Get curated fallback pool for a domain."""
         return self.FALLBACK_POOLS.get(domain, [])
+    
+    def generate_with_template(
+        self,
+        template: str,
+        size: int,
+        components: Dict[str, List[str]],
+    ) -> List[str]:
+        """Generate text by substituting template components.
+        
+        This creates more variety by combining parts rather than
+        picking from a fixed pool.
+        
+        Args:
+            template: String template with {component_name} placeholders
+            size: Number of values to generate
+            components: Dict mapping component names to value lists
+            
+        Returns:
+            List of generated strings
+            
+        Example:
+            template = "{first_name} {last_name}"
+            components = {
+                "first_name": ["John", "Jane", "Alex"],
+                "last_name": ["Smith", "Johnson", "Williams"],
+            }
+            values = gen.generate_with_template(template, 100, components)
+            # Returns: ["John Smith", "Jane Williams", "Alex Johnson", ...]
+        """
+        import random
+        
+        results = []
+        for _ in range(size):
+            text = template
+            for key, values in components.items():
+                if f"{{{key}}}" in text:
+                    text = text.replace(f"{{{key}}}", random.choice(values), 1)
+            results.append(text)
+        
+        return results
+    
+    def generate_composite_pool(
+        self,
+        domain: str,
+        size: int = 200,
+    ) -> List[str]:
+        """Generate larger pools using template composition.
+        
+        Instead of calling LLM for 200 values, we compose
+        templates with varied components.
+        
+        Args:
+            domain: Semantic domain
+            size: Target pool size
+            
+        Returns:
+            List of composed values
+        """
+        import random
+        
+        # Domain-specific templates
+        templates = {
+            "address": {
+                "template": "{number} {street_name} {street_type}, {city}, {state}",
+                "components": {
+                    "number": [str(i) for i in range(100, 10000)],
+                    "street_name": ["Oak", "Maple", "Cedar", "Pine", "Elm", "Birch", "Walnut", "Cherry", "Willow", "Aspen",
+                                   "Main", "First", "Second", "Third", "Park", "Lake", "River", "Hill", "Valley", "Spring"],
+                    "street_type": ["Street", "Avenue", "Boulevard", "Lane", "Drive", "Court", "Place", "Road", "Way", "Circle"],
+                    "city": ["Springfield", "Riverside", "Franklin", "Georgetown", "Clinton", "Salem", "Madison", "Bristol", "Fairview", "Newport"],
+                    "state": ["CA", "TX", "NY", "FL", "IL", "PA", "OH", "GA", "MI", "NC", "WA", "CO", "AZ", "MA", "VA"],
+                },
+            },
+            "email": {
+                "template": "{name_part}{separator}{domain_part}@{provider}.{tld}",
+                "components": {
+                    "name_part": ["john", "jane", "alex", "sam", "chris", "pat", "taylor", "jordan", "casey", "morgan",
+                                 "mike", "lisa", "david", "emma", "ryan", "kate", "nick", "amy", "steve", "jen"],
+                    "separator": ["", ".", "_", ""],
+                    "domain_part": ["smith", "jones", "work", "mail", "pro", "dev", "biz", "123", "2024", "online"],
+                    "provider": ["gmail", "yahoo", "outlook", "hotmail", "icloud", "proton", "fastmail", "zoho"],
+                    "tld": ["com", "com", "com", "org", "net", "io", "co"],
+                },
+            },
+            "product": {
+                "template": "{adjective} {material} {item_type} - {size_color}",
+                "components": {
+                    "adjective": ["Premium", "Ultra", "Pro", "Classic", "Modern", "Sleek", "Essential", "Deluxe", "Elite", "Smart"],
+                    "material": ["Stainless Steel", "Bamboo", "Ceramic", "Leather", "Cotton", "Titanium", "Wood", "Glass", "Silicone", "Carbon Fiber"],
+                    "item_type": ["Water Bottle", "Phone Case", "Backpack", "Wallet", "Watch Band", "Desk Lamp", "Speaker", "Charging Dock", "Notebook", "Organizer"],
+                    "size_color": ["Black/Large", "White/Medium", "Navy/Standard", "Gray/Compact", "Red/XL", "Brown/Regular", "Silver/Slim", "Green/Mini"],
+                },
+            },
+            "company_name": {
+                "template": "{prefix} {industry_word} {suffix}",
+                "components": {
+                    "prefix": ["Nova", "Apex", "Prime", "Vertex", "Quantum", "Fusion", "Nexus", "Stellar", "Vector", "Atlas",
+                              "Blue", "Red", "Green", "Global", "United", "First", "New", "Smart", "Tech", "Digital"],
+                    "industry_word": ["Solutions", "Systems", "Tech", "Labs", "Works", "Group", "Partners", "Dynamics", "Innovations", "Ventures",
+                                     "Digital", "Logic", "Flow", "Wave", "Net", "Cloud", "Data", "Edge", "Core", "Sync"],
+                    "suffix": ["Inc", "Corp", "LLC", "Co", "Ltd", "GmbH", "Technologies", "International", "Enterprises", "Holdings"],
+                },
+            },
+        }
+        
+        if domain in templates:
+            config = templates[domain]
+            return self.generate_with_template(
+                config["template"],
+                size,
+                config["components"]
+            )
+        
+        # Fall back to curated pool with random sampling
+        base_pool = self.FALLBACK_POOLS.get(domain, [])
+        if len(base_pool) >= size:
+            return random.sample(base_pool, size)
+        elif len(base_pool) > 0:
+            # Repeat with slight variations
+            result = []
+            for i in range(size):
+                base = random.choice(base_pool)
+                if random.random() < 0.3:  # 30% chance to add suffix
+                    suffix = random.choice([" (v2)", " Pro", " Plus", " - Updated", " 2.0", ""])
+                    base = base + suffix
+                result.append(base)
+            return result
+        
+        return []
+
+
+# ============ Template Registry ============
+
+COMPOSITION_TEMPLATES = {
+    "order_id": "{prefix}-{year}-{number}",
+    "invoice_number": "INV-{year}{month}-{number}",
+    "tracking_number": "{carrier}{number}{check}",
+    "sku": "{category}-{brand}-{variant}-{size}",
+    "username": "{adjective}{noun}{number}",
+}
+
+TEMPLATE_COMPONENTS = {
+    "order_id": {
+        "prefix": ["ORD", "SO", "PO", "WO", "REQ"],
+        "year": ["2023", "2024", "2025"],
+        "number": [str(i).zfill(6) for i in range(1, 1000)],
+    },
+    "invoice_number": {
+        "year": ["23", "24", "25"],
+        "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
+        "number": [str(i).zfill(4) for i in range(1, 10000)],
+    },
+    "tracking_number": {
+        "carrier": ["1Z", "9400", "92", "420"],
+        "number": [str(i).zfill(12) for i in range(100000000000, 100001000000)],
+        "check": [str(i) for i in range(10)],
+    },
+    "sku": {
+        "category": ["ELC", "CLO", "HOM", "SPT", "TOY", "BOK"],
+        "brand": ["APP", "SAM", "NIK", "ADI", "SON", "LG"],
+        "variant": ["BLK", "WHT", "RED", "BLU", "GRN", "GRY"],
+        "size": ["S", "M", "L", "XL", "XXL", "OS"],
+    },
+    "username": {
+        "adjective": ["cool", "super", "mega", "ultra", "epic", "pro", "fast", "swift", "bold", "smart"],
+        "noun": ["ninja", "tiger", "dragon", "wolf", "hawk", "bear", "lion", "eagle", "shark", "fox"],
+        "number": [str(i) for i in range(1, 1000)],
+    },
+}
 
 
 # Convenience function for quick testing
 def smart_generate(column_name: str, table_name: str = "", size: int = 10) -> List[str]:
     """Quick smart value generation for testing."""
     gen = SmartValueGenerator()
-    pool = gen.get_pool(column_name, table_name)
+    pool = gen.get_pool(column_name, table_name, size=max(size * 2, 50))
     if pool:
         import random
-        return random.sample(pool, min(size, len(pool)))
+        return random.choices(pool, k=size)
     return []

misata/streaming.py

@@ -0,0 +1,228 @@
+"""
+Streaming export utilities for Misata.
+
+Provides streaming CSV/Parquet export to handle large datasets
+without loading everything into memory.
+"""
+
+import csv
+import os
+from pathlib import Path
+from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from misata.exceptions import ExportError, FileWriteError
+
+
+class StreamingExporter:
+    """Export data in streaming fashion to handle large datasets.
+    
+    Instead of building a full DataFrame and then exporting, this writes
+    batches directly to files as they are generated.
+    
+    Example:
+        exporter = StreamingExporter(output_dir="./data")
+        
+        for table_name, batch_df in simulator.generate_all():
+            exporter.write_batch(table_name, batch_df)
+        
+        exporter.finalize()
+    """
+    
+    def __init__(
+        self,
+        output_dir: str,
+        format: str = "csv",
+        progress_callback: Optional[Callable[[str, int], None]] = None,
+    ):
+        """Initialize the exporter.
+        
+        Args:
+            output_dir: Directory to write files to
+            format: Export format ('csv' or 'parquet')
+            progress_callback: Optional callback(table_name, rows_written)
+        """
+        self.output_dir = Path(output_dir)
+        self.format = format.lower()
+        self.progress_callback = progress_callback
+        
+        self._file_handles: Dict[str, Any] = {}
+        self._csv_writers: Dict[str, csv.writer] = {}
+        self._rows_written: Dict[str, int] = {}
+        self._headers_written: Dict[str, bool] = {}
+        
+        # Create output directory
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def write_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch of data to the appropriate file.
+        
+        Args:
+            table_name: Name of the table
+            df: Batch DataFrame to write
+            
+        Returns:
+            Number of rows written
+        """
+        if self.format == "csv":
+            return self._write_csv_batch(table_name, df)
+        elif self.format == "parquet":
+            return self._write_parquet_batch(table_name, df)
+        else:
+            raise ExportError(f"Unsupported format: {self.format}")
+    
+    def _write_csv_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch to CSV file."""
+        file_path = self.output_dir / f"{table_name}.csv"
+        
+        try:
+            # First batch: write header
+            if table_name not in self._headers_written:
+                with open(file_path, 'w', newline='', encoding='utf-8') as f:
+                    writer = csv.writer(f)
+                    writer.writerow(df.columns.tolist())
+                self._headers_written[table_name] = True
+                self._rows_written[table_name] = 0
+            
+            # Append data
+            with open(file_path, 'a', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                for _, row in df.iterrows():
+                    writer.writerow(row.tolist())
+            
+            rows = len(df)
+            self._rows_written[table_name] = self._rows_written.get(table_name, 0) + rows
+            
+            if self.progress_callback:
+                self.progress_callback(table_name, self._rows_written[table_name])
+            
+            return rows
+            
+        except Exception as e:
+            raise FileWriteError(f"Failed to write CSV: {e}", path=str(file_path))
+    
+    def _write_parquet_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch to Parquet file using append mode."""
+        try:
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+        except ImportError:
+            raise ExportError(
+                "PyArrow required for Parquet export",
+                details={"suggestion": "pip install pyarrow"}
+            )
+        
+        file_path = self.output_dir / f"{table_name}.parquet"
+        
+        try:
+            table = pa.Table.from_pandas(df)
+            
+            if file_path.exists():
+                # Append to existing file
+                existing = pq.read_table(file_path)
+                combined = pa.concat_tables([existing, table])
+                pq.write_table(combined, file_path)
+            else:
+                pq.write_table(table, file_path)
+            
+            rows = len(df)
+            self._rows_written[table_name] = self._rows_written.get(table_name, 0) + rows
+            
+            if self.progress_callback:
+                self.progress_callback(table_name, self._rows_written[table_name])
+            
+            return rows
+            
+        except Exception as e:
+            raise FileWriteError(f"Failed to write Parquet: {e}", path=str(file_path))
+    
+    def finalize(self) -> Dict[str, int]:
+        """Finalize all exports and return summary.
+        
+        Returns:
+            Dict mapping table names to row counts
+        """
+        # Close any open file handles
+        for handle in self._file_handles.values():
+            try:
+                handle.close()
+            except Exception:
+                pass
+        
+        self._file_handles.clear()
+        self._csv_writers.clear()
+        
+        return self._rows_written.copy()
+    
+    def get_file_paths(self) -> Dict[str, Path]:
+        """Get paths to all exported files."""
+        ext = ".csv" if self.format == "csv" else ".parquet"
+        return {
+            table: self.output_dir / f"{table}{ext}"
+            for table in self._rows_written.keys()
+        }
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get export statistics."""
+        return {
+            "output_dir": str(self.output_dir),
+            "format": self.format,
+            "tables": self._rows_written.copy(),
+            "total_rows": sum(self._rows_written.values()),
+        }
+
+
+def stream_to_csv(
+    generator: Generator[tuple, None, None],
+    output_dir: str,
+    progress_callback: Optional[Callable[[str, int], None]] = None,
+) -> Dict[str, int]:
+    """Stream data from a generator directly to CSV files.
+    
+    Args:
+        generator: Iterator yielding (table_name, batch_df) tuples
+        output_dir: Directory to write files to
+        progress_callback: Optional callback(table_name, rows_written)
+        
+    Returns:
+        Dict mapping table names to final row counts
+    """
+    exporter = StreamingExporter(
+        output_dir=output_dir,
+        format="csv",
+        progress_callback=progress_callback,
+    )
+    
+    for table_name, batch_df in generator:
+        exporter.write_batch(table_name, batch_df)
+    
+    return exporter.finalize()
+
+
+def stream_to_parquet(
+    generator: Generator[tuple, None, None],
+    output_dir: str,
+    progress_callback: Optional[Callable[[str, int], None]] = None,
+) -> Dict[str, int]:
+    """Stream data from a generator directly to Parquet files.
+    
+    Args:
+        generator: Iterator yielding (table_name, batch_df) tuples
+        output_dir: Directory to write files to
+        progress_callback: Optional callback(table_name, rows_written)
+        
+    Returns:
+        Dict mapping table names to final row counts
+    """
+    exporter = StreamingExporter(
+        output_dir=output_dir,
+        format="parquet",
+        progress_callback=progress_callback,
+    )
+    
+    for table_name, batch_df in generator:
+        exporter.write_batch(table_name, batch_df)
+    
+    return exporter.finalize()

{misata-0.2.0b0.dist-info → misata-0.3.1b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.2.0b0
+Version: 0.3.1b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT

{misata-0.2.0b0.dist-info → misata-0.3.1b0.dist-info}/RECORD

@@ -1,29 +1,37 @@
-misata/__init__.py,sha256=rTXPG74KAXwnJD9TchtkbOr3kVPaC6W6CrhTKEeSSyk,1404
+misata/__init__.py,sha256=IMFNFb00vkOctRozJJt5HjfPZokACGCBsNin5tqNj5I,3144
 misata/api.py,sha256=Wq2H3iJzocNTsCzb9vhYJxDyag3Yiucvb-GVF0tdKhI,14999
 misata/audit.py,sha256=4eUCHT2STptemfakWeNODbVuBRhyD8Q32LlB2eufvuw,12291
 misata/benchmark.py,sha256=Y1-tuKegJyAlTneROQpPo276qnfmMmupGDbVDs9k5J8,12358
+misata/cache.py,sha256=fuLk7cQ7hOEmlqEWmm-O516L26btZ6zFO8FdrqFCRLg,7087
 misata/cli.py,sha256=a7YijZCUYrkCYGVYJ2nZSL9J3JfFqbXQQOad6bhy7zM,22642
 misata/codegen.py,sha256=m7ykTtLgITvaqzVB1cVhs1b9Puo2X4uyzngZ85wi6J0,5791
+misata/constraints.py,sha256=8jUKlA2VyVomDnl2zz0RDkUqxEkxlwUnBbHTXr_SA5g,9937
+misata/context.py,sha256=tjYrU67wjII07Pl3MKV_uCMl_s55DIOQZCouxAryyzE,8509
 misata/curve_fitting.py,sha256=gLj4BkIxNWKkfo3QKZFI_aq60bsXlI53K5yZX4hc9EU,4126
 misata/customization.py,sha256=pw-BEsPKN091hyOrQWWQoRhTrlmQ9_PXXopm2FZSEvs,8551
+misata/exceptions.py,sha256=C3IGMk8xAy9AmRVWeSAnLHHui7drv6rzgzvOmr6gh50,8335
 misata/feedback.py,sha256=HBEsoKi_vdRqwRzMoVFVj_cjfzQ5SUAaGz40s1HMD50,13313
 misata/formulas.py,sha256=KOTq5YN_19vv1ERd92bdzKot9yo9rrrwjOuWO13nFCg,11210
-misata/generators.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
+misata/generators_legacy.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
 misata/hybrid.py,sha256=5oopAdfOLWUYzdRWlc0plVeVEVg7Nu1CVGNNCDSjQt8,13104
 misata/llm_parser.py,sha256=2SVozbKtb0kaPaR4ERz9FtIIxK5jQVaYJ8L_xC6gU10,20662
 misata/noise.py,sha256=UO7MokzQ5Y5Vj7JaayDUG0JwCLnpHtnpQTcJ4UHWibo,10460
+misata/profiles.py,sha256=0djys8wWvH8VP74KmGn6cGLuOb64h9Hk0g0bkXOfxP4,9578
 misata/quality.py,sha256=VSntJfMnF1tVWJ05fvbVJOMcAPEB7QtuEg18k6aEwhA,11685
 misata/schema.py,sha256=zMYDPCgPfcy_STgANiS-Ow3dUETpW3Ayo02G88jmBe0,8954
 misata/semantic.py,sha256=0fauGWJ75wlbHVqT0hohYTN4m_nscdaMaVAIfkhTZXk,7087
-misata/simulator.py,sha256=nq9KxOS-4oUMNu7a2Ten0TQyhT2u_rTo2ImmvdkMRbU,34037
-misata/smart_values.py,sha256=_jVE3kqqSnFqfKchFDwlsuzFPmyJhtgmpJr4O-wpXrA,28274
+misata/simulator.py,sha256=dLAJf_Ko_3b27OwcVk-d6n7fBVvYmY2v-B_Qscq-m6c,41085
+misata/smart_values.py,sha256=8-TYBK5cVBst9tfGuQXXetOLSqgns_NKnIl14rpVrbk,35870
 misata/story_parser.py,sha256=7N7so3KWisl2UxkOtENQwP-4hN2cs9vTKsPHVRZB2Mc,15964
+misata/streaming.py,sha256=qbEnoFRfn9a7H_gWlq5C3TwbNUnP5U98OPo1EdU_cQ0,7578
 misata/validation.py,sha256=5yJSN7jecVNLJ8ss6y7l2U4hF1Ljn27Q6Xs9N1iDPcw,10791
+misata/generators/__init__.py,sha256=V4I_1IucuywRJZH3cLxKvBd2Ib7kE0WIJ7tq8y4lkx8,568
+misata/generators/base.py,sha256=iON9iAONMEQdbq2Fdric3V3bWn3caD1ITC16DTCK0Og,21329
 misata/templates/__init__.py,sha256=0RcZz9d4bmCqLAr77h0gpMfHncqAPeZCguqsuGCz7rE,25245
 misata/templates/library.py,sha256=eMex18ZKlzQqIkGFgs1uy9QGs7PmUN_VVL4txKvxynM,20930
-misata-0.2.0b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
-misata-0.2.0b0.dist-info/METADATA,sha256=t5yL_ZD7DNiH7TuCfaF4ZNT-5wYTiIaUSj8puQ4TROw,8114
-misata-0.2.0b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-misata-0.2.0b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
-misata-0.2.0b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
-misata-0.2.0b0.dist-info/RECORD,,
+misata-0.3.1b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
+misata-0.3.1b0.dist-info/METADATA,sha256=A-4ymJEBCSo-yl3CM8sNYr0aCdJfUBgtg7S-bABtBkw,8114
+misata-0.3.1b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+misata-0.3.1b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
+misata-0.3.1b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
+misata-0.3.1b0.dist-info/RECORD,,