misata 0.2.0b0__py3-none-any.whl → 0.3.0b0__py3-none-any.whl

misata/streaming.py

@@ -0,0 +1,228 @@
+"""
+Streaming export utilities for Misata.
+
+Provides streaming CSV/Parquet export to handle large datasets
+without loading everything into memory.
+"""
+
+import csv
+import os
+from pathlib import Path
+from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from misata.exceptions import ExportError, FileWriteError
+
+
+class StreamingExporter:
+    """Export data in streaming fashion to handle large datasets.
+    
+    Instead of building a full DataFrame and then exporting, this writes
+    batches directly to files as they are generated.
+    
+    Example:
+        exporter = StreamingExporter(output_dir="./data")
+        
+        for table_name, batch_df in simulator.generate_all():
+            exporter.write_batch(table_name, batch_df)
+        
+        exporter.finalize()
+    """
+    
+    def __init__(
+        self,
+        output_dir: str,
+        format: str = "csv",
+        progress_callback: Optional[Callable[[str, int], None]] = None,
+    ):
+        """Initialize the exporter.
+        
+        Args:
+            output_dir: Directory to write files to
+            format: Export format ('csv' or 'parquet')
+            progress_callback: Optional callback(table_name, rows_written)
+        """
+        self.output_dir = Path(output_dir)
+        self.format = format.lower()
+        self.progress_callback = progress_callback
+        
+        self._file_handles: Dict[str, Any] = {}
+        self._csv_writers: Dict[str, csv.writer] = {}
+        self._rows_written: Dict[str, int] = {}
+        self._headers_written: Dict[str, bool] = {}
+        
+        # Create output directory
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+    
+    def write_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch of data to the appropriate file.
+        
+        Args:
+            table_name: Name of the table
+            df: Batch DataFrame to write
+            
+        Returns:
+            Number of rows written
+        """
+        if self.format == "csv":
+            return self._write_csv_batch(table_name, df)
+        elif self.format == "parquet":
+            return self._write_parquet_batch(table_name, df)
+        else:
+            raise ExportError(f"Unsupported format: {self.format}")
+    
+    def _write_csv_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch to CSV file."""
+        file_path = self.output_dir / f"{table_name}.csv"
+        
+        try:
+            # First batch: write header
+            if table_name not in self._headers_written:
+                with open(file_path, 'w', newline='', encoding='utf-8') as f:
+                    writer = csv.writer(f)
+                    writer.writerow(df.columns.tolist())
+                self._headers_written[table_name] = True
+                self._rows_written[table_name] = 0
+            
+            # Append data
+            with open(file_path, 'a', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                for _, row in df.iterrows():
+                    writer.writerow(row.tolist())
+            
+            rows = len(df)
+            self._rows_written[table_name] = self._rows_written.get(table_name, 0) + rows
+            
+            if self.progress_callback:
+                self.progress_callback(table_name, self._rows_written[table_name])
+            
+            return rows
+            
+        except Exception as e:
+            raise FileWriteError(f"Failed to write CSV: {e}", path=str(file_path))
+    
+    def _write_parquet_batch(self, table_name: str, df: pd.DataFrame) -> int:
+        """Write a batch to Parquet file using append mode."""
+        try:
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+        except ImportError:
+            raise ExportError(
+                "PyArrow required for Parquet export",
+                details={"suggestion": "pip install pyarrow"}
+            )
+        
+        file_path = self.output_dir / f"{table_name}.parquet"
+        
+        try:
+            table = pa.Table.from_pandas(df)
+            
+            if file_path.exists():
+                # Append to existing file
+                existing = pq.read_table(file_path)
+                combined = pa.concat_tables([existing, table])
+                pq.write_table(combined, file_path)
+            else:
+                pq.write_table(table, file_path)
+            
+            rows = len(df)
+            self._rows_written[table_name] = self._rows_written.get(table_name, 0) + rows
+            
+            if self.progress_callback:
+                self.progress_callback(table_name, self._rows_written[table_name])
+            
+            return rows
+            
+        except Exception as e:
+            raise FileWriteError(f"Failed to write Parquet: {e}", path=str(file_path))
+    
+    def finalize(self) -> Dict[str, int]:
+        """Finalize all exports and return summary.
+        
+        Returns:
+            Dict mapping table names to row counts
+        """
+        # Close any open file handles
+        for handle in self._file_handles.values():
+            try:
+                handle.close()
+            except Exception:
+                pass
+        
+        self._file_handles.clear()
+        self._csv_writers.clear()
+        
+        return self._rows_written.copy()
+    
+    def get_file_paths(self) -> Dict[str, Path]:
+        """Get paths to all exported files."""
+        ext = ".csv" if self.format == "csv" else ".parquet"
+        return {
+            table: self.output_dir / f"{table}{ext}"
+            for table in self._rows_written.keys()
+        }
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get export statistics."""
+        return {
+            "output_dir": str(self.output_dir),
+            "format": self.format,
+            "tables": self._rows_written.copy(),
+            "total_rows": sum(self._rows_written.values()),
+        }
+
+
+def stream_to_csv(
+    generator: Generator[tuple, None, None],
+    output_dir: str,
+    progress_callback: Optional[Callable[[str, int], None]] = None,
+) -> Dict[str, int]:
+    """Stream data from a generator directly to CSV files.
+    
+    Args:
+        generator: Iterator yielding (table_name, batch_df) tuples
+        output_dir: Directory to write files to
+        progress_callback: Optional callback(table_name, rows_written)
+        
+    Returns:
+        Dict mapping table names to final row counts
+    """
+    exporter = StreamingExporter(
+        output_dir=output_dir,
+        format="csv",
+        progress_callback=progress_callback,
+    )
+    
+    for table_name, batch_df in generator:
+        exporter.write_batch(table_name, batch_df)
+    
+    return exporter.finalize()
+
+
+def stream_to_parquet(
+    generator: Generator[tuple, None, None],
+    output_dir: str,
+    progress_callback: Optional[Callable[[str, int], None]] = None,
+) -> Dict[str, int]:
+    """Stream data from a generator directly to Parquet files.
+    
+    Args:
+        generator: Iterator yielding (table_name, batch_df) tuples
+        output_dir: Directory to write files to
+        progress_callback: Optional callback(table_name, rows_written)
+        
+    Returns:
+        Dict mapping table names to final row counts
+    """
+    exporter = StreamingExporter(
+        output_dir=output_dir,
+        format="parquet",
+        progress_callback=progress_callback,
+    )
+    
+    for table_name, batch_df in generator:
+        exporter.write_batch(table_name, batch_df)
+    
+    return exporter.finalize()

{misata-0.2.0b0.dist-info → misata-0.3.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.2.0b0
+Version: 0.3.0b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT

{misata-0.2.0b0.dist-info → misata-0.3.0b0.dist-info}/RECORD

@@ -1,29 +1,37 @@
-misata/__init__.py,sha256=rTXPG74KAXwnJD9TchtkbOr3kVPaC6W6CrhTKEeSSyk,1404
+misata/__init__.py,sha256=Vra5zMkd5Y6HTzhGRc76jTv10Z0yuhw33MDUoLpACrE,3144
 misata/api.py,sha256=Wq2H3iJzocNTsCzb9vhYJxDyag3Yiucvb-GVF0tdKhI,14999
 misata/audit.py,sha256=4eUCHT2STptemfakWeNODbVuBRhyD8Q32LlB2eufvuw,12291
 misata/benchmark.py,sha256=Y1-tuKegJyAlTneROQpPo276qnfmMmupGDbVDs9k5J8,12358
+misata/cache.py,sha256=fuLk7cQ7hOEmlqEWmm-O516L26btZ6zFO8FdrqFCRLg,7087
 misata/cli.py,sha256=a7YijZCUYrkCYGVYJ2nZSL9J3JfFqbXQQOad6bhy7zM,22642
 misata/codegen.py,sha256=m7ykTtLgITvaqzVB1cVhs1b9Puo2X4uyzngZ85wi6J0,5791
+misata/constraints.py,sha256=8jUKlA2VyVomDnl2zz0RDkUqxEkxlwUnBbHTXr_SA5g,9937
+misata/context.py,sha256=tjYrU67wjII07Pl3MKV_uCMl_s55DIOQZCouxAryyzE,8509
 misata/curve_fitting.py,sha256=gLj4BkIxNWKkfo3QKZFI_aq60bsXlI53K5yZX4hc9EU,4126
 misata/customization.py,sha256=pw-BEsPKN091hyOrQWWQoRhTrlmQ9_PXXopm2FZSEvs,8551
+misata/exceptions.py,sha256=C3IGMk8xAy9AmRVWeSAnLHHui7drv6rzgzvOmr6gh50,8335
 misata/feedback.py,sha256=HBEsoKi_vdRqwRzMoVFVj_cjfzQ5SUAaGz40s1HMD50,13313
 misata/formulas.py,sha256=KOTq5YN_19vv1ERd92bdzKot9yo9rrrwjOuWO13nFCg,11210
 misata/generators.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
 misata/hybrid.py,sha256=5oopAdfOLWUYzdRWlc0plVeVEVg7Nu1CVGNNCDSjQt8,13104
 misata/llm_parser.py,sha256=2SVozbKtb0kaPaR4ERz9FtIIxK5jQVaYJ8L_xC6gU10,20662
 misata/noise.py,sha256=UO7MokzQ5Y5Vj7JaayDUG0JwCLnpHtnpQTcJ4UHWibo,10460
+misata/profiles.py,sha256=0djys8wWvH8VP74KmGn6cGLuOb64h9Hk0g0bkXOfxP4,9578
 misata/quality.py,sha256=VSntJfMnF1tVWJ05fvbVJOMcAPEB7QtuEg18k6aEwhA,11685
 misata/schema.py,sha256=zMYDPCgPfcy_STgANiS-Ow3dUETpW3Ayo02G88jmBe0,8954
 misata/semantic.py,sha256=0fauGWJ75wlbHVqT0hohYTN4m_nscdaMaVAIfkhTZXk,7087
 misata/simulator.py,sha256=nq9KxOS-4oUMNu7a2Ten0TQyhT2u_rTo2ImmvdkMRbU,34037
-misata/smart_values.py,sha256=_jVE3kqqSnFqfKchFDwlsuzFPmyJhtgmpJr4O-wpXrA,28274
+misata/smart_values.py,sha256=8-TYBK5cVBst9tfGuQXXetOLSqgns_NKnIl14rpVrbk,35870
 misata/story_parser.py,sha256=7N7so3KWisl2UxkOtENQwP-4hN2cs9vTKsPHVRZB2Mc,15964
+misata/streaming.py,sha256=qbEnoFRfn9a7H_gWlq5C3TwbNUnP5U98OPo1EdU_cQ0,7578
 misata/validation.py,sha256=5yJSN7jecVNLJ8ss6y7l2U4hF1Ljn27Q6Xs9N1iDPcw,10791
+misata/generators/__init__.py,sha256=V4I_1IucuywRJZH3cLxKvBd2Ib7kE0WIJ7tq8y4lkx8,568
+misata/generators/base.py,sha256=iON9iAONMEQdbq2Fdric3V3bWn3caD1ITC16DTCK0Og,21329
 misata/templates/__init__.py,sha256=0RcZz9d4bmCqLAr77h0gpMfHncqAPeZCguqsuGCz7rE,25245
 misata/templates/library.py,sha256=eMex18ZKlzQqIkGFgs1uy9QGs7PmUN_VVL4txKvxynM,20930
-misata-0.2.0b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
-misata-0.2.0b0.dist-info/METADATA,sha256=t5yL_ZD7DNiH7TuCfaF4ZNT-5wYTiIaUSj8puQ4TROw,8114
-misata-0.2.0b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-misata-0.2.0b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
-misata-0.2.0b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
-misata-0.2.0b0.dist-info/RECORD,,
+misata-0.3.0b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
+misata-0.3.0b0.dist-info/METADATA,sha256=Wxpa2V0Sum-CFOpNnmRd27eEDfyT9CKIy-4nGZnrCys,8114
+misata-0.3.0b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+misata-0.3.0b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
+misata-0.3.0b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
+misata-0.3.0b0.dist-info/RECORD,,