misata 0.3.0b0__py3-none-any.whl → 0.3.1b0__py3-none-any.whl

misata/__init__.py

@@ -15,7 +15,7 @@ Usage:
     config = load_template("ecommerce")
 """
 
-__version__ = "0.3.0b0"
+__version__ = "0.3.1b0"
 __author__ = "Muhammed Rasin"
 
 from misata.schema import (

misata/simulator.py

@@ -16,7 +16,9 @@ from typing import Any, Dict, List, Optional
 import numpy as np
 import pandas as pd
 
-from misata.generators import TextGenerator
+from misata.generators.base import TextGenerator as _FactoryTextGenerator  # Generator factory version
+# Use the original generators.py TextGenerator which supports seed
+from misata.generators_legacy import TextGenerator
 from misata.schema import Column, Relationship, ScenarioEvent, SchemaConfig
 
 
@@ -34,6 +36,10 @@ class DataSimulator:
         rng: NumPy random generator for reproducibility
     """
 
+    # Performance constants
+    MAX_CONTEXT_ROWS = 50000  # Cap context storage for memory efficiency
+    TEXT_POOL_SIZE = 10000    # Size of text value pools for vectorized sampling
+
     def __init__(self, config: SchemaConfig,
                  apply_semantic_fixes: bool = True, batch_size: int = 10_000,
                  smart_mode: bool = False, use_llm: bool = True):
@@ -57,6 +63,7 @@ class DataSimulator:
         self._unique_pools: Dict[str, np.ndarray] = {}  # Store pre-generated unique values
         self._unique_counters: Dict[str, int] = {}      # Track usage of unique pools
         self._smart_pools: Dict[str, np.ndarray] = {}   # Cache smart value pools
+        self._text_pools: Dict[str, np.ndarray] = {}    # Cache text pools for vectorized sampling
 
         # Apply semantic inference to fix column types
         if apply_semantic_fixes:
@@ -199,10 +206,24 @@ class DataSimulator:
         ctx_df = df[cols_to_store].copy()
 
         if table_name not in self.context:
+            # First batch: store up to MAX_CONTEXT_ROWS
+            if len(ctx_df) > self.MAX_CONTEXT_ROWS:
+                ctx_df = ctx_df.sample(n=self.MAX_CONTEXT_ROWS, random_state=self.config.seed)
             self.context[table_name] = ctx_df
         else:
-            # Append to existing context
-            self.context[table_name] = pd.concat([self.context[table_name], ctx_df], ignore_index=True)
+            # Append to existing context, but cap at MAX_CONTEXT_ROWS
+            current_len = len(self.context[table_name])
+            if current_len >= self.MAX_CONTEXT_ROWS:
+                # Already at capacity, use reservoir sampling for randomness
+                # Replace some existing rows with new ones (probability-based)
+                return  # Skip appending, we have enough IDs
+            
+            remaining_space = self.MAX_CONTEXT_ROWS - current_len
+            rows_to_add = ctx_df.iloc[:remaining_space]
+            self.context[table_name] = pd.concat(
+                [self.context[table_name], rows_to_add], 
+                ignore_index=True
+            )
 
     def generate_column(
         self,
@@ -225,6 +246,70 @@ class DataSimulator:
         """
         params = column.distribution_params
 
+        # ========== CORRELATED COLUMN GENERATION ==========
+        # If this column depends on another column's value, use conditional distribution
+        if "depends_on" in params and table_data is not None:
+            parent_col = params["depends_on"]
+            mapping = params.get("mapping", {})
+            
+            if parent_col in table_data.columns and mapping:
+                parent_values = table_data[parent_col].values
+                
+                # Check if it's numeric or categorical mapping
+                first_val = next(iter(mapping.values()))
+                if isinstance(first_val, dict) and "mean" in first_val:
+                    # Numeric conditional distribution (e.g., salary based on job_title)
+                    # mapping = {"Intern": {"mean": 40000, "std": 5000}, "CTO": {"mean": 200000, "std": 30000}}
+                    values = np.zeros(size)
+                    for key, dist in mapping.items():
+                        mask = parent_values == key
+                        count = mask.sum()
+                        if count > 0:
+                            mean = dist.get("mean", 50000)
+                            std = dist.get("std", mean * 0.1)
+                            values[mask] = self.rng.normal(mean, std, count)
+                    
+                    # Handle values that didn't match any key (use default)
+                    default = params.get("default", {"mean": 50000, "std": 10000})
+                    unmatched = ~np.isin(parent_values, list(mapping.keys()))
+                    if unmatched.sum() > 0:
+                        values[unmatched] = self.rng.normal(
+                            default.get("mean", 50000), 
+                            default.get("std", 10000), 
+                            unmatched.sum()
+                        )
+                    return values
+                    
+                elif isinstance(first_val, list):
+                    # Categorical conditional (e.g., state based on country)
+                    # mapping = {"USA": ["CA", "TX", "NY"], "UK": ["England", "Scotland"]}
+                    values = np.empty(size, dtype=object)
+                    for key, choices in mapping.items():
+                        mask = parent_values == key
+                        count = mask.sum()
+                        if count > 0:
+                            values[mask] = self.rng.choice(choices, count)
+                    
+                    # Default for unmatched
+                    default_choices = params.get("default", ["Unknown"])
+                    unmatched = values == None  # noqa
+                    if unmatched.sum() > 0:
+                        values[unmatched] = self.rng.choice(default_choices, unmatched.sum())
+                    return values
+                    
+                elif isinstance(first_val, (int, float)):
+                    # Probability-based boolean (e.g., churn probability based on plan)
+                    # mapping = {"free": 0.3, "pro": 0.1, "enterprise": 0.05}
+                    values = np.zeros(size, dtype=bool)
+                    for key, prob in mapping.items():
+                        mask = parent_values == key
+                        count = mask.sum()
+                        if count > 0:
+                            values[mask] = self.rng.random(count) < prob
+                    return values
+
+        # ========== STANDARD COLUMN GENERATION ==========
+        
         # CATEGORICAL
         if column.type == "categorical":
             choices = params.get("choices", ["A", "B", "C"])
@@ -469,23 +554,59 @@ class DataSimulator:
                         return values
 
             if text_type == "name":
-                values = np.array([self.text_gen.name() for _ in range(size)])
+                pool_key = "text_name"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.name() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "email":
-                values = np.array([self.text_gen.email() for _ in range(size)])
+                pool_key = "text_email"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.email() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "company":
-                values = np.array([self.text_gen.company() for _ in range(size)])
+                pool_key = "text_company"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.company() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "sentence":
-                values = np.array([self.text_gen.sentence() for _ in range(size)])
+                pool_key = "text_sentence"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.sentence() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "word":
-                values = np.array([self.text_gen.word() for _ in range(size)])
+                pool_key = "text_word"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.word() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "address":
-                values = np.array([self.text_gen.full_address() for _ in range(size)])
+                pool_key = "text_address"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.full_address() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "phone":
-                values = np.array([self.text_gen.phone_number() for _ in range(size)])
+                pool_key = "text_phone"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.phone_number() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             elif text_type == "url":
-                values = np.array([self.text_gen.url() for _ in range(size)])
+                pool_key = "text_url"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.url() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
             else:
-                values = np.array([self.text_gen.sentence() for _ in range(size)])
+                pool_key = "text_sentence"
+                if pool_key not in self._text_pools:
+                    pool_size = min(size, self.TEXT_POOL_SIZE)
+                    self._text_pools[pool_key] = np.array([self.text_gen.sentence() for _ in range(pool_size)])
+                values = self.rng.choice(self._text_pools[pool_key], size=size)
 
             return values
 

{misata-0.3.0b0.dist-info → misata-0.3.1b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: misata
-Version: 0.3.0b0
+Version: 0.3.1b0
 Summary: AI-Powered Synthetic Data Engine - Generate realistic multi-table datasets from natural language
 Author-email: Muhammed Rasin <rasinbinabdulla@gmail.com>
 License: MIT

{misata-0.3.0b0.dist-info → misata-0.3.1b0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-misata/__init__.py,sha256=Vra5zMkd5Y6HTzhGRc76jTv10Z0yuhw33MDUoLpACrE,3144
+misata/__init__.py,sha256=IMFNFb00vkOctRozJJt5HjfPZokACGCBsNin5tqNj5I,3144
 misata/api.py,sha256=Wq2H3iJzocNTsCzb9vhYJxDyag3Yiucvb-GVF0tdKhI,14999
 misata/audit.py,sha256=4eUCHT2STptemfakWeNODbVuBRhyD8Q32LlB2eufvuw,12291
 misata/benchmark.py,sha256=Y1-tuKegJyAlTneROQpPo276qnfmMmupGDbVDs9k5J8,12358
@@ -12,7 +12,7 @@ misata/customization.py,sha256=pw-BEsPKN091hyOrQWWQoRhTrlmQ9_PXXopm2FZSEvs,8551
 misata/exceptions.py,sha256=C3IGMk8xAy9AmRVWeSAnLHHui7drv6rzgzvOmr6gh50,8335
 misata/feedback.py,sha256=HBEsoKi_vdRqwRzMoVFVj_cjfzQ5SUAaGz40s1HMD50,13313
 misata/formulas.py,sha256=KOTq5YN_19vv1ERd92bdzKot9yo9rrrwjOuWO13nFCg,11210
-misata/generators.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
+misata/generators_legacy.py,sha256=NrMF12i6CB7K6fUsqcqurmZBBQ382ZhVnYB9oMBIZCE,8844
 misata/hybrid.py,sha256=5oopAdfOLWUYzdRWlc0plVeVEVg7Nu1CVGNNCDSjQt8,13104
 misata/llm_parser.py,sha256=2SVozbKtb0kaPaR4ERz9FtIIxK5jQVaYJ8L_xC6gU10,20662
 misata/noise.py,sha256=UO7MokzQ5Y5Vj7JaayDUG0JwCLnpHtnpQTcJ4UHWibo,10460
@@ -20,7 +20,7 @@ misata/profiles.py,sha256=0djys8wWvH8VP74KmGn6cGLuOb64h9Hk0g0bkXOfxP4,9578
 misata/quality.py,sha256=VSntJfMnF1tVWJ05fvbVJOMcAPEB7QtuEg18k6aEwhA,11685
 misata/schema.py,sha256=zMYDPCgPfcy_STgANiS-Ow3dUETpW3Ayo02G88jmBe0,8954
 misata/semantic.py,sha256=0fauGWJ75wlbHVqT0hohYTN4m_nscdaMaVAIfkhTZXk,7087
-misata/simulator.py,sha256=nq9KxOS-4oUMNu7a2Ten0TQyhT2u_rTo2ImmvdkMRbU,34037
+misata/simulator.py,sha256=dLAJf_Ko_3b27OwcVk-d6n7fBVvYmY2v-B_Qscq-m6c,41085
 misata/smart_values.py,sha256=8-TYBK5cVBst9tfGuQXXetOLSqgns_NKnIl14rpVrbk,35870
 misata/story_parser.py,sha256=7N7so3KWisl2UxkOtENQwP-4hN2cs9vTKsPHVRZB2Mc,15964
 misata/streaming.py,sha256=qbEnoFRfn9a7H_gWlq5C3TwbNUnP5U98OPo1EdU_cQ0,7578
@@ -29,9 +29,9 @@ misata/generators/__init__.py,sha256=V4I_1IucuywRJZH3cLxKvBd2Ib7kE0WIJ7tq8y4lkx8
 misata/generators/base.py,sha256=iON9iAONMEQdbq2Fdric3V3bWn3caD1ITC16DTCK0Og,21329
 misata/templates/__init__.py,sha256=0RcZz9d4bmCqLAr77h0gpMfHncqAPeZCguqsuGCz7rE,25245
 misata/templates/library.py,sha256=eMex18ZKlzQqIkGFgs1uy9QGs7PmUN_VVL4txKvxynM,20930
-misata-0.3.0b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
-misata-0.3.0b0.dist-info/METADATA,sha256=Wxpa2V0Sum-CFOpNnmRd27eEDfyT9CKIy-4nGZnrCys,8114
-misata-0.3.0b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-misata-0.3.0b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
-misata-0.3.0b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
-misata-0.3.0b0.dist-info/RECORD,,
+misata-0.3.1b0.dist-info/licenses/LICENSE,sha256=oagkechmfr9iT214N871zCm7TnB0KTfPjAUWxHsYJ4I,1071
+misata-0.3.1b0.dist-info/METADATA,sha256=A-4ymJEBCSo-yl3CM8sNYr0aCdJfUBgtg7S-bABtBkw,8114
+misata-0.3.1b0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+misata-0.3.1b0.dist-info/entry_points.txt,sha256=k3SDuju7VnqB4AcY0Vufw-j1tWU3Ay612G3DGqoNs0U,43
+misata-0.3.1b0.dist-info/top_level.txt,sha256=dpwR99XWKUAXqNg7WiNLu_XYd7WYGmZpJzrfQXbAZFs,7
+misata-0.3.1b0.dist-info/RECORD,,