vesper-wizard 2.3.1 → 2.3.2

package/src/python/quality_engine.py

@@ -1,267 +0,0 @@
-import sys
-import json
-import polars as pl
-import numpy as np
-
-def analyze_column(df, col_name, dtype):
-    stats = {
-        "name": col_name,
-        "type": str(dtype),
-        "inferred_type": str(dtype), # Default to actual
-        "missing_count": 0,
-        "missing_percentage": 0.0,
-        "unique_count": 0,
-        "is_constant": False,
-        "is_mixed_type": False
-    }
-
-    try:
-        col = df[col_name]
-        null_count = col.null_count()
-        row_count = len(col)
-        
-        stats["missing_count"] = null_count
-        stats["missing_percentage"] = (null_count / row_count) * 100 if row_count > 0 else 0
-        stats["unique_count"] = col.n_unique()
-        stats["is_constant"] = stats["unique_count"] <= 1 and row_count > 0
-
-        # Schema Inference & Validation
-        is_string = dtype == pl.Utf8 or dtype == pl.Object
-        
-        if is_string and row_count > 0:
-            # Try inferring Numeric
-            # Check if majority can be cast to float
-            try:
-                # Use strict=False to turn non-numbers into nulls
-                numeric_cast = col.str.strip_chars().cast(pl.Float64, strict=False)
-                numeric_nulls = numeric_cast.null_count()
-                
-                # If valid numbers are significantly more than original nulls, it might be numeric
-                valid_numbers = row_count - numeric_nulls
-                original_valid = row_count - null_count
-                
-                if valid_numbers > 0 and (valid_numbers / original_valid) > 0.9:
-                    stats["inferred_type"] = "Numeric (Stored as String)"
-                
-                # Mixed type check: If valid numbers exist but plenty of strings too
-                elif valid_numbers > 0 and (valid_numbers / original_valid) < 0.9:
-                    stats["is_mixed_type"] = True
-            except:
-                pass
-
-        # Numeric Analysis
-        if dtype in [pl.Int64, pl.Int32, pl.Float64, pl.Float32] or stats["inferred_type"].startswith("Numeric"):
-            clean_col = col
-            if is_string:
-                 # Cast for analysis if it was inferred
-                 clean_col = col.str.strip_chars().cast(pl.Float64, strict=False)
-            
-            clean_col = clean_col.drop_nulls()
-            
-            if len(clean_col) > 0:
-                stats["distribution"] = {
-                    "min": float(clean_col.min()),
-                    "max": float(clean_col.max()),
-                    "mean": float(clean_col.mean()),
-                    "std": float(clean_col.std()) if len(clean_col) > 1 else 0,
-                    "p25": float(clean_col.quantile(0.25)),
-                    "p50": float(clean_col.median()),
-                    "p75": float(clean_col.quantile(0.75))
-                }
-
-        # Categorical Analysis
-        if dtype == pl.Utf8 or dtype == pl.Categorical:
-            value_counts = col.value_counts(sort=True).head(5)
-            # Handle different polars versions return structure for value_counts
-            try:
-                # Format: struct with name/counts or columns
-                rows = value_counts.rows()
-                top_values = {}
-                for row in rows:
-                    val = str(row[0]) if row[0] is not None else "null"
-                    count = int(row[1])
-                    top_values[val] = count
-                stats["top_values"] = top_values
-            except:
-                pass
-
-    except Exception as e:
-        stats["error"] = str(e)
-
-    return stats
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"error": "No file path provided"}))
-        sys.exit(1)
-
-    file_path = sys.argv[1]
-    
-    try:
-        # Robust file reading with extension detection
-        file_path_lower = file_path.lower()
-        if file_path_lower.endswith(".csv"):
-            df = pl.read_csv(file_path, ignore_errors=True, n_rows=10000)
-        elif file_path_lower.endswith(".tsv"):
-            df = pl.read_csv(file_path, separator="\t", ignore_errors=True, n_rows=10000)
-        elif file_path_lower.endswith(".txt"):
-            sep = ","
-            try:
-                with open(file_path, "r", encoding="utf-8", errors="ignore") as fh:
-                    first_line = fh.readline()
-                    if "\t" in first_line:
-                        sep = "\t"
-            except Exception:
-                sep = ","
-            df = pl.read_csv(file_path, separator=sep, ignore_errors=True, n_rows=10000)
-        elif file_path_lower.endswith(".parquet"):
-            try:
-                # Try scanning first (faster for large files)
-                df = pl.scan_parquet(file_path).limit(10000).collect()
-            except:
-                df = pl.read_parquet(file_path)
-                if len(df) > 10000: df = df.head(10000)
-        elif file_path_lower.endswith(".jsonl") or file_path_lower.endswith(".ndjson"):
-            # Explicit NDJSON
-            df = pl.scan_ndjson(file_path).limit(10000).collect()
-        elif file_path_lower.endswith(".json"):
-            # Ambiguous .json: Try standard JSON first, then NDJSON fallback
-            try:
-                # read_json reads standard JSON array [{}, {}]
-                df = pl.read_json(file_path)
-                if len(df) > 10000: df = df.head(10000)
-            except Exception:
-                try:
-                     # Fallback to NDJSON (common for large datasets mislabeled as .json)
-                     df = pl.scan_ndjson(file_path).limit(10000).collect()
-                except Exception as e:
-                     print(json.dumps({"error": f"Failed to read JSON: {str(e)}"}))
-                     sys.exit(1)
-        else:
-            print(json.dumps({"error": f"Unsupported file extension: {file_path}"}))
-            sys.exit(1)
-
-        row_count = len(df)
-        column_count = len(df.columns)
-        
-        # Duplicate detection (exact)
-        # NOTE: Some Polars versions can panic on is_duplicated() for nested/null rows.
-        # Use a Python fallback that is slower but robust for the 10k sampled rows.
-        duplicate_count = 0
-        try:
-            seen = set()
-            for row in df.to_dicts():
-                row_key = json.dumps(row, sort_keys=True, default=str)
-                if row_key in seen:
-                    duplicate_count += 1
-                else:
-                    seen.add(row_key)
-        except Exception:
-            duplicate_count = 0
-        
-        columns_stats = []
-        text_cols = []
-        for col in df.columns:
-            stats = analyze_column(df, col, df.schema[col])
-            columns_stats.append(stats)
-            # Check for String type (Polars can return 'String' or 'Utf8' depending on version)
-            dtype_str = stats["type"]
-            if ("String" in dtype_str or "Utf8" in dtype_str) and stats["unique_count"] > 1:
-                text_cols.append(col)
-
-        report = {
-            "row_count": row_count,
-            "column_count": column_count,
-            "duplicate_rows": int(duplicate_count),
-            "duplicate_percentage": (duplicate_count / row_count * 100) if row_count > 0 else 0,
-            "columns": columns_stats,
-            "warnings": [],
-            "schema_warnings": [],
-            "overall_score": 100 
-        }
-
-        # Integrity Check 1: Text Duplicates (Fuzzyish Proxy)
-        # If duplicated rows are 0, check if main text content is duplicated
-        if duplicate_count == 0 and len(text_cols) > 0:
-            # Pick longest text column as likely "content"
-            # In real impl, we'd use heuristics. For now, first text col.
-            target_col = text_cols[0]
-            try:
-                text_dupes = df.select(pl.col(target_col)).is_duplicated().sum()
-                if text_dupes > 0:
-                    report["text_duplicates"] = int(text_dupes)
-                    if text_dupes > (row_count * 0.2):
-                        report["warnings"].append(f"High text duplication in '{target_col}' ({text_dupes} rows)")
-            except Exception:
-                # Skip text duplicate warning if backend cannot compute duplicates for this dtype
-                pass
-
-        # Integrity Check 2: Contamination / Leakage (Basic)
-        # (Skipping correlation for now)
-
-        report["class_imbalance_warnings"] = []
-        report["pii_warnings"] = []
-        
-        # PII Patterns (Regex)
-        import re
-        pii_patterns = {
-            "Email": r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}',
-            "Phone": r'\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}', # Basic US-ish pattern
-            "SSN": r'\d{3}-\d{2}-\d{4}',
-            "IPv4": r'\b(?:\d{1,3}\.){3}\d{1,3}\b'
-        }
-
-        # Bias & PII Analysis
-        for col_name, stats in zip(df.columns, columns_stats):
-            # Class Imbalance
-            if stats["unique_count"] > 1 and stats["unique_count"] < 50:
-                 try:
-                     col = df[col_name]
-                     top_val_count = col.value_counts().sort("count", descending=True).row(0)[1]
-                     total = len(col)
-                     if total > 0:
-                         ratio = top_val_count / total
-                         if ratio > 0.9:
-                             report["class_imbalance_warnings"].append(f"Severe imbalance in '{col_name}': Top class is {(ratio*100):.1f}% of data")
-                 except:
-                     pass
-            
-            # PII Detection (on Text Columns only)
-            if ("String" in stats["type"] or "Utf8" in stats["type"]):
-                try:
-                    # Sample for performance (check first 1000 non-null values)
-                    sample_text = df[col_name].drop_nulls().head(1000).to_list()
-                    # Join a subset to regex against (faster than row-by-row for simple checks)
-                    combined_text = " ".join([str(x) for x in sample_text])
-                    
-                    for pii_type, pattern in pii_patterns.items():
-                        if re.search(pattern, combined_text):
-                            # Ensure we don't flag column names like "email_address" but actual content
-                            # Double check with a strict count if trigger found
-                            matches = len(re.findall(pattern, combined_text))
-                            if matches > 0:
-                                report["pii_warnings"].append(f"Potential {pii_type} detected in column '{col_name}' ({matches} matches in sample)")
-                except:
-                    pass
-
-        # Basic warnings
-        if report["duplicate_percentage"] > 10:
-            report["warnings"].append("High duplication rate (>10%)")
-        if row_count < 50:
-            report["warnings"].append("Dataset is very small (<50 rows)")
-
-        # Schema warnings
-        for col in columns_stats:
-            if "Numeric" in col.get("inferred_type", "") and "Utf8" in col.get("type", ""):
-                 report["schema_warnings"].append(f"Column '{col['name']}' looks Numeric but is stored as String")
-            if col.get("is_mixed_type"):
-                 report["schema_warnings"].append(f"Column '{col['name']}' likely contains mixed types (numbers and strings)")
-
-        print(json.dumps(report))
-
-    except Exception as e:
-        print(json.dumps({"error": f"Analysis failed: {str(e)}"}))
-        sys.exit(1)
-
-if __name__ == "__main__":
-    main()

package/src/python/requirements.txt

@@ -1,12 +0,0 @@
-polars==1.2.0
-pandas==2.2.0
-numpy==1.26.0
-scikit-learn==1.4.0
-# Optional source/download extras:
-kaggle>=1.6.17
-aiohttp>=3.9.0
-aiofiles>=24.1.0
-datasets>=2.20.0
-webdataset>=0.2.86
-# Optional for secure key storage (preferred over file fallback):
-# keyring>=24.0.0

package/src/python/row_count.py

@@ -1,54 +0,0 @@
-import sys
-import json
-import os
-
-try:
-    import polars as pl
-except Exception:
-    print(json.dumps({"ok": False, "error": "polars is required"}))
-    sys.exit(1)
-
-
-def count_rows(path: str) -> int:
-    ext = os.path.splitext(path)[1].lower()
-
-    if ext == ".csv":
-        # Faster than full read for large csv
-        return int(pl.scan_csv(path, ignore_errors=True).select(pl.len()).collect().item())
-    if ext in [".parquet", ".pq"]:
-        return int(pl.scan_parquet(path).select(pl.len()).collect().item())
-    if ext in [".feather", ".ftr", ".arrow", ".ipc"]:
-        return int(pl.scan_ipc(path).select(pl.len()).collect().item())
-    if ext in [".jsonl", ".ndjson"]:
-        return int(pl.scan_ndjson(path).select(pl.len()).collect().item())
-    if ext == ".json":
-        # fallback to eager for plain JSON arrays
-        try:
-            return int(pl.read_json(path).height)
-        except Exception:
-            return int(pl.scan_ndjson(path).select(pl.len()).collect().item())
-
-    # unknown extension fallback
-    return int(pl.read_csv(path, ignore_errors=True).height)
-
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"ok": False, "error": "Usage: row_count.py <file_path>"}))
-        sys.exit(1)
-
-    p = sys.argv[1]
-    if not os.path.exists(p):
-        print(json.dumps({"ok": False, "error": f"File not found: {p}"}))
-        sys.exit(1)
-
-    try:
-        rows = count_rows(p)
-        print(json.dumps({"ok": True, "rows": rows}))
-    except Exception as e:
-        print(json.dumps({"ok": False, "error": str(e)}))
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/src/python/splitter_engine.py

@@ -1,283 +0,0 @@
-import sys
-import json
-import polars as pl
-import numpy as np
-from sklearn.model_selection import train_test_split
-
-def execute_split(file_path, config):
-    # Load Data
-    if file_path.endswith(".csv"):
-        df = pl.read_csv(file_path, ignore_errors=True)
-    elif file_path.endswith(".parquet"):
-        df = pl.read_parquet(file_path)
-    else:
-        raise ValueError("Unsupported format")
-
-    train_ratio = config["ratios"]["train"]
-    val_ratio = config["ratios"]["val"]
-    test_ratio = config["ratios"]["test"]
-    holdout_ratio = config["ratios"].get("holdout", 0)
-    seed = config.get("random_seed", 42)
-    shuffle = config.get("shuffle", True)
-    
-    # Strategy
-    strategy = config["type"]
-    target_col = config.get("target_column", None)
-    time_col = config.get("time_column", None)
-
-    train_df, val_df, test_df, holdout_df = None, None, None, None
-
-    # --- 1. RANDOM / STRATIFIED SPLIT ---
-    if strategy in ["random", "stratified"]:
-        if strategy == "random":
-            if shuffle:
-                df = df.sample(fraction=1.0, seed=seed, shuffle=True)
-            
-            n = len(df)
-            n_train = int(n * train_ratio)
-            n_val = int(n * val_ratio)
-            n_test = int(n * test_ratio)
-            
-            train_df = df.slice(0, n_train)
-            val_df = df.slice(n_train, n_val)
-            test_df = df.slice(n_train + n_val, n_test)
-            holdout_df = df.slice(n_train + n_val + n_test, n - (n_train + n_val + n_test))
-
-        elif strategy == "stratified":
-            if not target_col or target_col not in df.columns:
-                return {"error": f"Target column '{target_col}' not found needed for stratification"}
-            
-            y = df[target_col].to_list()
-            indices = np.arange(len(df))
-            
-            # Split 1: Train vs Others
-            others_ratio = val_ratio + test_ratio + holdout_ratio
-            if others_ratio == 0:
-                 train_idx, others_idx = indices, []
-            else:
-                 train_idx, others_idx = train_test_split(indices, test_size=others_ratio, stratify=y, random_state=seed, shuffle=True)
-            
-            train_df = df[train_idx]
-            
-            if len(others_idx) > 0:
-                y_others = [y[i] for i in others_idx]
-                
-                # Split 2: Val vs (Test + Holdout)
-                test_holdout_ratio = (test_ratio + holdout_ratio) / others_ratio
-                if test_holdout_ratio > 0 and test_holdout_ratio < 1:
-                    val_idx, test_holdout_idx = train_test_split(others_idx, test_size=test_holdout_ratio, stratify=y_others, random_state=seed, shuffle=True)
-                    val_df = df[val_idx]
-                    
-                    if len(test_holdout_idx) > 0:
-                        y_th = [y[i] for i in test_holdout_idx]
-                        relative_holdout_ratio = holdout_ratio / (test_ratio + holdout_ratio)
-                        
-                        if relative_holdout_ratio > 0 and relative_holdout_ratio < 1:
-                            test_idx, holdout_idx = train_test_split(test_holdout_idx, test_size=relative_holdout_ratio, stratify=y_th, random_state=seed, shuffle=True)
-                            test_df = df[test_idx]
-                            holdout_df = df[holdout_idx]
-                        elif relative_holdout_ratio >= 1:
-                            test_df = df.slice(0, 0)
-                            holdout_df = df[test_holdout_idx]
-                        else:
-                            test_df = df[test_holdout_idx]
-                            holdout_df = df.slice(0, 0)
-                elif test_holdout_ratio >= 1:
-                    val_df = df.slice(0, 0)
-                    # Chained split for Test/Holdout
-                    y_th = y_others
-                    relative_holdout_ratio = holdout_ratio / (test_ratio + holdout_ratio)
-                    if relative_holdout_ratio > 0 and relative_holdout_ratio < 1:
-                        test_idx, holdout_idx = train_test_split(others_idx, test_size=relative_holdout_ratio, stratify=y_th, random_state=seed, shuffle=True)
-                        test_df = df[test_idx]
-                        holdout_df = df[holdout_idx]
-                    else:
-                        test_df = df[others_idx]
-                        holdout_df = df.slice(0, 0)
-                else:
-                    val_df = df[others_idx]
-                    test_df = df.slice(0, 0)
-                    holdout_df = df.slice(0, 0)
-
-    # --- 2. TIME-BASED SPLIT ---
-    elif strategy == "time":
-        if not time_col or time_col not in df.columns:
-             return {"error": f"Time column '{time_col}' not found"}
-        
-        df = df.sort(time_col)
-        
-        n = len(df)
-        n_train = int(n * train_ratio)
-        n_val = int(n * val_ratio)
-        n_test = int(n * test_ratio)
-        
-        train_df = df.slice(0, n_train)
-        val_df = df.slice(n_train, n_val)
-        test_df = df.slice(n_train + n_val, n_test)
-        holdout_df = df.slice(n_train + n_val + n_test, n - (n_train + n_val + n_test))
-        
-    # --- 3. GROUP-BASED SPLIT ---
-    elif strategy == "group":
-        if not config.get("group_column") or config["group_column"] not in df.columns:
-            return {"error": f"Group column '{config.get('group_column')}' not found"}
-        
-        group_col = config["group_column"]
-        groups = df[group_col].unique().to_list()
-        
-        # Split groups first to ensure zero leakage
-        n_grps = len(groups)
-        n_train = int(n_grps * train_ratio)
-        n_val = int(n_grps * val_ratio)
-        n_test = int(n_grps * test_ratio)
-        
-        if shuffle:
-            np.random.seed(seed)
-            np.random.shuffle(groups)
-            
-        train_grps = set(groups[:n_train])
-        val_grps = set(groups[n_train:n_train+n_val])
-        test_grps = set(groups[n_train+n_val:n_train+n_val+n_test])
-        holdout_grps = set(groups[n_train+n_val+n_test:])
-        
-        train_df = df.filter(pl.col(group_col).is_in(train_grps))
-        val_df = df.filter(pl.col(group_col).is_in(val_grps))
-        test_df = df.filter(pl.col(group_col).is_in(test_grps))
-        holdout_df = df.filter(pl.col(group_col).is_in(holdout_grps))
-
-    else:
-        return {"error": f"Strategy {strategy} not implemented yet"}
-
-    # Save outputs
-    base_name = file_path.replace(".csv", "").replace(".parquet", "")
-    train_path = f"{base_name}_train.csv"
-    val_path = f"{base_name}_val.csv"
-    test_path = f"{base_name}_test.csv"
-    holdout_path = f"{base_name}_holdout.csv"
-    
-    train_df.write_csv(train_path)
-    val_df.write_csv(val_path)
-    test_df.write_csv(test_path)
-    holdout_df.write_csv(holdout_path)
-    
-    return {
-        "success": True,
-        "paths": { "train": train_path, "val": val_path, "test": test_path, "holdout": holdout_path },
-        "stats": {
-            "train_rows": len(train_df),
-            "val_rows": len(val_df),
-            "test_rows": len(test_df),
-            "holdout_rows": len(holdout_df)
-        }
-    }
-
-def validate_split(config):
-    # Config contains paths to check and optional ID column
-    train_path = config["paths"]["train"]
-    val_path = config["paths"]["val"]
-    test_path = config["paths"]["test"]
-    holdout_path = config["paths"].get("holdout")
-    id_col = config.get("id_column", "id") # Default to 'id' if exists
-    target_col = config.get("target_column", None)
-
-    # Load dfs
-    try:
-        train_df = pl.read_csv(train_path) if train_path.endswith(".csv") else pl.read_parquet(train_path)
-        val_df = pl.read_csv(val_path) if val_path.endswith(".csv") else pl.read_parquet(val_path)
-        test_df = pl.read_csv(test_path) if test_path.endswith(".csv") else pl.read_parquet(test_path)
-        holdout_df = None
-        if holdout_path:
-             holdout_df = pl.read_csv(holdout_path) if holdout_path.endswith(".csv") else pl.read_parquet(holdout_path)
-    except:
-        return {"error": "Failed to load split files for validation"}
-
-    report = {
-        "leakage_detected": False,
-        "leakage_count": 0,
-        "distribution_mismatch": False,
-        "warnings": []
-    }
-
-    # 1. Leakage Check (ID intersection)
-    if id_col in train_df.columns:
-        train_ids = set(train_df[id_col].to_list())
-        val_ids = set(val_df[id_col].to_list())
-        test_ids = set(test_df[id_col].to_list())
-        holdout_ids = set(holdout_df[id_col].to_list()) if holdout_df is not None else set()
-        
-        leakage_tv = len(train_ids.intersection(val_ids))
-        leakage_tt = len(train_ids.intersection(test_ids))
-        leakage_th = len(train_ids.intersection(holdout_ids))
-        leakage_vt = len(val_ids.intersection(test_ids))
-        leakage_vh = len(val_ids.intersection(holdout_ids))
-        leakage_th_val = len(test_ids.intersection(holdout_ids))
-        
-        total_leakage = leakage_tv + leakage_tt + leakage_th + leakage_vt + leakage_vh + leakage_th_val
-        
-        if total_leakage > 0:
-            report["leakage_detected"] = True
-            report["leakage_count"] = total_leakage
-            report["warnings"].append(f"Found {total_leakage} overlapping IDs between splits.")
-    else:
-        report["warnings"].append(f"ID column '{id_col}' not found. Skipping exact leakage check.")
-
-    # 2. Distribution Check (Target Distribution)
-    if target_col and target_col in train_df.columns:
-        try:
-           def get_ratios(df, col):
-               counts = df[col].value_counts()
-               total = len(df)
-               ratios = {}
-               for row in counts.rows():
-                   ratios[str(row[0])] = row[1] / total
-               return ratios
-
-           train_metrics = get_ratios(train_df, target_col)
-           val_metrics = get_ratios(val_df, target_col)
-           # test_metrics = get_ratios(test_df, target_col) # Optional: could check all
-           
-           for cls in train_metrics:
-               train_r = train_metrics[cls]
-               val_r = val_metrics.get(cls, 0)
-               diff = abs(train_r - val_r)
-               if diff > 0.1: # 10% drift
-                   report["distribution_mismatch"] = True
-                   report["warnings"].append(f"Class '{cls}' drift: Train={train_r:.2f}, Val={val_r:.2f}")
-        except:
-             pass
-
-    return report
-
-def main():
-    # Usage: 
-    #   split:   python splitter_engine.py split <file_path> <config_json>
-    #   validate: python splitter_engine.py validate <config_json> (dummy file arg ignored)
-    
-    if len(sys.argv) < 3:
-        print(json.dumps({"error": "Usage: splitter_engine.py <action> <arg1> [arg2]"}), file=sys.stderr)
-        sys.exit(1)
-
-    action = sys.argv[1]
-    
-    try:
-        if action == "split":
-            file_path = sys.argv[2]
-            config = json.loads(sys.argv[3])
-            result = execute_split(file_path, config)
-            print(json.dumps(result))
-        
-        elif action == "validate":
-            config = json.loads(sys.argv[2])
-            result = validate_split(config)
-            print(json.dumps(result))
-            
-        else:
-            # Fallback for old calls (implicit split) - if users used old signature
-            # But since we control the caller, we can just update the caller (DataSplitter.ts).
-            raise ValueError(f"Unknown action: {action}")
-
-    except Exception as e:
-        print(json.dumps({"success": False, "error": str(e)}))
-        sys.exit(1)
-
-if __name__ == "__main__":
-    main()