flasheda 0.4.0__tar.gz → 0.6.0__tar.gz

{flasheda-0.4.0 → flasheda-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flasheda
-Version: 0.4.0
+Version: 0.6.0
 Summary: Fixed-time EDA for any dataset size, plus validated cleaning, feature and training code for pandas, polars, DuckDB and PySpark.
 Home-page: https://github.com/sjapanjots/Flash-EDA
 Author: Japanjot Singh
@@ -9,7 +9,7 @@ License: MIT
 Project-URL: Source, https://github.com/sjapanjots/Flash-EDA
 Project-URL: Changelog, https://github.com/sjapanjots/Flash-EDA/blob/main/CHANGELOG.md
 Project-URL: Issues, https://github.com/sjapanjots/Flash-EDA/issues
-Keywords: eda,data-cleaning,pandas,polars,duckdb,pyspark,code-generation,feature-engineering,automl
+Keywords: eda,data-cleaning,pandas,polars,duckdb,pyspark,code-generation,feature-engineering,automl,data-quality,data-drift,time-series,forecasting,pii
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Developers
@@ -35,17 +35,24 @@ Requires-Dist: pyspark>=3.4; extra == "spark"
 Provides-Extra: ml
 Requires-Dist: scikit-learn>=1.2; extra == "ml"
 Requires-Dist: pyarrow>=12; extra == "ml"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy>=1.4; extra == "sql"
+Requires-Dist: pyarrow>=12; extra == "sql"
 Provides-Extra: all
 Requires-Dist: duckdb>=0.10; extra == "all"
 Requires-Dist: polars>=0.20; extra == "all"
 Requires-Dist: pyarrow>=12; extra == "all"
 Requires-Dist: scikit-learn>=1.2; extra == "all"
+Requires-Dist: sqlalchemy>=1.4; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=7; extra == "dev"
 Requires-Dist: duckdb>=0.10; extra == "dev"
 Requires-Dist: polars>=0.20; extra == "dev"
 Requires-Dist: pyarrow>=12; extra == "dev"
 Requires-Dist: scikit-learn>=1.2; extra == "dev"
+Requires-Dist: pandera; extra == "dev"
+Requires-Dist: sqlalchemy>=1.4; extra == "dev"
+Requires-Dist: nbformat; extra == "dev"
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Dynamic: author
@@ -72,7 +79,7 @@ Dynamic: summary
 [![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
 
-**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+**Current version: 0.6.0** · [Changelog](CHANGELOG.md)
 
 Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
 
@@ -84,6 +91,7 @@ FlashEDA analyses a fixed-size random sample, so analysis time barely changes be
 pip install flasheda            # core
 pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
 pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[sql]"     # + SQLAlchemy, for database tables
 pip install "flasheda[spark]"   # + PySpark
 pip install "flasheda[all]"     # everything except PySpark
 ```
@@ -106,6 +114,27 @@ code.save("clean_orders.py")       # then: python clean_orders.py
 result = fe.train("orders.csv", target="churn", engine="polars")
 result.show()                      # leaderboard
 result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+
+# 4. Guard: learn rules from good data, check every new file
+contract = fe.analyze("orders_jan.csv").contract()
+fe.check("orders_feb.csv", contract).show()
+open("check.py", "w").write(contract.script())   # python check.py new.csv → exit 1 on failure
+
+# 5. Compare: what changed between two datasets
+fe.compare("orders_jan.csv", "orders_feb.csv").show()
+
+# Databases work like files
+src = fe.sql("postgresql://user:pass@host/shop", table="orders")
+fe.analyze(src, target="churn")
+
+# 6. Time series: trend, seasonality, forecast
+ts = fe.timeseries("orders.parquet", time="created_at", value="amount")
+ts.show()
+fc = ts.forecast()                                 # backtested baselines
+open("forecast.py", "w").write(fc.script())        # python forecast.py → forecast.csv
+
+# 7. Everything as one runnable notebook
+report.to_notebook("orders.ipynb", train=True, time="created_at")
 ```
 
 ## What it does
@@ -120,6 +149,12 @@ result.save("prep.py", "train.py") # python prep.py && python train.py → model
 | **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
 | **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
 | **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Data contracts** | rules learned from good data; full-data checks; standalone check script for CI; pandera export |
+| **Drift** | schema changes and per-column drift (PSI, KS, new categories, null-rate change) between two datasets |
+| **Databases** | any SQLAlchemy URL; sampling runs in the database; passwords never stored |
+| **Text** | personal data (emails, phones, card numbers…) flagged; free-text stats; length/word features |
+| **Time series** | full-data aggregation per hour/day/week; gaps, trend, seasonality, anomalies; backtested forecast + script |
+| **Notebook** | analysis, cleaning, training, forecast and contract check as one runnable `.ipynb` |
 | **Validation** | generated code is run on the sample and compared with a reference result before you get it |
 
 ### Cleaning steps
@@ -141,6 +176,17 @@ Each step in the generated code has a comment saying why it was added:
 
 The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
 
+### Data contracts and drift
+A **contract** describes what good data looks like, learned from a sample with tolerances so normal variation passes:
+type, missing %, 1st/99th percentile range (catches unit changes like ×100), no negatives, allowed values, uniqueness of IDs.
+`fe.check()` checks every row of a file in one DuckDB pass; `contract.script()` writes the same check as a standalone script that exits with code 1 on failure, ready for a pipeline or CI job.
+
+`fe.compare(old, new)` reports added/removed columns, type changes and how much each column shifted (PSI: < 0.1 stable, 0.1–0.25 moderate, ≥ 0.25 major). Dates and IDs are shown as range shifts, since they always move.
+
+### Time series and forecasting
+A random sample breaks time order, so `fe.timeseries()` first aggregates **every row** per period (one DuckDB `GROUP BY`, about 0.7 s for 5M rows), then reports gaps, trend (% per year), daily/weekly/yearly seasonality, the busiest weekday or hour, and anomalies.
+`ts.forecast()` backtests five simple baselines (naive, seasonal naive, moving average, drift, trend + season) and keeps the most accurate. These are the numbers any fancier model has to beat. `fc.script()` writes it as a standalone script.
+
 ### Engine choice
 
 | Rows | Engine |
@@ -163,12 +209,20 @@ Measured on synthetic messy data:
 | Run the generated polars script on 1M rows | 0.9 s |
 | `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
 | Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+| Learn a contract from 1M rows / run the check script on 1M rows | 0.8 s / 0.6–0.9 s |
+| Compare two 1M-row files | 0.6 s |
+| Analyse a 1M-row SQLite table (5,000 rows transferred) | 0.3 s |
+| Time-series profile of a 5.3M-row Parquet file | 0.7 s |
+| Generated forecast script on 5.3M rows | 0.6 s |
+| Notebook with cleaning, features, training, forecast and contract | 5 s |
 
 ## Examples
 
 - [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
 - [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
 - [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+- [`examples/phase4_demo.py`](examples/phase4_demo.py): contracts, check script, drift and a database on 1M rows
+- [`examples/phase5_demo.py`](examples/phase5_demo.py): text, time series, forecast and notebook on 5M rows
 
 ## Development
 

{flasheda-0.4.0 → flasheda-0.6.0}/README.md

@@ -7,7 +7,7 @@
 [![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
 
-**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+**Current version: 0.6.0** · [Changelog](CHANGELOG.md)
 
 Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
 
@@ -19,6 +19,7 @@ FlashEDA analyses a fixed-size random sample, so analysis time barely changes be
 pip install flasheda            # core
 pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
 pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[sql]"     # + SQLAlchemy, for database tables
 pip install "flasheda[spark]"   # + PySpark
 pip install "flasheda[all]"     # everything except PySpark
 ```
@@ -41,6 +42,27 @@ code.save("clean_orders.py")       # then: python clean_orders.py
 result = fe.train("orders.csv", target="churn", engine="polars")
 result.show()                      # leaderboard
 result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+
+# 4. Guard: learn rules from good data, check every new file
+contract = fe.analyze("orders_jan.csv").contract()
+fe.check("orders_feb.csv", contract).show()
+open("check.py", "w").write(contract.script())   # python check.py new.csv → exit 1 on failure
+
+# 5. Compare: what changed between two datasets
+fe.compare("orders_jan.csv", "orders_feb.csv").show()
+
+# Databases work like files
+src = fe.sql("postgresql://user:pass@host/shop", table="orders")
+fe.analyze(src, target="churn")
+
+# 6. Time series: trend, seasonality, forecast
+ts = fe.timeseries("orders.parquet", time="created_at", value="amount")
+ts.show()
+fc = ts.forecast()                                 # backtested baselines
+open("forecast.py", "w").write(fc.script())        # python forecast.py → forecast.csv
+
+# 7. Everything as one runnable notebook
+report.to_notebook("orders.ipynb", train=True, time="created_at")
 ```
 
 ## What it does
@@ -55,6 +77,12 @@ result.save("prep.py", "train.py") # python prep.py && python train.py → model
 | **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
 | **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
 | **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Data contracts** | rules learned from good data; full-data checks; standalone check script for CI; pandera export |
+| **Drift** | schema changes and per-column drift (PSI, KS, new categories, null-rate change) between two datasets |
+| **Databases** | any SQLAlchemy URL; sampling runs in the database; passwords never stored |
+| **Text** | personal data (emails, phones, card numbers…) flagged; free-text stats; length/word features |
+| **Time series** | full-data aggregation per hour/day/week; gaps, trend, seasonality, anomalies; backtested forecast + script |
+| **Notebook** | analysis, cleaning, training, forecast and contract check as one runnable `.ipynb` |
 | **Validation** | generated code is run on the sample and compared with a reference result before you get it |
 
 ### Cleaning steps
@@ -76,6 +104,17 @@ Each step in the generated code has a comment saying why it was added:
 
 The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
 
+### Data contracts and drift
+A **contract** describes what good data looks like, learned from a sample with tolerances so normal variation passes:
+type, missing %, 1st/99th percentile range (catches unit changes like ×100), no negatives, allowed values, uniqueness of IDs.
+`fe.check()` checks every row of a file in one DuckDB pass; `contract.script()` writes the same check as a standalone script that exits with code 1 on failure, ready for a pipeline or CI job.
+
+`fe.compare(old, new)` reports added/removed columns, type changes and how much each column shifted (PSI: < 0.1 stable, 0.1–0.25 moderate, ≥ 0.25 major). Dates and IDs are shown as range shifts, since they always move.
+
+### Time series and forecasting
+A random sample breaks time order, so `fe.timeseries()` first aggregates **every row** per period (one DuckDB `GROUP BY`, about 0.7 s for 5M rows), then reports gaps, trend (% per year), daily/weekly/yearly seasonality, the busiest weekday or hour, and anomalies.
+`ts.forecast()` backtests five simple baselines (naive, seasonal naive, moving average, drift, trend + season) and keeps the most accurate. These are the numbers any fancier model has to beat. `fc.script()` writes it as a standalone script.
+
 ### Engine choice
 
 | Rows | Engine |
@@ -98,12 +137,20 @@ Measured on synthetic messy data:
 | Run the generated polars script on 1M rows | 0.9 s |
 | `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
 | Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+| Learn a contract from 1M rows / run the check script on 1M rows | 0.8 s / 0.6–0.9 s |
+| Compare two 1M-row files | 0.6 s |
+| Analyse a 1M-row SQLite table (5,000 rows transferred) | 0.3 s |
+| Time-series profile of a 5.3M-row Parquet file | 0.7 s |
+| Generated forecast script on 5.3M rows | 0.6 s |
+| Notebook with cleaning, features, training, forecast and contract | 5 s |
 
 ## Examples
 
 - [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
 - [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
 - [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+- [`examples/phase4_demo.py`](examples/phase4_demo.py): contracts, check script, drift and a database on 1M rows
+- [`examples/phase5_demo.py`](examples/phase5_demo.py): text, time series, forecast and notebook on 5M rows
 
 ## Development
 

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/__init__.py

@@ -10,21 +10,29 @@ import pandas as pd
 from . import codegen, engine
 from .clean import clean
 from .modeling import train
-from .engine import available_engines, choose_engine, inspect_source, sample, get_sample_size, set_sample_size
+from .contract import check
+from .compare import compare
+from .timeseries import timeseries
+from .engine import available_engines, choose_engine, inspect_source, sample, get_sample_size, set_sample_size, sql
 from .analyzers import overview, missing, numeric, categorical, correlation
 from .analyzers import target as target_analyzer
+from .analyzers import text as text_analyzer
 from .report import EDAReport
 
-__version__ = "0.4.0"
+__version__ = "0.6.0"
 __all__ = [
     "analyze",
     "available_engines",
+    "check",
     "clean",
+    "compare",
     "codegen",
     "engine",
     "get_sample_size",
     "inspect_source",
     "set_sample_size",
+    "sql",
+    "timeseries",
     "train",
     "EDAReport",
 ]
@@ -36,13 +44,17 @@ def analyze(
     *,
     show: bool = False,
     random_state: int = 42,
-    save_pdf: bool = True,
+    save_pdf: bool = False,
     pdf_path: str = "flasheda_report.pdf",
     engine: Optional[str] = None,
     target: Optional[str] = None,
 ) -> EDAReport:
     """
-    Analyse a fixed-size sample of `source` (DataFrame, CSV/TSV or Parquet path).
+    Analyse a fixed-size sample of `source`: a DataFrame, a CSV/TSV or Parquet
+    path, or a database table/query from flasheda.sql().
+
+    Nothing is written to disk unless you ask: pass save_pdf=True (to
+    pdf_path) or call report.save_pdf() / report.save_html().
 
     `target` names the column you want to predict; it adds task detection,
     class balance, per-feature predictive power and leakage checks.
@@ -65,12 +77,13 @@ def analyze(
         "numeric":     lambda: numeric.analyze(sampled),
         "categorical": lambda: categorical.analyze(sampled),
         "correlation": lambda: correlation.analyze(sampled),
+        "text":        lambda: text_analyzer.analyze(sampled),
     }
     if target is not None:
         tasks["target"] = lambda: target_analyzer.analyze(sampled, target, random_state)
 
     results: dict = {}
-    with ThreadPoolExecutor(max_workers=5) as pool:
+    with ThreadPoolExecutor(max_workers=len(tasks)) as pool:
         future_map = {pool.submit(fn): name for name, fn in tasks.items()}
         for future in as_completed(future_map):
             name = future_map[future]
@@ -92,6 +105,14 @@ def analyze(
     corr = results.get("correlation", {})
     if corr.get("numeric_strong_pairs"):
         warnings.append(f"{len(corr['numeric_strong_pairs'])} strongly correlated pair(s) found")
+    for p in corr.get("numeric_boolean_strong_pairs", []):
+        warnings.append(f"{p['numeric']!r} almost determines {p['boolean']!r} "
+                        f"(point-biserial r = {p['point_biserial_r']})")
+
+    txt = results.get("text", {})
+    for col in txt.get("personal_data_columns", []):
+        kinds = ", ".join(f"{k} {v}%" for k, v in txt["columns"][col]["personal_data"].items())
+        warnings.append(f"Possible personal data in {col!r}: {kinds} of non-empty values")
 
     tgt = results.get("target", {})
     if "error" in tgt:
@@ -115,13 +136,17 @@ def analyze(
         numeric=results.get("numeric", {}),
         categorical=results.get("categorical", {}),
         correlation=results.get("correlation", {}),
+        text=txt,
         sample_size=len(sampled),
         elapsed_seconds=elapsed,
         original_shape=original_shape,
         warnings=warnings,
         target=tgt,
         source=str(info.path) if info.path else None,
-        recommended_engine=choose_engine(info.n_rows, prefer=engine),
+        sql_source=info.sql,
+        # database sources are read with pandas (or exported with .to_parquet())
+        recommended_engine=choose_engine(
+            info.n_rows, prefer=engine or ("pandas" if info.sql is not None else None)),
         sample=sampled,
     )
 

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/analyzers/__init__.py

@@ -1,3 +1,3 @@
-from . import overview, missing, numeric, categorical, correlation, target
+from . import overview, missing, numeric, categorical, correlation, target, text
 
-__all__ = ["overview", "missing", "numeric", "categorical", "correlation", "target"]
+__all__ = ["overview", "missing", "numeric", "categorical", "correlation", "target", "text"]

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/analyzers/categorical.py

@@ -11,7 +11,7 @@ import pandas as pd
 import numpy as np
 from typing import Dict, Any, List
 
-from ..clean.detectors import looks_like_datetime
+from ..clean.detectors import is_free_text, looks_like_datetime
 
 
 HIGH_CARDINALITY_RATIO = 0.5   # unique/total > 50% → high cardinality warning
@@ -54,8 +54,9 @@ def analyze(df: pd.DataFrame) -> Dict[str, Any]:
         mode_pct = round(vc.iloc[0] / n * 100, 2) if len(vc) > 0 else 0.0
 
         # Detect potential ID columns (very high cardinality); unique
-        # timestamps are high-cardinality too, but they're not identifiers
-        likely_id = cardinality_ratio > 0.95 and not looks_like_datetime(df[col])
+        # timestamps and free text are high-cardinality too, but not identifiers
+        likely_id = (cardinality_ratio > 0.95 and not looks_like_datetime(df[col])
+                     and not is_free_text(df[col]))
 
         results[col] = {
             "count": n,

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/analyzers/correlation.py

@@ -4,7 +4,7 @@ flasheda.analyzers.correlation
 Computes:
   - Pearson correlation for numeric–numeric pairs
   - Cramér's V for categorical–categorical pairs
-  - Point-biserial for numeric–boolean pairs
+  - Point-biserial for numeric–boolean pairs (Pearson with the boolean as 0/1)
 
 Flags strongly correlated pairs (potential redundant features).
 Always operates on the fixed sample — constant time.
@@ -21,6 +21,7 @@ from itertools import combinations
 STRONG_CORR_THRESHOLD = 0.80
 MAX_COLS_FOR_HEATMAP = 20   # skip heatmap if too many cols (slow to render)
 MAX_CRAMER_LEVELS = 100     # Cramér's V is meaningless (and slow) for ID-like columns
+MAX_BOOL_COLS = 10
 
 
 def analyze(df: pd.DataFrame) -> Dict[str, Any]:
@@ -66,10 +67,20 @@ def analyze(df: pd.DataFrame) -> Dict[str, Any]:
             "matrix": [[round(float(hm_corr.loc[r, c]), 3) for c in heatmap_cols] for r in heatmap_cols],
         }
 
+    # ── Point-biserial for numeric–boolean pairs ────────────────────────────
+    biserial_pairs: List[Dict[str, Any]] = []
+    for b in [c for c in df.columns if _is_boolean(df[c])][:MAX_BOOL_COLS]:
+        flag = _as_01(df[b])
+        for a in num_cols[:MAX_COLS_FOR_HEATMAP]:
+            r = _point_biserial(df[a], flag)
+            if r is not None and abs(r) >= STRONG_CORR_THRESHOLD:
+                biserial_pairs.append({"numeric": a, "boolean": b, "point_biserial_r": round(r, 4)})
+
     return {
         "pearson_matrix": pearson_matrix,
         "numeric_strong_pairs": strong_pairs,
         "categorical_strong_pairs": cramer_pairs,
+        "numeric_boolean_strong_pairs": biserial_pairs,
         "heatmap": heatmap_data,
         "numeric_col_names": num_cols,
     }
@@ -92,4 +103,29 @@ def _cramers_v(a: pd.Series, b: pd.Series) -> float | None:
             return None
         return float(np.sqrt(phi2_corr / denom))
     except Exception:
-        return None
+        return None
+
+def _is_boolean(s: pd.Series) -> bool:
+    """bool dtype, or text holding only true/false."""
+    if pd.api.types.is_bool_dtype(s):
+        return True
+    values = s.dropna()
+    if not len(values) or pd.api.types.is_numeric_dtype(values):
+        return False
+    return bool(values.astype(str).str.strip().str.lower().isin(["true", "false"]).all())
+
+
+def _as_01(s: pd.Series) -> pd.Series:
+    if pd.api.types.is_bool_dtype(s):
+        return s.astype(float)
+    text = s.astype("string").str.strip().str.lower()
+    return text.map({"true": 1.0, "false": 0.0}).astype(float)
+
+
+def _point_biserial(x: pd.Series, flag: pd.Series) -> float | None:
+    """Point-biserial r: Pearson between a numeric column and a 0/1 flag."""
+    pair = pd.DataFrame({"x": pd.to_numeric(x, errors="coerce"), "b": flag}).dropna()
+    if len(pair) < 3 or pair["x"].std() == 0 or pair["b"].nunique() < 2:
+        return None
+    from scipy.stats import pointbiserialr
+    return float(pointbiserialr(pair["b"], pair["x"]).statistic)

flasheda-0.6.0/flasheda/analyzers/text.py

@@ -0,0 +1,134 @@
+"""
+flasheda.analyzers.text
+~~~~~~~~~~~~~~~~~~~~~~~
+Text columns:
+
+  personal data  emails, phone numbers, URLs, IP addresses and card numbers
+                 (Luhn-checked) found in any text column, as a share of the
+                 column's non-empty values
+  free text      for columns that read like sentences (≥ 3 words on average
+                 or ≥ 30 characters): length and word statistics, empty and
+                 duplicate values, all-caps share, most common words
+
+Regex-based and dependency-free; runs on the fixed sample.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Any, Dict, List
+
+import pandas as pd
+
+from ..clean.detectors import is_free_text, is_text
+
+PII_PATTERNS = {
+    "email": r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}",
+    "phone": r"(?<![\w.])\+?\d[\d\s().-]{7,}\d(?![\w.])",
+    "url": r"\bhttps?://[^\s]+|\bwww\.[^\s]+",
+    "ip_address": r"(?<![\d.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?![\d.])",
+    "card_number": r"\b(?:\d[ -]?){13,19}\b",
+}
+PII_MIN_PCT = 1.0                 # report a type when at least 1% of values contain it
+TOP_WORDS = 10
+STOPWORDS = frozenset("""a an and are as at be but by for from has have i in is it its of on or
+that the this to was were will with you your we our they their he she not no so if when what how
+my me all just can do did been than then there about up out very""".split())
+WORD = re.compile(r"[a-z][a-z']+")
+DATE_LIKE = re.compile(r"\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4}|\d:\d{2}")
+
+
+def analyze(df: pd.DataFrame) -> Dict[str, Any]:
+    columns: Dict[str, Dict[str, Any]] = {}
+    for col in df.columns:
+        s = df[col]
+        if not is_text(s):
+            continue
+        values = s.dropna().astype(str)
+        info: Dict[str, Any] = {}
+        pii = personal_data(values)
+        if pii:
+            info["personal_data"] = pii
+        if is_free_text(s):
+            info["free_text"] = _free_text_stats(values, values.str.len(), values.str.count(r"\S+"))
+        if info:
+            columns[str(col)] = info
+
+    return {
+        "columns": columns,
+        "personal_data_columns": [c for c, v in columns.items() if "personal_data" in v],
+        "free_text_columns": [c for c, v in columns.items() if "free_text" in v],
+    }
+
+
+def personal_data(values: pd.Series) -> Dict[str, float]:
+    """{kind: % of non-empty values containing it}, for kinds in ≥ PII_MIN_PCT of them."""
+    if not len(values):
+        return {}
+    found = {}
+    for kind, pattern in PII_PATTERNS.items():
+        hits = values.str.contains(pattern, regex=True)
+        if kind == "card_number" and hits.any():
+            hits = hits & values.map(_has_luhn_number)
+        if kind == "phone" and hits.any():
+            hits = hits & values.map(_looks_like_phone) & \
+                ~values.str.contains(PII_PATTERNS["ip_address"], regex=True)
+        pct = round(float(hits.mean() * 100), 2)
+        if pct >= PII_MIN_PCT:
+            found[kind] = pct
+    if "card_number" in found:
+        found.pop("phone", None)          # card numbers also match the phone pattern
+    return found
+
+
+def _looks_like_phone(text: str) -> bool:
+    """9–15 digits, written with a + or separators, or exactly 10 plain digits.
+    Plain digit runs of other lengths are usually order or account numbers."""
+    for m in re.finditer(PII_PATTERNS["phone"], text):
+        raw = m.group()
+        digits = sum(ch.isdigit() for ch in raw)
+        formatted = raw.startswith("+") or any(ch in raw for ch in " ().-")
+        if DATE_LIKE.search(raw) or re.fullmatch(r"\d+\.\d+", raw):
+            continue                       # dates, times and decimals have digit runs too
+        if 9 <= digits <= 15 and (formatted or digits == 10):
+            return True
+    return False
+
+
+def _has_luhn_number(text: str) -> bool:
+    for m in re.finditer(PII_PATTERNS["card_number"], text):
+        digits = [int(d) for d in re.sub(r"\D", "", m.group())]
+        if 13 <= len(digits) <= 19 and _luhn(digits):
+            return True
+    return False
+
+
+def _luhn(digits: List[int]) -> bool:
+    total = 0
+    for i, d in enumerate(reversed(digits)):
+        if i % 2:
+            d *= 2
+            d -= 9 if d > 9 else 0
+        total += d
+    return total % 10 == 0
+
+
+def _free_text_stats(values: pd.Series, lengths: pd.Series, words: pd.Series) -> Dict[str, Any]:
+    stripped = values.str.strip()
+    letters = stripped.str.replace(r"[^A-Za-z]", "", regex=True)
+    counter: Counter = Counter()
+    for text in values.str.lower().head(2_000):
+        counter.update(w for w in WORD.findall(text) if w not in STOPWORDS)
+    n = len(values)
+    return {
+        "mean_chars": round(float(lengths.mean()), 1),
+        "median_chars": float(lengths.median()),
+        "max_chars": int(lengths.max()),
+        "mean_words": round(float(words.mean()), 1),
+        "empty_pct": round(float((stripped == "").mean() * 100), 2),
+        "duplicate_pct": round(float(values.duplicated().mean() * 100), 2),
+        "all_caps_pct": round(float(((letters.str.len() > 3) & (letters == letters.str.upper())).mean() * 100), 2),
+        "top_words": [w for w, _ in counter.most_common(TOP_WORDS)],
+        "rows": n,
+    }

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/clean/detectors.py

@@ -41,6 +41,19 @@ def looks_like_datetime(s: pd.Series, min_ratio: float = 0.95) -> bool:
     return bool(parsed.notna().mean() >= min_ratio)
 
 
+FREE_TEXT_MIN_WORDS = 3.0
+FREE_TEXT_MIN_CHARS = 30.0
+
+
+def is_free_text(s: pd.Series) -> bool:
+    """Text that reads like sentences: ≥ 3 words or ≥ 30 characters on average."""
+    if not is_text(s):
+        return False
+    values = s.dropna().astype(str).head(2_000)
+    return bool(values.str.count(r"\S+").mean() >= FREE_TEXT_MIN_WORDS
+                or values.str.len().mean() >= FREE_TEXT_MIN_CHARS)
+
+
 def numeric_text_ratio(s: pd.Series) -> Optional[float]:
     """Share of non-empty text values that parse as numbers, or None if it's not text."""
     if not is_text(s):

{flasheda-0.4.0 → flasheda-0.6.0}/flasheda/codegen/__init__.py

@@ -18,7 +18,7 @@ from .planner import plan_from_report
 from .steps import (
     ClipOutliers, DateParts, DropColumns, DropDuplicates, DropNullRows, FillNull,
     FrequencyEncode, GroupRare, Log1p, MissingIndicator, NormalizeText, OneHot, Plan, Ratio,
-    ToNumeric,
+    TextStats, ToNumeric,
 )
 from .validate import ValidationResult, compare_frames, run_generated, validate_code
 from .writers import WRITERS, get_writer
@@ -58,10 +58,12 @@ def generate(
     output: str = "cleaned.parquet",
     sample: Optional[pd.DataFrame] = None,
     columns: Optional[List[str]] = None,
+    sql: Any = None,
 ) -> GeneratedCode:
+    """`sql` is a flasheda.sql() source to read from instead of a file."""
     if columns is None and sample is not None:
         columns = [str(c) for c in sample.columns]
-    code = get_writer(engine).render(plan, source, output, columns)
+    code = get_writer(engine).render(plan, source, output, columns, sql=sql)
     return GeneratedCode(engine, code, plan, sample)
 
 
@@ -81,6 +83,7 @@ __all__ = [
     "OneHot",
     "Plan",
     "Ratio",
+    "TextStats",
     "ToNumeric",
     "ValidationResult",
     "WRITERS",