flasheda 0.1.3__tar.gz → 0.4.0__tar.gz

flasheda-0.4.0/MIT.License

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flasheda-0.4.0/PKG-INFO

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: flasheda
+Version: 0.4.0
+Summary: Fixed-time EDA for any dataset size, plus validated cleaning, feature and training code for pandas, polars, DuckDB and PySpark.
+Home-page: https://github.com/sjapanjots/Flash-EDA
+Author: Japanjot Singh
+Author-email: sjapanjots@gmail.com
+License: MIT
+Project-URL: Source, https://github.com/sjapanjots/Flash-EDA
+Project-URL: Changelog, https://github.com/sjapanjots/Flash-EDA/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/sjapanjots/Flash-EDA/issues
+Keywords: eda,data-cleaning,pandas,polars,duckdb,pyspark,code-generation,feature-engineering,automl
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: MIT.License
+Requires-Dist: pandas>=2.0
+Requires-Dist: numpy>=1.22
+Requires-Dist: scipy>=1.7.0
+Requires-Dist: rich>=12.0.0
+Requires-Dist: jinja2>=3.0.0
+Requires-Dist: matplotlib>=3.4.0
+Requires-Dist: fpdf2>=2.7.0
+Provides-Extra: fast
+Requires-Dist: duckdb>=0.10; extra == "fast"
+Requires-Dist: polars>=0.20; extra == "fast"
+Requires-Dist: pyarrow>=12; extra == "fast"
+Provides-Extra: spark
+Requires-Dist: pyspark>=3.4; extra == "spark"
+Provides-Extra: ml
+Requires-Dist: scikit-learn>=1.2; extra == "ml"
+Requires-Dist: pyarrow>=12; extra == "ml"
+Provides-Extra: all
+Requires-Dist: duckdb>=0.10; extra == "all"
+Requires-Dist: polars>=0.20; extra == "all"
+Requires-Dist: pyarrow>=12; extra == "all"
+Requires-Dist: scikit-learn>=1.2; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: duckdb>=0.10; extra == "dev"
+Requires-Dist: polars>=0.20; extra == "dev"
+Requires-Dist: pyarrow>=12; extra == "dev"
+Requires-Dist: scikit-learn>=1.2; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<img width="1254" height="1254" alt="image" src="https://github.com/user-attachments/assets/a87d02e0-7a18-41e0-844d-a2b9c3d6f769" />
+
+# FlashEDA
+
+[![PyPI](https://img.shields.io/pypi/v/flasheda)](https://pypi.org/project/flasheda/)
+[![Python](https://img.shields.io/pypi/pyversions/flasheda)](https://pypi.org/project/flasheda/)
+[![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
+
+**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+
+Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
+
+FlashEDA analyses a fixed-size random sample, so analysis time barely changes between 100 thousand and 100 million rows.
+
+## Install
+
+```bash
+pip install flasheda            # core
+pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
+pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[spark]"   # + PySpark
+pip install "flasheda[all]"     # everything except PySpark
+```
+
+## Quick start
+
+```python
+import flasheda as fe
+
+# 1. Analyse: any size, a few seconds
+report = fe.analyze("orders.csv", target="churn", save_pdf=False)
+report.show()                      # console report
+report.save_html("report.html")    # or report.save_pdf("report.pdf")
+
+# 2. Clean: get a script for the full data, already checked on the sample
+code = fe.clean("orders.csv", target="churn", engine="polars")
+code.save("clean_orders.py")       # then: python clean_orders.py
+
+# 3. Model: compare models on the sample, get prep + training scripts
+result = fe.train("orders.csv", target="churn", engine="polars")
+result.show()                      # leaderboard
+result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+```
+
+## What it does
+
+| | |
+|---|---|
+| **Overview** | true row count (read from file metadata, no full load), types, memory, duplicates |
+| **Data quality** | missing values, outliers, skew, likely ID columns, numbers stored as text, spelling variants, rare categories |
+| **Target analysis** | task type, class balance, per-feature predictive power, **possible leakage**, recommended metric and validation split |
+| **Correlations** | Pearson (numeric) and Cramér's V (categorical) |
+| **Reports** | console, HTML, PDF, JSON |
+| **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
+| **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
+| **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Validation** | generated code is run on the sample and compared with a reference result before you get it |
+
+### Cleaning steps
+Each step in the generated code has a comment saying why it was added:
+
+- drop likely-ID, mostly-empty, constant and leaky columns
+- drop rows where the target is missing, and drop duplicate rows
+- convert numbers stored as text (`"12.5"`, `"N/A"`) to numbers
+- merge spelling variants (`"Delhi"`, `" delhi"`, `"DELHI"`)
+- group rare categories into `other`
+- add `<col>_was_missing` flags when missingness predicts the target
+- fill nulls (median / mode), log-transform skewed columns, clip outliers
+
+### Features and models
+`fe.train()` (or `report.train()`) adds features on top of the cleaning steps, compares models on the sample, and writes two scripts:
+
+- **`prep.py`**: cleaning + features for the full data, in the chosen engine
+- **`train.py`**: scikit-learn training with cross-validation, using the same metric and split, that saves `model.joblib`
+
+The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
+
+### Engine choice
+
+| Rows | Engine |
+|---|---|
+| < 1M | pandas |
+| 1M – 200M | polars or DuckDB |
+| > 200M | PySpark (if installed) |
+
+Override with `fe.analyze(..., engine="duckdb")` or `report.codegen("duckdb")`.
+
+## Performance
+
+Measured on synthetic messy data:
+
+| Task | Time |
+|---|---|
+| Analyse a 5.1M-row Parquet file | 0.56 s |
+| Sample a 1M-row CSV (0.1.x → 0.2.0) | 37.3 s → 0.65 s |
+| Analyse + plan + generate + validate, 1M-row CSV with target | 2.5 s |
+| Run the generated polars script on 1M rows | 0.9 s |
+| `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
+| Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+
+## Examples
+
+- [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
+- [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
+- [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT

flasheda-0.4.0/README.md

@@ -0,0 +1,117 @@
+<img width="1254" height="1254" alt="image" src="https://github.com/user-attachments/assets/a87d02e0-7a18-41e0-844d-a2b9c3d6f769" />
+
+# FlashEDA
+
+[![PyPI](https://img.shields.io/pypi/v/flasheda)](https://pypi.org/project/flasheda/)
+[![Python](https://img.shields.io/pypi/pyversions/flasheda)](https://pypi.org/project/flasheda/)
+[![Tests](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml/badge.svg)](https://github.com/sjapanjots/Flash-EDA/actions/workflows/tests.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](MIT.License)
+
+**Current version: 0.4.0** · [Changelog](CHANGELOG.md)
+
+Analyse any dataset in a few seconds, then get **validated cleaning, feature and training code** that runs on the full data, in the engine that fits its size: pandas, polars, DuckDB or PySpark.
+
+FlashEDA analyses a fixed-size random sample, so analysis time barely changes between 100 thousand and 100 million rows.
+
+## Install
+
+```bash
+pip install flasheda            # core
+pip install "flasheda[fast]"    # + polars, DuckDB, pyarrow (recommended for big files)
+pip install "flasheda[ml]"      # + scikit-learn, for flasheda.train
+pip install "flasheda[spark]"   # + PySpark
+pip install "flasheda[all]"     # everything except PySpark
+```
+
+## Quick start
+
+```python
+import flasheda as fe
+
+# 1. Analyse: any size, a few seconds
+report = fe.analyze("orders.csv", target="churn", save_pdf=False)
+report.show()                      # console report
+report.save_html("report.html")    # or report.save_pdf("report.pdf")
+
+# 2. Clean: get a script for the full data, already checked on the sample
+code = fe.clean("orders.csv", target="churn", engine="polars")
+code.save("clean_orders.py")       # then: python clean_orders.py
+
+# 3. Model: compare models on the sample, get prep + training scripts
+result = fe.train("orders.csv", target="churn", engine="polars")
+result.show()                      # leaderboard
+result.save("prep.py", "train.py") # python prep.py && python train.py → model.joblib
+```
+
+## What it does
+
+| | |
+|---|---|
+| **Overview** | true row count (read from file metadata, no full load), types, memory, duplicates |
+| **Data quality** | missing values, outliers, skew, likely ID columns, numbers stored as text, spelling variants, rare categories |
+| **Target analysis** | task type, class balance, per-feature predictive power, **possible leakage**, recommended metric and validation split |
+| **Correlations** | Pearson (numeric) and Cramér's V (categorical) |
+| **Reports** | console, HTML, PDF, JSON |
+| **Code generation** | one cleaning plan → pandas, polars (lazy, streaming), DuckDB SQL (larger than memory) or PySpark |
+| **Features** | date parts, one-hot / frequency encoding, ratios that beat their inputs |
+| **Models** | baseline, linear, random forest, gradient boosting compared with the right metric and split; training script for the winner |
+| **Validation** | generated code is run on the sample and compared with a reference result before you get it |
+
+### Cleaning steps
+Each step in the generated code has a comment saying why it was added:
+
+- drop likely-ID, mostly-empty, constant and leaky columns
+- drop rows where the target is missing, and drop duplicate rows
+- convert numbers stored as text (`"12.5"`, `"N/A"`) to numbers
+- merge spelling variants (`"Delhi"`, `" delhi"`, `"DELHI"`)
+- group rare categories into `other`
+- add `<col>_was_missing` flags when missingness predicts the target
+- fill nulls (median / mode), log-transform skewed columns, clip outliers
+
+### Features and models
+`fe.train()` (or `report.train()`) adds features on top of the cleaning steps, compares models on the sample, and writes two scripts:
+
+- **`prep.py`**: cleaning + features for the full data, in the chosen engine
+- **`train.py`**: scikit-learn training with cross-validation, using the same metric and split, that saves `model.joblib`
+
+The metric and split follow the target: PR-AUC for imbalanced binary targets, MAE for skewed regression targets, time-based splits when there's a date column, and so on.
+
+### Engine choice
+
+| Rows | Engine |
+|---|---|
+| < 1M | pandas |
+| 1M – 200M | polars or DuckDB |
+| > 200M | PySpark (if installed) |
+
+Override with `fe.analyze(..., engine="duckdb")` or `report.codegen("duckdb")`.
+
+## Performance
+
+Measured on synthetic messy data:
+
+| Task | Time |
+|---|---|
+| Analyse a 5.1M-row Parquet file | 0.56 s |
+| Sample a 1M-row CSV (0.1.x → 0.2.0) | 37.3 s → 0.65 s |
+| Analyse + plan + generate + validate, 1M-row CSV with target | 2.5 s |
+| Run the generated polars script on 1M rows | 0.9 s |
+| `fe.train()` on a 1M-row CSV (plan, features, 4 models, checks) | 6.7 s |
+| Generated `prep.py` + `train.py` on 1M rows | 1.3 s + 20 s |
+
+## Examples
+
+- [`examples/phase1_demo.py`](examples/phase1_demo.py): engine choice and code generation at 0.1M / 1M / 5M rows
+- [`examples/phase2_demo.py`](examples/phase2_demo.py): target analysis and cleaning on a 1M-row churn CSV
+- [`examples/phase3_demo.py`](examples/phase3_demo.py): features, model comparison and training on 1M rows
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License
+
+MIT

{flasheda-0.1.3 → flasheda-0.4.0}/flasheda/__init__.py

@@ -1,93 +1,134 @@
-from __future__ import annotations
-
-import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-from typing import Union, Optional
-
-import pandas as pd
-
-from .sampler import sample, get_sample_size, set_sample_size
-from .analyzers import overview, missing, numeric, categorical, correlation
-from .report import EDAReport
-
-__version__ = "0.1.2"
-__all__ = ["analyze", "set_sample_size", "get_sample_size", "EDAReport"]
-
-
-def analyze(
-    source: Union[pd.DataFrame, str, Path],
-    n: Optional[int] = None,
-    *,
-    show: bool = False,
-    random_state: int = 42,
-    save_pdf: bool = True,
-    pdf_path: str = "flasheda_report.pdf",
-) -> EDAReport:
-
-    t0 = time.perf_counter()
-    n = n or get_sample_size()
-
-    if isinstance(source, pd.DataFrame):
-        original_shape = source.shape
-    else:
-        original_shape = (-1, -1)
-
-    sampled = sample(source, n=n, random_state=random_state)
-
-    if original_shape == (-1, -1):
-        original_shape = sampled.shape
-
-    tasks = {
-        "overview":    lambda: overview.analyze(sampled, original_shape),
-        "missing":     lambda: missing.analyze(sampled),
-        "numeric":     lambda: numeric.analyze(sampled),
-        "categorical": lambda: categorical.analyze(sampled),
-        "correlation": lambda: correlation.analyze(sampled),
-    }
-
-    results: dict = {}
-    with ThreadPoolExecutor(max_workers=5) as pool:
-        future_map = {pool.submit(fn): name for name, fn in tasks.items()}
-        for future in as_completed(future_map):
-            name = future_map[future]
-            try:
-                results[name] = future.result()
-            except Exception as exc:
-                results[name] = {"error": str(exc)}
-
-    warnings: list[str] = []
-    miss = results.get("missing", {})
-    if miss.get("critical_columns"):
-        warnings.append(f"Columns with >50% missing: {', '.join(miss['critical_columns'])}")
-    num = results.get("numeric", {})
-    if num.get("skewed_columns"):
-        warnings.append(f"Highly skewed columns: {', '.join(num['skewed_columns'])}")
-    cat = results.get("categorical", {})
-    if cat.get("likely_id_columns"):
-        warnings.append(f"Likely ID columns: {', '.join(cat['likely_id_columns'])}")
-    corr = results.get("correlation", {})
-    if corr.get("numeric_strong_pairs"):
-        warnings.append(f"{len(corr['numeric_strong_pairs'])} strongly correlated pair(s) found")
-
-    elapsed = round(time.perf_counter() - t0, 3)
-
-    report = EDAReport(
-        overview=results.get("overview", {}),
-        missing=results.get("missing", {}),
-        numeric=results.get("numeric", {}),
-        categorical=results.get("categorical", {}),
-        correlation=results.get("correlation", {}),
-        sample_size=len(sampled),
-        elapsed_seconds=elapsed,
-        original_shape=original_shape,
-        warnings=warnings,
-    )
-
-    if show:
-        report.show()
-
-    if save_pdf:
-        report.save_pdf(pdf_path)
-
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Union, Optional
+
+import pandas as pd
+
+from . import codegen, engine
+from .clean import clean
+from .modeling import train
+from .engine import available_engines, choose_engine, inspect_source, sample, get_sample_size, set_sample_size
+from .analyzers import overview, missing, numeric, categorical, correlation
+from .analyzers import target as target_analyzer
+from .report import EDAReport
+
+__version__ = "0.4.0"
+__all__ = [
+    "analyze",
+    "available_engines",
+    "clean",
+    "codegen",
+    "engine",
+    "get_sample_size",
+    "inspect_source",
+    "set_sample_size",
+    "train",
+    "EDAReport",
+]
+
+
+def analyze(
+    source: Union[pd.DataFrame, str, Path],
+    n: Optional[int] = None,
+    *,
+    show: bool = False,
+    random_state: int = 42,
+    save_pdf: bool = True,
+    pdf_path: str = "flasheda_report.pdf",
+    engine: Optional[str] = None,
+    target: Optional[str] = None,
+) -> EDAReport:
+    """
+    Analyse a fixed-size sample of `source` (DataFrame, CSV/TSV or Parquet path).
+
+    `target` names the column you want to predict; it adds task detection,
+    class balance, per-feature predictive power and leakage checks.
+
+    `engine` forces the engine recommended for full-size work; by default it
+    is chosen from the dataset's row count.
+    """
+    t0 = time.perf_counter()
+    n = n or get_sample_size()
+
+    info = inspect_source(source)
+    if target is not None and target not in info.columns:
+        raise KeyError(f"Target column {target!r} not found. Columns: {info.columns}")
+    sampled = sample(source, n=n, random_state=random_state)
+    original_shape = info.shape
+
+    tasks = {
+        "overview":    lambda: overview.analyze(sampled, original_shape),
+        "missing":     lambda: missing.analyze(sampled),
+        "numeric":     lambda: numeric.analyze(sampled),
+        "categorical": lambda: categorical.analyze(sampled),
+        "correlation": lambda: correlation.analyze(sampled),
+    }
+    if target is not None:
+        tasks["target"] = lambda: target_analyzer.analyze(sampled, target, random_state)
+
+    results: dict = {}
+    with ThreadPoolExecutor(max_workers=5) as pool:
+        future_map = {pool.submit(fn): name for name, fn in tasks.items()}
+        for future in as_completed(future_map):
+            name = future_map[future]
+            try:
+                results[name] = future.result()
+            except Exception as exc:
+                results[name] = {"error": str(exc)}
+
+    warnings: list[str] = []
+    miss = results.get("missing", {})
+    if miss.get("critical_columns"):
+        warnings.append(f"Columns with >50% missing: {', '.join(miss['critical_columns'])}")
+    num = results.get("numeric", {})
+    if num.get("skewed_columns"):
+        warnings.append(f"Highly skewed columns: {', '.join(num['skewed_columns'])}")
+    cat = results.get("categorical", {})
+    if cat.get("likely_id_columns"):
+        warnings.append(f"Likely ID columns: {', '.join(cat['likely_id_columns'])}")
+    corr = results.get("correlation", {})
+    if corr.get("numeric_strong_pairs"):
+        warnings.append(f"{len(corr['numeric_strong_pairs'])} strongly correlated pair(s) found")
+
+    tgt = results.get("target", {})
+    if "error" in tgt:
+        raise RuntimeError(f"Target analysis failed: {tgt['error']}")
+    if tgt.get("leakage_columns"):
+        cols = ", ".join(str(l["column"]) for l in tgt["leakage_columns"])
+        warnings.append(f"Possible target leakage: {cols}")
+    if tgt.get("imbalanced"):
+        warnings.append(
+            f"Imbalanced target: minority class is {tgt['minority_pct']}% "
+            f"(ratio {tgt['imbalance_ratio']}:1)"
+        )
+    if tgt.get("null_count"):
+        warnings.append(f"Target {target!r} is null in {tgt['null_pct']}% of sampled rows")
+
+    elapsed = round(time.perf_counter() - t0, 3)
+
+    report = EDAReport(
+        overview=results.get("overview", {}),
+        missing=results.get("missing", {}),
+        numeric=results.get("numeric", {}),
+        categorical=results.get("categorical", {}),
+        correlation=results.get("correlation", {}),
+        sample_size=len(sampled),
+        elapsed_seconds=elapsed,
+        original_shape=original_shape,
+        warnings=warnings,
+        target=tgt,
+        source=str(info.path) if info.path else None,
+        recommended_engine=choose_engine(info.n_rows, prefer=engine),
+        sample=sampled,
+    )
+
+    if show:
+        report.show()
+
+    if save_pdf:
+        report.save_pdf(pdf_path)
+
     return report

{flasheda-0.1.3 → flasheda-0.4.0}/flasheda/analyzers/__init__.py

@@ -1,3 +1,3 @@
-from . import overview, missing, numeric, categorical, correlation
-
-__all__ = ["overview", "missing", "numeric", "categorical", "correlation"]
+from . import overview, missing, numeric, categorical, correlation, target
+
+__all__ = ["overview", "missing", "numeric", "categorical", "correlation", "target"]