elessar 1.0.0__py3-none-any.whl

elessar/__init__.py

@@ -0,0 +1,40 @@
+"""elessar: automated exploratory data analysis and data-quality profiling for pandas DataFrames.
+
+>>> import elessar
+>>> report = elessar.see(df, target="price")        # prints the report, writes plots + report.html
+>>> report = elessar.see("sales.parquet", target="ask")  # any file; pick the target in a window
+"""
+
+__version__ = "1.0.0"
+
+from .config import StoneConfig  # noqa: E402 - __version__ first: submodules read it
+from .core import see, see_column  # noqa: E402
+from .dates import parse_dates  # noqa: E402
+from .correlations import see_corr  # noqa: E402
+from .distributions import see_dist  # noqa: E402
+from .picker import pick_target, suggest_targets  # noqa: E402
+from .plots import see_plot  # noqa: E402
+from .quality import see_quality  # noqa: E402
+from .report import Vision  # noqa: E402
+from .sources import LoadError, Source, load  # noqa: E402
+from .stats import see_stats  # noqa: E402
+from .types import see_types  # noqa: E402
+
+__all__ = [
+    "LoadError",
+    "Source",
+    "StoneConfig",
+    "Vision",
+    "load",
+    "parse_dates",
+    "pick_target",
+    "see",
+    "see_column",
+    "see_corr",
+    "see_dist",
+    "see_plot",
+    "see_quality",
+    "see_stats",
+    "see_types",
+    "suggest_targets",
+]

elessar/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

elessar/_utils.py

@@ -0,0 +1,216 @@
+"""Small shared helpers (private)."""
+
+from __future__ import annotations
+
+import math
+import re
+import warnings
+from contextlib import contextmanager
+from typing import Any, Iterator
+
+import numpy as np
+import pandas as pd
+
+UNHASHABLE = (list, dict, set, np.ndarray)
+REPR_TYPES = (*UNHASHABLE, bytes, bytearray)
+
+
+@contextmanager
+def quiet() -> Iterator[None]:
+    """Silence numpy/scipy/pandas warnings inside the block."""
+    with warnings.catch_warnings(), np.errstate(all="ignore"):
+        warnings.simplefilter("ignore")
+        yield
+
+
+def is_text_dtype(s: pd.Series) -> bool:
+    """True for object columns and pandas string dtypes."""
+    return pd.api.types.is_object_dtype(s.dtype) or isinstance(s.dtype, pd.StringDtype)
+
+
+def hashable(s: pd.Series) -> pd.Series:
+    """Return ``s`` with unhashable cells (lists, dicts, ...) and raw bytes replaced by their repr.
+
+    Bytes are included because pandas decodes them as UTF-8 when casting to str, which can fail.
+    """
+    if not pd.api.types.is_object_dtype(s.dtype):
+        return s
+    if not s.map(lambda v: isinstance(v, REPR_TYPES)).any():
+        return s
+    return s.map(lambda v: repr(v) if isinstance(v, REPR_TYPES) else v)
+
+
+def sample(obj: pd.Series | pd.DataFrame, n: int, random_state: int) -> pd.Series | pd.DataFrame:
+    """Deterministic row sample that preserves the original row order."""
+    if len(obj) <= n:
+        return obj
+    rng = np.random.default_rng(random_state)
+    pos = np.sort(rng.choice(len(obj), size=n, replace=False))
+    return obj.iloc[pos]
+
+
+def sample_array(x: np.ndarray, n: int, random_state: int) -> np.ndarray:
+    if len(x) <= n:
+        return x
+    rng = np.random.default_rng(random_state)
+    return x[np.sort(rng.choice(len(x), size=n, replace=False))]
+
+
+def finite(x: Any) -> float | None:
+    """Convert to a plain float, mapping NaN/inf/errors to None."""
+    try:
+        f = float(x)
+    except (TypeError, ValueError):
+        return None
+    return f if math.isfinite(f) else None
+
+
+def jsonable(obj: Any) -> Any:
+    """Recursively convert numpy / pandas objects to JSON-serialisable Python."""
+    if isinstance(obj, dict):
+        return {str(k): jsonable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple, set)):
+        return [jsonable(v) for v in obj]
+    if isinstance(obj, pd.DataFrame):
+        return [jsonable(r) for r in obj.reset_index().to_dict(orient="records")]
+    if isinstance(obj, pd.Series):
+        return {str(k): jsonable(v) for k, v in obj.items()}
+    if isinstance(obj, np.ndarray):
+        return [jsonable(v) for v in obj.tolist()]
+    if isinstance(obj, (bool, np.bool_)):
+        return bool(obj)
+    if isinstance(obj, (int, np.integer)):
+        return int(obj)
+    if isinstance(obj, (float, np.floating)):
+        return finite(obj)
+    if isinstance(obj, (pd.Timestamp, pd.Timedelta)):
+        return str(obj)
+    if obj is None or isinstance(obj, str):
+        return obj
+    try:
+        if pd.isna(obj):
+            return None
+    except (TypeError, ValueError):
+        pass
+    return str(obj)
+
+
+def dedupe_names(names: list[Any]) -> list[str]:
+    """Stringify column names and make them unique (``a``, ``a.1``, ...)."""
+    out: list[str] = []
+    seen: dict[str, int] = {}
+    for name in names:
+        base = "|".join(map(str, name)) if isinstance(name, tuple) else str(name)
+        key = base
+        while key in seen:
+            seen[base] += 1
+            key = f"{base}.{seen[base]}"
+        seen.setdefault(key, 0)
+        out.append(key)
+    return out
+
+
+def _as_frame(df: pd.DataFrame | pd.Series) -> pd.DataFrame:
+    if isinstance(df, pd.Series):
+        return df.to_frame(name="value" if df.name is None else df.name)
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(f"expected a pandas DataFrame or Series, got {type(df).__name__}")
+    return df
+
+
+def prepare(df: pd.DataFrame | pd.Series) -> pd.DataFrame:
+    """Accept a Series or DataFrame; return a DataFrame safe to analyse.
+
+    Column names become unique strings (``a``, ``a.1``, tuples joined with ``|``), sparse columns
+    are densified, and a non-unique row index is replaced by row positions.
+    """
+    df = _as_frame(df)
+    names = dedupe_names(list(df.columns))
+    if names != list(df.columns):
+        df = df.set_axis(names, axis=1)
+    sparse = [c for c in df.columns if isinstance(df[c].dtype, pd.SparseDtype)]
+    if sparse:
+        df = df.assign(**{c: df[c].sparse.to_dense() for c in sparse})
+    if not df.index.is_unique:
+        df = df.reset_index(drop=True)
+    return df
+
+
+def _same_label(a: Any, b: Any) -> bool:
+    if a is b:
+        return True
+    try:
+        if bool(a == b):
+            return True
+    except (TypeError, ValueError):
+        pass
+    try:
+        return bool(pd.isna(a) and pd.isna(b))
+    except (TypeError, ValueError):
+        return False
+
+
+def resolve_target(df: pd.DataFrame | pd.Series, target: Any) -> str | None:
+    """Map a target given as an original column label (any hashable) to its name after :func:`prepare`."""
+    if target is None:
+        return None
+    cols = list(_as_frame(df).columns)
+    names = dedupe_names(cols)
+    for label, name in zip(cols, names):
+        if _same_label(label, target):
+            return name
+    if str(target) in names:
+        return str(target)
+    raise KeyError(f"target column {target!r} not found")
+
+
+def fmt_num(x: Any, digits: int = 4) -> str:
+    """Compact human formatting for numbers (thousands separators, sig. digits)."""
+    if x is None:
+        return "-"
+    if isinstance(x, (bool, np.bool_)):
+        return str(bool(x))
+    if isinstance(x, (int, np.integer)):
+        return f"{int(x):,}"
+    f = finite(x)
+    if f is None:
+        return str(x)
+    if f == 0:
+        return "0"
+    if abs(f) >= 1e15 or abs(f) < 1e-3:
+        return f"{f:.{digits - 1}e}"
+    if f.is_integer() and abs(f) < 1e15:
+        return f"{int(f):,}"
+    return f"{f:,.{digits}g}" if abs(f) < 1000 else f"{f:,.1f}"
+
+
+def fmt_p(p: Any) -> str:
+    """Compact p-value: 0.032, 4e-07, < 1e-300 (exact zero after underflow), n/a."""
+    f = finite(p)
+    if f is None:
+        return "n/a"
+    if f == 0:
+        return "< 1e-300"
+    return f"{f:.2g}" if f >= 1e-3 else f"{f:.0e}"
+
+
+def fmt_bytes(n: float | None) -> str:
+    if n is None:
+        return "-"
+    for unit in ("B", "KB", "MB", "GB"):
+        if abs(n) < 1024:
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+    return f"{n:.1f} TB"
+
+
+def safe_name(name: Any) -> str:
+    """Filesystem-safe version of a column name (used in plot file names)."""
+    return re.sub(r"[^\w.-]+", "_", str(name)).strip("_")[:40] or "col"
+
+
+def truncate(text: Any, width: int) -> str:
+    s = str(text).replace("\n", " ")
+    if width <= 1:
+        return s[:width]
+    return s if len(s) <= width else s[: width - 1] + "…"

elessar/cli.py

@@ -0,0 +1,124 @@
+"""Command line: ``elessar data.csv --target y`` (also ``python -m elessar``)."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__, recap
+from .config import StoneConfig
+from .core import see
+from .picker import pick_target
+from .sources import LoadError, load
+from .types import FORCEABLE_TYPES
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="elessar",
+        description="Automated exploratory data analysis and data-quality profiling: semantic types, statistics, "
+        "distributions, associations with significance, target analysis, data quality, charts and an HTML report.",
+    )
+    p.add_argument("source", help="file, URL, glob pattern ('logs/*.csv') or folder: CSV / TSV / TXT, JSON / JSONL, "
+                                  "Parquet, Feather, ORC, Excel / ODS, pickle, HDF5, SAS, Stata, SPSS, SQLite, XML "
+                                  "(compressed or not)")
+    p.add_argument("-V", "--version", action="version", version=f"elessar {__version__}")
+    target = p.add_mutually_exclusive_group()
+    target.add_argument("-t", "--target", help="target column to analyse features against ('ask' opens a picker)")
+    target.add_argument("--no-target", action="store_true", help="do not ask for a target")
+    p.add_argument("-o", "--out", help="output folder (default: elessar_report/<file>_<target-X or no-target>_<YYYY-MM-DD>/)")
+    p.add_argument("--type", action="append", default=[], metavar="COL=TYPE",
+                   help="set a column's type by hand (repeatable), e.g. --type region=categorical --type cylinders="
+                        "numeric_discrete")
+    p.add_argument("--no-plots", action="store_true", help="skip the charts (and the HTML recap)")
+    p.add_argument("--single-file", action="store_true", help="also write report_single.html with the charts inlined")
+    read = p.add_argument_group("reading")
+    read.add_argument("--sheet", help="Excel sheet name or number (default: the first sheet with data)")
+    read.add_argument("--table", help="SQLite table / HDF5 key (default: the biggest table)")
+    read.add_argument("--sep", help="CSV delimiter (default: sniffed)")
+    read.add_argument("--encoding", help="text encoding (default: sniffed)")
+    read.add_argument("--columns", help="comma-separated columns to read")
+    read.add_argument("--nrows", type=int, help="only read the first N rows")
+    read.add_argument("--max-rows", type=int, help="profile at most N rows; bigger tables are streamed and sampled, "
+                                                   "with exact counts over every row (default 1000000)")
+    read.add_argument("--engine", choices=["auto", "dask", "pandas"], default="auto",
+                      help="streaming engine for big files (auto: dask when installed)")
+    p.add_argument("-j", "--jobs", type=int, help="processes for the heavy steps (default: automatic; 1 = none)")
+    p.add_argument("--sample", type=int, help="max rows for the expensive tests (default 100000)")
+    p.add_argument("--alpha", type=float, help="significance level (default 0.05)")
+    p.add_argument("--width", type=int, help="report width in characters (default: terminal width)")
+    p.add_argument("--json", action="store_true", help="also write <out>/report.json")
+    p.add_argument("--markdown", action="store_true", help="also write <out>/report.md")
+    p.add_argument("--csv", action="store_true", help="also write <out>/summary.csv")
+    p.add_argument("-q", "--quiet", action="store_true", help="do not print the report")
+    p.add_argument("--open", choices=["auto", "html", "md", "none"], default="auto",
+                   help="open the recap when done: auto = HTML recap when run from a terminal (default)")
+    return p
+
+
+def _interactive() -> bool:
+    return bool(sys.stdin and sys.stdin.isatty() and sys.stdout.isatty())
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    cfg = StoneConfig()
+    for flag, attr in (("sample", "sample_size"), ("alpha", "alpha"), ("max_rows", "max_rows"), ("jobs", "n_jobs")):
+        if getattr(args, flag) is not None:
+            setattr(cfg, attr, getattr(args, flag))
+    cfg.single_file_report = args.single_file
+    for spec in args.type:
+        col, sep, stype = spec.partition("=")
+        if not sep or not col.strip() or not stype.strip():
+            print(f"elessar: --type expects COL=TYPE, got {spec!r}", file=sys.stderr)
+            return 2
+        if stype.strip() not in FORCEABLE_TYPES:
+            print(f"elessar: --type {spec!r}: choose one of {', '.join(FORCEABLE_TYPES)}", file=sys.stderr)
+            return 2
+        cfg.column_types[col.strip()] = stype.strip()
+    sheet = int(args.sheet) if args.sheet is not None and args.sheet.isdigit() else args.sheet
+    columns = [c.strip() for c in args.columns.split(",")] if args.columns else None
+    try:
+        src = load(args.source, cfg, sheet=sheet, table=args.table, sep=args.sep, encoding=args.encoding,
+                   columns=columns, nrows=args.nrows, engine=args.engine)
+    except (LoadError, OSError, ValueError, ImportError) as exc:
+        print(f"elessar: {exc}", file=sys.stderr)
+        return 2
+    target = args.target
+    if target == "ask" or (target is None and not args.no_target and _interactive()):
+        try:
+            target = pick_target(src.df)
+        except RuntimeError as exc:
+            print(f"elessar: {exc}", file=sys.stderr)
+            return 2
+    if target is not None and str(target) not in map(str, src.df.columns):
+        print(f"elessar: target {target!r} not in columns: {', '.join(map(str, src.df.columns))}", file=sys.stderr)
+        return 2
+    open_report = {"auto": None, "html": "html", "md": "md", "none": False}[args.open]
+    report = see(src, target=target, config=cfg, plots=not args.no_plots, out_dir=args.out, verbose=False,
+                 open_report=False)
+    if not args.quiet:
+        report.print(width=args.width)
+        sys.stdout.flush()  # keep the report before the "wrote ..." messages when both streams share a file
+    if recap.maybe_open(report, open_report, sys.stderr) is None and report.recap:
+        print(f"\nReport: {report.recap['html'].resolve()}", file=sys.stderr)
+    out = report.out_dir or Path(args.out or ".")
+    if args.json or args.markdown or args.csv:
+        out.mkdir(parents=True, exist_ok=True)
+    written = [
+        export(out / name)
+        for flag, name, export in (
+            (args.json, "report.json", report.to_json),
+            (args.markdown, "report.md", report.to_markdown),
+            (args.csv, "summary.csv", report.to_csv),
+        )
+        if flag
+    ]
+    for path in written:
+        print(f"wrote {path}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

elessar/config.py

@@ -0,0 +1,114 @@
+"""Configuration: every threshold used by elessar lives here."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+CONTINUOUS_CANDIDATES: tuple[str, ...] = (
+    "norm",
+    "lognorm",
+    "expon",
+    "gamma",
+    "weibull_min",
+    "beta",
+    "uniform",
+    "logistic",
+    "t",
+    "laplace",
+    "pareto",
+    "chi2",
+    "cauchy",
+)
+DISCRETE_CANDIDATES: tuple[str, ...] = ("poisson", "nbinom", "geom", "binom")
+
+PLACEHOLDER_STRINGS: tuple[str, ...] = (
+    "", "na", "n/a", "n.a.", "nan", "null", "none", "nil", "-", "--", "?", ".",
+    "missing", "unknown", "undefined", "#n/a", "#null!", "not available",
+    "0000-00-00", "00/00/0000", "00-00-0000", "0000/00/00", "0000-00-00 00:00:00",  # zero dates: no date
+)
+PLACEHOLDER_NUMBERS: tuple[float, ...] = (-999, -9999, -99999, 999, 9999, 99999, 999999)
+
+
+@dataclass
+class StoneConfig:
+    """Thresholds and knobs. Percentages are expressed in 0-100."""
+
+    # sampling
+    sample_size: int = 100_000
+    """Max rows used by expensive steps (correlations, Kendall, KDE, quality pairs)."""
+    fit_sample_size: int = 20_000
+    """Max values used for distribution fitting and normality tests."""
+    random_state: int = 42
+    max_rows: int = 1_000_000
+    """Larger tables are profiled on a uniform random sample of this many rows; row counts, missing
+    values, min/max/mean/std, distinct counts and duplicate rows stay exact over every row."""
+    probe_max_rows: int = 200_000
+    """Most rows progressive typing reads before the full pass (big tables only)."""
+    chunk_rows: int = 250_000
+    """Rows per chunk when a file is streamed (files too big to profile in one piece)."""
+    max_hashed_rows: int | None = None
+    """Duplicate rows are counted exactly up to this many rows (16 bytes of memory per row); None = as
+    many as a quarter of the available memory allows."""
+    memory_fraction: float = 0.5
+    """Share of the available RAM a table may take once loaded; bigger files are streamed."""
+
+    # parallelism
+    n_jobs: int | None = None
+    """Processes for the heavy steps: None = automatic (all cores for big tables), 1 = no
+    parallelism, -1 = all cores, k = at most k."""
+    parallel_min_cells: int = 20_000
+    """Below this many cells (rows x columns) the automatic mode stays in one process."""
+
+    # types
+    column_types: dict[str, str] = field(default_factory=dict)
+    """Types set by hand, e.g. ``{"cylinders": "numeric_discrete", "region": "categorical"}``: they replace
+    the decision tree for those columns (numeric_continuous, numeric_discrete, categorical, ordinal-candidate,
+    id or text)."""
+    id_unique_ratio: float = 0.95
+    pattern_match_ratio: float = 0.9
+    ordinal_max_levels: int = 12
+    ordinal_max_int_levels: int = 7
+    discrete_max_levels: int = 50
+    text_min_avg_words: float = 3.0
+    text_min_avg_length: float = 40.0
+
+    # stats
+    top_n: int = 10
+    rare_category_pct: float = 1.0
+    max_categories: int = 50
+    trim_fraction: float = 0.1
+
+    # distributions
+    alpha: float = 0.05
+    candidates_continuous: tuple[str, ...] = CONTINUOUS_CANDIDATES
+    candidates_discrete: tuple[str, ...] = DISCRETE_CANDIDATES
+    fit_timeout: float = 5.0
+    """Seconds allowed per candidate fit (enforced on POSIX main thread)."""
+    n_best_fits: int = 3
+
+    # correlations
+    corr_threshold: float = 0.5
+    """Pairs with |association| above this are listed as strong."""
+    high_corr: float = 0.95
+    """Pairs above this are flagged as redundant."""
+    leakage_threshold: float = 0.98
+    vif_threshold: float = 10.0
+    """Variance inflation factor above which a column is flagged as multicollinear."""
+
+    # quality
+    high_missing_pct: float = 50.0
+    medium_missing_pct: float = 20.0
+    quasi_constant_pct: float = 99.0
+    row_missing_pct: float = 50.0
+    co_missing_jaccard: float = 0.8
+    outlier_alert_pct: float = 5.0
+    placeholder_strings: tuple[str, ...] = PLACEHOLDER_STRINGS
+    placeholder_numbers: tuple[float, ...] = PLACEHOLDER_NUMBERS
+
+    # plots
+    plot_top_n: int = 15
+    n_scatter: int = 4
+    n_target_plots: int = 8
+    dpi: int = 144
+    single_file_report: bool = False
+    """Also write ``report_single.html`` with every chart inlined (one file to send or attach)."""