openstat-cli 1.0.0__py3-none-any.whl

openstat/commands/regex_cmds.py

@@ -0,0 +1,104 @@
+"""Regex column commands: regex extract/replace/match/split."""
+
+from __future__ import annotations
+
+import re as _re
+
+from openstat.commands.base import command, CommandArgs, friendly_error
+from openstat.session import Session
+
+
+@command("regex", usage="regex extract|replace|match|split <col> <pattern> [options]")
+def cmd_regex(session: Session, args: str) -> str:
+    """Apply regex operations to a string column.
+
+    Sub-commands:
+      regex extract <col> <pattern> [into(<newcol>)]
+          — extract first capture group; stores in new column
+      regex replace <col> <pattern> <replacement> [into(<newcol>)]
+          — replace all matches with replacement string
+      regex match <col> <pattern> [into(<newcol>)]
+          — add boolean column: 1 if row matches, 0 otherwise
+      regex split <col> <pattern> [into(<newcol>)]
+          — split on pattern, store list as string repr
+
+    Examples:
+      regex extract email "([^@]+)@" into(username)
+      regex replace phone "[^0-9]" "" into(phone_clean)
+      regex match address "\\bStreet\\b" into(is_street)
+      regex split tags "," into(tag_list)
+    """
+    import polars as pl
+
+    ca = CommandArgs(args)
+    if len(ca.positional) < 3:
+        return "Usage: regex extract|replace|match|split <col> <pattern> ..."
+
+    subcmd = ca.positional[0].lower()
+    col_name = ca.positional[1]
+    pattern = ca.positional[2]
+
+    try:
+        df = session.require_data()
+        if col_name not in df.columns:
+            return f"Column not found: {col_name}"
+
+        # Validate regex
+        try:
+            compiled = _re.compile(pattern)
+        except _re.error as exc:
+            return f"Invalid regex: {exc}"
+
+        into_raw = ca.rest_after("into")
+        new_col = into_raw.strip().strip("()") if into_raw else None
+
+        if subcmd == "extract":
+            new_col = new_col or f"{col_name}_extracted"
+            vals = df[col_name].cast(pl.Utf8).to_list()
+            results = []
+            for v in vals:
+                if v is None:
+                    results.append(None)
+                    continue
+                m = compiled.search(v)
+                if m:
+                    results.append(m.group(1) if m.lastindex else m.group(0))
+                else:
+                    results.append(None)
+            session.snapshot()
+            session.df = df.with_columns(pl.Series(new_col, results))
+            n_matched = sum(1 for r in results if r is not None)
+            return f"Extracted to '{new_col}': {n_matched}/{df.height} rows matched."
+
+        elif subcmd == "replace":
+            repl = ca.positional[3] if len(ca.positional) > 3 else ""
+            new_col = new_col or col_name
+            vals = df[col_name].cast(pl.Utf8).to_list()
+            results = [compiled.sub(repl, v) if v is not None else None for v in vals]
+            session.snapshot()
+            session.df = df.with_columns(pl.Series(new_col, results))
+            n_changed = sum(1 for orig, new in zip(vals, results) if orig != new)
+            return f"Replaced in '{new_col}': {n_changed}/{df.height} rows changed."
+
+        elif subcmd == "match":
+            new_col = new_col or f"{col_name}_match"
+            vals = df[col_name].cast(pl.Utf8).to_list()
+            results = [1 if (v is not None and compiled.search(v)) else 0 for v in vals]
+            session.snapshot()
+            session.df = df.with_columns(pl.Series(new_col, results, dtype=pl.Int8))
+            n_match = sum(results)
+            return f"Match column '{new_col}': {n_match}/{df.height} rows matched."
+
+        elif subcmd == "split":
+            new_col = new_col or f"{col_name}_split"
+            vals = df[col_name].cast(pl.Utf8).to_list()
+            results = [str(compiled.split(v)) if v is not None else None for v in vals]
+            session.snapshot()
+            session.df = df.with_columns(pl.Series(new_col, results))
+            return f"Split column '{new_col}' created."
+
+        else:
+            return f"Unknown sub-command: {subcmd}. Use extract, replace, match, or split."
+
+    except Exception as e:
+        return friendly_error(e, "regex")

openstat/commands/report_cmds.py

@@ -0,0 +1,48 @@
+"""Report and help commands."""
+
+from __future__ import annotations
+
+from openstat.session import Session
+from openstat.reporting.report import generate_report
+from openstat.commands.base import command, get_registry
+
+
+@command("report", usage="report [eda [path.html] | path.md]")
+def cmd_report(session: Session, args: str) -> str:
+    """Generate a Markdown report or automated EDA HTML report."""
+    stripped = args.strip()
+    if stripped.startswith("eda"):
+        path = stripped[3:].strip() or "outputs/eda_report.html"
+        try:
+            from openstat.reporting.eda import generate_eda_report
+            out = generate_eda_report(session, path)
+            return f"EDA report saved: {out}"
+        except Exception as e:
+            return f"EDA report error: {e}"
+
+    path = stripped or "outputs/report.md"
+    try:
+        out = generate_report(session, path)
+        return f"Report saved: {out}"
+    except Exception as e:
+        return f"Report error: {e}"
+
+
+@command("help", usage="help [command]")
+def cmd_help(session: Session, args: str) -> str:
+    """Show available commands or help for a specific command."""
+    registry = get_registry()
+    if args.strip() and args.strip() in registry:
+        handler = registry[args.strip()]
+        from openstat.commands.base import get_usage
+        usage = get_usage(args.strip())
+        doc = handler.__doc__ or "No description."
+        return f"{args.strip()}: {doc}\nUsage: {usage}"
+
+    lines = ["Available commands:", ""]
+    for name, handler in sorted(registry.items()):
+        doc = (handler.__doc__ or "").split("\n")[0]
+        lines.append(f"  {name:<25} {doc}")
+    lines.append("")
+    lines.append("Type 'help <command>' for details. Type 'quit' to exit.")
+    return "\n".join(lines)

openstat/commands/repro_cmds.py

@@ -0,0 +1,129 @@
+"""Reproducibility commands: set seed, session save/replay/info."""
+
+from __future__ import annotations
+
+import datetime
+import re
+from pathlib import Path
+
+from openstat.commands.base import command, get_registry
+from openstat.session import Session
+from openstat import __version__
+
+
+# Module-level seed tracking
+_current_seed: int | None = None
+
+
+def get_current_seed() -> int | None:
+    return _current_seed
+
+
+
+@command("session", usage="session info | session save <path> | session replay <path>")
+def cmd_session(session: Session, args: str) -> str:
+    """Session management: view info, save commands to script, replay a script.
+
+    Examples:
+      session info              — show session details
+      session save analysis.ost — save all commands to a script file
+      session replay script.ost — run an .ost script in current session
+    """
+    tokens = args.strip().split(None, 1)
+    subcmd = tokens[0].lower() if tokens else "info"
+
+    if subcmd == "info":
+        seed = getattr(session, "_repro_seed", _current_seed)
+        lines = [
+            "Session Information",
+            "=" * 50,
+            f"  OpenStat version : {__version__}",
+            f"  Dataset          : {session.dataset_name or '(none)'}",
+            f"  Shape            : {session.shape_str}",
+            f"  Random seed      : {seed if seed is not None else '(not set)'}",
+            f"  Commands run     : {len(session.history)}",
+            f"  Models fitted    : {len(session.results)}",
+            f"  Plots generated  : {len(session.plot_paths)}",
+            f"  Output dir       : {session.output_dir}",
+            f"  Log file         : {session._log_path or '(none)'}",
+            f"  Timestamp        : {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+        ]
+        return "\n".join(lines)
+
+    elif subcmd == "save":
+        path = tokens[1].strip() if len(tokens) > 1 else "session_script.ost"
+        path_obj = Path(path)
+        path_obj.parent.mkdir(parents=True, exist_ok=True)
+        seed = getattr(session, "_repro_seed", _current_seed)
+
+        with open(path_obj, "w", encoding="utf-8") as f:
+            f.write(f"# OpenStat script — saved {datetime.datetime.now().isoformat()}\n")
+            f.write(f"# OpenStat version: {__version__}\n")
+            if seed is not None:
+                f.write(f"# Random seed: {seed}\n")
+                f.write(f"set seed {seed}\n")
+            f.write(f"# Dataset: {session.dataset_name or '(none)'}\n")
+            f.write("\n")
+            for cmd_line in session.history:
+                # Skip the current 'session save' command
+                if cmd_line.strip().startswith("session save"):
+                    continue
+                f.write(f"{cmd_line}\n")
+
+        return f"Session saved to: {path_obj.absolute()} ({len(session.history)} commands)"
+
+    elif subcmd == "replay":
+        path = tokens[1].strip() if len(tokens) > 1 else None
+        if not path:
+            return "Usage: session replay <path.ost>"
+        if not Path(path).exists():
+            return f"File not found: {path}"
+        # Use run_script
+        from openstat.repl import run_script
+        try:
+            run_script(path, session)
+            return f"Replayed: {path}"
+        except SystemExit:
+            return f"Replay stopped due to error in: {path}"
+        except Exception as exc:
+            return f"Replay error: {exc}"
+
+    else:
+        return (
+            "Usage:\n"
+            "  session info              — view session details\n"
+            "  session save <path.ost>   — save commands to script\n"
+            "  session replay <path.ost> — run a script file"
+        )
+
+
+@command("version", usage="version")
+def cmd_version(session: Session, args: str) -> str:
+    """Show OpenStat version and environment information."""
+    import sys
+    import platform
+
+    lines = [
+        f"OpenStat {__version__}",
+        f"Python  {sys.version.split()[0]}",
+        f"Platform {platform.system()} {platform.machine()}",
+    ]
+    deps = [
+        ("polars", "polars"),
+        ("numpy", "numpy"),
+        ("statsmodels", "statsmodels"),
+        ("scipy", "scipy"),
+        ("matplotlib", "matplotlib"),
+    ]
+    for name, mod in deps:
+        try:
+            m = __import__(mod)
+            lines.append(f"  {name:<15} {m.__version__}")
+        except ImportError:
+            lines.append(f"  {name:<15} (not installed)")
+
+    seed = getattr(session, "_repro_seed", _current_seed)
+    if seed is not None:
+        lines.append(f"Random seed: {seed}")
+
+    return "\n".join(lines)

openstat/commands/resampling_cmds.py

@@ -0,0 +1,109 @@
+"""Bootstrap and permutation test commands."""
+
+from __future__ import annotations
+
+import re
+
+from openstat.commands.base import command
+from openstat.session import Session
+
+
+def _stata_opts(raw: str) -> tuple[list[str], dict[str, str]]:
+    opts: dict[str, str] = {}
+    for m in re.finditer(r'(\w+)\(([^)]*)\)', raw):
+        opts[m.group(1).lower()] = m.group(2)
+    rest = re.sub(r'\w+\([^)]*\)', '', raw)
+    positional = [t.strip(',') for t in rest.split() if t.strip(',')]
+    return positional, opts
+
+
+def _fmt(r: dict) -> str:
+    lines = [f"\n{r.get('test', 'Result')}", "=" * 55]
+    skip = {"test", "groups", "_model"}
+    for k, v in r.items():
+        if k in skip:
+            continue
+        if isinstance(v, float):
+            lines.append(f"  {k:<35} {v:.6f}")
+        elif isinstance(v, list):
+            lines.append(f"  {k:<35} {v}")
+        else:
+            lines.append(f"  {k:<35} {v}")
+    lines.append("=" * 55)
+    return "\n".join(lines)
+
+
+@command("bootstrap", usage="bootstrap var [by(groupvar)] [stat(mean)] [n(2000)] [ci(0.95)]")
+def cmd_bootstrap(session: Session, args: str) -> str:
+    """Bootstrap confidence interval. With by(): tests difference between groups."""
+    from openstat.stats.resampling import bootstrap_ci, bootstrap_diff
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: bootstrap var [by(group)] [stat(mean)] [n(2000)] [ci(0.95)]"
+    col = positional[0]
+    if col not in df.columns:
+        return f"Column '{col}' not found."
+    by = opts.get("by")
+    stat = opts.get("stat", "mean")
+    n_boot = int(opts.get("n", 2000))
+    ci = float(opts.get("ci", 0.95))
+    try:
+        if by:
+            if by not in df.columns:
+                return f"Group column '{by}' not found."
+            r = bootstrap_diff(df, col, by, stat=stat, n_boot=n_boot, ci=ci)
+        else:
+            r = bootstrap_ci(df, col, stat=stat, n_boot=n_boot, ci=ci)
+        return _fmt(r)
+    except Exception as exc:
+        return f"bootstrap error: {exc}"
+
+
+@command("permtest", usage="permtest var by(groupvar) [stat(mean)] [n(2000)] [--greater|--less]")
+def cmd_permtest(session: Session, args: str) -> str:
+    """Permutation test for difference between two groups."""
+    from openstat.stats.resampling import permutation_test
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: permtest var by(groupvar) [stat(mean)] [n(2000)]"
+    col = positional[0]
+    if col not in df.columns:
+        return f"Column '{col}' not found."
+    by = opts.get("by")
+    if not by:
+        return "Specify group variable: permtest var by(groupvar)"
+    if by not in df.columns:
+        return f"Group column '{by}' not found."
+    stat = opts.get("stat", "mean")
+    n_perm = int(opts.get("n", 2000))
+    alt = "two-sided"
+    if "--greater" in args:
+        alt = "greater"
+    elif "--less" in args:
+        alt = "less"
+    try:
+        r = permutation_test(df, col, by, stat=stat, n_perm=n_perm, alternative=alt)
+        return _fmt(r)
+    except Exception as exc:
+        return f"permtest error: {exc}"
+
+
+@command("jackknife", usage="jackknife var [stat(mean)]")
+def cmd_jackknife(session: Session, args: str) -> str:
+    """Jackknife bias and standard error estimation."""
+    from openstat.stats.resampling import jackknife_ci
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: jackknife var [stat(mean)]"
+    col = positional[0]
+    if col not in df.columns:
+        return f"Column '{col}' not found."
+    stat = opts.get("stat", "mean")
+    try:
+        r = jackknife_ci(df, col, stat=stat)
+        return _fmt(r)
+    except Exception as exc:
+        return f"jackknife error: {exc}"

openstat/commands/reshape_cmds.py

@@ -0,0 +1,223 @@
+"""reshape, collapse, encode, decode commands."""
+
+from __future__ import annotations
+
+import re
+
+import polars as pl
+
+from openstat.commands.base import command
+from openstat.session import Session
+
+
+def _stata_opts(raw: str) -> tuple[list[str], dict[str, str]]:
+    opts: dict[str, str] = {}
+    for m in re.finditer(r'(\w+)\(([^)]*)\)', raw):
+        opts[m.group(1).lower()] = m.group(2)
+    rest = re.sub(r'\w+\([^)]*\)', '', raw)
+    positional = [t.strip(',') for t in rest.split() if t.strip(',')]
+    return positional, opts
+
+
+# ── reshape ────────────────────────────────────────────────────────────────
+
+@command("reshape", usage="reshape wide|long varlist, i(id) j(timevar) [stub(prefix)]")
+def cmd_reshape(session: Session, args: str) -> str:
+    """Reshape data between wide and long format (Stata-style).
+
+    Wide→Long: reshape long prefix, i(id) j(time)
+    Long→Wide: reshape wide varlist, i(id) j(timevar)
+    """
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+
+    if len(positional) < 2:
+        return (
+            "Usage:\n"
+            "  reshape long stubname, i(idvar) j(timevar)\n"
+            "  reshape wide varlist, i(idvar) j(timevar)"
+        )
+
+    direction = positional[0].lower()
+    id_var = opts.get("i")
+    j_var = opts.get("j")
+    if not id_var or not j_var:
+        return "Specify: i(idvar) j(timevar)"
+
+    session.snapshot()
+
+    if direction == "long":
+        # wide → long: column prefix → stub
+        stub = positional[1] if len(positional) > 1 else ""
+        value_vars = [c for c in df.columns if c.startswith(stub) and c != id_var]
+        if not value_vars:
+            return f"No columns found with prefix '{stub}'"
+        try:
+            long_df = df.unpivot(
+                on=value_vars,
+                index=[id_var],
+                variable_name=j_var,
+                value_name=stub or "value",
+            )
+            # Extract numeric suffix from variable name
+            long_df = long_df.with_columns(
+                pl.col(j_var).str.replace(stub, "", literal=True).alias(j_var)
+            )
+            session.df = long_df
+            return f"Reshaped wide→long: {df.shape} → {long_df.shape}. {j_var} = {long_df[j_var].unique().to_list()}"
+        except Exception as exc:
+            return f"reshape long error: {exc}"
+
+    elif direction == "wide":
+        # long → wide
+        var_list = [c for c in positional[1:] if c in df.columns]
+        if not var_list:
+            return "No valid value variables found."
+        try:
+            wide_df = df.pivot(
+                on=j_var,
+                index=id_var,
+                values=var_list[0] if len(var_list) == 1 else var_list,
+                aggregate_function="first",
+            )
+            session.df = wide_df
+            return f"Reshaped long→wide: {df.shape} → {wide_df.shape}"
+        except Exception as exc:
+            return f"reshape wide error: {exc}"
+
+    else:
+        return f"Unknown reshape direction: {direction}. Use 'wide' or 'long'."
+
+
+# ── collapse ───────────────────────────────────────────────────────────────
+
+_COLLAPSE_FUNS = {
+    "mean":   lambda c: pl.col(c).mean(),
+    "sum":    lambda c: pl.col(c).sum(),
+    "count":  lambda c: pl.col(c).count(),
+    "min":    lambda c: pl.col(c).min(),
+    "max":    lambda c: pl.col(c).max(),
+    "median": lambda c: pl.col(c).median(),
+    "std":    lambda c: pl.col(c).std(),
+    "var":    lambda c: pl.col(c).var(),
+    "first":  lambda c: pl.col(c).first(),
+    "last":   lambda c: pl.col(c).last(),
+}
+
+
+@command("collapse", usage="collapse (stat) varlist [, by(groupvars)]")
+def cmd_collapse(session: Session, args: str) -> str:
+    """Collapse dataset to group-level aggregates (replaces df).
+
+    Examples:
+      collapse (mean) income age, by(region)
+      collapse (sum) sales, by(year region)
+    """
+    df = session.require_data()
+    session.snapshot()
+
+    # Parse (stat) from args
+    stat_m = re.search(r'\((\w+)\)', args)
+    stat = stat_m.group(1).lower() if stat_m else "mean"
+    if stat not in _COLLAPSE_FUNS:
+        return f"Unknown statistic: {stat}. Use: {', '.join(_COLLAPSE_FUNS)}"
+
+    # _stata_opts extracts key(value) pairs (including by(...)) from full args
+    positional_raw, opts = _stata_opts(args)
+    # positional_raw may include "(mean)" token — filter it out
+    value_vars = [c for c in positional_raw if c in df.columns]
+    by_raw = opts.get("by", "")
+    by_vars = [c.strip() for c in by_raw.split() if c.strip() in df.columns]
+
+    if not value_vars:
+        return "No valid value variables found."
+
+    agg_fn = _COLLAPSE_FUNS[stat]
+    agg_exprs = [agg_fn(c).alias(c) for c in value_vars]
+
+    try:
+        if by_vars:
+            result = df.group_by(by_vars).agg(agg_exprs).sort(by_vars)
+        else:
+            result = df.select(agg_exprs)
+
+        session.df = result
+        return (
+            f"Collapsed to {result.shape[0]} rows × {result.shape[1]} cols "
+            f"using {stat}({', '.join(value_vars)})"
+            + (f" by {', '.join(by_vars)}" if by_vars else "")
+        )
+    except Exception as exc:
+        return f"collapse error: {exc}"
+
+
+# ── encode ─────────────────────────────────────────────────────────────────
+
+@command("encode", usage="encode varname [, gen(newvar)]")
+def cmd_encode(session: Session, args: str) -> str:
+    """Encode a string/categorical column to integer codes (0-based)."""
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: encode varname [, gen(newvar)]"
+
+    var = positional[0]
+    if var not in df.columns:
+        return f"Column '{var}' not found."
+
+    new_var = opts.get("gen", var + "_encoded")
+    session.snapshot()
+
+    try:
+        # Map unique sorted values to integers
+        unique_vals = sorted(df[var].drop_nulls().unique().to_list(), key=str)
+        val_map = {v: i for i, v in enumerate(unique_vals)}
+        encoded = df[var].map_elements(
+            lambda x: val_map.get(x), return_dtype=pl.Int64
+        )
+        session.df = df.with_columns(encoded.alias(new_var))
+        mapping_str = "\n".join(f"  {i} = {v}" for i, v in enumerate(unique_vals[:20]))
+        if len(unique_vals) > 20:
+            mapping_str += f"\n  ... ({len(unique_vals)} total)"
+        return f"Encoded '{var}' → '{new_var}' ({len(unique_vals)} categories)\n{mapping_str}"
+    except Exception as exc:
+        return f"encode error: {exc}"
+
+
+# ── decode ─────────────────────────────────────────────────────────────────
+
+@command("decode", usage="decode encodedvar origvar [, gen(newvar)]")
+def cmd_decode(session: Session, args: str) -> str:
+    """Decode integer codes back to string labels.
+
+    Requires a reference (original) string column with the same row order.
+    decode encoded_col orig_col [, gen(newvar)]
+    """
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if len(positional) < 2:
+        return "Usage: decode encoded_col orig_col [, gen(newvar)]"
+
+    enc_var = positional[0]
+    orig_var = positional[1]
+    new_var = opts.get("gen", enc_var + "_decoded")
+
+    for v in (enc_var, orig_var):
+        if v not in df.columns:
+            return f"Column '{v}' not found."
+
+    session.snapshot()
+    try:
+        # Build map from int codes → original labels using orig_var
+        code_col = df[enc_var].cast(pl.Int64)
+        label_col = df[orig_var].cast(pl.Utf8)
+        pairs = list(zip(code_col.to_list(), label_col.to_list()))
+        code_map = {c: l for c, l in pairs if c is not None and l is not None}
+
+        decoded = df[enc_var].map_elements(
+            lambda x: code_map.get(x, None), return_dtype=pl.Utf8
+        )
+        session.df = df.with_columns(decoded.alias(new_var))
+        return f"Decoded '{enc_var}' → '{new_var}' ({len(code_map)} unique codes)"
+    except Exception as exc:
+        return f"decode error: {exc}"