openstat-cli 1.0.0__py3-none-any.whl

openstat/commands/automodel_cmds.py

@@ -0,0 +1,212 @@
+"""Automatic model selection: automodel command."""
+
+from __future__ import annotations
+
+import itertools
+from typing import NamedTuple
+
+import numpy as np
+import polars as pl
+
+from openstat.commands.base import command
+from openstat.session import Session
+from openstat.dsl.parser import parse_formula, ParseError
+
+
+class _Candidate(NamedTuple):
+    formula: str
+    model_type: str
+    aic: float
+    bic: float
+    r2: float | None
+    n: int
+    k: int  # number of predictors
+
+
+def _fit_candidate(
+    df: pl.DataFrame,
+    dep: str,
+    indeps: list[str],
+    model_type: str,
+) -> _Candidate | None:
+    """Fit a single candidate model, return metrics or None on failure."""
+    try:
+        if model_type == "ols":
+            from openstat.stats.models import fit_ols
+            result, _ = fit_ols(df, dep, indeps)
+            formula = f"{dep} ~ {' + '.join(indeps)}"
+            return _Candidate(
+                formula=formula, model_type="OLS",
+                aic=result.aic or float("inf"), bic=result.bic or float("inf"),
+                r2=result.r_squared, n=result.n_obs, k=len(indeps),
+            )
+        elif model_type == "logit":
+            from openstat.stats.models import fit_logit
+            result, _ = fit_logit(df, dep, indeps)
+            formula = f"{dep} ~ {' + '.join(indeps)}"
+            return _Candidate(
+                formula=formula, model_type="Logit",
+                aic=result.aic or float("inf"), bic=result.bic or float("inf"),
+                r2=result.pseudo_r2, n=result.n_obs, k=len(indeps),
+            )
+        elif model_type == "poisson":
+            from openstat.stats.models import fit_poisson
+            result, _ = fit_poisson(df, dep, indeps)
+            formula = f"{dep} ~ {' + '.join(indeps)}"
+            return _Candidate(
+                formula=formula, model_type="Poisson",
+                aic=result.aic or float("inf"), bic=result.bic or float("inf"),
+                r2=result.pseudo_r2, n=result.n_obs, k=len(indeps),
+            )
+    except Exception:
+        return None
+    return None
+
+
+@command("automodel", usage="automodel <depvar> ~ <x1> <x2> ... [--ols|--logit|--poisson] [--criterion=aic|bic] [--maxvars=N]")
+def cmd_automodel(session: Session, args: str) -> str:
+    """Automatic model selection: fits all variable subsets and ranks by AIC/BIC.
+
+    Uses exhaustive search for ≤ 8 predictors, forward stepwise for more.
+
+    Examples:
+      automodel score ~ age income education
+      automodel employed ~ age income score region --logit --criterion=bic
+      automodel score ~ age income education region --criterion=aic --maxvars=3
+    """
+    import re
+    df = session.require_data()
+
+    # Parse flags
+    use_logit = "--logit" in args
+    use_poisson = "--poisson" in args
+    model_type = "logit" if use_logit else "poisson" if use_poisson else "ols"
+
+    m_crit = re.search(r"--criterion[= ](\w+)", args)
+    criterion = m_crit.group(1).lower() if m_crit else "aic"
+    if criterion not in ("aic", "bic"):
+        criterion = "aic"
+
+    m_max = re.search(r"--maxvars[= ](\d+)", args)
+    max_vars = int(m_max.group(1)) if m_max else None
+
+    # Clean flags from formula
+    formula_str = re.sub(r"--\w+(?:[= ]\w+)?", "", args).strip()
+    if "~" not in formula_str:
+        return (
+            "Usage: automodel <depvar> ~ <x1> <x2> ... [--ols|--logit|--poisson]\n"
+            "Example: automodel score ~ age income education"
+        )
+
+    # Normalize: allow space-separated predictors (convert to + separated)
+    if "~" in formula_str:
+        lhs, rhs = formula_str.split("~", 1)
+        # If no + in rhs, convert spaces to +
+        if "+" not in rhs:
+            rhs = " + ".join(rhs.split())
+        formula_str = f"{lhs.strip()} ~ {rhs.strip()}"
+
+    try:
+        dep, indeps = parse_formula(formula_str)
+    except ParseError as e:
+        return f"Formula error: {e}"
+
+    if dep not in df.columns:
+        return f"Dependent variable not found: {dep}"
+    missing = [x for x in indeps if x not in df.columns]
+    if missing:
+        return f"Predictors not found: {', '.join(missing)}"
+
+    if max_vars:
+        indeps = indeps[:max_vars + 10]  # allow some buffer
+
+    k = len(indeps)
+    strategy = "exhaustive" if k <= 8 else "forward stepwise"
+
+    # Build candidates
+    candidates: list[_Candidate] = []
+
+    if strategy == "exhaustive":
+        total = 2 ** k - 1  # exclude empty model
+        for r in range(1, k + 1):
+            if max_vars and r > max_vars:
+                break
+            for subset in itertools.combinations(indeps, r):
+                c = _fit_candidate(df, dep, list(subset), model_type)
+                if c:
+                    candidates.append(c)
+    else:
+        # Forward stepwise
+        current = []
+        remaining = list(indeps)
+        while remaining and (max_vars is None or len(current) < max_vars):
+            best: _Candidate | None = None
+            for var in remaining:
+                trial = current + [var]
+                c = _fit_candidate(df, dep, trial, model_type)
+                if c:
+                    if best is None or getattr(c, criterion) < getattr(best, criterion):
+                        best = c
+            if best is None:
+                break
+            # Find which var was added
+            best_vars = best.formula.split("~")[1].strip().split(" + ")
+            added = [v for v in best_vars if v not in current]
+            current.extend(added)
+            remaining = [v for v in remaining if v not in current]
+            candidates.append(best)
+
+    if not candidates:
+        return "No valid models found. Check your data and variable names."
+
+    # Sort by criterion
+    candidates.sort(key=lambda c: getattr(c, criterion))
+    top_n = min(10, len(candidates))
+    top = candidates[:top_n]
+
+    # Store best model result in session
+    best = candidates[0]
+    try:
+        dep2, indeps2 = parse_formula(best.formula)
+        if model_type == "ols":
+            from openstat.stats.models import fit_ols
+            result, raw = fit_ols(df, dep2, indeps2)
+        elif model_type == "logit":
+            from openstat.stats.models import fit_logit
+            result, raw = fit_logit(df, dep2, indeps2)
+        else:
+            from openstat.stats.models import fit_poisson
+            result, raw = fit_poisson(df, dep2, indeps2)
+        session._last_model = raw
+        session._last_model_vars = (dep2, indeps2)
+        session._last_fit_result = result
+        session._last_fit_kwargs = {}
+    except Exception:
+        pass
+
+    crit_label = criterion.upper()
+    lines = [
+        f"Dependent: {dep}   Candidates: {len(candidates)}   Strategy: {strategy}",
+        f"Model type: {model_type.upper()}   Selection criterion: {crit_label}",
+        "",
+        f"Top {top_n} models by {crit_label}:",
+        f"  {'#':<3} {'AIC':>9}  {'BIC':>9}  {'R²/PseudoR²':>11}  k  Formula",
+        "  " + "-" * 76,
+    ]
+    for i, c in enumerate(top, 1):
+        r2_str = f"{c.r2:.4f}" if c.r2 is not None else "      —"
+        marker = " ← best" if i == 1 else ""
+        lines.append(
+            f"  {i:<3} {c.aic:>9.2f}  {c.bic:>9.2f}  "
+            f"{r2_str:>11}  {c.k}  {c.formula}{marker}"
+        )
+
+    lines += [
+        "",
+        f"Best model: {best.formula}",
+        f"  AIC = {best.aic:.2f}   BIC = {best.bic:.2f}",
+        "",
+        "Best model loaded. Use 'estimates', 'vif', 'residuals', 'plot coef' for diagnostics.",
+    ]
+
+    return "\n" + "=" * 60 + "\nAutomatic Model Selection\n" + "=" * 60 + "\n" + "\n".join(lines) + "\n" + "=" * 60

openstat/commands/backend_cmds.py

@@ -0,0 +1,82 @@
+"""Backend management commands: set backend, sql."""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.table import Table
+
+from openstat.session import Session
+from openstat.commands.base import command, CommandArgs, rich_to_str, friendly_error
+
+
+@command("set", usage="set seed <N> | set backend polars|duckdb")
+def cmd_set(session: Session, args: str) -> str:
+    """Change settings: random seed, backend."""
+    ca = CommandArgs(args)
+    if not ca.positional:
+        return "Usage: set seed <N> | set backend polars|duckdb"
+
+    subcmd = ca.positional[0].lower()
+
+    if subcmd == "seed":
+        if len(ca.positional) < 2:
+            seed = getattr(session, "_repro_seed", None)
+            return f"Current seed: {seed}" if seed is not None else "No seed set."
+        try:
+            seed = int(ca.positional[1])
+        except ValueError:
+            return f"Invalid seed: {ca.positional[1]}. Must be an integer."
+        import numpy as np
+        import random as _random
+        np.random.seed(seed)
+        _random.seed(seed)
+        session._repro_seed = seed  # type: ignore[attr-defined]
+        return f"Seed set to {seed}. Reproducible random operations enabled."
+
+    elif subcmd == "backend":
+        backend_name = ca.positional[1].lower() if len(ca.positional) > 1 else ""
+        if backend_name == "polars":
+            session._backend = "polars"
+            session._backend_obj = None
+            return "Backend set to: polars"
+        elif backend_name == "duckdb":
+            try:
+                from openstat.backends.duckdb_backend import DuckDBBackend
+                session._backend_obj = DuckDBBackend()
+                session._backend = "duckdb"
+                # If data already loaded, register it
+                if session.df is not None:
+                    session._backend_obj._conn.register("data", session.df.to_pandas())
+                    session._backend_obj._table_loaded = True
+                return "Backend set to: duckdb"
+            except ImportError as e:
+                return str(e)
+        else:
+            return f"Unknown backend: {backend_name}. Use 'polars' or 'duckdb'."
+    else:
+        return f"Unknown setting: {subcmd}. Available: seed, backend"
+
+
+@command("sql", usage='sql "SELECT * FROM data WHERE ..."')
+def cmd_sql(session: Session, args: str) -> str:
+    """Execute SQL query on the loaded dataset (DuckDB backend recommended)."""
+    query = args.strip().strip('"\'')
+    if not query:
+        return 'Usage: sql "SELECT * FROM data WHERE ..."'
+
+    try:
+        if session._backend == "duckdb" and session._backend_obj is not None:
+            result_df = session._backend_obj.sql(query)
+        elif session.df is not None:
+            # Use Polars SQL context as fallback
+            import polars as pl
+            ctx = pl.SQLContext({"data": session.df})
+            result_df = ctx.execute(query).collect()
+        else:
+            return "No data loaded."
+
+        session.snapshot()
+        session.df = result_df
+        return f"Query returned {session.shape_str}"
+    except Exception as e:
+        return friendly_error(e, "sql")

openstat/commands/base.py

@@ -0,0 +1,170 @@
+"""Command registration infrastructure.
+
+Provides a @command decorator that auto-registers handler functions,
+and a CommandArgs helper for standardized argument parsing.
+
+Usage:
+    from openstat.commands.base import command
+
+    @command("mycommand", usage="mycommand <arg>")
+    def cmd_mycommand(session, args):
+        '''One-line description shown in help.'''
+        ...
+        return "result text"
+"""
+
+from __future__ import annotations
+
+import io
+import re
+from typing import Callable
+
+from rich.console import Console
+
+from openstat.session import Session
+from openstat.logging_config import get_logger
+
+log = get_logger("commands")
+
+# Type alias for command handlers
+Handler = Callable[[Session, str], str]
+
+# Global registry — populated by @command decorator
+_REGISTRY: dict[str, Handler] = {}
+_USAGE: dict[str, str] = {}
+
+
+class CommandArgs:
+    """Standardized argument parser for commands.
+
+    Handles: positional args, --flags, key=value options.
+
+    Usage:
+        ca = CommandArgs(args)
+        ca.positional    # list of positional tokens
+        ca.has_flag("--robust")  # True/False
+        ca.get_option("how", "inner")  # key=value with default
+        ca.rest_after("on")  # everything after keyword "on"
+    """
+
+    def __init__(self, raw: str) -> None:
+        self.raw = raw
+        self._tokens = raw.split()
+        self.flags: set[str] = set()
+        self.options: dict[str, str] = {}
+        self.positional: list[str] = []
+
+        for tok in self._tokens:
+            if tok.startswith("--"):
+                if "=" in tok:
+                    k, v = tok.split("=", 1)
+                    self.options[k.lstrip("-")] = v
+                else:
+                    self.flags.add(tok)
+            elif "=" in tok and not tok.startswith('"') and not tok.startswith("'"):
+                k, v = tok.split("=", 1)
+                self.options[k] = v
+            else:
+                self.positional.append(tok)
+
+    def has_flag(self, flag: str) -> bool:
+        return flag in self.flags
+
+    def get_option(self, key: str, default: str | None = None) -> str | None:
+        return self.options.get(key, default)
+
+    def get_option_float(self, key: str, default: float) -> float:
+        val = self.options.get(key)
+        if val is None:
+            return default
+        try:
+            return float(val)
+        except ValueError:
+            return default
+
+    def rest_after(self, keyword: str) -> str | None:
+        """Return everything after a keyword (case-insensitive)."""
+        parts = re.split(rf"\b{keyword}\b", self.raw, maxsplit=1, flags=re.IGNORECASE)
+        if len(parts) < 2:
+            return None
+        return parts[1].strip()
+
+    def strip_flags_and_options(self) -> str:
+        """Return raw string with all --flags and key=value removed."""
+        result = self.raw
+        for flag in self.flags:
+            result = result.replace(flag, "")
+        for k, v in self.options.items():
+            result = result.replace(f"--{k}={v}", "")
+            result = result.replace(f"{k}={v}", "")
+        return result.strip()
+
+    def __bool__(self) -> bool:
+        return bool(self.raw.strip())
+
+
+def command(name: str, *, usage: str = "") -> Callable[[Handler], Handler]:
+    """Decorator to register a command handler.
+
+    Args:
+        name:  Command name as typed by the user.
+        usage: One-line usage example shown in help.
+    """
+    def decorator(fn: Handler) -> Handler:
+        if name in _REGISTRY:
+            log.warning("Command '%s' re-registered (overriding previous)", name)
+        _REGISTRY[name] = fn
+        _USAGE[name] = usage or f"{name} ..."
+        return fn
+    return decorator
+
+
+def get_registry() -> dict[str, Handler]:
+    """Return a live read-only view of the command registry.
+
+    The returned mapping always reflects the current state of the
+    registry, so commands registered after import time (e.g. plugins)
+    are visible automatically.
+    """
+    from types import MappingProxyType
+    return MappingProxyType(_REGISTRY)  # type: ignore[return-value]
+
+
+def get_usage(name: str) -> str:
+    return _USAGE.get(name, "")
+
+
+def run_command(session, line: str) -> str:
+    """Run a command line string against the session. Used by DSL loops."""
+    line = line.strip()
+    if not line:
+        return ""
+    parts = line.split(None, 1)
+    cmd_name = parts[0].lower()
+    args = parts[1] if len(parts) > 1 else ""
+    handler = _REGISTRY.get(cmd_name)
+    if handler is None:
+        return f"Unknown command: {cmd_name}"
+    return handler(session, args) or ""
+
+
+def rich_to_str(fn) -> str:
+    """Capture Rich output as plain text (no stdout side-effect)."""
+    buf = io.StringIO()
+    console = Console(file=buf, width=120, record=True)
+    fn(console)
+    return console.export_text().rstrip()
+
+
+def friendly_error(e: Exception, context: str) -> str:
+    """Convert common Polars/statsmodels errors to user-friendly messages."""
+    msg = str(e)
+    etype = type(e).__name__
+    if "not found" in msg.lower() or "ColumnNotFoundError" in etype:
+        return f"[red]Error:[/red] {context}: Column not found. Check column names with 'describe'."
+    if "type" in msg.lower() and ("str" in msg.lower() or "string" in msg.lower()):
+        return f"[red]Error:[/red] {context}: Type mismatch — cannot use arithmetic on text columns."
+    if "singular" in msg.lower() or "linalg" in msg.lower():
+        return f"[red]Error:[/red] {context}: Matrix is singular — check for perfect multicollinearity or constant columns."
+    log.debug("Unhandled error in %s: %s: %s", context, etype, msg)
+    return f"[red]Error:[/red] {context}: {e}"

openstat/commands/bayes_cmds.py

@@ -0,0 +1,71 @@
+"""Bayesian regression commands."""
+
+from __future__ import annotations
+
+import re
+
+from openstat.commands.base import command
+from openstat.session import Session
+
+
+def _stata_opts(raw: str) -> tuple[list[str], dict[str, str]]:
+    opts: dict[str, str] = {}
+    for m in re.finditer(r'(\w+)\(([^)]*)\)', raw):
+        opts[m.group(1).lower()] = m.group(2)
+    rest = re.sub(r'\w+\([^)]*\)', '', raw)
+    positional = [t.strip(',') for t in rest.split() if t.strip(',')]
+    return positional, opts
+
+
+@command("bayes", usage="bayes: ols depvar indepvars [, samples(4000) priorscale(10) ci(0.95)]")
+def cmd_bayes(session: Session, args: str) -> str:
+    """Bayesian OLS with conjugate Normal-Inverse-Gamma prior (no MCMC required)."""
+    df = session.require_data()
+
+    # strip "ols" or ": ols" prefix
+    clean = re.sub(r'^\s*:?\s*ols\s+', '', args, flags=re.IGNORECASE)
+    positional, opts = _stata_opts(clean)
+
+    dep = positional[0] if positional else ""
+    indeps = [c for c in positional[1:] if c in df.columns]
+
+    if not dep or not indeps:
+        return "Usage: bayes: ols depvar indepvar1 indepvar2 ... [, samples(4000)]"
+
+    n_samples = int(opts.get("samples", 4000))
+    prior_scale = float(opts.get("priorscale", 10.0))
+    ci = float(opts.get("ci", 0.95))
+
+    try:
+        from openstat.stats.bayesian import bayes_ols
+        result = bayes_ols(
+            df, dep, indeps,
+            n_samples=n_samples,
+            prior_scale=prior_scale,
+            credible_interval=ci,
+        )
+    except Exception as exc:
+        return f"bayes error: {exc}"
+
+    ci_pct = int(ci * 100)
+    lines = [f"\n{result['model']}", "=" * 70]
+    lines.append(f"  Dependent: {dep}   N = {result['n_obs']}   "
+                 f"Draws = {n_samples}   R² ≈ {result['r_squared']:.4f}")
+    lines.append(f"  σ̂ = {result['sigma_mean']:.4f} (±{result['sigma_std']:.4f})")
+    lines.append("")
+    lines.append(
+        f"  {'Variable':<20}  {'Post. Mean':>12}  {'Post. SD':>10}  "
+        f"{'CI Lo ({ci_pct}%)':>12}  {'CI Hi':>12}  {'P(β>0)':>8}"
+    )
+    lines.append("  " + "-" * 66)
+    for name, stats in result["coefficients"].items():
+        lo_key = f"ci_{ci_pct}_lo"
+        hi_key = f"ci_{ci_pct}_hi"
+        lines.append(
+            f"  {name:<20}  {stats['mean']:>12.4f}  {stats['std']:>10.4f}  "
+            f"  {stats[lo_key]:>12.4f}  {stats[hi_key]:>12.4f}  {stats['prob_positive']:>8.4f}"
+        )
+    lines.append("=" * 70)
+    lines.append(f"  Prior: Normal(0, {prior_scale}²) on coefficients  |  IG(0.001, 0.001) on σ²")
+    session._last_model = result
+    return "\n".join(lines)