openstat-cli 1.0.0__py3-none-any.whl

openstat/commands/network_cmds.py

@@ -0,0 +1,288 @@
+"""Network analysis commands: network descriptives, centrality, community detection."""
+
+from __future__ import annotations
+
+from openstat.commands.base import command
+from openstat.session import Session
+
+
+def _require_nx():
+    try:
+        import networkx as nx
+        return nx
+    except ImportError:
+        raise ImportError("Network analysis requires networkx. Install with: pip install networkx")
+
+
+@command("network", usage="network <subcommand> ...")
+def cmd_network(session: Session, args: str) -> str:
+    """Network analysis using NetworkX.
+
+    Subcommands:
+      network build   from <source_col> to <target_col> [weight=<col>]
+      network describe
+      network centrality  [--degree|--betweenness|--closeness|--eigenvector]
+      network community   [--louvain|--greedy]
+      network plot        [--layout=spring|circular|kamada]
+
+    Examples:
+      network build from sender to receiver
+      network build from from_node to to_node weight=strength
+      network describe
+      network centrality --degree
+      network community --greedy
+      network plot
+    """
+    from openstat.commands.base import CommandArgs
+    ca = CommandArgs(args)
+    if not ca.positional:
+        return cmd_network.__doc__ or "Usage: network <subcommand>"
+
+    subcmd = ca.positional[0].lower()
+
+    if subcmd == "build":
+        return _build_network(session, args)
+    elif subcmd == "describe":
+        return _describe_network(session)
+    elif subcmd == "centrality":
+        return _centrality(session, args)
+    elif subcmd == "community":
+        return _community(session, args)
+    elif subcmd == "plot":
+        return _plot_network(session, args)
+    else:
+        return (
+            f"Unknown subcommand: {subcmd}\n"
+            "Available: build, describe, centrality, community, plot"
+        )
+
+
+def _build_network(session: Session, args: str) -> str:
+    nx = _require_nx()
+    import re
+
+    df = session.require_data()
+
+    m_from = re.search(r"from\s+(\w+)", args)
+    m_to = re.search(r"to\s+(\w+)", args)
+    if not m_from or not m_to:
+        return "Usage: network build from <source_col> to <target_col> [weight=<col>]"
+
+    src_col = m_from.group(1)
+    tgt_col = m_to.group(1)
+
+    m_w = re.search(r"weight[= ](\w+)", args)
+    weight_col = m_w.group(1) if m_w else None
+
+    for c in [src_col, tgt_col] + ([weight_col] if weight_col else []):
+        if c not in df.columns:
+            return f"Column not found: {c}"
+
+    sub = df.select([c for c in [src_col, tgt_col, weight_col] if c]).drop_nulls()
+
+    G = nx.DiGraph() if "--directed" in args else nx.Graph()
+    for row in sub.iter_rows():
+        src, tgt = str(row[0]), str(row[1])
+        w = float(row[2]) if weight_col else 1.0
+        if G.has_edge(src, tgt):
+            G[src][tgt]["weight"] += w
+        else:
+            G.add_edge(src, tgt, weight=w)
+
+    session._network = G
+    session._network_weight_col = weight_col
+
+    directed_str = "directed" if G.is_directed() else "undirected"
+    return (
+        f"Network built: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges ({directed_str})\n"
+        f"Source: '{src_col}' → Target: '{tgt_col}'"
+        + (f"  Weight: '{weight_col}'" if weight_col else "")
+        + "\nUse 'network describe', 'network centrality', 'network community', 'network plot'"
+    )
+
+
+def _describe_network(session: Session) -> str:
+    nx = _require_nx()
+    G = getattr(session, "_network", None)
+    if G is None:
+        return "No network built. Use 'network build from <src> to <tgt>' first."
+
+    n = G.number_of_nodes()
+    e = G.number_of_edges()
+    density = nx.density(G)
+    is_connected = nx.is_connected(G.to_undirected()) if G.is_directed() else nx.is_connected(G)
+
+    lines = [
+        f"Nodes: {n}",
+        f"Edges: {e}",
+        f"Density: {density:.4f}",
+        f"Connected: {is_connected}",
+    ]
+
+    if not G.is_directed():
+        if n > 0 and e > 0:
+            try:
+                avg_clust = nx.average_clustering(G)
+                lines.append(f"Avg Clustering: {avg_clust:.4f}")
+            except Exception:
+                pass
+            try:
+                if nx.is_connected(G):
+                    avg_path = nx.average_shortest_path_length(G)
+                    lines.append(f"Avg Path Length: {avg_path:.4f}")
+                    diam = nx.diameter(G)
+                    lines.append(f"Diameter: {diam}")
+            except Exception:
+                pass
+
+    # Degree distribution summary
+    import numpy as np
+    degrees = [d for _, d in G.degree()]
+    if degrees:
+        lines += [
+            "",
+            f"Degree — Min: {min(degrees)}  Mean: {np.mean(degrees):.2f}  Max: {max(degrees)}",
+        ]
+
+    # Top 5 nodes by degree
+    top = sorted(G.degree(), key=lambda x: x[1], reverse=True)[:5]
+    if top:
+        lines.append("")
+        lines.append("Top nodes by degree:")
+        for node, deg in top:
+            lines.append(f"  {str(node):<20} degree = {deg}")
+
+    return "\n" + "=" * 50 + "\nNetwork Descriptives\n" + "=" * 50 + "\n" + "\n".join(lines) + "\n" + "=" * 50
+
+
+def _centrality(session: Session, args: str) -> str:
+    nx = _require_nx()
+    G = getattr(session, "_network", None)
+    if G is None:
+        return "No network built. Use 'network build' first."
+
+    if "--betweenness" in args:
+        label, scores = "Betweenness Centrality", nx.betweenness_centrality(G)
+    elif "--closeness" in args:
+        label, scores = "Closeness Centrality", nx.closeness_centrality(G)
+    elif "--eigenvector" in args:
+        try:
+            label, scores = "Eigenvector Centrality", nx.eigenvector_centrality(G, max_iter=1000)
+        except nx.PowerIterationFailedConvergence:
+            label, scores = "Degree Centrality (fallback)", nx.degree_centrality(G)
+    else:
+        label, scores = "Degree Centrality", nx.degree_centrality(G)
+
+    top_n = 20
+    sorted_nodes = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_n]
+
+    lines = [f"Top {min(top_n, len(sorted_nodes))} nodes by {label}:"]
+    lines.append(f"  {'Node':<25} {'Score':>10}")
+    lines.append("  " + "-" * 37)
+    for node, score in sorted_nodes:
+        lines.append(f"  {str(node):<25} {score:>10.4f}")
+
+    return "\n" + "=" * 50 + f"\n{label}\n" + "=" * 50 + "\n" + "\n".join(lines) + "\n" + "=" * 50
+
+
+def _community(session: Session, args: str) -> str:
+    nx = _require_nx()
+    G = getattr(session, "_network", None)
+    if G is None:
+        return "No network built. Use 'network build' first."
+
+    G_undir = G.to_undirected() if G.is_directed() else G
+
+    try:
+        if "--louvain" in args:
+            try:
+                import community as community_louvain
+                partition = community_louvain.best_partition(G_undir)
+                communities = {}
+                for node, comm_id in partition.items():
+                    communities.setdefault(comm_id, []).append(node)
+                method = "Louvain"
+            except ImportError:
+                return "Louvain requires python-louvain. Install with: pip install python-louvain"
+        else:
+            # Greedy modularity (built into networkx)
+            from networkx.algorithms.community import greedy_modularity_communities
+            comms = list(greedy_modularity_communities(G_undir))
+            communities = {i: list(c) for i, c in enumerate(comms)}
+            method = "Greedy Modularity"
+
+        n_comm = len(communities)
+        modularity = None
+        try:
+            from networkx.algorithms.community.quality import modularity as nx_mod
+            modularity = nx_mod(G_undir, [set(v) for v in communities.values()])
+        except Exception:
+            pass
+
+        lines = [
+            f"Method: {method}",
+            f"Communities found: {n_comm}",
+        ]
+        if modularity is not None:
+            lines.append(f"Modularity: {modularity:.4f}")
+        lines.append("")
+
+        for cid, members in sorted(communities.items(), key=lambda x: -len(x[1])):
+            sample = ", ".join(str(m) for m in members[:5])
+            if len(members) > 5:
+                sample += f", ... (+{len(members)-5})"
+            lines.append(f"  Community {cid+1:>3}: {len(members):>4} nodes  [{sample}]")
+
+        return "\n" + "=" * 55 + f"\nCommunity Detection ({method})\n" + "=" * 55 + "\n" + "\n".join(lines) + "\n" + "=" * 55
+
+    except Exception as exc:
+        return f"community detection error: {exc}"
+
+
+def _plot_network(session: Session, args: str) -> str:
+    nx = _require_nx()
+    G = getattr(session, "_network", None)
+    if G is None:
+        return "No network built. Use 'network build' first."
+
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    from openstat.plots.plotter import _unique_path
+
+    # Layout
+    if "--circular" in args:
+        pos = nx.circular_layout(G)
+        layout_name = "circular"
+    elif "--kamada" in args:
+        pos = nx.kamada_kawai_layout(G)
+        layout_name = "kamada-kawai"
+    else:
+        pos = nx.spring_layout(G, seed=42)
+        layout_name = "spring"
+
+    n = G.number_of_nodes()
+    fig_size = min(max(6, n * 0.3), 16)
+    fig, ax = plt.subplots(figsize=(fig_size, fig_size))
+
+    node_size = max(50, min(500, 2000 // (n + 1)))
+    nx.draw_networkx(
+        G, pos=pos, ax=ax,
+        node_size=node_size,
+        node_color="#4C72B0",
+        edge_color="#AAAAAA",
+        font_size=max(6, min(10, 120 // (n + 1))),
+        arrows=G.is_directed(),
+        width=0.8,
+        alpha=0.9,
+    )
+    ax.set_title(f"Network ({n} nodes, {G.number_of_edges()} edges) — {layout_name} layout")
+    ax.axis("off")
+    fig.tight_layout()
+
+    session.output_dir.mkdir(parents=True, exist_ok=True)
+    path = _unique_path(session.output_dir, "network_plot")
+    fig.savefig(path, dpi=150)
+    plt.close(fig)
+    session.plot_paths.append(str(path))
+    return f"Network plot saved: {path}"

openstat/commands/nlquery_cmds.py

@@ -0,0 +1,161 @@
+"""Natural language query: 'ask' command using OpenAI or Anthropic API."""
+
+from __future__ import annotations
+
+from openstat.commands.base import command, CommandArgs, friendly_error
+from openstat.session import Session
+
+
+def _build_context(session: Session) -> str:
+    """Build a compact dataset context string for the LLM."""
+    import polars as pl
+
+    if session.df is None:
+        return "No dataset loaded."
+
+    df = session.df
+    NUMERIC = (pl.Float32, pl.Float64, pl.Int8, pl.Int16, pl.Int32, pl.Int64,
+               pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64)
+
+    col_info = []
+    for c in df.columns[:30]:  # cap at 30 cols
+        dtype = str(df[c].dtype)
+        n_miss = df[c].null_count()
+        if df[c].dtype in NUMERIC:
+            col_data = df[c].drop_nulls()
+            if col_data.len() > 0:
+                extra = f"mean={col_data.mean():.2f}, sd={col_data.std():.2f}"
+            else:
+                extra = "all null"
+        else:
+            n_uniq = df[c].drop_nulls().n_unique()
+            extra = f"{n_uniq} unique values"
+        miss_str = f", {n_miss} missing" if n_miss else ""
+        col_info.append(f"  {c} ({dtype}{miss_str}): {extra}")
+
+    lines = [
+        f"Dataset: {session.dataset_name or 'unknown'}",
+        f"Shape: {df.height} rows × {df.width} columns",
+        "Columns:",
+    ] + col_info
+
+    if session.results:
+        lines.append("\nLast model: " + session.results[-1].name +
+                     " — " + session.results[-1].formula)
+
+    return "\n".join(lines)
+
+
+def _ask_openai(question: str, context: str, model: str) -> str:
+    try:
+        from openai import OpenAI
+    except ImportError:
+        return None  # type: ignore
+
+    client = OpenAI()  # uses OPENAI_API_KEY env var
+    resp = client.chat.completions.create(
+        model=model or "gpt-4o-mini",
+        messages=[
+            {
+                "role": "system",
+                "content": (
+                    "You are a helpful statistical analysis assistant for OpenStat, "
+                    "a Python-based data analysis REPL similar to Stata. "
+                    "Answer questions about the dataset concisely. "
+                    "When relevant, suggest the exact OpenStat command to use."
+                ),
+            },
+            {
+                "role": "user",
+                "content": f"Dataset context:\n{context}\n\nQuestion: {question}",
+            },
+        ],
+        max_tokens=500,
+    )
+    return resp.choices[0].message.content.strip()
+
+
+def _ask_anthropic(question: str, context: str, model: str) -> str:
+    try:
+        import anthropic
+    except ImportError:
+        return None  # type: ignore
+
+    client = anthropic.Anthropic()  # uses ANTHROPIC_API_KEY env var
+    resp = client.messages.create(
+        model=model or "claude-haiku-4-5-20251001",
+        max_tokens=500,
+        system=(
+            "You are a helpful statistical analysis assistant for OpenStat, "
+            "a Python-based data analysis REPL similar to Stata. "
+            "Answer questions about the dataset concisely. "
+            "When relevant, suggest the exact OpenStat command to use."
+        ),
+        messages=[
+            {
+                "role": "user",
+                "content": f"Dataset context:\n{context}\n\nQuestion: {question}",
+            }
+        ],
+    )
+    return resp.content[0].text.strip()
+
+
+@command("ask", usage='ask "<natural language question>"')
+def cmd_ask(session: Session, args: str) -> str:
+    """Ask a natural language question about your dataset (requires AI API key).
+
+    Uses OpenAI (OPENAI_API_KEY) or Anthropic (ANTHROPIC_API_KEY), whichever
+    is available. The assistant can suggest OpenStat commands, explain results,
+    and answer statistical questions.
+
+    Options:
+      --provider=openai|anthropic   (default: auto-detect from env)
+      --model=<model-name>          (default: gpt-4o-mini / claude-haiku-4-5)
+
+    Examples:
+      ask "What's the correlation between income and education?"
+      ask "Which variables have the most missing data?"
+      ask "What regression model should I use for this binary outcome?"
+      ask "Explain the OLS results" --provider=anthropic
+    """
+    import os
+
+    ca = CommandArgs(args)
+    # Question is the rest of args after stripping flags
+    question = ca.strip_flags_and_options().strip().strip('"\'')
+    if not question:
+        return 'Usage: ask "<question>" [--provider=openai|anthropic] [--model=<name>]'
+
+    provider = ca.options.get("provider", "").lower()
+    model = ca.options.get("model", "")
+
+    context = _build_context(session)
+
+    # Auto-detect provider
+    if not provider:
+        if os.getenv("ANTHROPIC_API_KEY"):
+            provider = "anthropic"
+        elif os.getenv("OPENAI_API_KEY"):
+            provider = "openai"
+        else:
+            return (
+                "No AI API key found.\n"
+                "Set OPENAI_API_KEY or ANTHROPIC_API_KEY environment variable.\n"
+                "Install: pip install openai  OR  pip install anthropic"
+            )
+
+    try:
+        if provider == "openai":
+            result = _ask_openai(question, context, model)
+            if result is None:
+                return "openai package not installed. Run: pip install openai"
+        elif provider == "anthropic":
+            result = _ask_anthropic(question, context, model)
+            if result is None:
+                return "anthropic package not installed. Run: pip install anthropic"
+        else:
+            return f"Unknown provider: {provider}. Use 'openai' or 'anthropic'."
+        return result
+    except Exception as e:
+        return friendly_error(e, "ask")

openstat/commands/nonparam_cmds.py

@@ -0,0 +1,149 @@
+"""Nonparametric test commands: ranksum, signrank, kwallis, spearman."""
+
+from __future__ import annotations
+
+import re
+
+from openstat.commands.base import command
+from openstat.session import Session
+from openstat.stats.nonparametric import (
+    spearman_corr,
+    ranksum_test,
+    signrank_test,
+    kruskal_wallis_test,
+)
+
+
+def _stata_opts(raw: str) -> tuple[list[str], dict[str, str]]:
+    opts: dict[str, str] = {}
+    for m in re.finditer(r'(\w+)\(([^)]*)\)', raw):
+        opts[m.group(1).lower()] = m.group(2)
+    rest = re.sub(r'\w+\([^)]*\)', '', raw)
+    positional = [t.strip(',') for t in rest.split() if t.strip(',')]
+    return positional, opts
+
+
+def _fmt(d: dict) -> str:
+    lines = [f"\n{d.get('test', 'Result')}", "-" * 55]
+    skip = {"test", "groups", "n_per_group", "_model"}
+    for k, v in d.items():
+        if k in skip:
+            continue
+        if isinstance(v, float):
+            lines.append(f"  {k:<30} {v:.6f}")
+        elif isinstance(v, list):
+            lines.append(f"  {k:<30} {v}")
+        else:
+            lines.append(f"  {k:<30} {v}")
+    lines.append("-" * 55)
+    return "\n".join(lines)
+
+
+@command("ranksum", usage="ranksum var by(groupvar) [--less|--greater]")
+def cmd_ranksum(session: Session, args: str) -> str:
+    """Wilcoxon rank-sum (Mann-Whitney U) test for two independent groups."""
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: ranksum var by(groupvar)"
+
+    var = positional[0]
+    by = opts.get("by")
+    if by is None:
+        return "Specify group variable: ranksum var by(groupvar)"
+
+    alt = "two-sided"
+    if "--less" in args:
+        alt = "less"
+    elif "--greater" in args:
+        alt = "greater"
+
+    try:
+        r = ranksum_test(df, var, by, alternative=alt)
+        return _fmt(r)
+    except Exception as exc:
+        return f"ranksum error: {exc}"
+
+
+@command("signrank", usage="signrank var1 [var2] [mu(0)]")
+def cmd_signrank(session: Session, args: str) -> str:
+    """Wilcoxon signed-rank test (one-sample or paired)."""
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: signrank var1 [var2] [mu(0)]"
+
+    mu = float(opts.get("mu", 0.0))
+    var1 = positional[0]
+    var2 = positional[1] if len(positional) > 1 and positional[1] in df.columns else None
+
+    try:
+        r = signrank_test(df, var1, var2, mu=mu)
+        return _fmt(r)
+    except Exception as exc:
+        return f"signrank error: {exc}"
+
+
+@command("kwallis", usage="kwallis var by(groupvar)")
+def cmd_kwallis(session: Session, args: str) -> str:
+    """Kruskal-Wallis H test for k independent groups."""
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    if not positional:
+        return "Usage: kwallis var by(groupvar)"
+
+    var = positional[0]
+    by = opts.get("by")
+    if by is None:
+        return "Specify group variable: kwallis var by(groupvar)"
+
+    try:
+        r = kruskal_wallis_test(df, var, by)
+        lines = [_fmt(r)]
+        lines.append("\nGroup counts:")
+        for g, n in zip(r["groups"], r["n_per_group"]):
+            lines.append(f"  {g!s:<20} n = {n}")
+        return "\n".join(lines)
+    except Exception as exc:
+        return f"kwallis error: {exc}"
+
+
+@command("spearman", usage="spearman var1 var2 [var3 ...]")
+def cmd_spearman(session: Session, args: str) -> str:
+    """Spearman rank correlation matrix."""
+    df = session.require_data()
+    positional, opts = _stata_opts(args)
+    cols = [c for c in positional if c in df.columns]
+    if len(cols) < 2:
+        return "spearman requires at least 2 numeric variables."
+
+    try:
+        r = spearman_corr(df, cols)
+    except Exception as exc:
+        return f"spearman error: {exc}"
+
+    rho = r["rho"]
+    pvals = r["pvalues"]
+    k = len(cols)
+    w = max(len(c) for c in cols) + 2
+
+    lines = ["\nSpearman Rank Correlation", "=" * (w + k * 9 + 2)]
+    header = " " * w + "".join(f"  {c[:7]:>7}" for c in cols)
+    lines.append(header)
+    lines.append("-" * (w + k * 9 + 2))
+    for i, ci in enumerate(cols):
+        row = f"{ci:<{w}}"
+        for j in range(k):
+            row += f"  {rho[i][j]:>7.4f}"
+        lines.append(row)
+    lines.append("")
+    lines.append("P-values:")
+    for i, ci in enumerate(cols):
+        row = f"{ci:<{w}}"
+        for j in range(k):
+            if i == j:
+                row += f"  {'  .':>7}"
+            else:
+                row += f"  {pvals[i][j]:>7.4f}"
+        lines.append(row)
+    return "\n".join(lines)