statchat-app 1.0.0__py3-none-any.whl

statchat/__main__.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Stat Chat — Data Cleaning, Normalization, Statistical Analysis & Iterative Adjustment
+
+Entry points:
+  GUI:  statchat
+  CLI:  statchat --cli --input data.csv [options]
+  Also: python -m statchat
+"""
+
+import argparse
+import sys
+
+
+# ── Dependency check ───────────────────────────────────────────────────────────
+
+REQUIRED = {
+    "pandas":      "pandas",
+    "numpy":       "numpy",
+    "scipy":       "scipy",
+    "sklearn":     "scikit-learn",
+    "reportlab":   "reportlab",
+    "matplotlib":  "matplotlib",
+    "openpyxl":    "openpyxl",
+    "requests":    "requests",
+    "PIL":         "Pillow",
+}
+
+
+def check_dependencies():
+    missing = []
+    for module, pip_name in REQUIRED.items():
+        try:
+            __import__(module)
+        except ImportError:
+            missing.append(pip_name)
+    return missing
+
+
+def prompt_install(missing):
+    cmd = "pip install " + " ".join(missing)
+    print("=" * 60)
+    print("  Stat Chat — Missing Dependencies")
+    print("=" * 60)
+    print(f"\n  The following packages are required but not installed:\n")
+    for p in missing:
+        print(f"    • {p}")
+    print(f"\n  Install them by running:\n\n    {cmd}\n")
+
+    try:
+        import tkinter as tk
+        from tkinter import messagebox
+        root = tk.Tk()
+        root.withdraw()
+        answer = messagebox.askyesno(
+            "Stat Chat — Missing packages",
+            "The following required packages are missing:\n\n"
+            + "\n".join(f"  • {p}" for p in missing)
+            + f"\n\nInstall them now?\n\n{cmd}",
+        )
+        root.destroy()
+        if answer:
+            import subprocess
+            subprocess.check_call([sys.executable, "-m", "pip", "install"] + missing)
+            print("\n  ✓  Installation complete. Relaunch Stat Chat.")
+    except Exception:
+        pass
+
+    sys.exit(1)
+
+
+# ── Launchers ──────────────────────────────────────────────────────────────────
+
+def run_gui():
+    try:
+        import tkinter as tk
+    except ImportError:
+        print("[ERROR] Tkinter is not available.")
+        print("  Linux:        sudo apt install python3-tk")
+        print("  macOS/Windows: reinstall Python from python.org")
+        sys.exit(1)
+
+    from statchat.gui.app import StatChatApp
+    root = tk.Tk()
+    StatChatApp(root)
+    root.mainloop()
+
+
+def run_cli(args):
+    from statchat.cli.runner import CLIRunner
+    CLIRunner(args).run()
+
+
+# ── CLI argument parser ────────────────────────────────────────────────────────
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog="statchat",
+        description="Stat Chat — Data Cleaning, Normalization & Statistical Analysis",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument("--cli", action="store_true", help="Run in CLI mode (no GUI)")
+    parser.add_argument("--input",  "-i", type=str, help="Input CSV or Excel file")
+    parser.add_argument("--output", "-o", type=str, help="Path to save processed data")
+    parser.add_argument("--output-format", choices=["csv", "xlsx", "json"], default="csv",
+                        help="Output format (default: csv)")
+    parser.add_argument("--report", "-r", type=str, help="Path to save PDF report")
+
+    cg = parser.add_argument_group("Cleaning Options")
+    cg.add_argument("--drop-duplicates", action="store_true", help="Remove duplicate rows")
+    cg.add_argument("--drop-nulls",      action="store_true", help="Drop rows with null values")
+    cg.add_argument("--fill-nulls", choices=["mean", "median", "mode", "zero"],
+                    help="Fill null values with strategy")
+
+    ng = parser.add_argument_group("Normalization Options")
+    ng.add_argument("--normalize", choices=["zscore", "minmax", "robust"],
+                    help="Normalization method")
+    ng.add_argument("--norm-mean", type=float, default=0.0,
+                    help="Z-score target mean (default: 0.0)")
+    ng.add_argument("--norm-std",  type=float, default=1.0,
+                    help="Z-score target std  (default: 1.0)")
+
+    ag = parser.add_argument_group("Analysis Metrics")
+    ag.add_argument("--central-tendency", action="store_true", help="Mean, median, mode")
+    ag.add_argument("--dispersion",       action="store_true", help="Std dev, variance, IQR, range")
+    ag.add_argument("--shape",            action="store_true", help="Skewness & kurtosis")
+    ag.add_argument("--correlation",      action="store_true", help="Correlation matrix")
+    ag.add_argument("--percentiles",      action="store_true", help="P5–P95 percentiles")
+    ag.add_argument("--roc-auc", type=str, metavar="TARGET_COL",
+                    help="ROC-AUC vs a binary target column")
+    ag.add_argument("--all-metrics", action="store_true", help="Run all metrics")
+
+    adj = parser.add_argument_group("Adjustment Options (requires LLM backend)")
+    adj.add_argument("--adjust", action="append", metavar="INSTRUCTION",
+                     help="Natural-language adjustment (repeatable).\n"
+                          "e.g. --adjust 'Add 1000 to spend'")
+    adj.add_argument("--adjust-image", type=str, metavar="IMAGE_PATH",
+                     help="Annotated report image — vision model extracts instructions")
+
+    bg = parser.add_argument_group("LLM Backend Options")
+    bg.add_argument("--backend", choices=["claude", "lmstudio", "lmstudio_vision"],
+                    default="claude", help="LLM provider (default: claude)")
+    bg.add_argument("--lmstudio-url",          type=str, default="http://localhost:1234",
+                    metavar="URL",      help="LM Studio server URL")
+    bg.add_argument("--lmstudio-model",        type=str, default="", metavar="MODEL_ID",
+                    help="LM Studio text model ID")
+    bg.add_argument("--lmstudio-vision-model", type=str, default="", metavar="MODEL_ID",
+                    help="LM Studio vision model ID")
+
+    return parser
+
+
+# ── Main entry point ───────────────────────────────────────────────────────────
+
+def main():
+    missing = check_dependencies()
+    if missing:
+        prompt_install(missing)
+
+    parser = build_parser()
+    args   = parser.parse_args()
+
+    if args.cli:
+        if not args.input:
+            parser.error("--input is required in CLI mode")
+        run_cli(args)
+    else:
+        run_gui()
+
+
+if __name__ == "__main__":
+    main()

statchat/cli/runner.py

@@ -0,0 +1,197 @@
+"""CLI mode runner."""
+
+import sys
+import base64
+from pathlib import Path
+from statchat.core.loader import load_file, save_file, get_file_info
+from statchat.core.cleaner import apply_cleaning, apply_normalization
+from statchat.core.analyzer import run_analysis
+from statchat.core.reporter import generate_report
+
+
+class CLIRunner:
+    def __init__(self, args):
+        self.args = args
+
+    def run(self):
+        args = self.args
+        print(f"\n── Stat Chat CLI ──────────────────────────────────")
+        print(f"Loading: {args.input}")
+
+        try:
+            original_df = load_file(args.input)
+        except Exception as e:
+            print(f"[ERROR] {e}")
+            sys.exit(1)
+
+        info = get_file_info(original_df)
+        print(f"  Rows: {info['rows']}  |  Columns: {info['columns']}")
+        print(f"  Numeric cols: {info['numeric_columns']}")
+
+        # ── Configure LLM backend ─────────────────────────────────────────────
+        if args.adjust or args.adjust_image:
+            from statchat.core.llm_backend import LLMConfig, set_config
+            provider = getattr(args, "backend", "claude")
+            cfg = LLMConfig(
+                provider=provider,
+                lmstudio_base_url=getattr(args, "lmstudio_url", "http://localhost:1234"),
+                lmstudio_model=getattr(args, "lmstudio_model", ""),
+                lmstudio_vision_model=getattr(args, "lmstudio_vision_model", ""),
+            )
+            set_config(cfg)
+            print(f"  [backend] {provider}"
+                  + (f" @ {cfg.lmstudio_base_url}" if "lmstudio" in provider else ""))
+
+        # ── Cleaning ──────────────────────────────────────────────────────────
+        clean_opts = {
+            "drop_duplicates": args.drop_duplicates,
+            "drop_nulls":      args.drop_nulls,
+            "fill_nulls":      args.fill_nulls,
+        }
+        cleaned_df, clean_log = apply_cleaning(original_df.copy(), clean_opts)
+        for msg in clean_log:
+            print(f"  [clean] {msg}")
+
+        # ── Normalization ─────────────────────────────────────────────────────
+        norm_opts = {
+            "normalize": args.normalize,
+            "norm_mean": args.norm_mean,
+            "norm_std":  args.norm_std,
+        }
+        cleaned_df, norm_log = apply_normalization(cleaned_df, norm_opts)
+        for msg in norm_log:
+            print(f"  [norm]  {msg}")
+
+        # ── Text adjustments ──────────────────────────────────────────────────
+        adj_history = []
+        working_df  = cleaned_df.copy()
+
+        if args.adjust:
+            from statchat.core.adjuster import apply_instructions
+            import datetime
+            print(f"\n── Applying Text Adjustments ──────────────────────")
+            for instruction in args.adjust:
+                print(f"  Instruction: \"{instruction}\"")
+                try:
+                    working_df, descriptions, ops = apply_instructions(working_df, instruction)
+                    for d in descriptions:
+                        print(f"  [adjust] {d}")
+                    adj_history.append({
+                        "version":     len(adj_history) + 1,
+                        "df":          working_df.copy(),
+                        "description": "; ".join(descriptions),
+                        "ops":         ops,
+                        "timestamp":   datetime.datetime.now().strftime("%H:%M:%S"),
+                    })
+                except Exception as e:
+                    print(f"  [ERROR] Could not apply: {e}")
+                    sys.exit(1)
+
+        # ── Image annotation ──────────────────────────────────────────────────
+        if args.adjust_image:
+            from statchat.core.adjuster import apply_instructions_from_image
+            import datetime
+            print(f"\n── Parsing Annotated Image ────────────────────────")
+            img_path = Path(args.adjust_image)
+            if not img_path.exists():
+                print(f"  [ERROR] Image not found: {img_path}")
+                sys.exit(1)
+            print(f"  Image: {img_path.name}")
+            try:
+                with open(img_path, "rb") as f:
+                    image_b64 = base64.b64encode(f.read()).decode("utf-8")
+                working_df, descriptions, ops = apply_instructions_from_image(
+                    working_df, image_b64
+                )
+                for d in descriptions:
+                    print(f"  [adjust] {d}")
+                adj_history.append({
+                    "version":     len(adj_history) + 1,
+                    "df":          working_df.copy(),
+                    "description": f"From image '{img_path.name}': " + "; ".join(descriptions),
+                    "ops":         ops,
+                    "timestamp":   datetime.datetime.now().strftime("%H:%M:%S"),
+                })
+            except Exception as e:
+                print(f"  [ERROR] Could not parse image: {e}")
+                sys.exit(1)
+
+        # ── Save output data ──────────────────────────────────────────────────
+        final_df = working_df  # post-adjustment (or just cleaned if no adjustments)
+
+        if args.output:
+            out_path = save_file(final_df, args.output, fmt=args.output_format)
+            print(f"\n  [saved] Data → {out_path}")
+
+        # ── Analysis ─────────────────────────────────────────────────────────
+        df_for_analysis = final_df.copy().reset_index(drop=True)
+        if args.roc_auc and args.roc_auc in original_df.columns:
+            df_for_analysis[args.roc_auc] = original_df[args.roc_auc].reset_index(drop=True)
+
+        analysis_opts = {
+            "central_tendency": args.central_tendency,
+            "dispersion":       args.dispersion,
+            "shape":            args.shape,
+            "correlation":      args.correlation,
+            "percentiles":      args.percentiles,
+            "normality":        False,
+            "roc_auc":          args.roc_auc,
+            "all_metrics":      args.all_metrics,
+            "original_df":      original_df,
+        }
+        results = run_analysis(df_for_analysis, analysis_opts)
+
+        if results:
+            print(f"\n── Analysis Results ───────────────────────────────")
+            for section, data in results.items():
+                if section == "error":
+                    print(f"  [error] {data}")
+                elif section == "correlation":
+                    print(f"  [correlation] Matrix computed "
+                          f"({len(data.columns)} x {len(data.columns)})")
+                elif section == "roc_auc":
+                    for col, v in data.items():
+                        if "auc" in v:
+                            print(f"  [roc_auc] {col}: AUC = {v['auc']:.4f}")
+                        elif "error" in v:
+                            print(f"  [roc_auc] {col}: ERROR - {v['error']}")
+                else:
+                    print(f"\n  [{section}]")
+                    for col, vals in data.items():
+                        summary = "  ".join(
+                            f"{k}={v:.4f}" if isinstance(v, float) else f"{k}={v}"
+                            for k, v in vals.items()
+                        )
+                        print(f"    {col}: {summary}")
+
+        # ── PDF Report ────────────────────────────────────────────────────────
+        if args.report:
+            print(f"\n── Generating PDF Report ──────────────────────────")
+            try:
+                # Seed history with original if adjustments were made
+                full_history = []
+                if adj_history:
+                    import datetime
+                    full_history = [{
+                        "version":     0,
+                        "df":          original_df.copy(),
+                        "description": "Original dataset",
+                        "ops":         [],
+                        "timestamp":   datetime.datetime.now().strftime("%H:%M:%S"),
+                    }] + adj_history
+
+                out = generate_report(
+                    filepath=args.report,
+                    original_df=original_df,
+                    cleaned_df=final_df,
+                    analysis_results=results,
+                    clean_log=clean_log,
+                    norm_log=norm_log,
+                    source_file=args.input,
+                    adjustment_history=full_history if full_history else None,
+                )
+                print(f"  [saved] Report → {out}")
+            except Exception as e:
+                print(f"  [ERROR] Could not generate report: {e}")
+
+        print("\n── Done ───────────────────────────────────────────\n")

statchat/core/adjuster.py

@@ -0,0 +1,265 @@
+"""
+core/adjuster.py
+----------------
+Interprets natural-language adjustment instructions via the configured LLM
+backend (Claude API or LM Studio) and applies them to a pandas DataFrame.
+
+Adjustment instructions are translated into a small safe DSL:
+  { "op": "add"|"subtract"|"multiply"|"divide"|"set"|"clip"|"round"|
+          "rename"|"drop_col"|"filter_rows"|"fillna"|"abs"|"log"|"sqrt",
+    "column": "<col>",           # target column (or "*" for all numeric)
+    "value":  <number|str>,      # operand (where relevant)
+    "condition": "<expr>",       # optional pandas .query() string
+    "description": "<human summary>"
+  }
+
+Multiple operations are returned as a list.
+"""
+
+import json
+import math
+from typing import Optional
+import pandas as pd
+import numpy as np
+
+from statchat.core.llm_backend import chat_json, get_config, LLMConfig
+
+
+# ── System prompt ─────────────────────────────────────────────────────────────
+
+_SYSTEM = """You are a data transformation assistant. The user will describe adjustments
+they want to make to a pandas DataFrame in plain English.
+
+You must respond with ONLY a JSON array of operation objects — no prose, no markdown fences.
+
+Each object has these keys:
+  op          : one of add | subtract | multiply | divide | set | clip | round |
+                rename | drop_col | filter_rows | fillna | abs | log | sqrt
+  column      : the column name to operate on (use "*" to mean all numeric columns)
+  value       : numeric value, new name string, or clip bounds [min, max]
+  condition   : (optional) a pandas DataFrame.query() expression string to limit rows
+  description : a short human-readable summary of this operation
+
+Examples:
+  User: "Add $1000 to each value in spend"
+  -> [{{"op":"add","column":"spend","value":1000,"description":"Add 1000 to spend"}}]
+
+  User: "Multiply income and age by 1.1"
+  -> [{{"op":"multiply","column":"income","value":1.1,"description":"Multiply income by 1.1"}},
+     {{"op":"multiply","column":"age","value":1.1,"description":"Multiply age by 1.1"}}]
+
+  User: "Rename 'score' to 'risk_score' and drop the 'region' column"
+  -> [{{"op":"rename","column":"score","value":"risk_score","description":"Rename score to risk_score"}},
+     {{"op":"drop_col","column":"region","description":"Drop column region"}}]
+
+  User: "Clip spend between 0 and 5000"
+  -> [{{"op":"clip","column":"spend","value":[0,5000],"description":"Clip spend to [0, 5000]"}}]
+
+  User: "Set all income values below 20000 to 20000"
+  -> [{{"op":"clip","column":"income","value":[20000,null],"description":"Floor income at 20000"}}]
+
+  User: "Fill nulls in income with 0"
+  -> [{{"op":"fillna","column":"income","value":0,"description":"Fill income nulls with 0"}}]
+
+  User: "Round age to 0 decimal places"
+  -> [{{"op":"round","column":"age","value":0,"description":"Round age to 0 decimals"}}]
+
+  User: "Remove rows where spend is negative"
+  -> [{{"op":"filter_rows","condition":"spend >= 0","description":"Remove rows where spend < 0"}}]
+
+  User: "Log-transform spend"
+  -> [{{"op":"log","column":"spend","description":"Natural log of spend"}}]
+
+Columns available: {columns}
+Numeric columns: {numeric_columns}
+
+Only use column names from the list above. Respond with ONLY the JSON array."""
+
+
+# ── Vision system prompt (for image/PDF annotation workflow) ──────────────────
+
+_VISION_SYSTEM = """You are a data transformation assistant. The user has annotated a
+data report (shown as an image) with handwritten notes, highlights, or typed
+comments describing changes they want to make to the underlying dataset.
+
+Extract ALL adjustment instructions visible in the image and translate them
+into the same JSON operation format.
+
+You must respond with ONLY a JSON array of operation objects — no prose, no markdown fences.
+
+Each object has these keys:
+  op          : one of add | subtract | multiply | divide | set | clip | round |
+                rename | drop_col | filter_rows | fillna | abs | log | sqrt
+  column      : the column name to operate on (use "*" for all numeric columns)
+  value       : numeric value, new name string, or clip bounds [min, max]
+  condition   : (optional) a pandas .query() expression string
+  description : a short human-readable summary of this operation
+
+Columns available: {columns}
+Numeric columns: {numeric_columns}
+
+Respond with ONLY the JSON array. If no clear instructions are found, return []."""
+
+
+# ── Parsing ───────────────────────────────────────────────────────────────────
+
+def parse_instructions(instruction: str, df: pd.DataFrame,
+                       cfg: Optional[LLMConfig] = None) -> list[dict]:
+    """Send a text instruction to the LLM and return operation dicts."""
+    columns         = list(df.columns)
+    numeric_columns = list(df.select_dtypes(include="number").columns)
+    system = _SYSTEM.format(
+        columns=json.dumps(columns),
+        numeric_columns=json.dumps(numeric_columns),
+    )
+    return chat_json(system, instruction, cfg=cfg or get_config())
+
+
+def parse_instructions_from_image(image_b64: str, df: pd.DataFrame,
+                                   cfg: Optional[LLMConfig] = None) -> list[dict]:
+    """
+    Send an annotated report image to a vision LLM and return operation dicts.
+
+    image_b64 : base64-encoded PNG or JPEG of the annotated report page.
+    df        : current DataFrame (used for column hints in the system prompt).
+    """
+    columns         = list(df.columns)
+    numeric_columns = list(df.select_dtypes(include="number").columns)
+    system = _VISION_SYSTEM.format(
+        columns=json.dumps(columns),
+        numeric_columns=json.dumps(numeric_columns),
+    )
+    vision_cfg = cfg or get_config()
+    # If the global config is text-only lmstudio, auto-switch to vision variant
+    if vision_cfg.provider == "lmstudio":
+        from dataclasses import replace
+        vision_cfg = replace(vision_cfg, provider="lmstudio_vision")
+
+    return chat_json(system, "Please extract all adjustment instructions from this annotated report.",
+                     image_b64=image_b64, cfg=vision_cfg)
+
+
+# ── Operation executor ────────────────────────────────────────────────────────
+
+def _resolve_mask(df: pd.DataFrame, condition: Optional[str]) -> pd.Series:
+    if not condition:
+        return pd.Series([True] * len(df), index=df.index)
+    return df.eval(condition)
+
+
+def apply_operation(df: pd.DataFrame, op: dict) -> tuple[pd.DataFrame, str]:
+    """Apply a single operation dict to df. Returns (new_df, description)."""
+    df      = df.copy()
+    o       = op.get("op", "").lower()
+    col     = op.get("column", "")
+    val     = op.get("value")
+    cond    = op.get("condition")
+    desc    = op.get("description", f"{o} on {col}")
+    numeric = list(df.select_dtypes(include="number").columns)
+
+    target_cols = numeric if col == "*" else [col]
+    for tc in target_cols:
+        if tc not in df.columns and o not in ("filter_rows", "rename"):
+            raise ValueError(f"Column '{tc}' not found in dataset.")
+
+    mask = _resolve_mask(df, cond)
+
+    if o == "add":
+        for tc in target_cols:
+            df.loc[mask, tc] = df.loc[mask, tc] + float(val)
+
+    elif o == "subtract":
+        for tc in target_cols:
+            df.loc[mask, tc] = df.loc[mask, tc] - float(val)
+
+    elif o == "multiply":
+        for tc in target_cols:
+            df.loc[mask, tc] = df.loc[mask, tc] * float(val)
+
+    elif o == "divide":
+        if float(val) == 0:
+            raise ValueError("Division by zero.")
+        for tc in target_cols:
+            df.loc[mask, tc] = df.loc[mask, tc] / float(val)
+
+    elif o == "set":
+        for tc in target_cols:
+            df.loc[mask, tc] = float(val) if isinstance(val, (int, float)) else val
+
+    elif o == "clip":
+        lo, hi = (val[0], val[1]) if isinstance(val, list) else (None, None)
+        lo = None if lo is None or (isinstance(lo, float) and math.isnan(lo)) else float(lo)
+        hi = None if hi is None or (isinstance(hi, float) and math.isnan(hi)) else float(hi)
+        for tc in target_cols:
+            df[tc] = df[tc].clip(lower=lo, upper=hi)
+
+    elif o == "round":
+        decimals = int(val) if val is not None else 0
+        for tc in target_cols:
+            df[tc] = df[tc].round(decimals)
+
+    elif o == "rename":
+        if col not in df.columns:
+            raise ValueError(f"Column '{col}' not found.")
+        df = df.rename(columns={col: str(val)})
+
+    elif o == "drop_col":
+        for tc in target_cols:
+            df = df.drop(columns=[tc], errors="ignore")
+
+    elif o == "filter_rows":
+        if not cond:
+            raise ValueError("filter_rows requires a condition.")
+        df = df.query(cond).reset_index(drop=True)
+
+    elif o == "fillna":
+        for tc in target_cols:
+            df[tc] = df[tc].fillna(float(val) if isinstance(val, (int, float)) else val)
+
+    elif o == "abs":
+        for tc in target_cols:
+            df[tc] = df[tc].abs()
+
+    elif o == "log":
+        for tc in target_cols:
+            df[tc] = np.log(df[tc].replace(0, np.nan))
+
+    elif o == "sqrt":
+        for tc in target_cols:
+            df[tc] = np.sqrt(df[tc].clip(lower=0))
+
+    else:
+        raise ValueError(f"Unknown operation: '{o}'")
+
+    return df, desc
+
+
+def apply_instructions(df: pd.DataFrame, instruction: str,
+                       cfg: Optional[LLMConfig] = None) -> tuple[pd.DataFrame, list[str], list[dict]]:
+    """
+    Parse and apply all operations from a natural-language instruction.
+    Returns (modified_df, descriptions_list, ops_list).
+    """
+    ops = parse_instructions(instruction, df, cfg=cfg)
+    descriptions = []
+    for op in ops:
+        df, desc = apply_operation(df, op)
+        descriptions.append(desc)
+    return df, descriptions, ops
+
+
+def apply_instructions_from_image(df: pd.DataFrame, image_b64: str,
+                                   cfg: Optional[LLMConfig] = None
+                                   ) -> tuple[pd.DataFrame, list[str], list[dict]]:
+    """
+    Parse and apply operations extracted from an annotated report image.
+    Returns (modified_df, descriptions_list, ops_list).
+    """
+    ops = parse_instructions_from_image(image_b64, df, cfg=cfg)
+    if not ops:
+        raise ValueError("No adjustment instructions found in the image.")
+    descriptions = []
+    for op in ops:
+        df, desc = apply_operation(df, op)
+        descriptions.append(desc)
+    return df, descriptions, ops