learnstat 0.2.0__py3-none-any.whl

learnstat/config.py

@@ -0,0 +1,223 @@
+"""learnstat's own AI settings: saved profiles of provider, API key, endpoint and model.
+
+Run ``learnstat setup`` for a guided choice of company, key and model. Several
+profiles can be saved (one per company / key) and switched between::
+
+    learnstat setup                  # guided wizard: company -> key -> model -> check -> start
+    learnstat config list            # saved profiles
+    learnstat config use anthropic   # switch the active profile
+    learnstat config set model gpt-4o   # edit the active profile
+
+Environment variables (``LEARNSTAT_API_KEY``, ``LEARNSTAT_BASE_URL``,
+``LEARNSTAT_MODEL``) override the active profile for one run.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import providers
+
+DEFAULT_BASE_URL = "https://api.openai.com/v1"
+DEFAULT_MODEL = "gpt-4o-mini"
+
+# setting name -> environment variable
+ENV_VARS = {
+    "api_key": "LEARNSTAT_API_KEY",
+    "base_url": "LEARNSTAT_BASE_URL",
+    "model": "LEARNSTAT_MODEL",
+}
+KEYS = tuple(ENV_VARS)
+_PROFILE_FIELDS = ("provider", "base_url", "api_key", "model")
+
+
+def config_path() -> Path:
+    """Location of the config file (``LEARNSTAT_CONFIG`` overrides the default)."""
+    override = os.environ.get("LEARNSTAT_CONFIG")
+    if override:
+        return Path(override).expanduser()
+    return Path.home() / ".learnstat" / "config.json"
+
+
+# ---------------------------------------------------------------------------
+# storage
+
+
+def _read() -> dict:
+    """Load the file as ``{"active": str|None, "profiles": {...}, "skip_setup": bool}``."""
+    try:
+        raw = json.loads(config_path().read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        raw = {}
+    if not isinstance(raw, dict):
+        raw = {}
+    profiles = {
+        name: {k: str(v) for k, v in p.items() if k in _PROFILE_FIELDS and v not in (None, "")}
+        for name, p in (raw.get("profiles") or {}).items()
+        if isinstance(p, dict)
+    }
+    active = raw.get("active") if raw.get("active") in profiles else None
+    # an older flat file ({"api_key": ..., "model": ...}) becomes a "custom" profile
+    flat = {k: str(raw[k]) for k in KEYS if raw.get(k)}
+    if flat and not profiles:
+        profiles["custom"] = {**flat, "provider": "custom"}
+        active = "custom"
+    if active is None and profiles:
+        active = next(iter(profiles))
+    return {"active": active, "profiles": profiles, "skip_setup": bool(raw.get("skip_setup"))}
+
+
+def _write(data: dict) -> Path:
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+    try:  # the file holds API keys: keep it private where the OS supports it
+        path.chmod(stat.S_IRUSR | stat.S_IWUSR)
+    except OSError:
+        pass
+    return path
+
+
+def profiles() -> dict[str, dict]:
+    return _read()["profiles"]
+
+
+def active_name() -> str | None:
+    return _read()["active"]
+
+
+def save_profile(name: str, provider: str, base_url: str, api_key: str, model: str, activate: bool = True) -> Path:
+    data = _read()
+    data["profiles"][name] = {k: v for k, v in
+                              {"provider": provider, "base_url": base_url, "api_key": api_key, "model": model}.items() if v}
+    if activate or data["active"] is None:
+        data["active"] = name
+    return _write(data)
+
+
+def set_active(name: str) -> Path:
+    data = _read()
+    if name not in data["profiles"]:
+        raise KeyError(f"no saved profile {name!r}; saved: {', '.join(data['profiles']) or 'none'}")
+    data["active"] = name
+    return _write(data)
+
+
+def remove_profile(name: str) -> Path:
+    data = _read()
+    if name not in data["profiles"]:
+        raise KeyError(f"no saved profile {name!r}; saved: {', '.join(data['profiles']) or 'none'}")
+    del data["profiles"][name]
+    if data["active"] == name:
+        data["active"] = next(iter(data["profiles"]), None)
+    return _write(data)
+
+
+def set_skip_setup(skip: bool) -> Path:
+    data = _read()
+    data["skip_setup"] = skip
+    return _write(data)
+
+
+def skip_setup() -> bool:
+    return _read()["skip_setup"]
+
+
+def set_value(key: str, value: str) -> Path:
+    """Edit one setting of the active profile (a ``custom`` profile is created if none exists)."""
+    if key not in KEYS:
+        raise KeyError(f"unknown setting {key!r}; choose from {', '.join(KEYS)}")
+    data = _read()
+    if data["active"] is None:
+        data["profiles"]["custom"] = {"provider": "custom"}
+        data["active"] = "custom"
+    data["profiles"][data["active"]][key] = value.strip()
+    return _write(data)
+
+
+def unset_value(key: str) -> Path:
+    if key not in KEYS:
+        raise KeyError(f"unknown setting {key!r}; choose from {', '.join(KEYS)}")
+    data = _read()
+    if data["active"] is not None:
+        data["profiles"][data["active"]].pop(key, None)
+    return _write(data)
+
+
+# ---------------------------------------------------------------------------
+# resolved settings
+
+
+@dataclass
+class AIConfig:
+    api_key: str
+    base_url: str
+    model: str
+    provider: str = "custom"
+    profile: str | None = None
+
+    @property
+    def chat_url(self) -> str:
+        base = self.base_url.rstrip("/")
+        return base if base.endswith("/chat/completions") else base + "/chat/completions"
+
+    @property
+    def models_url(self) -> str:
+        base = self.base_url.rstrip("/")
+        if base.endswith("/chat/completions"):
+            base = base[: -len("/chat/completions")]
+        return base + "/models"
+
+    @property
+    def provider_name(self) -> str:
+        p = providers.get(self.provider)
+        return p.name.split(" (")[0] if p and p.id != "custom" else (self.profile or "custom endpoint")
+
+    @property
+    def needs_key(self) -> bool:
+        """Hosted endpoints need a key; local ones (Ollama, LM Studio) and custom URLs may not."""
+        p = providers.get(self.provider)
+        if p is not None and p.id != "custom":
+            return p.needs_key
+        return self.base_url.rstrip("/") == DEFAULT_BASE_URL
+
+    @property
+    def headers(self) -> dict[str, str]:
+        h = {"Content-Type": "application/json"}
+        if self.api_key:
+            h["Authorization"] = f"Bearer {self.api_key}"
+            if self.provider == "anthropic":  # its native auth headers (needed for /models)
+                h["x-api-key"] = self.api_key
+                h["anthropic-version"] = "2023-06-01"
+        return h
+
+    @property
+    def is_configured(self) -> bool:
+        return bool(self.api_key) or not self.needs_key
+
+
+def resolve() -> AIConfig:
+    data = _read()
+    prof = data["profiles"].get(data["active"], {}) if data["active"] else {}
+
+    def pick(key: str, default: str = "") -> str:
+        return os.environ.get(ENV_VARS[key], "").strip() or prof.get(key, "") or default
+
+    preset = providers.get(prof.get("provider"))
+    return AIConfig(
+        api_key=pick("api_key"),
+        base_url=pick("base_url", preset.base_url if preset and preset.base_url else DEFAULT_BASE_URL),
+        model=pick("model", DEFAULT_MODEL),
+        provider=prof.get("provider", "custom"),
+        profile=data["active"],
+    )
+
+
+def mask(secret: str) -> str:
+    if not secret:
+        return "(not set)"
+    return secret[:3] + "..." + secret[-4:] if len(secret) > 10 else "***"

learnstat/ipython_ext.py

@@ -0,0 +1,104 @@
+"""Jupyter / IPython integration: ``%load_ext learnstat``.
+
+* Before each cell runs, its code is checked against the dataset profiles and
+  suggestions are printed (the cell still runs exactly as written).
+* A cell with a syntax error gets a diagnosis and, where possible, a corrected
+  version placed in a new cell for the user to review and run.
+* When a cell raises, the failing statement is diagnosed and - for mechanical
+  mistakes - corrected and retried; statements that already ran are not re-run.
+* Lines starting with ``/describe``, ``/fix``, ``/memory`` ... are learnstat commands.
+* ``/accept`` places the suggested code in a new cell rather than editing yours.
+"""
+
+from __future__ import annotations
+
+import ast
+import traceback
+
+from .commands import is_command
+from .recovery import _fix_syntax, diagnose
+from .session import Session, set_session
+
+_state: dict = {}
+
+
+def _confirm(question: str) -> bool:
+    try:
+        return input(f"{question} [y/N] ").strip().lower() in ("y", "yes")
+    except (EOFError, OSError):
+        return False
+
+
+def load_ipython_extension(ip) -> None:
+    if "session" in _state:
+        return
+    session = Session(namespace=ip.user_ns, confirm=_confirm)
+    session.on_accept = lambda code: ip.set_next_input(code, replace=False)
+    set_session(session)
+    ip.learnstat_session = session
+    _state["session"] = session
+
+    def command_transformer(lines: list[str]) -> list[str]:
+        if lines and is_command(lines[0]):
+            text = "".join(lines).strip()
+            return [f"print(get_ipython().learnstat_session.command({text!r}))\n"]
+        return lines
+
+    def pre_run_cell(info) -> None:
+        raw = getattr(info, "raw_cell", "") or ""
+        if not raw.strip() or is_command(raw.splitlines()[0]):
+            _state["cell"] = None
+            return
+        code = ip.transform_cell(raw)
+        _state["cell"] = code
+        try:
+            ast.parse(code)
+        except SyntaxError as e:
+            d = diagnose(e, code, e.lineno, ip.user_ns)
+            print(f"[learnstat] {d.exc_type} ({d.label}): {d.message}\n  {d.explanation}")
+            fix = _fix_syntax(e, raw)
+            if fix:
+                try:
+                    ast.parse(ip.transform_cell(fix.fixed))
+                    print(f"  Proposed fix: {fix.description}. A corrected copy was placed in the next cell.")
+                    ip.set_next_input(fix.fixed, replace=False)
+                except SyntaxError:
+                    print(f"  Next step: {d.next_step}")
+            else:
+                print(f"  Next step: {d.next_step}")
+            return
+        session.check(code)
+
+    def on_exception(shell, etype, evalue, tb, tb_offset=None):
+        code = _state.get("cell")
+        frames = traceback.extract_tb(tb)
+        cell_frame = next((f for f in frames if f.name == "<module>"), None)
+        if not code or cell_frame is None:
+            shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset)
+            return None
+        print(f"[learnstat] {etype.__name__}: {evalue}")
+        result = session.executor(filename=cell_frame.filename).run(
+            code, start_line=cell_frame.lineno, initial_exc=evalue
+        )
+        session._after_run(code, result)
+        if result.ok and result.value is not None:
+            shell.displayhook(result.value)
+        elif not result.ok and not result.attempts:
+            shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset)
+        return None
+
+    ip.input_transformers_cleanup.append(command_transformer)
+    ip.events.register("pre_run_cell", pre_run_cell)
+    ip.set_custom_exc((Exception,), on_exception)
+    _state.update(transformer=command_transformer, pre=pre_run_cell)
+    print(f"[learnstat] active. AI: {session.ai_status}. Load data with learnstat.read_csv(...); type /help for commands.")
+
+
+def unload_ipython_extension(ip) -> None:
+    if "session" not in _state:
+        return
+    if _state["transformer"] in ip.input_transformers_cleanup:
+        ip.input_transformers_cleanup.remove(_state["transformer"])
+    ip.events.unregister("pre_run_cell", _state["pre"])
+    ip.set_custom_exc((), None)
+    _state.clear()

learnstat/knowledge.py

@@ -0,0 +1,233 @@
+"""Offline explanations used by /describe, /alternatives and /learn.
+
+Each concept explains the statistical idea, the purpose of the suggested
+function, when it is appropriate and when something else would be better.
+When the AI backend is available these are used as grounding and then
+tailored to the user's dataset; without AI they are shown as-is, filled in with
+the dataset facts recorded on the suggestion.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class Concept:
+    title: str
+    concept: str
+    purpose: str
+    appropriate_when: str
+    alternatives: list[str] = field(default_factory=list)
+    lesson: str = ""
+    docs: list[str] = field(default_factory=list)
+
+
+CONCEPTS: dict[str, Concept] = {
+    "part_to_whole_charts": Concept(
+        title="Pie charts vs bar charts",
+        concept=(
+            "People judge angles and areas much less accurately than lengths along a common axis. "
+            "A pie chart asks the reader to compare slice angles; with more than about five slices, or "
+            "slices of similar size, the comparison becomes guesswork."
+        ),
+        purpose=(
+            "`Series.value_counts().plot.barh()` draws one bar per category on a shared baseline, so "
+            "differences in length are easy to compare and long labels stay readable."
+        ),
+        appropriate_when=(
+            "Use a bar chart whenever the goal is to compare categories. A pie chart is reasonable only "
+            "when there are 2-5 parts of a single whole and the message is 'one part dominates'."
+        ),
+        alternatives=[
+            "Horizontal bar chart sorted by count: `s.value_counts().plot.barh()`",
+            "Group rare levels into 'Other' first: `s.where(s.isin(s.value_counts().index[:8]), 'Other')`",
+            "A dot plot (`plt.plot(counts.values, counts.index, 'o')`) for a lighter look",
+            "A stacked 100% bar when comparing proportions across several groups",
+        ],
+        lesson=(
+            "Exercise: plot the same `value_counts()` as a pie and as a sorted horizontal bar. Try to "
+            "rank the 3rd-6th categories from each chart - notice which one you can do without reading labels."
+        ),
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.Series.plot.barh.html"],
+    ),
+    "center_of_skewed_data": Concept(
+        title="Mean vs median for skewed data",
+        concept=(
+            "The mean is pulled toward long tails and extreme values; the median (the 50th percentile) "
+            "is not. In a right-skewed variable such as income, the mean can sit far above what a "
+            "typical observation looks like."
+        ),
+        purpose=(
+            "`Series.median()` and the IQR (`quantile(0.75) - quantile(0.25)`) summarise the centre and "
+            "spread robustly. `describe()` shows both mean and quartiles so you can compare them."
+        ),
+        appropriate_when=(
+            "Report the median/IQR when |skew| is about 1 or more, or when there are outliers. The mean is "
+            "still the right choice when you need totals (mean x n), or for roughly symmetric data."
+        ),
+        alternatives=[
+            "`df[col].describe()` to see mean and quartiles together",
+            "A trimmed mean: `scipy.stats.trim_mean(x, 0.1)`",
+            "Analyse `np.log1p(x)` if the variable is positive and multiplicative in nature",
+        ],
+        lesson="Exercise: compute mean and median before and after adding one huge value to the column.",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.Series.median.html"],
+    ),
+    "welch_t_test": Concept(
+        title="Student's vs Welch's t-test",
+        concept=(
+            "`scipy.stats.ttest_ind` assumes equal variances by default (Student's t-test). When group "
+            "variances or sizes differ, that assumption inflates the false-positive rate. Welch's test "
+            "drops the assumption and loses almost nothing when variances are in fact equal."
+        ),
+        purpose="`ttest_ind(a, b, equal_var=False)` runs Welch's t-test on the same data.",
+        appropriate_when=(
+            "Welch's is a sensible default for comparing two independent means. For strongly skewed data "
+            "or small samples, a rank-based test (Mann-Whitney U) may be preferable; for paired measurements "
+            "use `ttest_rel`."
+        ),
+        alternatives=[
+            "`scipy.stats.mannwhitneyu(a, b)` - compares distributions without assuming normality",
+            "`scipy.stats.ttest_rel(a, b)` - for paired/before-after data",
+            "`scipy.stats.f_oneway(*groups)` / `kruskal(*groups)` - for 3+ groups",
+        ],
+        lesson="Exercise: simulate two normal groups with SDs 1 and 4 and sizes 10 and 100; compare p-values.",
+        docs=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html"],
+    ),
+    "correlation_method": Concept(
+        title="Pearson vs Spearman correlation",
+        concept=(
+            "Pearson's r measures linear association and is sensitive to outliers and skew. Spearman's rho "
+            "is Pearson on ranks: it measures monotonic association and is robust to extreme values."
+        ),
+        purpose="`df.corr(method='spearman')` or `scipy.stats.spearmanr(x, y)`.",
+        appropriate_when=(
+            "Use Spearman when variables are skewed, ordinal, or contain outliers, or when the relationship "
+            "is monotonic but curved. Use Pearson when both variables are roughly symmetric and the "
+            "relationship looks linear in a scatter plot."
+        ),
+        alternatives=[
+            "Look first: `df.plot.scatter(x=..., y=...)`",
+            "Kendall's tau (`method='kendall'`) for small samples with many ties",
+            "Pearson on log-transformed variables",
+        ],
+        lesson="Exercise: compute both coefficients for x and x**3 - why does only one equal 1?",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.corr.html"],
+    ),
+    "missing_values_in_tests": Concept(
+        title="Missing values in statistical functions",
+        concept=(
+            "pandas methods skip NaN by default, but many SciPy functions do not: they return `nan` or "
+            "raise an error. How missing values are handled is an analytical decision that changes your "
+            "sample, so it should be explicit and reported."
+        ),
+        purpose="`.dropna()` on the series passed to the test, or `nan_policy='omit'` where SciPy supports it.",
+        appropriate_when=(
+            "Dropping is fine when values are missing at random and few in number. If many are missing or "
+            "missingness relates to the outcome, investigate first and consider imputation."
+        ),
+        alternatives=[
+            "`nan_policy='omit'` (supported by many scipy.stats functions)",
+            "Report n per group after dropping: `a.dropna().size`",
+            "Imputation (e.g. median) - only with a documented justification",
+        ],
+        lesson="Exercise: compare `df[col].isna().groupby(df[group]).mean()` across groups - is missingness balanced?",
+        docs=["https://docs.scipy.org/doc/scipy/dev/api-dev/nan_policy.html"],
+    ),
+    "categorical_axis": Concept(
+        title="Plotting categorical variables",
+        concept=(
+            "Line plots imply order and continuity between x values; scatter plots and histograms assume "
+            "numeric scales. A nominal category (e.g. country) has neither, so these charts can mislead."
+        ),
+        purpose="Bar charts for counts, boxplots/strip plots for a numeric variable across categories.",
+        appropriate_when=(
+            "Use a line only when x is ordered (time, dose levels). For numeric-by-category comparisons "
+            "use `df.boxplot(column=num, by=cat)`."
+        ),
+        alternatives=[
+            "`df.boxplot(column='y', by='group')`",
+            "`df.groupby('group')['y'].mean().plot.bar()` (show error bars too)",
+            "`pd.crosstab(df.a, df.b).plot.bar(stacked=True)` for two categoricals",
+        ],
+        lesson="Exercise: compare a line plot and a bar chart of group means - what does the line falsely suggest?",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.boxplot.html"],
+    ),
+    "listwise_deletion": Concept(
+        title="Dropping rows with missing values",
+        concept=(
+            "`df.dropna()` removes every row with any missing value (listwise deletion). If missingness is "
+            "concentrated in one column you may lose far more data than you need to, and the remaining rows "
+            "may no longer represent the population."
+        ),
+        purpose="`df.dropna(subset=[...])` drops rows only where the columns you analyse are missing.",
+        appropriate_when=(
+            "Drop only on the columns used by a specific analysis, and record how many rows were removed."
+        ),
+        alternatives=[
+            "`df.dropna(subset=['col_a', 'col_b'])`",
+            "`df.isna().sum()` first, to see where values are missing",
+            "Keep the original: `clean = df.dropna(subset=...)` rather than overwriting `df`",
+        ],
+        lesson="Exercise: compare `len(df.dropna())` with `len(df.dropna(subset=[col]))` for each column.",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.dropna.html"],
+    ),
+    "column_names": Concept(
+        title="Selecting columns by name",
+        concept="Column labels are exact strings: case, spaces and underscores all matter.",
+        purpose="`df.columns` lists the exact names; `df[['a', 'b']]` selects several at once.",
+        appropriate_when="Prefer `df['name']` over `df.name` - it works for every label, including ones with spaces.",
+        alternatives=["`df.columns = df.columns.str.strip().str.lower()` to normalise names once, explicitly"],
+        lesson="Exercise: print `list(df.columns)` and compare it with the names you typed.",
+        docs=["https://pandas.pydata.org/docs/user_guide/indexing.html"],
+    ),
+    "text_numbers": Concept(
+        title="Numbers stored as text",
+        concept=(
+            "If a numeric column contains stray characters ('1,200', '$5', 'n/a'), pandas stores it as text "
+            "and arithmetic or statistics fail or behave unexpectedly."
+        ),
+        purpose="`pd.to_numeric(s, errors='coerce')` converts what it can and turns the rest into NaN.",
+        appropriate_when=(
+            "Always inspect which values failed first: `s[pd.to_numeric(s, errors='coerce').isna() & s.notna()]`. "
+            "Converting is a data-cleaning decision, so record it."
+        ),
+        alternatives=["`s.str.replace(',', '')` before converting", "`pd.read_csv(..., thousands=',')`"],
+        lesson="Exercise: find the rows that fail to convert and decide, case by case, how to treat them.",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.to_numeric.html"],
+    ),
+    "histogram_categorical": Concept(
+        title="Histograms are for numeric variables",
+        concept=(
+            "A histogram bins a continuous scale. Category labels have no scale, so you need counts per "
+            "level instead."
+        ),
+        purpose="`s.value_counts().plot.bar()` shows the count for each level.",
+        appropriate_when="Use histograms for numeric columns, bar charts of counts for categorical ones.",
+        alternatives=["`s.value_counts(normalize=True)` for proportions"],
+        lesson="Exercise: draw the bar chart and sort it by count vs by label - which is easier to read?",
+        docs=["https://pandas.pydata.org/docs/reference/api/pandas.Series.value_counts.html"],
+    ),
+}
+
+
+def describe_concept(key: str) -> Concept | None:
+    return CONCEPTS.get(key)
+
+
+def find_topic(query: str) -> Concept | None:
+    q = query.lower().strip()
+    if not q:
+        return None
+    for key, c in CONCEPTS.items():
+        if q in key or q in c.title.lower():
+            return c
+    words = set(q.split())
+    best, best_score = None, 0
+    for c in CONCEPTS.values():
+        text = (c.title + " " + c.concept + " " + c.purpose).lower()
+        score = sum(1 for w in words if w in text)
+        if score > best_score:
+            best, best_score = c, score
+    return best