learnstat 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- learnstat/__init__.py +80 -0
- learnstat/advisor.py +502 -0
- learnstat/cli.py +338 -0
- learnstat/commands.py +317 -0
- learnstat/config.py +223 -0
- learnstat/ipython_ext.py +104 -0
- learnstat/knowledge.py +233 -0
- learnstat/llm.py +280 -0
- learnstat/memory.py +193 -0
- learnstat/profile.py +344 -0
- learnstat/providers.py +78 -0
- learnstat/recovery.py +1021 -0
- learnstat/safety.py +100 -0
- learnstat/session.py +307 -0
- learnstat/setup.py +308 -0
- learnstat/suggestions.py +64 -0
- learnstat-0.2.0.dist-info/METADATA +199 -0
- learnstat-0.2.0.dist-info/RECORD +22 -0
- learnstat-0.2.0.dist-info/WHEEL +5 -0
- learnstat-0.2.0.dist-info/entry_points.txt +2 -0
- learnstat-0.2.0.dist-info/licenses/LICENSE +21 -0
- learnstat-0.2.0.dist-info/top_level.txt +1 -0
learnstat/setup.py
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"""Guided AI setup in the terminal: company -> API key -> model -> live check -> save.
|
|
2
|
+
|
|
3
|
+
learnstat setup (or /ai inside a session)
|
|
4
|
+
|
|
5
|
+
All input/output goes through :class:`IO` so the wizard is easy to test and works
|
|
6
|
+
in a plain terminal, in the learnstat REPL and in Jupyter.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import getpass
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Callable
|
|
15
|
+
|
|
16
|
+
from . import config, providers
|
|
17
|
+
from .llm import AIError, LLMAssistant, fetch_models
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Cancelled(Exception):
|
|
21
|
+
"""The user left the wizard (q, Ctrl+C or end of input)."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class IO:
|
|
26
|
+
ask: Callable[[str], str] = input
|
|
27
|
+
secret: Callable[[str], str] = getpass.getpass
|
|
28
|
+
say: Callable[[str], None] = print
|
|
29
|
+
|
|
30
|
+
def line(self, prompt: str) -> str:
|
|
31
|
+
try:
|
|
32
|
+
return self.ask(prompt).strip()
|
|
33
|
+
except (EOFError, KeyboardInterrupt):
|
|
34
|
+
raise Cancelled from None
|
|
35
|
+
|
|
36
|
+
def hidden(self, prompt: str) -> str:
|
|
37
|
+
try:
|
|
38
|
+
return self.secret(prompt).strip()
|
|
39
|
+
except (EOFError, KeyboardInterrupt):
|
|
40
|
+
raise Cancelled from None
|
|
41
|
+
|
|
42
|
+
def yes(self, prompt: str, default: bool = True) -> bool:
|
|
43
|
+
answer = self.line(f"{prompt} [{'Y/n' if default else 'y/N'}] ").lower()
|
|
44
|
+
return default if not answer else answer in ("y", "yes")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class Draft:
|
|
49
|
+
provider: providers.Provider
|
|
50
|
+
base_url: str
|
|
51
|
+
api_key: str = ""
|
|
52
|
+
model: str = ""
|
|
53
|
+
models: list[str] = field(default_factory=list)
|
|
54
|
+
|
|
55
|
+
def config(self) -> config.AIConfig:
|
|
56
|
+
return config.AIConfig(api_key=self.api_key, base_url=self.base_url, model=self.model, provider=self.provider.id)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
RULE = "-" * 60
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# steps
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def choose_provider(io: IO) -> providers.Provider:
|
|
67
|
+
io.say("\nWhich AI company do you want to use?\n")
|
|
68
|
+
choices = providers.all_choices()
|
|
69
|
+
for i, p in enumerate(choices, 1):
|
|
70
|
+
io.say(f" {i:>2}) {p.name}")
|
|
71
|
+
while True:
|
|
72
|
+
answer = io.line("\nNumber (or q to quit): ").lower()
|
|
73
|
+
if answer in ("q", "quit", "exit"):
|
|
74
|
+
raise Cancelled
|
|
75
|
+
if answer.isdigit() and 1 <= int(answer) <= len(choices):
|
|
76
|
+
return choices[int(answer) - 1]
|
|
77
|
+
by_name = [p for p in choices if answer and (answer == p.id or answer in p.name.lower())]
|
|
78
|
+
if len(by_name) == 1:
|
|
79
|
+
return by_name[0]
|
|
80
|
+
io.say(f"Please type a number from 1 to {len(choices)}.")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def ask_endpoint(io: IO, provider: providers.Provider) -> str:
|
|
84
|
+
if provider.id != "custom":
|
|
85
|
+
if provider.note:
|
|
86
|
+
io.say(f"\n{provider.note}")
|
|
87
|
+
return provider.base_url
|
|
88
|
+
io.say("\nAny OpenAI-compatible server works (it must offer /chat/completions).")
|
|
89
|
+
while True:
|
|
90
|
+
url = io.line("Base URL (for example https://api.example.com/v1): ")
|
|
91
|
+
if url.startswith(("http://", "https://")):
|
|
92
|
+
return url
|
|
93
|
+
io.say("The address must start with http:// or https://")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def ask_key(io: IO, provider: providers.Provider) -> str:
|
|
97
|
+
if not provider.needs_key:
|
|
98
|
+
return ""
|
|
99
|
+
if provider.key_url:
|
|
100
|
+
io.say(f"\nCreate a key at {provider.key_url}")
|
|
101
|
+
io.say("Paste your API key (it is hidden while you type; right-click or Ctrl+V pastes in most terminals).")
|
|
102
|
+
while True:
|
|
103
|
+
key = io.hidden("API key: ")
|
|
104
|
+
if key:
|
|
105
|
+
return key
|
|
106
|
+
io.say("The key is empty - paste it and press Enter (or Ctrl+C to quit).")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def load_models(io: IO, d: Draft) -> str | None:
|
|
110
|
+
"""Fetch the model list. Returns an error message when the key itself is the problem."""
|
|
111
|
+
io.say("\nChecking your key and loading the available models ...")
|
|
112
|
+
try:
|
|
113
|
+
d.models = fetch_models(d.config())
|
|
114
|
+
io.say(f" OK - {len(d.models)} models available.")
|
|
115
|
+
return None
|
|
116
|
+
except AIError as e:
|
|
117
|
+
if e.fatal:
|
|
118
|
+
return str(e)
|
|
119
|
+
d.models = list(d.provider.models)
|
|
120
|
+
io.say(f" Could not load the model list ({e}).")
|
|
121
|
+
if d.models:
|
|
122
|
+
io.say(" Showing the usual models instead - you can also type any model name.")
|
|
123
|
+
else:
|
|
124
|
+
io.say(" Type the model name you want to use.")
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def choose_model(io: IO, d: Draft) -> str:
|
|
129
|
+
shown = [m for m in d.models if providers.is_chat_model(m)] or list(d.models)
|
|
130
|
+
query = ""
|
|
131
|
+
while True:
|
|
132
|
+
view = [m for m in shown if query.lower() in m.lower()] if query else shown
|
|
133
|
+
if view:
|
|
134
|
+
io.say(f"\nModels{f' matching {query!r}' if query else ''}:\n")
|
|
135
|
+
for i, m in enumerate(view[:40], 1):
|
|
136
|
+
io.say(f" {i:>2}) {m}")
|
|
137
|
+
if len(view) > 40:
|
|
138
|
+
io.say(f" ... and {len(view) - 40} more - type part of a name to filter.")
|
|
139
|
+
answer = io.line("\nPick a number, type part of a name to filter, or type a full model name: ")
|
|
140
|
+
if not answer:
|
|
141
|
+
continue
|
|
142
|
+
if answer.isdigit() and 1 <= int(answer) <= len(view):
|
|
143
|
+
return view[int(answer) - 1]
|
|
144
|
+
exact = [m for m in d.models if m.lower() == answer.lower()]
|
|
145
|
+
if exact:
|
|
146
|
+
return exact[0]
|
|
147
|
+
matches = [m for m in shown if answer.lower() in m.lower()]
|
|
148
|
+
if len(matches) == 1:
|
|
149
|
+
return matches[0]
|
|
150
|
+
if matches:
|
|
151
|
+
query = answer
|
|
152
|
+
continue
|
|
153
|
+
if io.yes(f"'{answer}' is not in the list. Use it as the model name anyway?", default=False):
|
|
154
|
+
return answer
|
|
155
|
+
query = ""
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def verify(io: IO, d: Draft) -> str | None:
|
|
159
|
+
"""Send one tiny request. Returns None when it works, else the error message."""
|
|
160
|
+
label = d.provider.name.split(" (")[0] if d.provider.id != "custom" else d.base_url
|
|
161
|
+
io.say(f"\nTesting {label} / {d.model} ...")
|
|
162
|
+
assistant = LLMAssistant(d.config())
|
|
163
|
+
started = time.monotonic()
|
|
164
|
+
reply = assistant.test()
|
|
165
|
+
if reply is None:
|
|
166
|
+
return assistant.last_error or "the test request failed"
|
|
167
|
+
io.say(f" OK - the model answered in {time.monotonic() - started:.1f}s.")
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def profile_name(io: IO, d: Draft) -> str:
|
|
172
|
+
name = d.provider.id
|
|
173
|
+
if d.provider.id == "custom":
|
|
174
|
+
name = io.line("Name for this endpoint (default: custom): ") or "custom"
|
|
175
|
+
return name
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _save(io: IO, d: Draft, tested: bool) -> str:
|
|
179
|
+
name = profile_name(io, d)
|
|
180
|
+
path = config.save_profile(name, d.provider.id, d.base_url, d.api_key, d.model, activate=True)
|
|
181
|
+
if tested:
|
|
182
|
+
io.say("")
|
|
183
|
+
io.say(f"Ready. Saved '{name}' ({d.model}) and made it the active AI.")
|
|
184
|
+
else:
|
|
185
|
+
io.say("")
|
|
186
|
+
io.say(f"Saved '{name}' ({d.model}) without a successful test.")
|
|
187
|
+
io.say(f"Settings file: {path}")
|
|
188
|
+
return name
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def add_provider(io: IO) -> str:
|
|
192
|
+
"""Run the full flow for one new provider. Returns the saved profile name."""
|
|
193
|
+
provider = choose_provider(io)
|
|
194
|
+
while True: # restarts here when the user picks another company
|
|
195
|
+
d = Draft(provider=provider, base_url=ask_endpoint(io, provider))
|
|
196
|
+
d.api_key = ask_key(io, provider)
|
|
197
|
+
reload_models = True
|
|
198
|
+
while True:
|
|
199
|
+
if reload_models:
|
|
200
|
+
problem = load_models(io, d)
|
|
201
|
+
reload_models = False
|
|
202
|
+
if problem: # the key was rejected
|
|
203
|
+
io.say("")
|
|
204
|
+
io.say(f"X {problem}")
|
|
205
|
+
action = io.line("[r] re-enter key [p] pick another company [q] quit: ").lower()
|
|
206
|
+
if action == "q":
|
|
207
|
+
raise Cancelled
|
|
208
|
+
if action == "p":
|
|
209
|
+
break
|
|
210
|
+
d.api_key = ask_key(io, provider)
|
|
211
|
+
reload_models = True
|
|
212
|
+
continue
|
|
213
|
+
if not d.model:
|
|
214
|
+
d.model = choose_model(io, d)
|
|
215
|
+
problem = verify(io, d)
|
|
216
|
+
if problem is None:
|
|
217
|
+
return _save(io, d, tested=True)
|
|
218
|
+
io.say("")
|
|
219
|
+
io.say(f"X {problem}")
|
|
220
|
+
action = io.line(
|
|
221
|
+
"[m] choose another model [k] re-enter key [r] retry [p] another company "
|
|
222
|
+
"[s] save anyway [q] quit: ").lower()
|
|
223
|
+
if action == "q":
|
|
224
|
+
raise Cancelled
|
|
225
|
+
if action == "p":
|
|
226
|
+
break
|
|
227
|
+
if action == "s":
|
|
228
|
+
return _save(io, d, tested=False)
|
|
229
|
+
if action == "k":
|
|
230
|
+
d.api_key = ask_key(io, provider)
|
|
231
|
+
reload_models = True
|
|
232
|
+
d.model = ""
|
|
233
|
+
elif action != "r": # "m" or anything else: pick a model again
|
|
234
|
+
d.model = ""
|
|
235
|
+
provider = choose_provider(io)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def use_saved(io: IO, name: str) -> bool:
|
|
239
|
+
"""Activate a saved profile after checking it still works. Returns True on success."""
|
|
240
|
+
prof = config.profiles()[name]
|
|
241
|
+
cfg = config.AIConfig(api_key=prof.get("api_key", ""), base_url=prof.get("base_url", config.DEFAULT_BASE_URL),
|
|
242
|
+
model=prof.get("model", config.DEFAULT_MODEL), provider=prof.get("provider", "custom"),
|
|
243
|
+
profile=name)
|
|
244
|
+
io.say(f"\nTesting {name} / {cfg.model} ...")
|
|
245
|
+
assistant = LLMAssistant(cfg)
|
|
246
|
+
started = time.monotonic()
|
|
247
|
+
if assistant.test() is None:
|
|
248
|
+
io.say(f"X {assistant.last_error}")
|
|
249
|
+
if not io.yes("Use it anyway?", default=False):
|
|
250
|
+
return False
|
|
251
|
+
else:
|
|
252
|
+
io.say(f" OK - answered in {time.monotonic() - started:.1f}s.")
|
|
253
|
+
config.set_active(name)
|
|
254
|
+
io.say(f"Active profile: {name}")
|
|
255
|
+
return True
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def describe_profile(name: str, prof: dict, active: bool) -> str:
|
|
259
|
+
company = providers.get(prof.get("provider"))
|
|
260
|
+
label = company.name.split(" (")[0] if company and company.id != "custom" else prof.get("base_url", "custom endpoint")
|
|
261
|
+
star = " <- active" if active else ""
|
|
262
|
+
return f"{name}: {label}, model {prof.get('model', config.DEFAULT_MODEL)}, key {config.mask(prof.get('api_key', ''))}{star}"
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ---------------------------------------------------------------------------
|
|
266
|
+
# entry point
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def run_wizard(io: IO | None = None) -> str | None:
|
|
270
|
+
"""Interactive AI setup. Returns the active profile name, or None if the user left without one."""
|
|
271
|
+
io = io or IO()
|
|
272
|
+
try:
|
|
273
|
+
io.say(f"\n{RULE}\n learnstat - AI assistant setup\n{RULE}")
|
|
274
|
+
while True:
|
|
275
|
+
saved = config.profiles()
|
|
276
|
+
if not saved:
|
|
277
|
+
return add_provider(io)
|
|
278
|
+
active = config.active_name()
|
|
279
|
+
io.say("\nSaved AI accounts:\n")
|
|
280
|
+
names = list(saved)
|
|
281
|
+
for i, n in enumerate(names, 1):
|
|
282
|
+
io.say(f" {i:>2}) {describe_profile(n, saved[n], n == active)}")
|
|
283
|
+
io.say("\n n) add a new company / key d) delete one s) skip, use offline rules q) quit")
|
|
284
|
+
answer = io.line("\nNumber to switch, or a letter: ").lower()
|
|
285
|
+
if answer == "n":
|
|
286
|
+
return add_provider(io)
|
|
287
|
+
if answer in ("s", "skip"):
|
|
288
|
+
config.set_skip_setup(True)
|
|
289
|
+
io.say("OK - learnstat will use offline rules. Run `learnstat setup` any time to add an AI.")
|
|
290
|
+
return None
|
|
291
|
+
if answer in ("q", "quit"):
|
|
292
|
+
raise Cancelled
|
|
293
|
+
if answer == "d":
|
|
294
|
+
pick = io.line("Number to delete: ")
|
|
295
|
+
if pick.isdigit() and 1 <= int(pick) <= len(names):
|
|
296
|
+
target = names[int(pick) - 1]
|
|
297
|
+
if io.yes(f"Delete '{target}' and its stored key?", default=False):
|
|
298
|
+
config.remove_profile(target)
|
|
299
|
+
io.say(f"Deleted '{target}'.")
|
|
300
|
+
continue
|
|
301
|
+
if answer.isdigit() and 1 <= int(answer) <= len(names):
|
|
302
|
+
if use_saved(io, names[int(answer) - 1]):
|
|
303
|
+
return names[int(answer) - 1]
|
|
304
|
+
continue
|
|
305
|
+
io.say("Please type a number, n, d, s or q.")
|
|
306
|
+
except Cancelled:
|
|
307
|
+
io.say("\nSetup cancelled - nothing was changed after the last saved step.")
|
|
308
|
+
return config.active_name()
|
learnstat/suggestions.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Suggestion objects and the per-session suggestion history."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class Suggestion:
|
|
10
|
+
rule: str # machine name of the rule that fired
|
|
11
|
+
concept: str # key into knowledge.CONCEPTS for /describe and /learn
|
|
12
|
+
title: str
|
|
13
|
+
problem: str # what may be inappropriate about the current approach
|
|
14
|
+
alternative: str # which function / chart would be more suitable
|
|
15
|
+
rationale: str # why, for this dataset and objective
|
|
16
|
+
code: str | None = None # how to implement it
|
|
17
|
+
line: int | None = None
|
|
18
|
+
severity: str = "info" # info | warning
|
|
19
|
+
columns: list[str] = field(default_factory=list)
|
|
20
|
+
source_code: str = "" # the user's line that triggered this
|
|
21
|
+
status: str = "pending" # pending | accepted | rejected
|
|
22
|
+
origin: str = "rules" # rules | ai
|
|
23
|
+
id: int = 0 # assigned by SuggestionHistory in display order
|
|
24
|
+
|
|
25
|
+
def render(self, color: bool = False) -> str:
|
|
26
|
+
tag = "!" if self.severity == "warning" else "i"
|
|
27
|
+
where = f" (line {self.line})" if self.line else ""
|
|
28
|
+
out = [
|
|
29
|
+
f"[{tag}] Suggestion #{self.id}{where}: {self.title}",
|
|
30
|
+
f" Issue: {self.problem}",
|
|
31
|
+
f" Instead: {self.alternative}",
|
|
32
|
+
f" Why: {self.rationale}",
|
|
33
|
+
]
|
|
34
|
+
if self.code:
|
|
35
|
+
out.append(" How:")
|
|
36
|
+
out += [f" {ln}" for ln in self.code.splitlines()]
|
|
37
|
+
out.append(f" -> /describe {self.id} to learn more, /accept {self.id} or /reject {self.id}")
|
|
38
|
+
return "\n".join(out)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SuggestionHistory:
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self.items: list[Suggestion] = []
|
|
44
|
+
|
|
45
|
+
def add(self, s: Suggestion) -> Suggestion:
|
|
46
|
+
s.id = len(self.items) + 1
|
|
47
|
+
self.items.append(s)
|
|
48
|
+
return s
|
|
49
|
+
|
|
50
|
+
def extend(self, items: list[Suggestion]) -> None:
|
|
51
|
+
for s in items:
|
|
52
|
+
self.add(s)
|
|
53
|
+
|
|
54
|
+
def get(self, sid: int | None = None) -> Suggestion | None:
|
|
55
|
+
if sid is None:
|
|
56
|
+
return self.items[-1] if self.items else None
|
|
57
|
+
return next((s for s in self.items if s.id == sid), None)
|
|
58
|
+
|
|
59
|
+
def latest_pending(self) -> Suggestion | None:
|
|
60
|
+
return next((s for s in reversed(self.items) if s.status == "pending"), None)
|
|
61
|
+
|
|
62
|
+
def seen(self, rule: str, source_code: str) -> bool:
|
|
63
|
+
"""Avoid repeating the same advice for the same line in a session."""
|
|
64
|
+
return any(s.rule == rule and s.source_code == source_code for s in self.items)
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: learnstat
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: An AI-assisted learning companion for statistical analysis with pandas, NumPy, SciPy and Matplotlib.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/umorfaruksupto/leanstat
|
|
7
|
+
Project-URL: Repository, https://github.com/umorfaruksupto/leanstat
|
|
8
|
+
Project-URL: PyPI, https://pypi.org/project/learnstat/
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pandas>=2.0
|
|
13
|
+
Requires-Dist: numpy>=1.24
|
|
14
|
+
Provides-Extra: viz
|
|
15
|
+
Requires-Dist: matplotlib>=3.7; extra == "viz"
|
|
16
|
+
Provides-Extra: stats
|
|
17
|
+
Requires-Dist: scipy>=1.10; extra == "stats"
|
|
18
|
+
Provides-Extra: jupyter
|
|
19
|
+
Requires-Dist: ipython>=8.0; extra == "jupyter"
|
|
20
|
+
Provides-Extra: all
|
|
21
|
+
Requires-Dist: matplotlib>=3.7; extra == "all"
|
|
22
|
+
Requires-Dist: scipy>=1.10; extra == "all"
|
|
23
|
+
Requires-Dist: ipython>=8.0; extra == "all"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
26
|
+
Requires-Dist: matplotlib>=3.7; extra == "dev"
|
|
27
|
+
Requires-Dist: scipy>=1.10; extra == "dev"
|
|
28
|
+
Requires-Dist: ipython>=8.0; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# learnstat
|
|
32
|
+
|
|
33
|
+
An AI-assisted learning companion for statistical analysis in Python.
|
|
34
|
+
|
|
35
|
+
You write ordinary **pandas / NumPy / SciPy / Matplotlib** code. learnstat:
|
|
36
|
+
|
|
37
|
+
1. **Understands your dataset.** It profiles the data once and keeps a Markdown memory file of it.
|
|
38
|
+
2. **Suggests better approaches as you write.** For example, it flags a pie chart of 12 categories and suggests a sorted bar chart, explaining why.
|
|
39
|
+
3. **Explains itself** through `/describe`, `/alternatives`, `/learn`, `/memory` and `/fix`.
|
|
40
|
+
4. **Recovers from mechanical errors.** It fixes typos in column names, methods and keywords, plus missing imports and unclosed brackets, and shows you every change.
|
|
41
|
+
5. **Leaves analytical decisions to you.** Data-quality and statistical-method problems are explained but never "fixed" automatically.
|
|
42
|
+
|
|
43
|
+
It's not a new language. Everything you learn uses the real pandas, SciPy and Matplotlib APIs, so the skills carry over to the rest of the ecosystem.
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/learnstat/)
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
From PyPI: <https://pypi.org/project/learnstat/>
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install "learnstat[all]" # from PyPI
|
|
53
|
+
pip install -e ".[all]" # pandas, numpy, matplotlib, scipy, ipython
|
|
54
|
+
pip install -e . # core only: profiling, rules, recovery
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Add an AI (optional)
|
|
58
|
+
|
|
59
|
+
Just run `learnstat`. If no AI is set up yet it offers a guided setup, and you can repeat it any time with `learnstat setup` (or `/ai` inside a session):
|
|
60
|
+
|
|
61
|
+
1. **Choose the company**: OpenAI, Anthropic, Google Gemini, xAI, Mistral, DeepSeek, Groq, Together, OpenRouter, Ollama, LM Studio, or any other OpenAI-compatible endpoint.
|
|
62
|
+
2. **Paste your API key**. The input is hidden, and local servers such as Ollama need no key.
|
|
63
|
+
3. **Pick a model** from the list your key can actually use. Type part of a name to filter.
|
|
64
|
+
4. **learnstat checks it** with a live test request and shows `OK` or the exact problem (rejected key, unknown model, no connection, quota), with options to fix it.
|
|
65
|
+
5. It saves the account and starts the interactive session.
|
|
66
|
+
|
|
67
|
+
Several accounts can be saved and switched: run `learnstat setup` and pick a number, or use `/ai use <name>` in a session. The AI features need no extra package.
|
|
68
|
+
|
|
69
|
+
For scripts and CI there is a non-interactive route:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
learnstat config set api_key sk-your-key
|
|
73
|
+
learnstat config set base_url https://openrouter.ai/api/v1 # default: the active company's endpoint
|
|
74
|
+
learnstat config set model openai/gpt-4o-mini
|
|
75
|
+
learnstat config list # saved accounts learnstat config use <name> # switch
|
|
76
|
+
learnstat config test # one tiny request learnstat config show # key is masked
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Accounts are stored in `~/.learnstat/config.json` (`learnstat config path` prints the location), readable only by you where the OS allows it. The environment variables `LEARNSTAT_API_KEY`, `LEARNSTAT_BASE_URL` and `LEARNSTAT_MODEL` override the active account for one run.
|
|
80
|
+
|
|
81
|
+
With an AI configured, explanations are tailored to your dataset, `/review` checks code for issues the rules miss, and the AI can propose fixes for errors the rules can't handle. Without it, everything else still works offline. Pass `--no-ai` to switch it off for one run.
|
|
82
|
+
|
|
83
|
+
## Use it
|
|
84
|
+
|
|
85
|
+
### Jupyter
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
%load_ext learnstat
|
|
89
|
+
import learnstat as ls
|
|
90
|
+
df = ls.read_csv("survey.csv") # a normal DataFrame, now profiled into .learnstat/survey.md
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
df["country"].value_counts().plot.pie()
|
|
95
|
+
# [!] Suggestion #1 (line 1): Pie chart with 12 categories
|
|
96
|
+
# Issue: `country` has 12 categories; slices this numerous are hard to compare by angle.
|
|
97
|
+
# Instead: A horizontal bar chart of the counts, sorted by size.
|
|
98
|
+
# Why: Bars share a common baseline ...
|
|
99
|
+
# How: counts = df['country'].value_counts() ...
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
/describe <- the concept, why it fits this data, when another method is better
|
|
104
|
+
/accept <- puts the suggested code in a NEW cell; your cell is untouched
|
|
105
|
+
/reject only 3 countries matter here <- recorded in memory, not suggested again
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
When a cell raises an error, learnstat diagnoses the failing statement. If it's a mechanical mistake, learnstat corrects it, re-runs only that statement and the ones after it, and prints the diff. A cell with a syntax error gets a corrected copy placed in the next cell for you to review.
|
|
109
|
+
|
|
110
|
+
### Command line
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
cd examples && python make_survey.py
|
|
114
|
+
learnstat profile survey.csv # writes .learnstat/survey.md and prints it
|
|
115
|
+
learnstat check analysis.py # suggestions only, nothing is executed
|
|
116
|
+
learnstat watch analysis.py # re-checks every time you save the file
|
|
117
|
+
learnstat run analysis.py # runs with suggestions and guarded recovery
|
|
118
|
+
learnstat repl --data survey.csv # interactive, with /commands
|
|
119
|
+
learnstat config show # AI settings (see "Add an AI")
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Plain Python
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
import learnstat as ls
|
|
126
|
+
df = ls.read_csv("survey.csv")
|
|
127
|
+
ls.check("df['income'].mean()") # suggestions without running
|
|
128
|
+
print(ls.command("/describe"))
|
|
129
|
+
|
|
130
|
+
s = ls.Session(confirm=lambda q: input(q + " [y/N] ") == "y")
|
|
131
|
+
s.ns["df"] = s.read_csv("survey.csv")
|
|
132
|
+
result = s.run("df.groupBy('group')['score'].median()")
|
|
133
|
+
print(result.final_code) # df.groupby(...)... your source is never modified
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Commands
|
|
137
|
+
|
|
138
|
+
| Command | What it does |
|
|
139
|
+
|---|---|
|
|
140
|
+
| `/describe [id]` | Explains the latest (or given) suggestion, or the latest error: the concept, the purpose of the function, why it fits this data, and when something else is better |
|
|
141
|
+
| `/alternatives [id]` | Other ways to reach the same goal, with trade-offs |
|
|
142
|
+
| `/learn [topic]` | A short lesson with an exercise on your own data |
|
|
143
|
+
| `/fix` | The last error diagnosis; in interactive mode, a retry of the failed statement |
|
|
144
|
+
| `/memory [show\|note\|decide\|refresh]` | View or update the dataset's persistent profile |
|
|
145
|
+
| `/accept [id]`, `/reject [id] [why]` | Accept or reject a suggestion; the decision is saved in memory |
|
|
146
|
+
| `/suggestions`, `/review`, `/ask`, `/status`, `/help` | |
|
|
147
|
+
|
|
148
|
+
## Dataset memory
|
|
149
|
+
|
|
150
|
+
`ls.read_csv` profiles the data and writes two files to `.learnstat/`:
|
|
151
|
+
|
|
152
|
+
- `survey.md`: columns, types, missing values, likely column meanings (marked as guesses), distributions, relationships, data-quality issues, suggested analyses, assumptions, **your analytical decisions** and notes.
|
|
153
|
+
- `survey.json`: the machine-readable cache. It includes a content fingerprint, so the data is only re-profiled when it actually changes. Decisions and notes carry over when that happens.
|
|
154
|
+
|
|
155
|
+
## How error recovery stays safe
|
|
156
|
+
|
|
157
|
+
- Code runs **one top-level statement at a time**. When statement *k* fails, only statement *k* is corrected and retried, so earlier side effects are never repeated.
|
|
158
|
+
- Errors are classified as **syntax, name, attribute, argument, column, data-quality, statistical-method, environment** or other runtime errors.
|
|
159
|
+
- Only mechanical mistakes are corrected automatically, and only to a unique, high-similarity match.
|
|
160
|
+
- Before a fix runs, it must compile. It must not introduce file, shell or network operations, must not change which variables are assigned, must not add data-altering calls (`dropna`, `fillna`, `astype` and so on), and must not change the statistical method.
|
|
161
|
+
- AI-proposed fixes always need your confirmation. So does retrying a loop or block, since that could repeat side effects.
|
|
162
|
+
- Retries are bounded (`max_attempts`, 3 by default). If the same error comes back, recovery stops and gives you a concrete next step.
|
|
163
|
+
- Code that deletes or overwrites files, runs shell commands or opens network connections only runs after you confirm.
|
|
164
|
+
- **This is a policy layer, not a sandbox.** Your code runs in your Python process with your permissions, exactly as if you had run it yourself.
|
|
165
|
+
|
|
166
|
+
## Project layout
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
learnstat/
|
|
170
|
+
profile.py dataset profiling (no AI)
|
|
171
|
+
memory.py Markdown + JSON dataset memory
|
|
172
|
+
advisor.py AST-based, data-aware suggestion rules
|
|
173
|
+
knowledge.py offline explanations behind /describe, /alternatives, /learn
|
|
174
|
+
recovery.py error diagnosis, fix proposal, validation, bounded retry
|
|
175
|
+
safety.py detection of destructive side effects
|
|
176
|
+
llm.py optional AI backend (OpenAI-compatible endpoint)
|
|
177
|
+
config.py saved AI accounts: provider, api_key, base_url, model
|
|
178
|
+
providers.py presets for the AI companies
|
|
179
|
+
setup.py guided terminal setup (company -> key -> model -> check)
|
|
180
|
+
session.py ties it together; read_csv/track/check/run
|
|
181
|
+
commands.py slash commands
|
|
182
|
+
ipython_ext.py %load_ext learnstat
|
|
183
|
+
cli.py learnstat profile|check|watch|run|repl
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Add a rule by writing a function `(call, resolver, code) -> Suggestion | None` in `advisor.py`, appending it to `RULES`, and adding its concept to `knowledge.py`.
|
|
187
|
+
|
|
188
|
+
## Roadmap
|
|
189
|
+
|
|
190
|
+
- A language-server (LSP) frontend for suggestions while typing in VS Code and JupyterLab. The advisor is already editor-agnostic.
|
|
191
|
+
- More rules: multiple-comparison corrections, chi-square expected counts, normality checks before parametric tests, time-series plotting.
|
|
192
|
+
- Learning-progress tracking, so advice fades as the user masters a concept.
|
|
193
|
+
|
|
194
|
+
## Tests
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install -e ".[dev]"
|
|
198
|
+
pytest
|
|
199
|
+
```
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
learnstat/__init__.py,sha256=C3f7siCc35sZLVFnJsRHI-8AerlASu-V1EGjrFnQKCY,2496
|
|
2
|
+
learnstat/advisor.py,sha256=96fxyfxARVtQFNcEn4n1pp1UkgXJWGwBgJXE4rnAYUo,21211
|
|
3
|
+
learnstat/cli.py,sha256=Nwht__ssdKU8D2LA5luZvkmXmRdcamYZD9OlNmPtirk,13306
|
|
4
|
+
learnstat/commands.py,sha256=UTwRJAXw-95qjwYJKuSNFbwDwAQxCHppFmxjc4WBP58,13099
|
|
5
|
+
learnstat/config.py,sha256=5voQrI5yH4tcMzvtDdWmY0bEiDdNwkoF023hTc_n7pY,7339
|
|
6
|
+
learnstat/ipython_ext.py,sha256=OMSZQAY3lw9YGuppCQJhMUO-OO0G5y4fCHjDsxnoMKc,4172
|
|
7
|
+
learnstat/knowledge.py,sha256=GHuDob0aGhzCTwQSXvYyDwVQofzsXTfEonOfGiRIkww,12422
|
|
8
|
+
learnstat/llm.py,sha256=N_ShNvNAi64bE_VKwA35m8HGBd_8rz_SvreKIHcjv2o,13356
|
|
9
|
+
learnstat/memory.py,sha256=xRJUZ0Bv18WtjwD8PHWXt2NClbjW2D__M2s42u1qflI,7501
|
|
10
|
+
learnstat/profile.py,sha256=LohZ4E_HLsnXr-ncNDza1EC_a2KIDv2S3RPIEvFM2JQ,13377
|
|
11
|
+
learnstat/providers.py,sha256=Z7g2cd_8F2gmoDpCDSSTnGnUCDCTFlDUX6pH3iJSRyY,3440
|
|
12
|
+
learnstat/recovery.py,sha256=70ALYY-sjB0cRg16MGOFPND55sMlxW7DwhEw78r-nBo,48474
|
|
13
|
+
learnstat/safety.py,sha256=yolj4foYo03aPXnM802TMjms-JuO4oQ6feXC-Q4YRnw,4254
|
|
14
|
+
learnstat/session.py,sha256=ulIjOPeCggHjManShhxxqKiZ8IEUFqTyrFaD0sIiWBw,13292
|
|
15
|
+
learnstat/setup.py,sha256=5JvYDhdGBCt4mSPAK1seDO2NzaGFWIEROvWpTbvuJ38,12280
|
|
16
|
+
learnstat/suggestions.py,sha256=UteehmSJHGlbh2j-wvu6rB107kVSzI6EOShmMTROppQ,2507
|
|
17
|
+
learnstat-0.2.0.dist-info/licenses/LICENSE,sha256=H3PxNCq3f9tgoYAk-9yQU36dCVjsYFcj4Sc97kdqvRY,1094
|
|
18
|
+
learnstat-0.2.0.dist-info/METADATA,sha256=mbUkyMintdWCxdAL-gqSZHeYDYbdQfV32HfAylcKOUo,10494
|
|
19
|
+
learnstat-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
20
|
+
learnstat-0.2.0.dist-info/entry_points.txt,sha256=2GGW9VTdCu1dXtpv0OvrLVGm9kQzsRgCvQ9xy1XHfOg,49
|
|
21
|
+
learnstat-0.2.0.dist-info/top_level.txt,sha256=D9hHzVC0PYxXVk2MaIrkuCRd59BKcHJZzVbhhCsJZZ8,10
|
|
22
|
+
learnstat-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Umor Faruk Supto
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
learnstat
|