jaxforge 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jax_gemini/__init__.py +98 -0
- jax_gemini/agent.py +5 -0
- jax_gemini/config.py +5 -0
- jax_gemini/exceptions.py +23 -0
- jaxforge/__init__.py +80 -0
- jaxforge/agent.py +778 -0
- jaxforge/cli.py +112 -0
- jaxforge/codegen/__init__.py +15 -0
- jaxforge/codegen/diff.py +92 -0
- jaxforge/codegen/fixer.py +90 -0
- jaxforge/codegen/parser.py +58 -0
- jaxforge/codegen/validator.py +370 -0
- jaxforge/config.py +211 -0
- jaxforge/contrib/__init__.py +39 -0
- jaxforge/contrib/flax.py +49 -0
- jaxforge/contrib/optax.py +67 -0
- jaxforge/contrib/sharding.py +51 -0
- jaxforge/contrib/tracking.py +63 -0
- jaxforge/exceptions.py +60 -0
- jaxforge/exporter.py +63 -0
- jaxforge/llm/__init__.py +22 -0
- jaxforge/llm/base.py +35 -0
- jaxforge/llm/gemini.py +221 -0
- jaxforge/llm/providers.py +361 -0
- jaxforge/mcp_server.py +59 -0
- jaxforge/memory/__init__.py +5 -0
- jaxforge/memory/conversation.py +92 -0
- jaxforge/pipeline/__init__.py +13 -0
- jaxforge/pipeline/builder.py +19 -0
- jaxforge/pipeline/evaluator.py +16 -0
- jaxforge/pipeline/exporter.py +20 -0
- jaxforge/pipeline/trainer.py +21 -0
- jaxforge/plan.py +153 -0
- jaxforge/prompts/__init__.py +15 -0
- jaxforge/prompts/context_builder.py +142 -0
- jaxforge/prompts/intent.py +77 -0
- jaxforge/prompts/system_prompt.py +201 -0
- jaxforge/sandbox/__init__.py +25 -0
- jaxforge/sandbox/backends.py +385 -0
- jaxforge/sandbox/executor.py +80 -0
- jaxforge/sandbox/namespace.py +459 -0
- jaxforge-0.2.0.dist-info/METADATA +188 -0
- jaxforge-0.2.0.dist-info/RECORD +46 -0
- jaxforge-0.2.0.dist-info/WHEEL +4 -0
- jaxforge-0.2.0.dist-info/entry_points.txt +2 -0
- jaxforge-0.2.0.dist-info/licenses/LICENSE +21 -0
jaxforge/agent.py
ADDED
|
@@ -0,0 +1,778 @@
|
|
|
1
|
+
"""Main JaxForge session orchestrator supporting preview, multi-provider routing, and explore."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import copy
|
|
7
|
+
import os
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, cast
|
|
11
|
+
|
|
12
|
+
import flax.nnx as nnx
|
|
13
|
+
|
|
14
|
+
from jaxforge.codegen.diff import build_unified_diff
|
|
15
|
+
from jaxforge.codegen.fixer import CodeFixer
|
|
16
|
+
from jaxforge.codegen.parser import CodeParser
|
|
17
|
+
from jaxforge.codegen.validator import CodeValidator
|
|
18
|
+
from jaxforge.config import JaxForgeConfig
|
|
19
|
+
from jaxforge.contrib.tracking import InMemoryTracker, MetricsTracker
|
|
20
|
+
from jaxforge.exceptions import (
|
|
21
|
+
JaxForgeApprovalError,
|
|
22
|
+
JaxForgeCheckpointError,
|
|
23
|
+
JaxForgeError,
|
|
24
|
+
JaxForgeExecutionError,
|
|
25
|
+
JaxForgeIntentError,
|
|
26
|
+
JaxForgeLLMError,
|
|
27
|
+
JaxForgeValidationError,
|
|
28
|
+
)
|
|
29
|
+
from jaxforge.exporter import export_project as write_project_package
|
|
30
|
+
from jaxforge.llm.providers import RouterLLM, create_llm
|
|
31
|
+
from jaxforge.memory.conversation import ConversationMemory
|
|
32
|
+
from jaxforge.plan import (
|
|
33
|
+
DatasetProfile,
|
|
34
|
+
ExperimentCandidate,
|
|
35
|
+
ExperimentResult,
|
|
36
|
+
GeneratedPlan,
|
|
37
|
+
)
|
|
38
|
+
from jaxforge.prompts.context_builder import MAX_CONTEXT_PROMPT_CHARS, ContextBuilder
|
|
39
|
+
from jaxforge.prompts.intent import IntentClassifier
|
|
40
|
+
from jaxforge.sandbox.executor import SandboxExecutor
|
|
41
|
+
|
|
42
|
+
ApprovalCallback = Callable[[GeneratedPlan], bool]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _snapshot_runtime_args(runtime_args: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
46
|
+
"""Create an isolated copy of mutable runtime_args so failed retries do not mutate state."""
|
|
47
|
+
if runtime_args is None:
|
|
48
|
+
return None
|
|
49
|
+
snapshot: dict[str, Any] = {}
|
|
50
|
+
for key, val in runtime_args.items():
|
|
51
|
+
try:
|
|
52
|
+
snapshot[key] = copy.deepcopy(val)
|
|
53
|
+
except Exception:
|
|
54
|
+
snapshot[key] = val
|
|
55
|
+
return snapshot
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _sanitize_validation_feedback(err: JaxForgeValidationError) -> str:
|
|
59
|
+
"""Avoid exposing internal AST security policy details as an interactive bypass oracle."""
|
|
60
|
+
msg = str(err)
|
|
61
|
+
if "syntax error" in msg.lower() or "must define a top-level function" in msg.lower():
|
|
62
|
+
return msg
|
|
63
|
+
return "Generated code violated sandbox validation rules; use only allowed ML operations."
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class JaxForge:
|
|
67
|
+
"""Natural language-driven, inspect-first JAX engineering workbench."""
|
|
68
|
+
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
config: JaxForgeConfig | None = None,
|
|
72
|
+
tracker: MetricsTracker | None = None,
|
|
73
|
+
):
|
|
74
|
+
self.config = config or JaxForgeConfig()
|
|
75
|
+
self._llm: RouterLLM | None = None
|
|
76
|
+
self._llm_cache_key: tuple[str, str, str | None, str | None, str | None] | None = None
|
|
77
|
+
self._memory = ConversationMemory()
|
|
78
|
+
self._executor = SandboxExecutor()
|
|
79
|
+
self._tracker: MetricsTracker = tracker or InMemoryTracker()
|
|
80
|
+
self._last_code: str | None = None
|
|
81
|
+
self._last_build_code: str | None = None
|
|
82
|
+
self._last_train_code: str | None = None
|
|
83
|
+
self._last_eval_code: str | None = None
|
|
84
|
+
self._last_response: dict[str, Any] | None = None
|
|
85
|
+
self._current_model: Any = None
|
|
86
|
+
|
|
87
|
+
def _get_llm(self) -> RouterLLM:
|
|
88
|
+
self.config.validate()
|
|
89
|
+
effective_key = self.config.get_effective_api_key()
|
|
90
|
+
cache_key = (
|
|
91
|
+
self.config.resolve_provider(),
|
|
92
|
+
self.config.resolve_model_id(),
|
|
93
|
+
self.config.fast_model_name,
|
|
94
|
+
self.config.base_url,
|
|
95
|
+
effective_key,
|
|
96
|
+
)
|
|
97
|
+
if self._llm is None or self._llm_cache_key != cache_key:
|
|
98
|
+
primary = create_llm(self.config)
|
|
99
|
+
fast = (
|
|
100
|
+
create_llm(self.config, model_override=self.config.fast_model_name)
|
|
101
|
+
if self.config.fast_model_name
|
|
102
|
+
else None
|
|
103
|
+
)
|
|
104
|
+
self._llm = RouterLLM(primary=primary, fast=fast)
|
|
105
|
+
self._llm_cache_key = cache_key
|
|
106
|
+
return self._llm
|
|
107
|
+
|
|
108
|
+
def _log(self, message: str) -> None:
|
|
109
|
+
if self.config.verbose:
|
|
110
|
+
print(f"[jaxforge] {message}")
|
|
111
|
+
|
|
112
|
+
def _allowed_dirs(self, intent: str) -> tuple[Path, ...]:
|
|
113
|
+
ckpt_dir = Path(self.config.checkpoint_dir)
|
|
114
|
+
if intent in ("save", "load"):
|
|
115
|
+
return (ckpt_dir,)
|
|
116
|
+
return (Path.cwd(), ckpt_dir)
|
|
117
|
+
|
|
118
|
+
def _record_success(
|
|
119
|
+
self,
|
|
120
|
+
intent: str,
|
|
121
|
+
user_prompt: str,
|
|
122
|
+
code: str,
|
|
123
|
+
response: dict[str, Any],
|
|
124
|
+
) -> None:
|
|
125
|
+
self._last_code = code
|
|
126
|
+
self._last_response = response
|
|
127
|
+
if intent == "build":
|
|
128
|
+
self._last_build_code = code
|
|
129
|
+
elif intent == "train":
|
|
130
|
+
self._last_train_code = code
|
|
131
|
+
elif intent == "evaluate":
|
|
132
|
+
self._last_eval_code = code
|
|
133
|
+
self._memory.add_turn("user", user_prompt)
|
|
134
|
+
self._memory.add_turn("model", str(response.get("description", code)))
|
|
135
|
+
if intent in ("build", "train"):
|
|
136
|
+
self._memory.add_model_state(
|
|
137
|
+
code=code,
|
|
138
|
+
description=str(response.get("description", "")),
|
|
139
|
+
intent=intent,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
def preview(
|
|
143
|
+
self,
|
|
144
|
+
user_prompt: str,
|
|
145
|
+
intent: str | None = None,
|
|
146
|
+
dataset: Any = None,
|
|
147
|
+
diff_mode: bool = False,
|
|
148
|
+
) -> GeneratedPlan:
|
|
149
|
+
"""Generate and validate code cleanly without executing it in the sandbox."""
|
|
150
|
+
ContextBuilder.validate_prompt(user_prompt)
|
|
151
|
+
resolved_intent = intent or IntentClassifier.classify(user_prompt)
|
|
152
|
+
framework = IntentClassifier.detect_framework(user_prompt, default=self.config.framework)
|
|
153
|
+
profile = DatasetProfile.from_dataset(dataset) if dataset is not None else None
|
|
154
|
+
prev_state = self._memory.get_current_model_state()
|
|
155
|
+
allow_diff = diff_mode and resolved_intent == "build" and prev_state is not None
|
|
156
|
+
|
|
157
|
+
llm = self._get_llm()
|
|
158
|
+
context = ContextBuilder.build(
|
|
159
|
+
intent=resolved_intent,
|
|
160
|
+
user_prompt=user_prompt,
|
|
161
|
+
memory=self._memory,
|
|
162
|
+
current_model_state=prev_state,
|
|
163
|
+
dataset_profile=profile,
|
|
164
|
+
framework=framework,
|
|
165
|
+
diff_mode=allow_diff,
|
|
166
|
+
)
|
|
167
|
+
schema = ContextBuilder.get_schema(resolved_intent, allow_diff=allow_diff)
|
|
168
|
+
response = llm.generate(context, schema)
|
|
169
|
+
previous_code = prev_state.code if (allow_diff and prev_state is not None) else None
|
|
170
|
+
code = CodeParser.extract(response, resolved_intent, previous_code=previous_code)
|
|
171
|
+
CodeValidator.validate(code, resolved_intent)
|
|
172
|
+
diff_text = (
|
|
173
|
+
build_unified_diff(prev_state.code, code)
|
|
174
|
+
if (prev_state is not None and resolved_intent == "build")
|
|
175
|
+
else ""
|
|
176
|
+
)
|
|
177
|
+
raw_warnings = response.get("warnings") or []
|
|
178
|
+
warnings = [str(w) for w in raw_warnings] if isinstance(raw_warnings, list) else []
|
|
179
|
+
return GeneratedPlan(
|
|
180
|
+
intent=resolved_intent,
|
|
181
|
+
code=code,
|
|
182
|
+
description=str(response.get("description", "")),
|
|
183
|
+
provider=self.config.resolve_provider(),
|
|
184
|
+
model_name=self.config.resolve_model_id(),
|
|
185
|
+
framework=framework,
|
|
186
|
+
validation_passed=True,
|
|
187
|
+
unified_diff=diff_text,
|
|
188
|
+
warnings=warnings,
|
|
189
|
+
metadata={"raw_response": response},
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def execute_plan(
|
|
193
|
+
self,
|
|
194
|
+
plan: GeneratedPlan,
|
|
195
|
+
runtime_args: dict[str, Any] | None = None,
|
|
196
|
+
user_prompt: str = "",
|
|
197
|
+
approve: ApprovalCallback | None = None,
|
|
198
|
+
) -> Any:
|
|
199
|
+
"""Run a previously previewed plan after checking approval policy."""
|
|
200
|
+
if approve is not None:
|
|
201
|
+
if not approve(plan):
|
|
202
|
+
raise JaxForgeApprovalError(
|
|
203
|
+
f"Execution of intent '{plan.intent}' rejected by approval callback.",
|
|
204
|
+
code=plan.code,
|
|
205
|
+
)
|
|
206
|
+
elif not self.config.auto_approve:
|
|
207
|
+
raise JaxForgeApprovalError(
|
|
208
|
+
"auto_approve is False; pass an explicit approve callback or call execute_plan().",
|
|
209
|
+
code=plan.code,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
attempt_args = _snapshot_runtime_args(runtime_args)
|
|
213
|
+
result = self._executor.run(
|
|
214
|
+
plan.code,
|
|
215
|
+
plan.intent,
|
|
216
|
+
attempt_args,
|
|
217
|
+
allowed_dirs=self._allowed_dirs(plan.intent),
|
|
218
|
+
timeout_seconds=self.config.timeout_seconds,
|
|
219
|
+
backend_name=self.config.execution_backend,
|
|
220
|
+
max_memory_mb=self.config.max_memory_mb,
|
|
221
|
+
)
|
|
222
|
+
raw_resp = plan.metadata.get("raw_response") or {
|
|
223
|
+
"intent": plan.intent,
|
|
224
|
+
"code": plan.code,
|
|
225
|
+
"description": plan.description,
|
|
226
|
+
}
|
|
227
|
+
self._record_success(
|
|
228
|
+
plan.intent,
|
|
229
|
+
user_prompt or plan.description or plan.intent,
|
|
230
|
+
plan.code,
|
|
231
|
+
raw_resp,
|
|
232
|
+
)
|
|
233
|
+
if plan.intent == "build":
|
|
234
|
+
self._current_model = result
|
|
235
|
+
elif plan.intent == "train" and isinstance(result, tuple):
|
|
236
|
+
self._current_model = result[0]
|
|
237
|
+
if isinstance(result[1], dict):
|
|
238
|
+
self._tracker.log_metrics(result[1])
|
|
239
|
+
elif plan.intent == "evaluate" and isinstance(result, dict):
|
|
240
|
+
self._tracker.log_metrics(result)
|
|
241
|
+
elif plan.intent == "load":
|
|
242
|
+
self._current_model = result
|
|
243
|
+
return result
|
|
244
|
+
|
|
245
|
+
def _generate_and_execute(
|
|
246
|
+
self,
|
|
247
|
+
user_prompt: str,
|
|
248
|
+
intent: str,
|
|
249
|
+
runtime_args: dict[str, Any] | None = None,
|
|
250
|
+
dataset: Any = None,
|
|
251
|
+
dry_run: bool = False,
|
|
252
|
+
approve: ApprovalCallback | None = None,
|
|
253
|
+
diff_mode: bool = False,
|
|
254
|
+
) -> Any:
|
|
255
|
+
ContextBuilder.validate_prompt(user_prompt)
|
|
256
|
+
llm = self._get_llm()
|
|
257
|
+
framework = IntentClassifier.detect_framework(user_prompt, default=self.config.framework)
|
|
258
|
+
profile = DatasetProfile.from_dataset(dataset) if dataset is not None else None
|
|
259
|
+
prev_state = self._memory.get_current_model_state()
|
|
260
|
+
allow_diff = diff_mode and intent == "build" and prev_state is not None
|
|
261
|
+
|
|
262
|
+
context = ContextBuilder.build(
|
|
263
|
+
intent=intent,
|
|
264
|
+
user_prompt=user_prompt,
|
|
265
|
+
memory=self._memory,
|
|
266
|
+
current_model_state=prev_state,
|
|
267
|
+
dataset_profile=profile,
|
|
268
|
+
framework=framework,
|
|
269
|
+
diff_mode=allow_diff,
|
|
270
|
+
)
|
|
271
|
+
schema = ContextBuilder.get_schema(intent, allow_diff=allow_diff)
|
|
272
|
+
last_error: Exception | None = None
|
|
273
|
+
last_feedback = ""
|
|
274
|
+
last_code = ""
|
|
275
|
+
|
|
276
|
+
for attempt in range(1, self.config.max_retries + 1):
|
|
277
|
+
self._log(f"Attempt {attempt}/{self.config.max_retries} for intent '{intent}'...")
|
|
278
|
+
|
|
279
|
+
if attempt > 1 and last_error is not None:
|
|
280
|
+
fix_prompt = CodeFixer.build_fix_prompt(
|
|
281
|
+
original_prompt=user_prompt,
|
|
282
|
+
failed_code=last_code,
|
|
283
|
+
error_message=last_feedback or str(last_error),
|
|
284
|
+
attempt=attempt,
|
|
285
|
+
intent=intent,
|
|
286
|
+
)
|
|
287
|
+
context = ContextBuilder.build(
|
|
288
|
+
intent=intent,
|
|
289
|
+
user_prompt=fix_prompt,
|
|
290
|
+
memory=self._memory,
|
|
291
|
+
current_model_state=prev_state,
|
|
292
|
+
max_prompt_chars=MAX_CONTEXT_PROMPT_CHARS,
|
|
293
|
+
dataset_profile=profile,
|
|
294
|
+
framework=framework,
|
|
295
|
+
diff_mode=False,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
try:
|
|
299
|
+
response = llm.generate(context, schema, use_fast=(attempt > 1))
|
|
300
|
+
previous_code = (
|
|
301
|
+
prev_state.code if (allow_diff and attempt == 1 and prev_state) else None
|
|
302
|
+
)
|
|
303
|
+
code = CodeParser.extract(response, intent, previous_code=previous_code)
|
|
304
|
+
CodeValidator.validate(code, intent)
|
|
305
|
+
|
|
306
|
+
plan = GeneratedPlan(
|
|
307
|
+
intent=intent,
|
|
308
|
+
code=code,
|
|
309
|
+
description=str(response.get("description", "")),
|
|
310
|
+
provider=self.config.resolve_provider(),
|
|
311
|
+
model_name=self.config.resolve_model_id(),
|
|
312
|
+
framework=framework,
|
|
313
|
+
validation_passed=True,
|
|
314
|
+
unified_diff=(
|
|
315
|
+
build_unified_diff(prev_state.code, code)
|
|
316
|
+
if (prev_state is not None and intent == "build")
|
|
317
|
+
else ""
|
|
318
|
+
),
|
|
319
|
+
warnings=[
|
|
320
|
+
str(w) for w in (response.get("warnings") or []) if isinstance(w, str)
|
|
321
|
+
],
|
|
322
|
+
metadata={"raw_response": response},
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
if dry_run:
|
|
326
|
+
return plan
|
|
327
|
+
if approve is not None:
|
|
328
|
+
if not approve(plan):
|
|
329
|
+
raise JaxForgeApprovalError(
|
|
330
|
+
f"Execution of '{intent}' rejected during preview.", code=code
|
|
331
|
+
)
|
|
332
|
+
elif not self.config.auto_approve:
|
|
333
|
+
raise JaxForgeApprovalError(
|
|
334
|
+
"auto_approve is False; use preview()/execute_plan() or pass approve=.",
|
|
335
|
+
code=code,
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
attempt_args = _snapshot_runtime_args(runtime_args)
|
|
339
|
+
result = self._executor.run(
|
|
340
|
+
code,
|
|
341
|
+
intent,
|
|
342
|
+
attempt_args,
|
|
343
|
+
allowed_dirs=self._allowed_dirs(intent),
|
|
344
|
+
timeout_seconds=self.config.timeout_seconds,
|
|
345
|
+
backend_name=self.config.execution_backend,
|
|
346
|
+
max_memory_mb=self.config.max_memory_mb,
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
self._record_success(intent, user_prompt, code, response)
|
|
350
|
+
self._log(f"Attempt {attempt} succeeded.")
|
|
351
|
+
return result
|
|
352
|
+
|
|
353
|
+
except JaxForgeApprovalError:
|
|
354
|
+
raise
|
|
355
|
+
|
|
356
|
+
except JaxForgeValidationError as e:
|
|
357
|
+
last_error = e
|
|
358
|
+
last_feedback = _sanitize_validation_feedback(e)
|
|
359
|
+
last_code = getattr(e, "code", "")
|
|
360
|
+
self._log(f"Attempt {attempt} failed validation: {e}")
|
|
361
|
+
|
|
362
|
+
except JaxForgeExecutionError as e:
|
|
363
|
+
last_error = e
|
|
364
|
+
last_feedback = str(e)
|
|
365
|
+
last_code = getattr(e, "code", "")
|
|
366
|
+
self._log(f"Attempt {attempt} failed execution: {e}")
|
|
367
|
+
|
|
368
|
+
except JaxForgeLLMError as e:
|
|
369
|
+
last_error = e
|
|
370
|
+
last_feedback = (
|
|
371
|
+
"Previous response was malformed or failed to parse as valid JSON; "
|
|
372
|
+
"respond strictly with the requested JSON schema."
|
|
373
|
+
)
|
|
374
|
+
last_code = ""
|
|
375
|
+
self._log(f"Attempt {attempt} failed LLM response parsing: {e}")
|
|
376
|
+
|
|
377
|
+
if isinstance(last_error, JaxForgeLLMError):
|
|
378
|
+
raise last_error
|
|
379
|
+
if isinstance(last_error, JaxForgeValidationError):
|
|
380
|
+
raise last_error
|
|
381
|
+
|
|
382
|
+
prompt_preview = user_prompt[:160] + ("..." if len(user_prompt) > 160 else "")
|
|
383
|
+
raise JaxForgeExecutionError(
|
|
384
|
+
f"All {self.config.max_retries} attempts failed for '{prompt_preview}'.\n"
|
|
385
|
+
f"Last error: {last_error}",
|
|
386
|
+
code=last_code,
|
|
387
|
+
attempts=self.config.max_retries,
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
def build(
|
|
391
|
+
self,
|
|
392
|
+
prompt: str,
|
|
393
|
+
dry_run: bool = False,
|
|
394
|
+
approve: ApprovalCallback | None = None,
|
|
395
|
+
) -> Any:
|
|
396
|
+
"""Build a JAX neural network module from natural language (or return GeneratedPlan)."""
|
|
397
|
+
result = self._generate_and_execute(
|
|
398
|
+
prompt,
|
|
399
|
+
intent="build",
|
|
400
|
+
runtime_args={"rngs": nnx.Rngs(0)},
|
|
401
|
+
dry_run=dry_run,
|
|
402
|
+
approve=approve,
|
|
403
|
+
)
|
|
404
|
+
if dry_run:
|
|
405
|
+
return result
|
|
406
|
+
self._current_model = result
|
|
407
|
+
return result
|
|
408
|
+
|
|
409
|
+
def modify(
|
|
410
|
+
self,
|
|
411
|
+
prompt: str,
|
|
412
|
+
dry_run: bool = False,
|
|
413
|
+
approve: ApprovalCallback | None = None,
|
|
414
|
+
use_diff: bool = True,
|
|
415
|
+
) -> Any:
|
|
416
|
+
"""Refine the current model iteratively using diff or full-code updates."""
|
|
417
|
+
result = self._generate_and_execute(
|
|
418
|
+
prompt,
|
|
419
|
+
intent="build",
|
|
420
|
+
runtime_args={"rngs": nnx.Rngs(0)},
|
|
421
|
+
dry_run=dry_run,
|
|
422
|
+
approve=approve,
|
|
423
|
+
diff_mode=use_diff,
|
|
424
|
+
)
|
|
425
|
+
if dry_run:
|
|
426
|
+
return result
|
|
427
|
+
self._current_model = result
|
|
428
|
+
return result
|
|
429
|
+
|
|
430
|
+
def train(
|
|
431
|
+
self,
|
|
432
|
+
prompt: str,
|
|
433
|
+
dataset: Any = None,
|
|
434
|
+
dry_run: bool = False,
|
|
435
|
+
approve: ApprovalCallback | None = None,
|
|
436
|
+
) -> Any:
|
|
437
|
+
"""Train the active model cleanly returning ``(trained_model, metrics_dict)``."""
|
|
438
|
+
args: dict[str, Any] = {"dataset": dataset} if dataset is not None else {}
|
|
439
|
+
if self._current_model is not None:
|
|
440
|
+
args["model"] = self._current_model
|
|
441
|
+
result = self._generate_and_execute(
|
|
442
|
+
prompt,
|
|
443
|
+
intent="train",
|
|
444
|
+
runtime_args=args,
|
|
445
|
+
dataset=dataset,
|
|
446
|
+
dry_run=dry_run,
|
|
447
|
+
approve=approve,
|
|
448
|
+
)
|
|
449
|
+
if dry_run:
|
|
450
|
+
return result
|
|
451
|
+
if isinstance(result, tuple):
|
|
452
|
+
self._current_model = result[0]
|
|
453
|
+
if len(result) > 1 and isinstance(result[1], dict):
|
|
454
|
+
self._tracker.log_metrics(result[1])
|
|
455
|
+
return cast("tuple[Any, dict[str, Any]]", result)
|
|
456
|
+
return result, {}
|
|
457
|
+
|
|
458
|
+
def evaluate(
|
|
459
|
+
self,
|
|
460
|
+
prompt: str,
|
|
461
|
+
dataset: Any = None,
|
|
462
|
+
dry_run: bool = False,
|
|
463
|
+
approve: ApprovalCallback | None = None,
|
|
464
|
+
) -> Any:
|
|
465
|
+
"""Evaluate the current model and return a metrics dictionary."""
|
|
466
|
+
args: dict[str, Any] = {"dataset": dataset} if dataset is not None else {}
|
|
467
|
+
if self._current_model is not None:
|
|
468
|
+
args["model"] = self._current_model
|
|
469
|
+
result = self._generate_and_execute(
|
|
470
|
+
prompt,
|
|
471
|
+
intent="evaluate",
|
|
472
|
+
runtime_args=args,
|
|
473
|
+
dataset=dataset,
|
|
474
|
+
dry_run=dry_run,
|
|
475
|
+
approve=approve,
|
|
476
|
+
)
|
|
477
|
+
if not dry_run and isinstance(result, dict):
|
|
478
|
+
self._tracker.log_metrics(result)
|
|
479
|
+
return result
|
|
480
|
+
|
|
481
|
+
def explore(
|
|
482
|
+
self,
|
|
483
|
+
prompt: str,
|
|
484
|
+
dataset: Any,
|
|
485
|
+
metric: str = "accuracy",
|
|
486
|
+
maximize: bool = True,
|
|
487
|
+
candidates: int = 3,
|
|
488
|
+
) -> ExperimentResult:
|
|
489
|
+
"""Run bounded metric-guided tree-search exploration across training variants."""
|
|
490
|
+
if candidates < 1 or candidates > 10:
|
|
491
|
+
raise ValueError("candidates must be between 1 and 10.")
|
|
492
|
+
variants = (
|
|
493
|
+
"Optimize for baseline clean convergence with AdamW.",
|
|
494
|
+
"Explore stronger regularization and learning-rate warmup cosine decay.",
|
|
495
|
+
"Try wider or deeper residual block capacity with gradient clipping.",
|
|
496
|
+
)
|
|
497
|
+
records: list[ExperimentCandidate] = []
|
|
498
|
+
best: ExperimentCandidate | None = None
|
|
499
|
+
saved_model = self._current_model
|
|
500
|
+
|
|
501
|
+
for idx in range(candidates):
|
|
502
|
+
hint = variants[idx % len(variants)]
|
|
503
|
+
variant_prompt = f"{prompt} ({hint} Trial {idx + 1}/{candidates})"
|
|
504
|
+
self._current_model = saved_model
|
|
505
|
+
try:
|
|
506
|
+
model, metrics = self.train(variant_prompt, dataset=dataset)
|
|
507
|
+
raw_score = metrics.get(metric)
|
|
508
|
+
score = float(raw_score) if isinstance(raw_score, (int, float)) else None
|
|
509
|
+
cand = ExperimentCandidate(
|
|
510
|
+
index=idx + 1,
|
|
511
|
+
prompt_variant=hint,
|
|
512
|
+
code=self._last_code or "",
|
|
513
|
+
description=self.explain(),
|
|
514
|
+
metrics=metrics,
|
|
515
|
+
score=score,
|
|
516
|
+
model=model,
|
|
517
|
+
)
|
|
518
|
+
except Exception as exc:
|
|
519
|
+
cand = ExperimentCandidate(
|
|
520
|
+
index=idx + 1,
|
|
521
|
+
prompt_variant=hint,
|
|
522
|
+
code=self._last_code or "",
|
|
523
|
+
description="",
|
|
524
|
+
metrics={},
|
|
525
|
+
score=None,
|
|
526
|
+
error=str(exc),
|
|
527
|
+
model=None,
|
|
528
|
+
)
|
|
529
|
+
finally:
|
|
530
|
+
self._current_model = saved_model
|
|
531
|
+
records.append(cand)
|
|
532
|
+
if cand.score is not None and (
|
|
533
|
+
best is None
|
|
534
|
+
or best.score is None
|
|
535
|
+
or (cand.score > best.score if maximize else cand.score < best.score)
|
|
536
|
+
):
|
|
537
|
+
best = cand
|
|
538
|
+
|
|
539
|
+
if best is None:
|
|
540
|
+
best = records[0]
|
|
541
|
+
if best.model is not None:
|
|
542
|
+
self._current_model = best.model
|
|
543
|
+
return ExperimentResult(
|
|
544
|
+
best_candidate=best,
|
|
545
|
+
candidates=records,
|
|
546
|
+
metric=metric,
|
|
547
|
+
maximize=maximize,
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
def _resolve_checkpoint_path(self, name: str) -> str:
|
|
551
|
+
"""Resolve a checkpoint name strictly inside self.config.checkpoint_dir."""
|
|
552
|
+
if not isinstance(name, str):
|
|
553
|
+
raise JaxForgeCheckpointError("Checkpoint name must be a string.")
|
|
554
|
+
cleaned = name.strip()
|
|
555
|
+
if (
|
|
556
|
+
not cleaned
|
|
557
|
+
or "\x00" in cleaned
|
|
558
|
+
or "\n" in cleaned
|
|
559
|
+
or "\r" in cleaned
|
|
560
|
+
or "://" in cleaned
|
|
561
|
+
):
|
|
562
|
+
raise JaxForgeCheckpointError(f"Invalid checkpoint name: '{name}'")
|
|
563
|
+
raw = Path(cleaned)
|
|
564
|
+
if raw.is_absolute() or os.path.isabs(cleaned) or ".." in raw.parts:
|
|
565
|
+
raise JaxForgeCheckpointError(
|
|
566
|
+
f"Checkpoint name '{name}' must be a relative path within checkpoint_dir."
|
|
567
|
+
)
|
|
568
|
+
base_dir = Path(self.config.checkpoint_dir).resolve()
|
|
569
|
+
candidate = (base_dir / raw).resolve()
|
|
570
|
+
if candidate == base_dir or not candidate.is_relative_to(base_dir):
|
|
571
|
+
raise JaxForgeCheckpointError(
|
|
572
|
+
f"Checkpoint path '{name}' escapes checkpoint_dir '{self.config.checkpoint_dir}'."
|
|
573
|
+
)
|
|
574
|
+
return str(candidate)
|
|
575
|
+
|
|
576
|
+
def save(
|
|
577
|
+
self, name: str, dry_run: bool = False, approve: ApprovalCallback | None = None
|
|
578
|
+
) -> Any:
|
|
579
|
+
"""Save the active model checkpoint under ``name`` inside checkpoint_dir."""
|
|
580
|
+
path = self._resolve_checkpoint_path(name)
|
|
581
|
+
result = self._generate_and_execute(
|
|
582
|
+
"Save the model checkpoint to the provided path argument.",
|
|
583
|
+
intent="save",
|
|
584
|
+
runtime_args={"model": self._current_model, "path": path},
|
|
585
|
+
dry_run=dry_run,
|
|
586
|
+
approve=approve,
|
|
587
|
+
)
|
|
588
|
+
return result if dry_run else str(result)
|
|
589
|
+
|
|
590
|
+
def load(
|
|
591
|
+
self, name: str, dry_run: bool = False, approve: ApprovalCallback | None = None
|
|
592
|
+
) -> Any:
|
|
593
|
+
"""Load a model checkpoint by relative name within checkpoint_dir."""
|
|
594
|
+
path = self._resolve_checkpoint_path(name)
|
|
595
|
+
result = self._generate_and_execute(
|
|
596
|
+
"Load the model checkpoint from the provided path argument.",
|
|
597
|
+
intent="load",
|
|
598
|
+
runtime_args={"path": path},
|
|
599
|
+
dry_run=dry_run,
|
|
600
|
+
approve=approve,
|
|
601
|
+
)
|
|
602
|
+
if dry_run:
|
|
603
|
+
return result
|
|
604
|
+
self._current_model = result
|
|
605
|
+
return result
|
|
606
|
+
|
|
607
|
+
def load_data(self, prompt: str, dry_run: bool = False) -> Any:
|
|
608
|
+
"""Load or generate a dataset based on natural language description."""
|
|
609
|
+
return self._generate_and_execute(prompt, intent="load_data", dry_run=dry_run)
|
|
610
|
+
|
|
611
|
+
def preprocess_data(self, prompt: str, dataset: Any, dry_run: bool = False) -> Any:
|
|
612
|
+
"""Preprocess a given dataset based on natural language instruction."""
|
|
613
|
+
return self._generate_and_execute(
|
|
614
|
+
prompt,
|
|
615
|
+
intent="preprocess_data",
|
|
616
|
+
runtime_args={"dataset": dataset},
|
|
617
|
+
dataset=dataset,
|
|
618
|
+
dry_run=dry_run,
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
def analyze_data(self, prompt: str, dataset: Any, dry_run: bool = False) -> Any:
|
|
622
|
+
"""Analyze a given dataset and return computed summary statistics."""
|
|
623
|
+
return self._generate_and_execute(
|
|
624
|
+
prompt,
|
|
625
|
+
intent="analyze_data",
|
|
626
|
+
runtime_args={"dataset": dataset},
|
|
627
|
+
dataset=dataset,
|
|
628
|
+
dry_run=dry_run,
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
def visualize_data(self, prompt: str, dataset: Any, dry_run: bool = False) -> Any:
|
|
632
|
+
"""Visualize a given dataset by generating confined plots."""
|
|
633
|
+
return self._generate_and_execute(
|
|
634
|
+
prompt,
|
|
635
|
+
intent="visualize_data",
|
|
636
|
+
runtime_args={"dataset": dataset},
|
|
637
|
+
dataset=dataset,
|
|
638
|
+
dry_run=dry_run,
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
def run(
|
|
642
|
+
self,
|
|
643
|
+
prompt: str,
|
|
644
|
+
dataset: Any = None,
|
|
645
|
+
name: str | None = None,
|
|
646
|
+
dry_run: bool = False,
|
|
647
|
+
approve: ApprovalCallback | None = None,
|
|
648
|
+
) -> Any:
|
|
649
|
+
"""Route a natural-language prompt automatically via IntentClassifier."""
|
|
650
|
+
if dry_run:
|
|
651
|
+
intent = IntentClassifier.classify(prompt)
|
|
652
|
+
return self.preview(prompt, intent=intent, dataset=dataset)
|
|
653
|
+
intent = IntentClassifier.classify(prompt)
|
|
654
|
+
if intent == "build":
|
|
655
|
+
return (
|
|
656
|
+
self.modify(prompt, approve=approve)
|
|
657
|
+
if self._current_model is not None
|
|
658
|
+
else self.build(prompt, approve=approve)
|
|
659
|
+
)
|
|
660
|
+
if intent == "train":
|
|
661
|
+
return self.train(prompt, dataset=dataset, approve=approve)
|
|
662
|
+
if intent == "evaluate":
|
|
663
|
+
return self.evaluate(prompt, dataset=dataset, approve=approve)
|
|
664
|
+
if intent == "save":
|
|
665
|
+
if not name:
|
|
666
|
+
raise JaxForgeCheckpointError("Saving a model requires a checkpoint 'name'.")
|
|
667
|
+
return self.save(name, approve=approve)
|
|
668
|
+
if intent == "load":
|
|
669
|
+
if not name:
|
|
670
|
+
raise JaxForgeCheckpointError("Loading a model requires a checkpoint 'name'.")
|
|
671
|
+
return self.load(name, approve=approve)
|
|
672
|
+
if intent == "load_data":
|
|
673
|
+
return self.load_data(prompt)
|
|
674
|
+
if dataset is None:
|
|
675
|
+
raise JaxForgeIntentError(f"Intent '{intent}' requires a 'dataset' argument.")
|
|
676
|
+
if intent == "preprocess_data":
|
|
677
|
+
return self.preprocess_data(prompt, dataset)
|
|
678
|
+
if intent == "analyze_data":
|
|
679
|
+
return self.analyze_data(prompt, dataset)
|
|
680
|
+
return self.visualize_data(prompt, dataset)
|
|
681
|
+
|
|
682
|
+
async def apreview(
|
|
683
|
+
self, prompt: str, intent: str | None = None, dataset: Any = None
|
|
684
|
+
) -> GeneratedPlan:
|
|
685
|
+
"""Asynchronous wrapper for ``preview()``."""
|
|
686
|
+
return await asyncio.to_thread(self.preview, prompt, intent, dataset)
|
|
687
|
+
|
|
688
|
+
async def abuild(
|
|
689
|
+
self, prompt: str, dry_run: bool = False, approve: ApprovalCallback | None = None
|
|
690
|
+
) -> Any:
|
|
691
|
+
"""Asynchronous wrapper for ``build()``."""
|
|
692
|
+
return await asyncio.to_thread(self.build, prompt, dry_run, approve)
|
|
693
|
+
|
|
694
|
+
async def amodify(
|
|
695
|
+
self, prompt: str, dry_run: bool = False, approve: ApprovalCallback | None = None
|
|
696
|
+
) -> Any:
|
|
697
|
+
"""Asynchronous wrapper for ``modify()``."""
|
|
698
|
+
return await asyncio.to_thread(self.modify, prompt, dry_run, approve)
|
|
699
|
+
|
|
700
|
+
async def atrain(
|
|
701
|
+
self,
|
|
702
|
+
prompt: str,
|
|
703
|
+
dataset: Any = None,
|
|
704
|
+
dry_run: bool = False,
|
|
705
|
+
approve: ApprovalCallback | None = None,
|
|
706
|
+
) -> Any:
|
|
707
|
+
"""Asynchronous wrapper for ``train()``."""
|
|
708
|
+
return await asyncio.to_thread(self.train, prompt, dataset, dry_run, approve)
|
|
709
|
+
|
|
710
|
+
async def aevaluate(
|
|
711
|
+
self,
|
|
712
|
+
prompt: str,
|
|
713
|
+
dataset: Any = None,
|
|
714
|
+
dry_run: bool = False,
|
|
715
|
+
approve: ApprovalCallback | None = None,
|
|
716
|
+
) -> Any:
|
|
717
|
+
"""Asynchronous wrapper for ``evaluate()``."""
|
|
718
|
+
return await asyncio.to_thread(self.evaluate, prompt, dataset, dry_run, approve)
|
|
719
|
+
|
|
720
|
+
def export_project(
|
|
721
|
+
self,
|
|
722
|
+
output_dir: str | Path,
|
|
723
|
+
package_name: str = "generated_model",
|
|
724
|
+
) -> Path:
|
|
725
|
+
"""Export the last built, training, and evaluation code as a standalone Python package."""
|
|
726
|
+
model_code = self._last_build_code or self._last_code
|
|
727
|
+
if not model_code:
|
|
728
|
+
raise JaxForgeError("No generated model code available to export. Call build() first.")
|
|
729
|
+
return write_project_package(
|
|
730
|
+
output_dir=output_dir,
|
|
731
|
+
model_code=model_code,
|
|
732
|
+
train_code=self._last_train_code,
|
|
733
|
+
eval_code=self._last_eval_code,
|
|
734
|
+
package_name=package_name,
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
def usage_summary(self) -> dict[str, Any]:
|
|
738
|
+
"""Return aggregated token and latency telemetry for LLM calls in this session."""
|
|
739
|
+
if self._llm is None:
|
|
740
|
+
return {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
|
741
|
+
hist = self._llm.usage_history
|
|
742
|
+
return {
|
|
743
|
+
"calls": len(hist),
|
|
744
|
+
"prompt_tokens": sum(u.prompt_tokens for u in hist),
|
|
745
|
+
"completion_tokens": sum(u.completion_tokens for u in hist),
|
|
746
|
+
"total_tokens": sum(u.total_tokens for u in hist),
|
|
747
|
+
"total_latency_ms": round(sum(u.latency_ms for u in hist), 2),
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
def explain(self) -> str:
|
|
751
|
+
"""Return a plain-English explanation of the last generated code."""
|
|
752
|
+
if not self._last_code:
|
|
753
|
+
return "No code has been generated yet. Call jf.build() first."
|
|
754
|
+
if self._last_response and "description" in self._last_response:
|
|
755
|
+
return str(self._last_response["description"])
|
|
756
|
+
return "No explanation available for the last operation."
|
|
757
|
+
|
|
758
|
+
def show_code(self) -> str:
|
|
759
|
+
"""Return the raw generated Python code from the last operation."""
|
|
760
|
+
if not self._last_code:
|
|
761
|
+
return "No code has been generated yet."
|
|
762
|
+
return self._last_code
|
|
763
|
+
|
|
764
|
+
def reset(self) -> None:
|
|
765
|
+
"""Clear all conversation history, cached LLM client, and model state."""
|
|
766
|
+
self._memory.reset()
|
|
767
|
+
self._current_model = None
|
|
768
|
+
self._last_code = None
|
|
769
|
+
self._last_build_code = None
|
|
770
|
+
self._last_train_code = None
|
|
771
|
+
self._last_eval_code = None
|
|
772
|
+
self._last_response = None
|
|
773
|
+
self._llm = None
|
|
774
|
+
self._llm_cache_key = None
|
|
775
|
+
self._log("Session reset. Memory and cached LLM client cleared.")
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
JaxGemini = JaxForge
|