corethink 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. corethink/__init__.py +17 -0
  2. corethink/__main__.py +3 -0
  3. corethink/_version.py +3 -0
  4. corethink/account.py +470 -0
  5. corethink/cli.py +1058 -0
  6. corethink/client.py +253 -0
  7. corethink/commands/__init__.py +0 -0
  8. corethink/commands/admin.py +703 -0
  9. corethink/commands/api.py +59 -0
  10. corethink/commands/archive.py +433 -0
  11. corethink/commands/auth.py +375 -0
  12. corethink/commands/billing.py +314 -0
  13. corethink/commands/capture.py +564 -0
  14. corethink/commands/connection.py +1266 -0
  15. corethink/commands/endpoint.py +1124 -0
  16. corethink/commands/env.py +1966 -0
  17. corethink/commands/hub.py +269 -0
  18. corethink/commands/library.py +347 -0
  19. corethink/commands/models.py +258 -0
  20. corethink/commands/train.py +2655 -0
  21. corethink/commands/workspace.py +622 -0
  22. corethink/config.py +181 -0
  23. corethink/context.py +46 -0
  24. corethink/env_template.py +117 -0
  25. corethink/environment_http_store.py +96 -0
  26. corethink/experiment_cli.py +2187 -0
  27. corethink/hosted_profiles.py +44 -0
  28. corethink/hosted_rl.py +688 -0
  29. corethink/hosted_rl_archive.py +74 -0
  30. corethink/hosted_rl_attribution.py +44 -0
  31. corethink/hosted_rl_billing.py +103 -0
  32. corethink/hosted_rl_state.py +182 -0
  33. corethink/jobs_cli.py +839 -0
  34. corethink/main.py +204 -0
  35. corethink/output.py +201 -0
  36. corethink/py.typed +0 -0
  37. corethink/rl_cli.py +997 -0
  38. corethink/rl_inspect.py +521 -0
  39. corethink/rl_playground.py +129 -0
  40. corethink/rl_reassessment.py +214 -0
  41. corethink/rl_settings.py +43 -0
  42. corethink/sdk.py +331 -0
  43. corethink/settings.py +162 -0
  44. corethink-1.0.0.dist-info/METADATA +120 -0
  45. corethink-1.0.0.dist-info/RECORD +48 -0
  46. corethink-1.0.0.dist-info/WHEEL +5 -0
  47. corethink-1.0.0.dist-info/entry_points.txt +2 -0
  48. corethink-1.0.0.dist-info/top_level.txt +1 -0
corethink/rl_cli.py ADDED
@@ -0,0 +1,997 @@
1
+ """Persistent, stepwise RL workflow shared by the CLI and Workbench.
2
+
3
+ The underlying experiment and jobs commands own all scoring and release rules.
4
+ This module records handoffs, never manufactures a score or a gate record.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import math
11
+ import os
12
+ import re
13
+ import signal
14
+ import subprocess
15
+ import sys
16
+ import time
17
+ import uuid
18
+ from datetime import UTC, datetime
19
+ from pathlib import Path
20
+
21
+ from corethink_experiments.errors import ExperimentError
22
+ from corethink_experiments.providers import redact_text
23
+ from corethink_experiments.service import load_project
24
+
25
+ from corethink.rl_settings import apply_run_settings
26
+
27
+ try:
28
+ import fcntl
29
+ except ImportError: # Do not disable the rest of the CLI on non-POSIX hosts.
30
+ fcntl = None
31
+
32
+ # Installed only by the hosted worker. Cloud checkpoint failure stops the next
33
+ # external effect; local CLI/UI have no cloud dependency or storage side effect.
34
+ _checkpoint_hook = None
35
+
36
+
37
+ def register(commands):
38
+ parser = commands.add_parser("rl", help="collect, train, evaluate and release a LoRA adapter")
39
+ actions = parser.add_subparsers(dest="rl_action", required=True)
40
+ for action in (
41
+ "start",
42
+ "resume",
43
+ "list",
44
+ "get",
45
+ "inspect",
46
+ "train",
47
+ "evaluate",
48
+ "gate",
49
+ "reassess-gate",
50
+ "promote",
51
+ "rollback",
52
+ "infer",
53
+ "test",
54
+ "recheck",
55
+ "cancel",
56
+ ):
57
+ sub = actions.add_parser(action)
58
+ sub.add_argument("--root", default=".corethink")
59
+ sub.add_argument("--json", action="store_true")
60
+ if action == "start":
61
+ sub.add_argument("--config", required=True, type=Path)
62
+ sub.add_argument("--group", type=int)
63
+ sub.add_argument("--limit", type=int)
64
+ sub.add_argument(
65
+ "--temperature",
66
+ type=float,
67
+ help="Sampling temperature for collection and feedback rounds (0.1–2)",
68
+ )
69
+ sub.add_argument(
70
+ "--learning-rate", type=float, help="Optimizer learning rate (1e-6–5e-4)"
71
+ )
72
+ sub.add_argument(
73
+ "--rounds", type=int, help="Feedback rounds (2–8, online profiles only)"
74
+ )
75
+ elif action != "list":
76
+ sub.add_argument("run_id")
77
+ if action in {"get", "list"}:
78
+ sub.add_argument("--refresh", action="store_true")
79
+ if action == "inspect":
80
+ sub.add_argument("--split", choices=("training", "development"), default="development")
81
+ sub.add_argument("--round", type=int, dest="round_number")
82
+ sub.add_argument(
83
+ "--outcome",
84
+ choices=("all", "success", "failure", "error", "improved", "regressed"),
85
+ default="all",
86
+ )
87
+ sub.add_argument("--query", default="")
88
+ sub.add_argument("--offset", type=int, default=0)
89
+ sub.add_argument("--limit", type=int, default=50)
90
+ if action in {"promote", "rollback"}:
91
+ sub.add_argument("--approved-by", required=True)
92
+ sub.add_argument("--reason", required=True)
93
+ if action == "reassess-gate":
94
+ sub.add_argument("--min-score", type=float, required=True)
95
+ sub.add_argument("--approved-by", required=True)
96
+ sub.add_argument("--reason", required=True)
97
+ if action in {"promote", "rollback", "infer"}:
98
+ sub.add_argument("--alias", choices=("staging", "production"), default="staging")
99
+ if action in {"infer", "test"}:
100
+ sub.add_argument("--message", required=True)
101
+ if action == "test":
102
+ sub.add_argument("--compare-base", action="store_true")
103
+ if action == "train":
104
+ sub.add_argument(
105
+ "--retry",
106
+ action="store_true",
107
+ help="explicitly rent one T4 again after a failed job",
108
+ )
109
+ sub.set_defaults(handler=run)
110
+
111
+
112
+ def _directory(root: Path, run_id: str) -> Path:
113
+ if not re.fullmatch(r"rl-[a-f0-9]{32}", run_id):
114
+ raise ValueError("invalid RL run ID")
115
+ return root / "rl" / run_id
116
+
117
+
118
+ def _read(root, run_id):
119
+ return json.loads((_directory(root, run_id) / "run.json").read_text())
120
+
121
+
122
+ def _use_frozen_project(root, state):
123
+ """Keep existing runs on their exact manifest after packaged defaults change."""
124
+ from corethink_experiments.storage import ExperimentStore
125
+
126
+ config = state["config"]
127
+ try:
128
+ current = load_project(Path(config["project"]))
129
+ except (FileNotFoundError, ExperimentError):
130
+ current = None
131
+ if current is not None and current.content_hash == config["project_hash"]:
132
+ return
133
+ store = ExperimentStore(root)
134
+ try:
135
+ frozen = store.read_project(state["project_id"], config["project_hash"])
136
+ except FileNotFoundError:
137
+ raise ValueError(
138
+ "Original project manifest is missing; restore this run's checkpoint"
139
+ ) from None
140
+ if frozen.content_hash != config["project_hash"] or frozen.project_id != state["project_id"]:
141
+ raise ValueError("Original project manifest does not match the run's saved identity")
142
+ # This is an already-existing immutable record, not a modified policy.
143
+ config["project"] = str(
144
+ (Path(root) / "projects" / frozen.project_id / f"{frozen.content_hash}.json").resolve()
145
+ )
146
+
147
+
148
+ def _save(root, state):
149
+ directory = _directory(root, state["run_id"])
150
+ directory.mkdir(parents=True, exist_ok=True)
151
+ if (directory / "cancel.requested").exists():
152
+ cloud_pending = state.get("job_id") and state.get("job", {}).get("status") not in {
153
+ "succeeded",
154
+ "failed",
155
+ "cancelled",
156
+ }
157
+ state["status"] = (
158
+ "cancel_requested" if cloud_pending or state.get("submission_pending") else "cancelled"
159
+ )
160
+ state["updated_at"] = datetime.now(UTC).isoformat()
161
+ temporary = directory / f".run-{uuid.uuid4().hex}.json"
162
+ temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
163
+ temporary.replace(directory / "run.json")
164
+ if _checkpoint_hook is not None:
165
+ _checkpoint_hook(root, state)
166
+
167
+
168
+ def _cli(argv, *, config, root=None, cancel_path=None):
169
+ from corethink.jobs_cli import _identity_token
170
+
171
+ environment = dict(os.environ)
172
+ key = config.get("api_key_env", "CORETHINK_MODEL_API_KEY")
173
+ identity = environment.get(key) or _identity_token(config["sampling_url"])
174
+ if identity:
175
+ environment["CORETHINK_RL_MODEL_TOKEN"] = identity
176
+ command = [sys.executable, "-m", "corethink", *map(str, argv), "--json"]
177
+ if root is not None:
178
+ command += ["--root", str(root)]
179
+ process = subprocess.Popen(
180
+ command,
181
+ env=environment,
182
+ stdout=subprocess.PIPE,
183
+ stderr=subprocess.PIPE,
184
+ text=True,
185
+ start_new_session=True,
186
+ )
187
+ deadline = time.monotonic() + 1800
188
+ try:
189
+ while True:
190
+ try:
191
+ stdout, stderr = process.communicate(timeout=1)
192
+ break
193
+ except subprocess.TimeoutExpired:
194
+ if time.monotonic() >= deadline:
195
+ raise ExperimentError("operation exceeded its 30 minute timeout") from None
196
+ if cancel_path and cancel_path.exists() and list(argv[:2]) != ["jobs", "submit"]:
197
+ raise ExperimentError("run cancelled; local operation stopped") from None
198
+ finally:
199
+ if process.poll() is None:
200
+ os.killpg(process.pid, signal.SIGTERM)
201
+ try:
202
+ process.communicate(timeout=10)
203
+ except subprocess.TimeoutExpired:
204
+ os.killpg(process.pid, signal.SIGKILL)
205
+ process.communicate()
206
+ if process.returncode:
207
+ secrets = tuple(
208
+ value
209
+ for name, value in environment.items()
210
+ if any(word in name.upper() for word in ("TOKEN", "KEY", "SECRET", "PASSWORD"))
211
+ )
212
+ detail = redact_text(stderr or stdout, extra_secrets=secrets)[-2000:]
213
+ raise ExperimentError(f"{' '.join(map(str, argv[:2]))} failed: {detail.strip()}")
214
+ return json.loads(stdout)
215
+
216
+
217
+ def _eval_options(config, *, holdout=False):
218
+ return [
219
+ "--project",
220
+ config["project"],
221
+ "--taskset",
222
+ config["holdout"] if holdout else config.get("evaluation", config["development"]),
223
+ "--taskset-model",
224
+ config["base_model"],
225
+ "--base-model",
226
+ config["base_model"] + "@" + config.get("revision", "main"),
227
+ "--taskset-model-base-url",
228
+ config["sampling_url"],
229
+ "--taskset-model-api-key-env",
230
+ "CORETHINK_RL_MODEL_TOKEN",
231
+ ]
232
+
233
+
234
+ def _validate_collection_config(config):
235
+ sampling = config.get("collection_config", {})
236
+ if not isinstance(sampling, dict):
237
+ raise ValueError("collection_config must be an object")
238
+ bounds = {
239
+ "temperature": (0, 2, False),
240
+ "top_p": (0, 1, False),
241
+ "repetition_penalty": (0, 2, False),
242
+ "max_tokens": (1, 2048, True),
243
+ "top_k": (-1, 100000, True),
244
+ }
245
+ for key, value in sampling.items():
246
+ if key not in bounds:
247
+ raise ValueError(f"unsupported collection_config field: {key}")
248
+ lower, upper, integer = bounds[key]
249
+ if (
250
+ type(value) not in (int, float)
251
+ or not math.isfinite(value)
252
+ or (integer and type(value) is not int)
253
+ or not lower <= value <= upper
254
+ or (key in {"temperature", "top_p", "repetition_penalty"} and value == 0)
255
+ or (key == "top_k" and value == 0)
256
+ ):
257
+ raise ValueError(f"invalid collection_config {key}")
258
+
259
+
260
+ def _training_config(config):
261
+ values = {
262
+ "lora_rank": 8,
263
+ "lora_alpha": 16,
264
+ "epochs": 1,
265
+ "learning_rate": 0.0001,
266
+ "lora_targets": "attention",
267
+ "batching": "grouped-8",
268
+ }
269
+ overrides = config.get("training_config", {})
270
+ if not isinstance(overrides, dict) or set(overrides) - (
271
+ set(values) | {"online_reward", "online_rounds", "online_temperature"}
272
+ ):
273
+ raise ValueError(
274
+ "training_config must contain only rank, alpha, targets, batching, "
275
+ "epochs, learning_rate, online_reward, online_rounds and online_temperature"
276
+ )
277
+ values.update(overrides)
278
+ temperature = values.get("online_temperature", 1.0)
279
+ if (
280
+ type(temperature) not in (int, float)
281
+ or not math.isfinite(temperature)
282
+ or not 0.1 <= temperature <= 2
283
+ ):
284
+ raise ValueError("training_config online_temperature must be between 0.1 and 2")
285
+ if "online_temperature" in values and not values.get("online_reward"):
286
+ raise ValueError("online_temperature requires online training")
287
+ if values["batching"] not in ("grouped-8", "shuffled-64"):
288
+ raise ValueError("training_config batching must be grouped-8 or shuffled-64")
289
+ if values["lora_targets"] not in ("attention", "all-linear"):
290
+ raise ValueError("training_config lora_targets must be attention or all-linear")
291
+ for key in ("lora_rank", "lora_alpha", "epochs"):
292
+ if type(values[key]) is not int:
293
+ raise ValueError(f"training_config {key} must be an integer")
294
+ if values["lora_rank"] not in {8, 16, 32} or not 1 <= values["lora_alpha"] <= 128:
295
+ raise ValueError("training_config requires rank 8/16/32 and alpha 1–128")
296
+ if not 1 <= values["epochs"] <= 8:
297
+ raise ValueError("training_config epochs must be between 1 and 8")
298
+ rate = values["learning_rate"]
299
+ if type(rate) not in (int, float) or not math.isfinite(rate) or not 1e-6 <= rate <= 5e-4:
300
+ raise ValueError("training_config learning_rate must be between 1e-6 and 5e-4")
301
+ if "online_reward" in values or "online_rounds" in values:
302
+ # The CLI does not require installation of the hosted server package.
303
+ # Contract tests compare this client guard with TrainingSpec's guard.
304
+ rounds = values.get("online_rounds")
305
+ if (
306
+ values.get("online_reward") != "carry-addition-v1"
307
+ or type(rounds) is not int
308
+ or not 2 <= rounds <= 8
309
+ or values["epochs"] != 1
310
+ or config.get("base_model") != "Qwen/Qwen2.5-0.5B-Instruct"
311
+ ):
312
+ raise ValueError(
313
+ "training_config online mode requires carry-addition-v1, 2–8 rounds, "
314
+ "one epoch per round and Qwen/Qwen2.5-0.5B-Instruct"
315
+ )
316
+ return values
317
+
318
+
319
+ def _collection_program(config, directory):
320
+ """Freeze sampling overrides without changing the evaluated baseline program."""
321
+ if not config.get("collection_config"):
322
+ return config["baseline"]
323
+ _validate_collection_config(config)
324
+ definition = json.loads(Path(config["baseline"]).read_text())
325
+ definition["config"] = {**definition.get("config", {}), **config["collection_config"]}
326
+ payload = json.dumps(definition, indent=2, sort_keys=True) + "\n"
327
+ path = directory / "collection-program.json"
328
+ if path.exists():
329
+ if path.read_text() != payload:
330
+ raise ValueError("collection program changed since this run; start a new run")
331
+ else:
332
+ path.write_text(payload)
333
+ return path
334
+
335
+
336
+ def _validate_action(action, state):
337
+ allowed = {
338
+ "train": {"ready_to_train"},
339
+ "evaluate": {"ready_to_evaluate"},
340
+ "gate": {"ready_to_gate", "ready_to_promote"},
341
+ "promote": {
342
+ "ready_to_evaluate",
343
+ "ready_to_gate",
344
+ "ready_to_promote",
345
+ "gate_passed",
346
+ "gate_failed",
347
+ "promoted",
348
+ },
349
+ "rollback": {"promoted"},
350
+ "infer": {"promoted"},
351
+ "test": {
352
+ "ready_to_evaluate",
353
+ "ready_to_gate",
354
+ "ready_to_promote",
355
+ "gate_passed",
356
+ "gate_failed",
357
+ "promoted",
358
+ },
359
+ "recheck": {
360
+ "ready_to_evaluate",
361
+ "ready_to_gate",
362
+ "ready_to_promote",
363
+ "gate_passed",
364
+ "gate_failed",
365
+ "promoted",
366
+ },
367
+ "warm": {"ready_to_evaluate"},
368
+ }
369
+ if action in allowed and state["status"] not in allowed[action]:
370
+ raise ValueError(f"cannot {action} a run in {state['status']}")
371
+
372
+
373
+ def _refresh(root, state):
374
+ with (root / "rl" / "workflow.lock").open("a") as lock:
375
+ try:
376
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
377
+ except BlockingIOError:
378
+ return state
379
+ return _refresh_locked(root, _read(root, state["run_id"]))
380
+
381
+
382
+ def _refresh_locked(root, state):
383
+ if state["status"] in {"training", "cancel_requested"}:
384
+ if not state.get("job_id"):
385
+ # A submission still owns the lock while its job ID is being recorded.
386
+ # After an abrupt process exit, keep the uncertain outcome visible.
387
+ return state
388
+ config = state["config"]
389
+ job = _cli(["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config)
390
+ state["job"] = job
391
+ if job["status"] == "succeeded":
392
+ cancelled = state["status"] == "cancel_requested"
393
+ state["status"] = "ready_to_evaluate"
394
+ if os.environ.get("CORETHINK_INFERENCE_WARM_API") and not cancelled:
395
+ # Adopt and verify the real trained artifact, but do not score,
396
+ # gate or promote it. A warmup failure is not a training failure.
397
+ state["warmup"] = {"status": "warming"}
398
+ _save(root, state)
399
+ try:
400
+ _execute("warm", None, root, _directory(root, state["run_id"]), state)
401
+ state["warmup"] = {"status": "ready", "at": datetime.now(UTC).isoformat()}
402
+ except Exception:
403
+ state["warmup"] = {
404
+ "status": "failed",
405
+ "diagnostic": (
406
+ "Training succeeded, but inference warmup failed. "
407
+ "Use load & compare to retry."
408
+ ),
409
+ }
410
+ finally:
411
+ state.pop("operation", None)
412
+ elif job["status"] in {"failed", "cancelled"}:
413
+ state["status"] = "failed" if job["status"] == "failed" else "cancelled"
414
+ if job["status"] == "failed":
415
+ state["failed_step"] = "train"
416
+ state["diagnostic"] = (
417
+ f"Training job {job['status']}: {job.get('error_kind') or ''} "
418
+ f"{job.get('error_message') or 'inspect jobs logs ' + state['job_id']}"
419
+ )
420
+ _save(root, state)
421
+ return state
422
+
423
+
424
+ def run(args):
425
+ if fcntl is None:
426
+ raise ValueError("the RL workflow currently requires macOS or Linux")
427
+ root = Path(args.root).resolve()
428
+ action = args.rl_action
429
+ if action == "inspect":
430
+ from corethink.rl_inspect import inspect_run
431
+
432
+ print(
433
+ json.dumps(
434
+ inspect_run(
435
+ root,
436
+ args.run_id,
437
+ split=args.split,
438
+ round_number=args.round_number,
439
+ outcome=args.outcome,
440
+ query=args.query,
441
+ offset=args.offset,
442
+ limit=args.limit,
443
+ ),
444
+ indent=2,
445
+ )
446
+ )
447
+ return 0
448
+ if action == "list":
449
+ states = [
450
+ json.loads(path.read_text()) for path in sorted((root / "rl").glob("rl-*/run.json"))
451
+ ]
452
+ if args.refresh:
453
+ states = [_refresh(root, state) for state in states]
454
+ print(json.dumps({"runs": states}, indent=2))
455
+ return 0
456
+ if action == "start":
457
+ config = json.loads(args.config.read_text())
458
+ apply_run_settings(config, args)
459
+ args.group = config.get("default_group", 4) if args.group is None else args.group
460
+ args.limit = config.get("default_limit", 8) if args.limit is None else args.limit
461
+ if (
462
+ type(args.group) is not int
463
+ or type(args.limit) is not int
464
+ or not 2 <= args.group <= 8
465
+ or not 2 <= args.limit <= 128
466
+ ):
467
+ raise ValueError("the minimal flow requires group 2–8 and limit 2–128")
468
+ minimum = config.get("min_usable_groups", 1)
469
+ if type(minimum) is not int or not 1 <= minimum <= args.limit:
470
+ raise ValueError("min_usable_groups must be an integer between 1 and the task limit")
471
+ if _training_config(config).get("online_reward") and (
472
+ args.group != 8 or args.limit < 16 or minimum != 16
473
+ ):
474
+ raise ValueError("online carry addition requires eight samples and 16 mixed groups")
475
+ _validate_collection_config(config)
476
+ _training_config(config)
477
+ if config.get("training_config", {}).get("online_reward") == "carry-addition-v1":
478
+ config["label"] = (
479
+ f"Carry addition · {args.limit} tasks · "
480
+ f"{config['training_config']['online_rounds']} fresh-feedback rounds"
481
+ )
482
+ paths = ["project", "development", "holdout", "baseline"]
483
+ if "evaluation" in config:
484
+ paths.append("evaluation")
485
+ for key in paths:
486
+ config[key] = str((args.config.resolve().parent / config[key]).resolve())
487
+ spec = load_project(Path(config["project"]))
488
+ if spec.adapter != "lora_adapter":
489
+ raise ValueError("RL profile requires a lora_adapter project")
490
+ config["project_hash"] = spec.content_hash
491
+ state = {
492
+ "run_id": getattr(args, "hosted_run_id", None) or "rl-" + uuid.uuid4().hex,
493
+ "status": "collecting",
494
+ "config": config,
495
+ "project_id": spec.project_id,
496
+ "created_at": datetime.now(UTC).isoformat(),
497
+ "steps": {},
498
+ "diagnostic": "",
499
+ "group": args.group,
500
+ "limit": args.limit,
501
+ }
502
+ _save(root, state)
503
+ else:
504
+ state = _read(root, args.run_id)
505
+ config = state["config"]
506
+ if action == "resume":
507
+ if state["status"] != "failed" or state.get("job_id"):
508
+ raise ValueError(
509
+ "resume is for interrupted collection; other stages can be retried directly"
510
+ )
511
+ action = "start"
512
+ if action == "get":
513
+ if args.refresh:
514
+ state = _refresh(root, state)
515
+ print(json.dumps(state, indent=2))
516
+ return 0
517
+ directory = _directory(root, state["run_id"])
518
+ if action == "cancel":
519
+ # The marker is visible to a currently running command before its next handoff.
520
+ (directory / "cancel.requested").touch()
521
+ if state.get("job_id"):
522
+ state["job"] = _cli(
523
+ ["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config
524
+ )
525
+ if state["job"]["status"] not in {"succeeded", "failed", "cancelled"}:
526
+ state["job"] = _cli(
527
+ ["jobs", "cancel", state["job_id"], "--api", config["jobs_url"]], config=config
528
+ )
529
+ state["status"] = "cancelled"
530
+ _save(root, state)
531
+ print(json.dumps(state, indent=2))
532
+ return 0
533
+ with (root / "rl" / "workflow.lock").open("a") as lock:
534
+ try:
535
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
536
+ except BlockingIOError:
537
+ raise ValueError("an operation is already running for this RL run") from None
538
+ if action != "start":
539
+ state = _read(root, args.run_id)
540
+ if action == "reassess-gate":
541
+ from corethink.rl_reassessment import reassess_gate
542
+
543
+ revised = reassess_gate(
544
+ root,
545
+ state,
546
+ minimum=args.min_score,
547
+ approved_by=args.approved_by,
548
+ reason=args.reason,
549
+ )
550
+ print(json.dumps(revised, indent=2))
551
+ return 0
552
+ _use_frozen_project(root, state)
553
+ previous_status = state["status"]
554
+ if not (action == "train" and args.retry):
555
+ # Invalid transitions are not a new run failure. Preserve the original
556
+ # diagnostic (for example why collection failed its signal check).
557
+ _validate_action(action, state)
558
+ try:
559
+ if action == "train" and args.retry:
560
+ if state["status"] != "failed" or not state.get("job_id"):
561
+ raise ValueError("retry requires a failed training job")
562
+ job = _cli(
563
+ ["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config
564
+ )
565
+ if job["status"] != "failed":
566
+ raise ValueError("the previous job must be terminal and failed before retrying")
567
+ if not state.get("submission_pending"):
568
+ state.setdefault("training_attempts", []).append(job)
569
+ state["submission_key"] = (
570
+ state["run_id"] + "-" + str(len(state["training_attempts"]))
571
+ )
572
+ state.pop("submission_version", None)
573
+ state["status"] = "ready_to_train"
574
+ _execute(action, args, root, directory, state)
575
+ except Exception as error:
576
+ state["diagnostic"] = str(error)
577
+ state["failed_step"] = state.get("operation") or action
578
+ state.pop("operation", None)
579
+ state["status"] = "failed" if action == "start" else previous_status
580
+ if (directory / "cancel.requested").exists():
581
+ state["status"] = "cancelled"
582
+ _save(root, state)
583
+ if state["status"] in {"cancelled", "cancel_requested"}:
584
+ print(json.dumps(state, indent=2))
585
+ return 0
586
+ raise
587
+ if (directory / "cancel.requested").exists():
588
+ state["status"] = "cancelled"
589
+ _save(root, state)
590
+ print(json.dumps(state, indent=2))
591
+ return 0
592
+
593
+
594
+ def _execute(action, args, root, directory, state):
595
+ config = state["config"]
596
+ _validate_action(action, state)
597
+ state["diagnostic"] = ""
598
+ state.pop("failed_step", None)
599
+
600
+ def step(name, argv, *, store=True):
601
+ if (directory / "cancel.requested").exists():
602
+ raise ExperimentError("run cancelled before the next handoff")
603
+ if action in {"start", "evaluate", "warm"} and name in state["steps"]:
604
+ return state["steps"][name]
605
+ state["operation"] = name
606
+ _save(root, state)
607
+ result = _cli(
608
+ argv,
609
+ config=config,
610
+ root=root if store else None,
611
+ cancel_path=directory / "cancel.requested",
612
+ )
613
+ state["steps"][name] = result
614
+ _save(root, state)
615
+ return result
616
+
617
+ if action == "start":
618
+ step(
619
+ "prepare",
620
+ [
621
+ "project",
622
+ "prepare",
623
+ config["project"],
624
+ *(["--allow-local-source"] if config.get("allow_local_source") else []),
625
+ ],
626
+ )
627
+ created = step(
628
+ "create", ["experiment", "create", config["project"], "--candidate-budget", "2"]
629
+ )
630
+ state["experiment_id"] = created["experiment_id"]
631
+ baseline = step(
632
+ "baseline",
633
+ [
634
+ "experiment",
635
+ "baseline",
636
+ state["experiment_id"],
637
+ "--candidate",
638
+ config["baseline"],
639
+ *_eval_options(config),
640
+ ],
641
+ )
642
+ if baseline["primary_score"] is None:
643
+ raise ExperimentError("baseline did not produce a complete score")
644
+ collection = directory / "collection"
645
+ step(
646
+ "collect",
647
+ [
648
+ "eval",
649
+ config["development"],
650
+ "--program",
651
+ _collection_program(config, directory),
652
+ "--base-url",
653
+ config["sampling_url"],
654
+ "--model",
655
+ config["base_model"],
656
+ "--model-api-key-env",
657
+ "CORETHINK_RL_MODEL_TOKEN",
658
+ "--group",
659
+ str(state["group"]),
660
+ "--limit",
661
+ str(state["limit"]),
662
+ "--temperature",
663
+ str(config.get("collection_config", {}).get("temperature", 0.9)),
664
+ "--capture-logprobs",
665
+ "--out",
666
+ str(collection),
667
+ ],
668
+ store=False,
669
+ )
670
+ runs = sorted(collection.glob("runs/*"))
671
+ if len(runs) != state["group"]:
672
+ raise ExperimentError("collection did not finish every requested attempt")
673
+ report = step(
674
+ "corpus",
675
+ [
676
+ "experiment",
677
+ "export-training-data",
678
+ *runs,
679
+ "--output",
680
+ directory / "bundle" / "corpus.jsonl",
681
+ ],
682
+ )
683
+ advantages = step(
684
+ "advantages", ["experiment", "advantages", directory / "bundle" / "corpus.jsonl"]
685
+ )
686
+ minimum = config.get("min_usable_groups", 1)
687
+ if not report["trainable_for_policy_gradient"] or advantages["usable_groups"] < minimum:
688
+ raise ExperimentError(
689
+ f"corpus needs token IDs/logprobs and at least {minimum} non-flat reward groups"
690
+ )
691
+ state["status"] = "ready_to_train"
692
+ elif action == "train":
693
+ training = _training_config(config)
694
+ if state.get("submission_pending") and not state.get("submission_version"):
695
+ raise ExperimentError(
696
+ "legacy training submission outcome is uncertain; reconcile the existing "
697
+ "hosted job before retrying (its adapter version was not recorded)"
698
+ )
699
+ state.setdefault("submission_key", state["run_id"] + "-0")
700
+ # The API compares the entire request when replaying an idempotency key.
701
+ # Regenerating a timestamp here made a lost-response retry conflict with
702
+ # the already accepted job. Persist both values before the network effect.
703
+ state.setdefault("submission_version", "0.1." + str(int(datetime.now(UTC).timestamp())))
704
+ state["submission_pending"] = True
705
+ job = step(
706
+ "submit",
707
+ [
708
+ "jobs",
709
+ "submit",
710
+ directory / "bundle",
711
+ "--endpoint",
712
+ config["sampling_url"],
713
+ "--api",
714
+ config["jobs_url"],
715
+ "--gpus",
716
+ "1",
717
+ "--gpu-type",
718
+ "T4",
719
+ "--gpu-project",
720
+ config["gpu_project"],
721
+ "--train-on",
722
+ "bundle/corpus.jsonl",
723
+ "--taskset-path",
724
+ "bundle",
725
+ "--base-model",
726
+ config["base_model"],
727
+ "--trainer-id",
728
+ "corethink/grpo@v1",
729
+ "--adapter-id",
730
+ state["project_id"],
731
+ "--adapter-version",
732
+ state["submission_version"],
733
+ "--lora-rank",
734
+ str(training["lora_rank"]),
735
+ "--lora-alpha",
736
+ str(training["lora_alpha"]),
737
+ "--epochs",
738
+ str(training["epochs"]),
739
+ "--learning-rate",
740
+ str(training["learning_rate"]),
741
+ # Omit the default so replaying a saved submission preserves
742
+ # its original request body and idempotency identity.
743
+ *(
744
+ ["--lora-targets", training["lora_targets"]]
745
+ if training["lora_targets"] != "attention"
746
+ else []
747
+ ),
748
+ *(
749
+ ["--batching", training["batching"]]
750
+ if training["batching"] != "grouped-8"
751
+ else []
752
+ ),
753
+ *(
754
+ [
755
+ "--online-reward",
756
+ training["online_reward"],
757
+ "--online-rounds",
758
+ str(training["online_rounds"]),
759
+ *(
760
+ ["--online-temperature", str(training["online_temperature"])]
761
+ if "online_temperature" in training
762
+ else []
763
+ ),
764
+ ]
765
+ if training.get("online_reward")
766
+ else []
767
+ ),
768
+ "--idempotency-key",
769
+ state["submission_key"],
770
+ ],
771
+ store=False,
772
+ )
773
+ state["job_id"] = job["job_id"]
774
+ state["job"] = job
775
+ state["submission_pending"] = False
776
+ state["status"] = "training"
777
+ if (directory / "cancel.requested").exists():
778
+ state["job"] = _cli(
779
+ ["jobs", "cancel", state["job_id"], "--api", config["jobs_url"]], config=config
780
+ )
781
+ elif action in {"evaluate", "warm"}:
782
+ if action == "warm":
783
+ _use_frozen_project(root, state)
784
+ adapter = directory / "adapter"
785
+ if "adopt" not in state["steps"]:
786
+ step(
787
+ "adopt",
788
+ ["jobs", "adopt", state["job_id"], "--api", config["jobs_url"], "--into", adapter],
789
+ store=False,
790
+ )
791
+ if "candidate" not in state["steps"]:
792
+ candidate = step(
793
+ "candidate",
794
+ [
795
+ "experiment",
796
+ "candidate",
797
+ "add",
798
+ state["experiment_id"],
799
+ adapter / "adapter.json",
800
+ "--project",
801
+ config["project"],
802
+ ],
803
+ )
804
+ state["candidate_hash"] = candidate["candidate_hash"]
805
+ else:
806
+ state["candidate_hash"] = state["steps"]["candidate"]["candidate_hash"]
807
+ step(
808
+ "load",
809
+ [
810
+ "program",
811
+ "load-adapter",
812
+ state["candidate_hash"],
813
+ "--directory",
814
+ adapter,
815
+ "--base-url",
816
+ config["sampling_url"],
817
+ "--api-key-env",
818
+ "CORETHINK_RL_MODEL_TOKEN",
819
+ ],
820
+ )
821
+ previous_warmup = state.get("warmup", {})
822
+ state["warmup"] = {"status": "ready", "at": datetime.now(UTC).isoformat()}
823
+ if previous_warmup.get("status") == "failed":
824
+ state["warmup"]["recovered_from"] = previous_warmup
825
+ elif "recovered_from" in previous_warmup:
826
+ state["warmup"]["recovered_from"] = previous_warmup["recovered_from"]
827
+ _save(root, state)
828
+ if action == "warm":
829
+ return # A loaded candidate is not a measured or released one.
830
+ measured = step(
831
+ "evaluate",
832
+ [
833
+ "experiment",
834
+ "run",
835
+ state["experiment_id"],
836
+ "--provider",
837
+ "manual",
838
+ "--candidate",
839
+ adapter / "adapter.json",
840
+ *_eval_options(config),
841
+ ],
842
+ )
843
+ if not any(
844
+ item["candidate_hash"] == state["candidate_hash"] and item["primary_score"] is not None
845
+ for item in measured["results"]
846
+ ):
847
+ raise ExperimentError("candidate did not produce a complete development score")
848
+ step("compare", ["experiment", "compare", state["experiment_id"]])
849
+ state["status"] = "ready_to_promote"
850
+ elif action == "gate":
851
+ gate = step(
852
+ "gate",
853
+ [
854
+ "experiment",
855
+ "gate",
856
+ state["experiment_id"],
857
+ "--candidate",
858
+ state["candidate_hash"],
859
+ "--cleanup-verified",
860
+ *_eval_options(config, holdout=True),
861
+ ],
862
+ )
863
+ state["status"] = "gate_passed" if gate["passed"] else "gate_failed"
864
+ elif action == "promote":
865
+ if not state.get("candidate_hash") or not state["steps"].get("load"):
866
+ # Prepare these exact trained weights, without running a quality test.
867
+ # A failed preparation still stops promotion; a low score never does.
868
+ _execute("warm", args, root, directory, state)
869
+ step(
870
+ "promote",
871
+ [
872
+ "program",
873
+ "promote",
874
+ state["candidate_hash"],
875
+ "--project",
876
+ config["project"],
877
+ "--experiment",
878
+ state["experiment_id"],
879
+ "--alias",
880
+ args.alias,
881
+ "--from-alias",
882
+ "candidate" if args.alias == "staging" else "staging",
883
+ "--approved-by",
884
+ args.approved_by,
885
+ "--reason",
886
+ args.reason,
887
+ ],
888
+ )
889
+ state["status"] = "promoted"
890
+ state["alias"] = args.alias
891
+ elif action == "rollback":
892
+ step(
893
+ "rollback",
894
+ [
895
+ "program",
896
+ "rollback",
897
+ args.alias,
898
+ "--project-id",
899
+ state["project_id"],
900
+ "--approved-by",
901
+ args.approved_by,
902
+ "--reason",
903
+ args.reason,
904
+ ],
905
+ )
906
+ elif action == "test":
907
+ from corethink.rl_playground import test_model
908
+
909
+ test_model(args, root, state, _save)
910
+ elif action == "recheck":
911
+ from corethink_experiments.adapters.adapter_files import adapter_model_name
912
+
913
+ candidate = state.get("candidate_hash", "")
914
+ model = adapter_model_name(candidate)
915
+ if state["steps"].get("load", {}).get("model") != model:
916
+ raise ExperimentError("load this exact candidate before re-evaluating it")
917
+ record = {
918
+ "id": "recheck-" + uuid.uuid4().hex,
919
+ "candidate_hash": candidate,
920
+ "model": model,
921
+ "created_at": datetime.now(UTC).isoformat(),
922
+ "status": "running",
923
+ "split": "development",
924
+ }
925
+ state.setdefault("rechecks", []).append(record)
926
+ state["operation"] = "recheck"
927
+ _save(root, state)
928
+ try:
929
+ result = _cli(
930
+ [
931
+ "eval",
932
+ config.get("evaluation", config["development"]),
933
+ "--program",
934
+ config["baseline"],
935
+ "--base-url",
936
+ config["sampling_url"],
937
+ "--model",
938
+ model,
939
+ "--model-api-key-env",
940
+ "CORETHINK_RL_MODEL_TOKEN",
941
+ "--model-timeout",
942
+ "600",
943
+ "--task-timeout",
944
+ "620",
945
+ "--limit",
946
+ "64",
947
+ "--out",
948
+ directory / "rechecks" / record["id"],
949
+ ],
950
+ config=config,
951
+ cancel_path=directory / "cancel.requested",
952
+ )
953
+ value = result.get("primary_score")
954
+ if (
955
+ type(value) not in {int, float}
956
+ or not math.isfinite(value)
957
+ or result.get("errored_task_count", 0)
958
+ or not result.get("task_results")
959
+ ):
960
+ raise ExperimentError("re-evaluation did not produce a complete development score")
961
+ record.update(
962
+ status="completed",
963
+ primary_score=value,
964
+ task_count=len(result["task_results"]),
965
+ evaluation_run_id=result["run_id"],
966
+ )
967
+ except Exception:
968
+ record["status"] = "failed"
969
+ _save(root, state)
970
+ raise
971
+ # New evidence only: never replace the original compare, gate or release.
972
+ elif action == "infer":
973
+ # Alias resolution happens in the shared serving layer, not in the browser.
974
+ step(
975
+ "infer",
976
+ [
977
+ "program",
978
+ "call",
979
+ args.message,
980
+ "--project-id",
981
+ state["project_id"],
982
+ "--alias",
983
+ args.alias,
984
+ "--base-url",
985
+ config["sampling_url"],
986
+ "--base-model",
987
+ config["base_model"] + "@" + config.get("revision", "main"),
988
+ "--model-api-key-env",
989
+ "CORETHINK_RL_MODEL_TOKEN",
990
+ # Cloud Run may need several minutes to start vLLM and restore
991
+ # immutable adapters. The ordinary chat default was shorter
992
+ # than this measured cold start; keep the wait bounded.
993
+ "--timeout",
994
+ "600",
995
+ ],
996
+ )
997
+ state.pop("operation", None)