corethink 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corethink/__init__.py +17 -0
- corethink/__main__.py +3 -0
- corethink/_version.py +3 -0
- corethink/account.py +470 -0
- corethink/cli.py +1058 -0
- corethink/client.py +253 -0
- corethink/commands/__init__.py +0 -0
- corethink/commands/admin.py +703 -0
- corethink/commands/api.py +59 -0
- corethink/commands/archive.py +433 -0
- corethink/commands/auth.py +375 -0
- corethink/commands/billing.py +314 -0
- corethink/commands/capture.py +564 -0
- corethink/commands/connection.py +1266 -0
- corethink/commands/endpoint.py +1124 -0
- corethink/commands/env.py +1966 -0
- corethink/commands/hub.py +269 -0
- corethink/commands/library.py +347 -0
- corethink/commands/models.py +258 -0
- corethink/commands/train.py +2655 -0
- corethink/commands/workspace.py +622 -0
- corethink/config.py +181 -0
- corethink/context.py +46 -0
- corethink/env_template.py +117 -0
- corethink/environment_http_store.py +96 -0
- corethink/experiment_cli.py +2187 -0
- corethink/hosted_profiles.py +44 -0
- corethink/hosted_rl.py +688 -0
- corethink/hosted_rl_archive.py +74 -0
- corethink/hosted_rl_attribution.py +44 -0
- corethink/hosted_rl_billing.py +103 -0
- corethink/hosted_rl_state.py +182 -0
- corethink/jobs_cli.py +839 -0
- corethink/main.py +204 -0
- corethink/output.py +201 -0
- corethink/py.typed +0 -0
- corethink/rl_cli.py +997 -0
- corethink/rl_inspect.py +521 -0
- corethink/rl_playground.py +129 -0
- corethink/rl_reassessment.py +214 -0
- corethink/rl_settings.py +43 -0
- corethink/sdk.py +331 -0
- corethink/settings.py +162 -0
- corethink-1.0.0.dist-info/METADATA +120 -0
- corethink-1.0.0.dist-info/RECORD +48 -0
- corethink-1.0.0.dist-info/WHEEL +5 -0
- corethink-1.0.0.dist-info/entry_points.txt +2 -0
- corethink-1.0.0.dist-info/top_level.txt +1 -0
corethink/rl_cli.py
ADDED
|
@@ -0,0 +1,997 @@
|
|
|
1
|
+
"""Persistent, stepwise RL workflow shared by the CLI and Workbench.
|
|
2
|
+
|
|
3
|
+
The underlying experiment and jobs commands own all scoring and release rules.
|
|
4
|
+
This module records handoffs, never manufactures a score or a gate record.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import signal
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
import uuid
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from corethink_experiments.errors import ExperimentError
|
|
22
|
+
from corethink_experiments.providers import redact_text
|
|
23
|
+
from corethink_experiments.service import load_project
|
|
24
|
+
|
|
25
|
+
from corethink.rl_settings import apply_run_settings
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
import fcntl
|
|
29
|
+
except ImportError: # Do not disable the rest of the CLI on non-POSIX hosts.
|
|
30
|
+
fcntl = None
|
|
31
|
+
|
|
32
|
+
# Installed only by the hosted worker. Cloud checkpoint failure stops the next
|
|
33
|
+
# external effect; local CLI/UI have no cloud dependency or storage side effect.
|
|
34
|
+
_checkpoint_hook = None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def register(commands):
|
|
38
|
+
parser = commands.add_parser("rl", help="collect, train, evaluate and release a LoRA adapter")
|
|
39
|
+
actions = parser.add_subparsers(dest="rl_action", required=True)
|
|
40
|
+
for action in (
|
|
41
|
+
"start",
|
|
42
|
+
"resume",
|
|
43
|
+
"list",
|
|
44
|
+
"get",
|
|
45
|
+
"inspect",
|
|
46
|
+
"train",
|
|
47
|
+
"evaluate",
|
|
48
|
+
"gate",
|
|
49
|
+
"reassess-gate",
|
|
50
|
+
"promote",
|
|
51
|
+
"rollback",
|
|
52
|
+
"infer",
|
|
53
|
+
"test",
|
|
54
|
+
"recheck",
|
|
55
|
+
"cancel",
|
|
56
|
+
):
|
|
57
|
+
sub = actions.add_parser(action)
|
|
58
|
+
sub.add_argument("--root", default=".corethink")
|
|
59
|
+
sub.add_argument("--json", action="store_true")
|
|
60
|
+
if action == "start":
|
|
61
|
+
sub.add_argument("--config", required=True, type=Path)
|
|
62
|
+
sub.add_argument("--group", type=int)
|
|
63
|
+
sub.add_argument("--limit", type=int)
|
|
64
|
+
sub.add_argument(
|
|
65
|
+
"--temperature",
|
|
66
|
+
type=float,
|
|
67
|
+
help="Sampling temperature for collection and feedback rounds (0.1–2)",
|
|
68
|
+
)
|
|
69
|
+
sub.add_argument(
|
|
70
|
+
"--learning-rate", type=float, help="Optimizer learning rate (1e-6–5e-4)"
|
|
71
|
+
)
|
|
72
|
+
sub.add_argument(
|
|
73
|
+
"--rounds", type=int, help="Feedback rounds (2–8, online profiles only)"
|
|
74
|
+
)
|
|
75
|
+
elif action != "list":
|
|
76
|
+
sub.add_argument("run_id")
|
|
77
|
+
if action in {"get", "list"}:
|
|
78
|
+
sub.add_argument("--refresh", action="store_true")
|
|
79
|
+
if action == "inspect":
|
|
80
|
+
sub.add_argument("--split", choices=("training", "development"), default="development")
|
|
81
|
+
sub.add_argument("--round", type=int, dest="round_number")
|
|
82
|
+
sub.add_argument(
|
|
83
|
+
"--outcome",
|
|
84
|
+
choices=("all", "success", "failure", "error", "improved", "regressed"),
|
|
85
|
+
default="all",
|
|
86
|
+
)
|
|
87
|
+
sub.add_argument("--query", default="")
|
|
88
|
+
sub.add_argument("--offset", type=int, default=0)
|
|
89
|
+
sub.add_argument("--limit", type=int, default=50)
|
|
90
|
+
if action in {"promote", "rollback"}:
|
|
91
|
+
sub.add_argument("--approved-by", required=True)
|
|
92
|
+
sub.add_argument("--reason", required=True)
|
|
93
|
+
if action == "reassess-gate":
|
|
94
|
+
sub.add_argument("--min-score", type=float, required=True)
|
|
95
|
+
sub.add_argument("--approved-by", required=True)
|
|
96
|
+
sub.add_argument("--reason", required=True)
|
|
97
|
+
if action in {"promote", "rollback", "infer"}:
|
|
98
|
+
sub.add_argument("--alias", choices=("staging", "production"), default="staging")
|
|
99
|
+
if action in {"infer", "test"}:
|
|
100
|
+
sub.add_argument("--message", required=True)
|
|
101
|
+
if action == "test":
|
|
102
|
+
sub.add_argument("--compare-base", action="store_true")
|
|
103
|
+
if action == "train":
|
|
104
|
+
sub.add_argument(
|
|
105
|
+
"--retry",
|
|
106
|
+
action="store_true",
|
|
107
|
+
help="explicitly rent one T4 again after a failed job",
|
|
108
|
+
)
|
|
109
|
+
sub.set_defaults(handler=run)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _directory(root: Path, run_id: str) -> Path:
|
|
113
|
+
if not re.fullmatch(r"rl-[a-f0-9]{32}", run_id):
|
|
114
|
+
raise ValueError("invalid RL run ID")
|
|
115
|
+
return root / "rl" / run_id
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _read(root, run_id):
|
|
119
|
+
return json.loads((_directory(root, run_id) / "run.json").read_text())
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _use_frozen_project(root, state):
|
|
123
|
+
"""Keep existing runs on their exact manifest after packaged defaults change."""
|
|
124
|
+
from corethink_experiments.storage import ExperimentStore
|
|
125
|
+
|
|
126
|
+
config = state["config"]
|
|
127
|
+
try:
|
|
128
|
+
current = load_project(Path(config["project"]))
|
|
129
|
+
except (FileNotFoundError, ExperimentError):
|
|
130
|
+
current = None
|
|
131
|
+
if current is not None and current.content_hash == config["project_hash"]:
|
|
132
|
+
return
|
|
133
|
+
store = ExperimentStore(root)
|
|
134
|
+
try:
|
|
135
|
+
frozen = store.read_project(state["project_id"], config["project_hash"])
|
|
136
|
+
except FileNotFoundError:
|
|
137
|
+
raise ValueError(
|
|
138
|
+
"Original project manifest is missing; restore this run's checkpoint"
|
|
139
|
+
) from None
|
|
140
|
+
if frozen.content_hash != config["project_hash"] or frozen.project_id != state["project_id"]:
|
|
141
|
+
raise ValueError("Original project manifest does not match the run's saved identity")
|
|
142
|
+
# This is an already-existing immutable record, not a modified policy.
|
|
143
|
+
config["project"] = str(
|
|
144
|
+
(Path(root) / "projects" / frozen.project_id / f"{frozen.content_hash}.json").resolve()
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _save(root, state):
|
|
149
|
+
directory = _directory(root, state["run_id"])
|
|
150
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
if (directory / "cancel.requested").exists():
|
|
152
|
+
cloud_pending = state.get("job_id") and state.get("job", {}).get("status") not in {
|
|
153
|
+
"succeeded",
|
|
154
|
+
"failed",
|
|
155
|
+
"cancelled",
|
|
156
|
+
}
|
|
157
|
+
state["status"] = (
|
|
158
|
+
"cancel_requested" if cloud_pending or state.get("submission_pending") else "cancelled"
|
|
159
|
+
)
|
|
160
|
+
state["updated_at"] = datetime.now(UTC).isoformat()
|
|
161
|
+
temporary = directory / f".run-{uuid.uuid4().hex}.json"
|
|
162
|
+
temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
|
|
163
|
+
temporary.replace(directory / "run.json")
|
|
164
|
+
if _checkpoint_hook is not None:
|
|
165
|
+
_checkpoint_hook(root, state)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _cli(argv, *, config, root=None, cancel_path=None):
|
|
169
|
+
from corethink.jobs_cli import _identity_token
|
|
170
|
+
|
|
171
|
+
environment = dict(os.environ)
|
|
172
|
+
key = config.get("api_key_env", "CORETHINK_MODEL_API_KEY")
|
|
173
|
+
identity = environment.get(key) or _identity_token(config["sampling_url"])
|
|
174
|
+
if identity:
|
|
175
|
+
environment["CORETHINK_RL_MODEL_TOKEN"] = identity
|
|
176
|
+
command = [sys.executable, "-m", "corethink", *map(str, argv), "--json"]
|
|
177
|
+
if root is not None:
|
|
178
|
+
command += ["--root", str(root)]
|
|
179
|
+
process = subprocess.Popen(
|
|
180
|
+
command,
|
|
181
|
+
env=environment,
|
|
182
|
+
stdout=subprocess.PIPE,
|
|
183
|
+
stderr=subprocess.PIPE,
|
|
184
|
+
text=True,
|
|
185
|
+
start_new_session=True,
|
|
186
|
+
)
|
|
187
|
+
deadline = time.monotonic() + 1800
|
|
188
|
+
try:
|
|
189
|
+
while True:
|
|
190
|
+
try:
|
|
191
|
+
stdout, stderr = process.communicate(timeout=1)
|
|
192
|
+
break
|
|
193
|
+
except subprocess.TimeoutExpired:
|
|
194
|
+
if time.monotonic() >= deadline:
|
|
195
|
+
raise ExperimentError("operation exceeded its 30 minute timeout") from None
|
|
196
|
+
if cancel_path and cancel_path.exists() and list(argv[:2]) != ["jobs", "submit"]:
|
|
197
|
+
raise ExperimentError("run cancelled; local operation stopped") from None
|
|
198
|
+
finally:
|
|
199
|
+
if process.poll() is None:
|
|
200
|
+
os.killpg(process.pid, signal.SIGTERM)
|
|
201
|
+
try:
|
|
202
|
+
process.communicate(timeout=10)
|
|
203
|
+
except subprocess.TimeoutExpired:
|
|
204
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
205
|
+
process.communicate()
|
|
206
|
+
if process.returncode:
|
|
207
|
+
secrets = tuple(
|
|
208
|
+
value
|
|
209
|
+
for name, value in environment.items()
|
|
210
|
+
if any(word in name.upper() for word in ("TOKEN", "KEY", "SECRET", "PASSWORD"))
|
|
211
|
+
)
|
|
212
|
+
detail = redact_text(stderr or stdout, extra_secrets=secrets)[-2000:]
|
|
213
|
+
raise ExperimentError(f"{' '.join(map(str, argv[:2]))} failed: {detail.strip()}")
|
|
214
|
+
return json.loads(stdout)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _eval_options(config, *, holdout=False):
|
|
218
|
+
return [
|
|
219
|
+
"--project",
|
|
220
|
+
config["project"],
|
|
221
|
+
"--taskset",
|
|
222
|
+
config["holdout"] if holdout else config.get("evaluation", config["development"]),
|
|
223
|
+
"--taskset-model",
|
|
224
|
+
config["base_model"],
|
|
225
|
+
"--base-model",
|
|
226
|
+
config["base_model"] + "@" + config.get("revision", "main"),
|
|
227
|
+
"--taskset-model-base-url",
|
|
228
|
+
config["sampling_url"],
|
|
229
|
+
"--taskset-model-api-key-env",
|
|
230
|
+
"CORETHINK_RL_MODEL_TOKEN",
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _validate_collection_config(config):
|
|
235
|
+
sampling = config.get("collection_config", {})
|
|
236
|
+
if not isinstance(sampling, dict):
|
|
237
|
+
raise ValueError("collection_config must be an object")
|
|
238
|
+
bounds = {
|
|
239
|
+
"temperature": (0, 2, False),
|
|
240
|
+
"top_p": (0, 1, False),
|
|
241
|
+
"repetition_penalty": (0, 2, False),
|
|
242
|
+
"max_tokens": (1, 2048, True),
|
|
243
|
+
"top_k": (-1, 100000, True),
|
|
244
|
+
}
|
|
245
|
+
for key, value in sampling.items():
|
|
246
|
+
if key not in bounds:
|
|
247
|
+
raise ValueError(f"unsupported collection_config field: {key}")
|
|
248
|
+
lower, upper, integer = bounds[key]
|
|
249
|
+
if (
|
|
250
|
+
type(value) not in (int, float)
|
|
251
|
+
or not math.isfinite(value)
|
|
252
|
+
or (integer and type(value) is not int)
|
|
253
|
+
or not lower <= value <= upper
|
|
254
|
+
or (key in {"temperature", "top_p", "repetition_penalty"} and value == 0)
|
|
255
|
+
or (key == "top_k" and value == 0)
|
|
256
|
+
):
|
|
257
|
+
raise ValueError(f"invalid collection_config {key}")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _training_config(config):
|
|
261
|
+
values = {
|
|
262
|
+
"lora_rank": 8,
|
|
263
|
+
"lora_alpha": 16,
|
|
264
|
+
"epochs": 1,
|
|
265
|
+
"learning_rate": 0.0001,
|
|
266
|
+
"lora_targets": "attention",
|
|
267
|
+
"batching": "grouped-8",
|
|
268
|
+
}
|
|
269
|
+
overrides = config.get("training_config", {})
|
|
270
|
+
if not isinstance(overrides, dict) or set(overrides) - (
|
|
271
|
+
set(values) | {"online_reward", "online_rounds", "online_temperature"}
|
|
272
|
+
):
|
|
273
|
+
raise ValueError(
|
|
274
|
+
"training_config must contain only rank, alpha, targets, batching, "
|
|
275
|
+
"epochs, learning_rate, online_reward, online_rounds and online_temperature"
|
|
276
|
+
)
|
|
277
|
+
values.update(overrides)
|
|
278
|
+
temperature = values.get("online_temperature", 1.0)
|
|
279
|
+
if (
|
|
280
|
+
type(temperature) not in (int, float)
|
|
281
|
+
or not math.isfinite(temperature)
|
|
282
|
+
or not 0.1 <= temperature <= 2
|
|
283
|
+
):
|
|
284
|
+
raise ValueError("training_config online_temperature must be between 0.1 and 2")
|
|
285
|
+
if "online_temperature" in values and not values.get("online_reward"):
|
|
286
|
+
raise ValueError("online_temperature requires online training")
|
|
287
|
+
if values["batching"] not in ("grouped-8", "shuffled-64"):
|
|
288
|
+
raise ValueError("training_config batching must be grouped-8 or shuffled-64")
|
|
289
|
+
if values["lora_targets"] not in ("attention", "all-linear"):
|
|
290
|
+
raise ValueError("training_config lora_targets must be attention or all-linear")
|
|
291
|
+
for key in ("lora_rank", "lora_alpha", "epochs"):
|
|
292
|
+
if type(values[key]) is not int:
|
|
293
|
+
raise ValueError(f"training_config {key} must be an integer")
|
|
294
|
+
if values["lora_rank"] not in {8, 16, 32} or not 1 <= values["lora_alpha"] <= 128:
|
|
295
|
+
raise ValueError("training_config requires rank 8/16/32 and alpha 1–128")
|
|
296
|
+
if not 1 <= values["epochs"] <= 8:
|
|
297
|
+
raise ValueError("training_config epochs must be between 1 and 8")
|
|
298
|
+
rate = values["learning_rate"]
|
|
299
|
+
if type(rate) not in (int, float) or not math.isfinite(rate) or not 1e-6 <= rate <= 5e-4:
|
|
300
|
+
raise ValueError("training_config learning_rate must be between 1e-6 and 5e-4")
|
|
301
|
+
if "online_reward" in values or "online_rounds" in values:
|
|
302
|
+
# The CLI does not require installation of the hosted server package.
|
|
303
|
+
# Contract tests compare this client guard with TrainingSpec's guard.
|
|
304
|
+
rounds = values.get("online_rounds")
|
|
305
|
+
if (
|
|
306
|
+
values.get("online_reward") != "carry-addition-v1"
|
|
307
|
+
or type(rounds) is not int
|
|
308
|
+
or not 2 <= rounds <= 8
|
|
309
|
+
or values["epochs"] != 1
|
|
310
|
+
or config.get("base_model") != "Qwen/Qwen2.5-0.5B-Instruct"
|
|
311
|
+
):
|
|
312
|
+
raise ValueError(
|
|
313
|
+
"training_config online mode requires carry-addition-v1, 2–8 rounds, "
|
|
314
|
+
"one epoch per round and Qwen/Qwen2.5-0.5B-Instruct"
|
|
315
|
+
)
|
|
316
|
+
return values
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _collection_program(config, directory):
|
|
320
|
+
"""Freeze sampling overrides without changing the evaluated baseline program."""
|
|
321
|
+
if not config.get("collection_config"):
|
|
322
|
+
return config["baseline"]
|
|
323
|
+
_validate_collection_config(config)
|
|
324
|
+
definition = json.loads(Path(config["baseline"]).read_text())
|
|
325
|
+
definition["config"] = {**definition.get("config", {}), **config["collection_config"]}
|
|
326
|
+
payload = json.dumps(definition, indent=2, sort_keys=True) + "\n"
|
|
327
|
+
path = directory / "collection-program.json"
|
|
328
|
+
if path.exists():
|
|
329
|
+
if path.read_text() != payload:
|
|
330
|
+
raise ValueError("collection program changed since this run; start a new run")
|
|
331
|
+
else:
|
|
332
|
+
path.write_text(payload)
|
|
333
|
+
return path
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _validate_action(action, state):
|
|
337
|
+
allowed = {
|
|
338
|
+
"train": {"ready_to_train"},
|
|
339
|
+
"evaluate": {"ready_to_evaluate"},
|
|
340
|
+
"gate": {"ready_to_gate", "ready_to_promote"},
|
|
341
|
+
"promote": {
|
|
342
|
+
"ready_to_evaluate",
|
|
343
|
+
"ready_to_gate",
|
|
344
|
+
"ready_to_promote",
|
|
345
|
+
"gate_passed",
|
|
346
|
+
"gate_failed",
|
|
347
|
+
"promoted",
|
|
348
|
+
},
|
|
349
|
+
"rollback": {"promoted"},
|
|
350
|
+
"infer": {"promoted"},
|
|
351
|
+
"test": {
|
|
352
|
+
"ready_to_evaluate",
|
|
353
|
+
"ready_to_gate",
|
|
354
|
+
"ready_to_promote",
|
|
355
|
+
"gate_passed",
|
|
356
|
+
"gate_failed",
|
|
357
|
+
"promoted",
|
|
358
|
+
},
|
|
359
|
+
"recheck": {
|
|
360
|
+
"ready_to_evaluate",
|
|
361
|
+
"ready_to_gate",
|
|
362
|
+
"ready_to_promote",
|
|
363
|
+
"gate_passed",
|
|
364
|
+
"gate_failed",
|
|
365
|
+
"promoted",
|
|
366
|
+
},
|
|
367
|
+
"warm": {"ready_to_evaluate"},
|
|
368
|
+
}
|
|
369
|
+
if action in allowed and state["status"] not in allowed[action]:
|
|
370
|
+
raise ValueError(f"cannot {action} a run in {state['status']}")
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _refresh(root, state):
|
|
374
|
+
with (root / "rl" / "workflow.lock").open("a") as lock:
|
|
375
|
+
try:
|
|
376
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
377
|
+
except BlockingIOError:
|
|
378
|
+
return state
|
|
379
|
+
return _refresh_locked(root, _read(root, state["run_id"]))
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _refresh_locked(root, state):
|
|
383
|
+
if state["status"] in {"training", "cancel_requested"}:
|
|
384
|
+
if not state.get("job_id"):
|
|
385
|
+
# A submission still owns the lock while its job ID is being recorded.
|
|
386
|
+
# After an abrupt process exit, keep the uncertain outcome visible.
|
|
387
|
+
return state
|
|
388
|
+
config = state["config"]
|
|
389
|
+
job = _cli(["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config)
|
|
390
|
+
state["job"] = job
|
|
391
|
+
if job["status"] == "succeeded":
|
|
392
|
+
cancelled = state["status"] == "cancel_requested"
|
|
393
|
+
state["status"] = "ready_to_evaluate"
|
|
394
|
+
if os.environ.get("CORETHINK_INFERENCE_WARM_API") and not cancelled:
|
|
395
|
+
# Adopt and verify the real trained artifact, but do not score,
|
|
396
|
+
# gate or promote it. A warmup failure is not a training failure.
|
|
397
|
+
state["warmup"] = {"status": "warming"}
|
|
398
|
+
_save(root, state)
|
|
399
|
+
try:
|
|
400
|
+
_execute("warm", None, root, _directory(root, state["run_id"]), state)
|
|
401
|
+
state["warmup"] = {"status": "ready", "at": datetime.now(UTC).isoformat()}
|
|
402
|
+
except Exception:
|
|
403
|
+
state["warmup"] = {
|
|
404
|
+
"status": "failed",
|
|
405
|
+
"diagnostic": (
|
|
406
|
+
"Training succeeded, but inference warmup failed. "
|
|
407
|
+
"Use load & compare to retry."
|
|
408
|
+
),
|
|
409
|
+
}
|
|
410
|
+
finally:
|
|
411
|
+
state.pop("operation", None)
|
|
412
|
+
elif job["status"] in {"failed", "cancelled"}:
|
|
413
|
+
state["status"] = "failed" if job["status"] == "failed" else "cancelled"
|
|
414
|
+
if job["status"] == "failed":
|
|
415
|
+
state["failed_step"] = "train"
|
|
416
|
+
state["diagnostic"] = (
|
|
417
|
+
f"Training job {job['status']}: {job.get('error_kind') or ''} "
|
|
418
|
+
f"{job.get('error_message') or 'inspect jobs logs ' + state['job_id']}"
|
|
419
|
+
)
|
|
420
|
+
_save(root, state)
|
|
421
|
+
return state
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def run(args):
|
|
425
|
+
if fcntl is None:
|
|
426
|
+
raise ValueError("the RL workflow currently requires macOS or Linux")
|
|
427
|
+
root = Path(args.root).resolve()
|
|
428
|
+
action = args.rl_action
|
|
429
|
+
if action == "inspect":
|
|
430
|
+
from corethink.rl_inspect import inspect_run
|
|
431
|
+
|
|
432
|
+
print(
|
|
433
|
+
json.dumps(
|
|
434
|
+
inspect_run(
|
|
435
|
+
root,
|
|
436
|
+
args.run_id,
|
|
437
|
+
split=args.split,
|
|
438
|
+
round_number=args.round_number,
|
|
439
|
+
outcome=args.outcome,
|
|
440
|
+
query=args.query,
|
|
441
|
+
offset=args.offset,
|
|
442
|
+
limit=args.limit,
|
|
443
|
+
),
|
|
444
|
+
indent=2,
|
|
445
|
+
)
|
|
446
|
+
)
|
|
447
|
+
return 0
|
|
448
|
+
if action == "list":
|
|
449
|
+
states = [
|
|
450
|
+
json.loads(path.read_text()) for path in sorted((root / "rl").glob("rl-*/run.json"))
|
|
451
|
+
]
|
|
452
|
+
if args.refresh:
|
|
453
|
+
states = [_refresh(root, state) for state in states]
|
|
454
|
+
print(json.dumps({"runs": states}, indent=2))
|
|
455
|
+
return 0
|
|
456
|
+
if action == "start":
|
|
457
|
+
config = json.loads(args.config.read_text())
|
|
458
|
+
apply_run_settings(config, args)
|
|
459
|
+
args.group = config.get("default_group", 4) if args.group is None else args.group
|
|
460
|
+
args.limit = config.get("default_limit", 8) if args.limit is None else args.limit
|
|
461
|
+
if (
|
|
462
|
+
type(args.group) is not int
|
|
463
|
+
or type(args.limit) is not int
|
|
464
|
+
or not 2 <= args.group <= 8
|
|
465
|
+
or not 2 <= args.limit <= 128
|
|
466
|
+
):
|
|
467
|
+
raise ValueError("the minimal flow requires group 2–8 and limit 2–128")
|
|
468
|
+
minimum = config.get("min_usable_groups", 1)
|
|
469
|
+
if type(minimum) is not int or not 1 <= minimum <= args.limit:
|
|
470
|
+
raise ValueError("min_usable_groups must be an integer between 1 and the task limit")
|
|
471
|
+
if _training_config(config).get("online_reward") and (
|
|
472
|
+
args.group != 8 or args.limit < 16 or minimum != 16
|
|
473
|
+
):
|
|
474
|
+
raise ValueError("online carry addition requires eight samples and 16 mixed groups")
|
|
475
|
+
_validate_collection_config(config)
|
|
476
|
+
_training_config(config)
|
|
477
|
+
if config.get("training_config", {}).get("online_reward") == "carry-addition-v1":
|
|
478
|
+
config["label"] = (
|
|
479
|
+
f"Carry addition · {args.limit} tasks · "
|
|
480
|
+
f"{config['training_config']['online_rounds']} fresh-feedback rounds"
|
|
481
|
+
)
|
|
482
|
+
paths = ["project", "development", "holdout", "baseline"]
|
|
483
|
+
if "evaluation" in config:
|
|
484
|
+
paths.append("evaluation")
|
|
485
|
+
for key in paths:
|
|
486
|
+
config[key] = str((args.config.resolve().parent / config[key]).resolve())
|
|
487
|
+
spec = load_project(Path(config["project"]))
|
|
488
|
+
if spec.adapter != "lora_adapter":
|
|
489
|
+
raise ValueError("RL profile requires a lora_adapter project")
|
|
490
|
+
config["project_hash"] = spec.content_hash
|
|
491
|
+
state = {
|
|
492
|
+
"run_id": getattr(args, "hosted_run_id", None) or "rl-" + uuid.uuid4().hex,
|
|
493
|
+
"status": "collecting",
|
|
494
|
+
"config": config,
|
|
495
|
+
"project_id": spec.project_id,
|
|
496
|
+
"created_at": datetime.now(UTC).isoformat(),
|
|
497
|
+
"steps": {},
|
|
498
|
+
"diagnostic": "",
|
|
499
|
+
"group": args.group,
|
|
500
|
+
"limit": args.limit,
|
|
501
|
+
}
|
|
502
|
+
_save(root, state)
|
|
503
|
+
else:
|
|
504
|
+
state = _read(root, args.run_id)
|
|
505
|
+
config = state["config"]
|
|
506
|
+
if action == "resume":
|
|
507
|
+
if state["status"] != "failed" or state.get("job_id"):
|
|
508
|
+
raise ValueError(
|
|
509
|
+
"resume is for interrupted collection; other stages can be retried directly"
|
|
510
|
+
)
|
|
511
|
+
action = "start"
|
|
512
|
+
if action == "get":
|
|
513
|
+
if args.refresh:
|
|
514
|
+
state = _refresh(root, state)
|
|
515
|
+
print(json.dumps(state, indent=2))
|
|
516
|
+
return 0
|
|
517
|
+
directory = _directory(root, state["run_id"])
|
|
518
|
+
if action == "cancel":
|
|
519
|
+
# The marker is visible to a currently running command before its next handoff.
|
|
520
|
+
(directory / "cancel.requested").touch()
|
|
521
|
+
if state.get("job_id"):
|
|
522
|
+
state["job"] = _cli(
|
|
523
|
+
["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config
|
|
524
|
+
)
|
|
525
|
+
if state["job"]["status"] not in {"succeeded", "failed", "cancelled"}:
|
|
526
|
+
state["job"] = _cli(
|
|
527
|
+
["jobs", "cancel", state["job_id"], "--api", config["jobs_url"]], config=config
|
|
528
|
+
)
|
|
529
|
+
state["status"] = "cancelled"
|
|
530
|
+
_save(root, state)
|
|
531
|
+
print(json.dumps(state, indent=2))
|
|
532
|
+
return 0
|
|
533
|
+
with (root / "rl" / "workflow.lock").open("a") as lock:
|
|
534
|
+
try:
|
|
535
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
536
|
+
except BlockingIOError:
|
|
537
|
+
raise ValueError("an operation is already running for this RL run") from None
|
|
538
|
+
if action != "start":
|
|
539
|
+
state = _read(root, args.run_id)
|
|
540
|
+
if action == "reassess-gate":
|
|
541
|
+
from corethink.rl_reassessment import reassess_gate
|
|
542
|
+
|
|
543
|
+
revised = reassess_gate(
|
|
544
|
+
root,
|
|
545
|
+
state,
|
|
546
|
+
minimum=args.min_score,
|
|
547
|
+
approved_by=args.approved_by,
|
|
548
|
+
reason=args.reason,
|
|
549
|
+
)
|
|
550
|
+
print(json.dumps(revised, indent=2))
|
|
551
|
+
return 0
|
|
552
|
+
_use_frozen_project(root, state)
|
|
553
|
+
previous_status = state["status"]
|
|
554
|
+
if not (action == "train" and args.retry):
|
|
555
|
+
# Invalid transitions are not a new run failure. Preserve the original
|
|
556
|
+
# diagnostic (for example why collection failed its signal check).
|
|
557
|
+
_validate_action(action, state)
|
|
558
|
+
try:
|
|
559
|
+
if action == "train" and args.retry:
|
|
560
|
+
if state["status"] != "failed" or not state.get("job_id"):
|
|
561
|
+
raise ValueError("retry requires a failed training job")
|
|
562
|
+
job = _cli(
|
|
563
|
+
["jobs", "get", state["job_id"], "--api", config["jobs_url"]], config=config
|
|
564
|
+
)
|
|
565
|
+
if job["status"] != "failed":
|
|
566
|
+
raise ValueError("the previous job must be terminal and failed before retrying")
|
|
567
|
+
if not state.get("submission_pending"):
|
|
568
|
+
state.setdefault("training_attempts", []).append(job)
|
|
569
|
+
state["submission_key"] = (
|
|
570
|
+
state["run_id"] + "-" + str(len(state["training_attempts"]))
|
|
571
|
+
)
|
|
572
|
+
state.pop("submission_version", None)
|
|
573
|
+
state["status"] = "ready_to_train"
|
|
574
|
+
_execute(action, args, root, directory, state)
|
|
575
|
+
except Exception as error:
|
|
576
|
+
state["diagnostic"] = str(error)
|
|
577
|
+
state["failed_step"] = state.get("operation") or action
|
|
578
|
+
state.pop("operation", None)
|
|
579
|
+
state["status"] = "failed" if action == "start" else previous_status
|
|
580
|
+
if (directory / "cancel.requested").exists():
|
|
581
|
+
state["status"] = "cancelled"
|
|
582
|
+
_save(root, state)
|
|
583
|
+
if state["status"] in {"cancelled", "cancel_requested"}:
|
|
584
|
+
print(json.dumps(state, indent=2))
|
|
585
|
+
return 0
|
|
586
|
+
raise
|
|
587
|
+
if (directory / "cancel.requested").exists():
|
|
588
|
+
state["status"] = "cancelled"
|
|
589
|
+
_save(root, state)
|
|
590
|
+
print(json.dumps(state, indent=2))
|
|
591
|
+
return 0
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _execute(action, args, root, directory, state):
|
|
595
|
+
config = state["config"]
|
|
596
|
+
_validate_action(action, state)
|
|
597
|
+
state["diagnostic"] = ""
|
|
598
|
+
state.pop("failed_step", None)
|
|
599
|
+
|
|
600
|
+
def step(name, argv, *, store=True):
|
|
601
|
+
if (directory / "cancel.requested").exists():
|
|
602
|
+
raise ExperimentError("run cancelled before the next handoff")
|
|
603
|
+
if action in {"start", "evaluate", "warm"} and name in state["steps"]:
|
|
604
|
+
return state["steps"][name]
|
|
605
|
+
state["operation"] = name
|
|
606
|
+
_save(root, state)
|
|
607
|
+
result = _cli(
|
|
608
|
+
argv,
|
|
609
|
+
config=config,
|
|
610
|
+
root=root if store else None,
|
|
611
|
+
cancel_path=directory / "cancel.requested",
|
|
612
|
+
)
|
|
613
|
+
state["steps"][name] = result
|
|
614
|
+
_save(root, state)
|
|
615
|
+
return result
|
|
616
|
+
|
|
617
|
+
if action == "start":
|
|
618
|
+
step(
|
|
619
|
+
"prepare",
|
|
620
|
+
[
|
|
621
|
+
"project",
|
|
622
|
+
"prepare",
|
|
623
|
+
config["project"],
|
|
624
|
+
*(["--allow-local-source"] if config.get("allow_local_source") else []),
|
|
625
|
+
],
|
|
626
|
+
)
|
|
627
|
+
created = step(
|
|
628
|
+
"create", ["experiment", "create", config["project"], "--candidate-budget", "2"]
|
|
629
|
+
)
|
|
630
|
+
state["experiment_id"] = created["experiment_id"]
|
|
631
|
+
baseline = step(
|
|
632
|
+
"baseline",
|
|
633
|
+
[
|
|
634
|
+
"experiment",
|
|
635
|
+
"baseline",
|
|
636
|
+
state["experiment_id"],
|
|
637
|
+
"--candidate",
|
|
638
|
+
config["baseline"],
|
|
639
|
+
*_eval_options(config),
|
|
640
|
+
],
|
|
641
|
+
)
|
|
642
|
+
if baseline["primary_score"] is None:
|
|
643
|
+
raise ExperimentError("baseline did not produce a complete score")
|
|
644
|
+
collection = directory / "collection"
|
|
645
|
+
step(
|
|
646
|
+
"collect",
|
|
647
|
+
[
|
|
648
|
+
"eval",
|
|
649
|
+
config["development"],
|
|
650
|
+
"--program",
|
|
651
|
+
_collection_program(config, directory),
|
|
652
|
+
"--base-url",
|
|
653
|
+
config["sampling_url"],
|
|
654
|
+
"--model",
|
|
655
|
+
config["base_model"],
|
|
656
|
+
"--model-api-key-env",
|
|
657
|
+
"CORETHINK_RL_MODEL_TOKEN",
|
|
658
|
+
"--group",
|
|
659
|
+
str(state["group"]),
|
|
660
|
+
"--limit",
|
|
661
|
+
str(state["limit"]),
|
|
662
|
+
"--temperature",
|
|
663
|
+
str(config.get("collection_config", {}).get("temperature", 0.9)),
|
|
664
|
+
"--capture-logprobs",
|
|
665
|
+
"--out",
|
|
666
|
+
str(collection),
|
|
667
|
+
],
|
|
668
|
+
store=False,
|
|
669
|
+
)
|
|
670
|
+
runs = sorted(collection.glob("runs/*"))
|
|
671
|
+
if len(runs) != state["group"]:
|
|
672
|
+
raise ExperimentError("collection did not finish every requested attempt")
|
|
673
|
+
report = step(
|
|
674
|
+
"corpus",
|
|
675
|
+
[
|
|
676
|
+
"experiment",
|
|
677
|
+
"export-training-data",
|
|
678
|
+
*runs,
|
|
679
|
+
"--output",
|
|
680
|
+
directory / "bundle" / "corpus.jsonl",
|
|
681
|
+
],
|
|
682
|
+
)
|
|
683
|
+
advantages = step(
|
|
684
|
+
"advantages", ["experiment", "advantages", directory / "bundle" / "corpus.jsonl"]
|
|
685
|
+
)
|
|
686
|
+
minimum = config.get("min_usable_groups", 1)
|
|
687
|
+
if not report["trainable_for_policy_gradient"] or advantages["usable_groups"] < minimum:
|
|
688
|
+
raise ExperimentError(
|
|
689
|
+
f"corpus needs token IDs/logprobs and at least {minimum} non-flat reward groups"
|
|
690
|
+
)
|
|
691
|
+
state["status"] = "ready_to_train"
|
|
692
|
+
elif action == "train":
|
|
693
|
+
training = _training_config(config)
|
|
694
|
+
if state.get("submission_pending") and not state.get("submission_version"):
|
|
695
|
+
raise ExperimentError(
|
|
696
|
+
"legacy training submission outcome is uncertain; reconcile the existing "
|
|
697
|
+
"hosted job before retrying (its adapter version was not recorded)"
|
|
698
|
+
)
|
|
699
|
+
state.setdefault("submission_key", state["run_id"] + "-0")
|
|
700
|
+
# The API compares the entire request when replaying an idempotency key.
|
|
701
|
+
# Regenerating a timestamp here made a lost-response retry conflict with
|
|
702
|
+
# the already accepted job. Persist both values before the network effect.
|
|
703
|
+
state.setdefault("submission_version", "0.1." + str(int(datetime.now(UTC).timestamp())))
|
|
704
|
+
state["submission_pending"] = True
|
|
705
|
+
job = step(
|
|
706
|
+
"submit",
|
|
707
|
+
[
|
|
708
|
+
"jobs",
|
|
709
|
+
"submit",
|
|
710
|
+
directory / "bundle",
|
|
711
|
+
"--endpoint",
|
|
712
|
+
config["sampling_url"],
|
|
713
|
+
"--api",
|
|
714
|
+
config["jobs_url"],
|
|
715
|
+
"--gpus",
|
|
716
|
+
"1",
|
|
717
|
+
"--gpu-type",
|
|
718
|
+
"T4",
|
|
719
|
+
"--gpu-project",
|
|
720
|
+
config["gpu_project"],
|
|
721
|
+
"--train-on",
|
|
722
|
+
"bundle/corpus.jsonl",
|
|
723
|
+
"--taskset-path",
|
|
724
|
+
"bundle",
|
|
725
|
+
"--base-model",
|
|
726
|
+
config["base_model"],
|
|
727
|
+
"--trainer-id",
|
|
728
|
+
"corethink/grpo@v1",
|
|
729
|
+
"--adapter-id",
|
|
730
|
+
state["project_id"],
|
|
731
|
+
"--adapter-version",
|
|
732
|
+
state["submission_version"],
|
|
733
|
+
"--lora-rank",
|
|
734
|
+
str(training["lora_rank"]),
|
|
735
|
+
"--lora-alpha",
|
|
736
|
+
str(training["lora_alpha"]),
|
|
737
|
+
"--epochs",
|
|
738
|
+
str(training["epochs"]),
|
|
739
|
+
"--learning-rate",
|
|
740
|
+
str(training["learning_rate"]),
|
|
741
|
+
# Omit the default so replaying a saved submission preserves
|
|
742
|
+
# its original request body and idempotency identity.
|
|
743
|
+
*(
|
|
744
|
+
["--lora-targets", training["lora_targets"]]
|
|
745
|
+
if training["lora_targets"] != "attention"
|
|
746
|
+
else []
|
|
747
|
+
),
|
|
748
|
+
*(
|
|
749
|
+
["--batching", training["batching"]]
|
|
750
|
+
if training["batching"] != "grouped-8"
|
|
751
|
+
else []
|
|
752
|
+
),
|
|
753
|
+
*(
|
|
754
|
+
[
|
|
755
|
+
"--online-reward",
|
|
756
|
+
training["online_reward"],
|
|
757
|
+
"--online-rounds",
|
|
758
|
+
str(training["online_rounds"]),
|
|
759
|
+
*(
|
|
760
|
+
["--online-temperature", str(training["online_temperature"])]
|
|
761
|
+
if "online_temperature" in training
|
|
762
|
+
else []
|
|
763
|
+
),
|
|
764
|
+
]
|
|
765
|
+
if training.get("online_reward")
|
|
766
|
+
else []
|
|
767
|
+
),
|
|
768
|
+
"--idempotency-key",
|
|
769
|
+
state["submission_key"],
|
|
770
|
+
],
|
|
771
|
+
store=False,
|
|
772
|
+
)
|
|
773
|
+
state["job_id"] = job["job_id"]
|
|
774
|
+
state["job"] = job
|
|
775
|
+
state["submission_pending"] = False
|
|
776
|
+
state["status"] = "training"
|
|
777
|
+
if (directory / "cancel.requested").exists():
|
|
778
|
+
state["job"] = _cli(
|
|
779
|
+
["jobs", "cancel", state["job_id"], "--api", config["jobs_url"]], config=config
|
|
780
|
+
)
|
|
781
|
+
elif action in {"evaluate", "warm"}:
|
|
782
|
+
if action == "warm":
|
|
783
|
+
_use_frozen_project(root, state)
|
|
784
|
+
adapter = directory / "adapter"
|
|
785
|
+
if "adopt" not in state["steps"]:
|
|
786
|
+
step(
|
|
787
|
+
"adopt",
|
|
788
|
+
["jobs", "adopt", state["job_id"], "--api", config["jobs_url"], "--into", adapter],
|
|
789
|
+
store=False,
|
|
790
|
+
)
|
|
791
|
+
if "candidate" not in state["steps"]:
|
|
792
|
+
candidate = step(
|
|
793
|
+
"candidate",
|
|
794
|
+
[
|
|
795
|
+
"experiment",
|
|
796
|
+
"candidate",
|
|
797
|
+
"add",
|
|
798
|
+
state["experiment_id"],
|
|
799
|
+
adapter / "adapter.json",
|
|
800
|
+
"--project",
|
|
801
|
+
config["project"],
|
|
802
|
+
],
|
|
803
|
+
)
|
|
804
|
+
state["candidate_hash"] = candidate["candidate_hash"]
|
|
805
|
+
else:
|
|
806
|
+
state["candidate_hash"] = state["steps"]["candidate"]["candidate_hash"]
|
|
807
|
+
step(
|
|
808
|
+
"load",
|
|
809
|
+
[
|
|
810
|
+
"program",
|
|
811
|
+
"load-adapter",
|
|
812
|
+
state["candidate_hash"],
|
|
813
|
+
"--directory",
|
|
814
|
+
adapter,
|
|
815
|
+
"--base-url",
|
|
816
|
+
config["sampling_url"],
|
|
817
|
+
"--api-key-env",
|
|
818
|
+
"CORETHINK_RL_MODEL_TOKEN",
|
|
819
|
+
],
|
|
820
|
+
)
|
|
821
|
+
previous_warmup = state.get("warmup", {})
|
|
822
|
+
state["warmup"] = {"status": "ready", "at": datetime.now(UTC).isoformat()}
|
|
823
|
+
if previous_warmup.get("status") == "failed":
|
|
824
|
+
state["warmup"]["recovered_from"] = previous_warmup
|
|
825
|
+
elif "recovered_from" in previous_warmup:
|
|
826
|
+
state["warmup"]["recovered_from"] = previous_warmup["recovered_from"]
|
|
827
|
+
_save(root, state)
|
|
828
|
+
if action == "warm":
|
|
829
|
+
return # A loaded candidate is not a measured or released one.
|
|
830
|
+
measured = step(
|
|
831
|
+
"evaluate",
|
|
832
|
+
[
|
|
833
|
+
"experiment",
|
|
834
|
+
"run",
|
|
835
|
+
state["experiment_id"],
|
|
836
|
+
"--provider",
|
|
837
|
+
"manual",
|
|
838
|
+
"--candidate",
|
|
839
|
+
adapter / "adapter.json",
|
|
840
|
+
*_eval_options(config),
|
|
841
|
+
],
|
|
842
|
+
)
|
|
843
|
+
if not any(
|
|
844
|
+
item["candidate_hash"] == state["candidate_hash"] and item["primary_score"] is not None
|
|
845
|
+
for item in measured["results"]
|
|
846
|
+
):
|
|
847
|
+
raise ExperimentError("candidate did not produce a complete development score")
|
|
848
|
+
step("compare", ["experiment", "compare", state["experiment_id"]])
|
|
849
|
+
state["status"] = "ready_to_promote"
|
|
850
|
+
elif action == "gate":
|
|
851
|
+
gate = step(
|
|
852
|
+
"gate",
|
|
853
|
+
[
|
|
854
|
+
"experiment",
|
|
855
|
+
"gate",
|
|
856
|
+
state["experiment_id"],
|
|
857
|
+
"--candidate",
|
|
858
|
+
state["candidate_hash"],
|
|
859
|
+
"--cleanup-verified",
|
|
860
|
+
*_eval_options(config, holdout=True),
|
|
861
|
+
],
|
|
862
|
+
)
|
|
863
|
+
state["status"] = "gate_passed" if gate["passed"] else "gate_failed"
|
|
864
|
+
elif action == "promote":
|
|
865
|
+
if not state.get("candidate_hash") or not state["steps"].get("load"):
|
|
866
|
+
# Prepare these exact trained weights, without running a quality test.
|
|
867
|
+
# A failed preparation still stops promotion; a low score never does.
|
|
868
|
+
_execute("warm", args, root, directory, state)
|
|
869
|
+
step(
|
|
870
|
+
"promote",
|
|
871
|
+
[
|
|
872
|
+
"program",
|
|
873
|
+
"promote",
|
|
874
|
+
state["candidate_hash"],
|
|
875
|
+
"--project",
|
|
876
|
+
config["project"],
|
|
877
|
+
"--experiment",
|
|
878
|
+
state["experiment_id"],
|
|
879
|
+
"--alias",
|
|
880
|
+
args.alias,
|
|
881
|
+
"--from-alias",
|
|
882
|
+
"candidate" if args.alias == "staging" else "staging",
|
|
883
|
+
"--approved-by",
|
|
884
|
+
args.approved_by,
|
|
885
|
+
"--reason",
|
|
886
|
+
args.reason,
|
|
887
|
+
],
|
|
888
|
+
)
|
|
889
|
+
state["status"] = "promoted"
|
|
890
|
+
state["alias"] = args.alias
|
|
891
|
+
elif action == "rollback":
|
|
892
|
+
step(
|
|
893
|
+
"rollback",
|
|
894
|
+
[
|
|
895
|
+
"program",
|
|
896
|
+
"rollback",
|
|
897
|
+
args.alias,
|
|
898
|
+
"--project-id",
|
|
899
|
+
state["project_id"],
|
|
900
|
+
"--approved-by",
|
|
901
|
+
args.approved_by,
|
|
902
|
+
"--reason",
|
|
903
|
+
args.reason,
|
|
904
|
+
],
|
|
905
|
+
)
|
|
906
|
+
elif action == "test":
|
|
907
|
+
from corethink.rl_playground import test_model
|
|
908
|
+
|
|
909
|
+
test_model(args, root, state, _save)
|
|
910
|
+
elif action == "recheck":
|
|
911
|
+
from corethink_experiments.adapters.adapter_files import adapter_model_name
|
|
912
|
+
|
|
913
|
+
candidate = state.get("candidate_hash", "")
|
|
914
|
+
model = adapter_model_name(candidate)
|
|
915
|
+
if state["steps"].get("load", {}).get("model") != model:
|
|
916
|
+
raise ExperimentError("load this exact candidate before re-evaluating it")
|
|
917
|
+
record = {
|
|
918
|
+
"id": "recheck-" + uuid.uuid4().hex,
|
|
919
|
+
"candidate_hash": candidate,
|
|
920
|
+
"model": model,
|
|
921
|
+
"created_at": datetime.now(UTC).isoformat(),
|
|
922
|
+
"status": "running",
|
|
923
|
+
"split": "development",
|
|
924
|
+
}
|
|
925
|
+
state.setdefault("rechecks", []).append(record)
|
|
926
|
+
state["operation"] = "recheck"
|
|
927
|
+
_save(root, state)
|
|
928
|
+
try:
|
|
929
|
+
result = _cli(
|
|
930
|
+
[
|
|
931
|
+
"eval",
|
|
932
|
+
config.get("evaluation", config["development"]),
|
|
933
|
+
"--program",
|
|
934
|
+
config["baseline"],
|
|
935
|
+
"--base-url",
|
|
936
|
+
config["sampling_url"],
|
|
937
|
+
"--model",
|
|
938
|
+
model,
|
|
939
|
+
"--model-api-key-env",
|
|
940
|
+
"CORETHINK_RL_MODEL_TOKEN",
|
|
941
|
+
"--model-timeout",
|
|
942
|
+
"600",
|
|
943
|
+
"--task-timeout",
|
|
944
|
+
"620",
|
|
945
|
+
"--limit",
|
|
946
|
+
"64",
|
|
947
|
+
"--out",
|
|
948
|
+
directory / "rechecks" / record["id"],
|
|
949
|
+
],
|
|
950
|
+
config=config,
|
|
951
|
+
cancel_path=directory / "cancel.requested",
|
|
952
|
+
)
|
|
953
|
+
value = result.get("primary_score")
|
|
954
|
+
if (
|
|
955
|
+
type(value) not in {int, float}
|
|
956
|
+
or not math.isfinite(value)
|
|
957
|
+
or result.get("errored_task_count", 0)
|
|
958
|
+
or not result.get("task_results")
|
|
959
|
+
):
|
|
960
|
+
raise ExperimentError("re-evaluation did not produce a complete development score")
|
|
961
|
+
record.update(
|
|
962
|
+
status="completed",
|
|
963
|
+
primary_score=value,
|
|
964
|
+
task_count=len(result["task_results"]),
|
|
965
|
+
evaluation_run_id=result["run_id"],
|
|
966
|
+
)
|
|
967
|
+
except Exception:
|
|
968
|
+
record["status"] = "failed"
|
|
969
|
+
_save(root, state)
|
|
970
|
+
raise
|
|
971
|
+
# New evidence only: never replace the original compare, gate or release.
|
|
972
|
+
elif action == "infer":
|
|
973
|
+
# Alias resolution happens in the shared serving layer, not in the browser.
|
|
974
|
+
step(
|
|
975
|
+
"infer",
|
|
976
|
+
[
|
|
977
|
+
"program",
|
|
978
|
+
"call",
|
|
979
|
+
args.message,
|
|
980
|
+
"--project-id",
|
|
981
|
+
state["project_id"],
|
|
982
|
+
"--alias",
|
|
983
|
+
args.alias,
|
|
984
|
+
"--base-url",
|
|
985
|
+
config["sampling_url"],
|
|
986
|
+
"--base-model",
|
|
987
|
+
config["base_model"] + "@" + config.get("revision", "main"),
|
|
988
|
+
"--model-api-key-env",
|
|
989
|
+
"CORETHINK_RL_MODEL_TOKEN",
|
|
990
|
+
# Cloud Run may need several minutes to start vLLM and restore
|
|
991
|
+
# immutable adapters. The ordinary chat default was shorter
|
|
992
|
+
# than this measured cold start; keep the wait bounded.
|
|
993
|
+
"--timeout",
|
|
994
|
+
"600",
|
|
995
|
+
],
|
|
996
|
+
)
|
|
997
|
+
state.pop("operation", None)
|