flexlock 0.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flexlock/__init__.py +87 -0
- flexlock/api.py +1444 -0
- flexlock/backends/__init__.py +11 -0
- flexlock/backends/base.py +41 -0
- flexlock/backends/pbs.py +219 -0
- flexlock/backends/slurm.py +333 -0
- flexlock/cli.py +953 -0
- flexlock/config.py +121 -0
- flexlock/context.py +7 -0
- flexlock/data_hash.py +310 -0
- flexlock/debug.py +473 -0
- flexlock/diff.py +253 -0
- flexlock/diff_cli.py +155 -0
- flexlock/exceptions.py +50 -0
- flexlock/export.py +134 -0
- flexlock/fingerprint.py +129 -0
- flexlock/flexcli.py +163 -0
- flexlock/freeze.py +291 -0
- flexlock/git_utils.py +196 -0
- flexlock/index.py +300 -0
- flexlock/load_stage.py +60 -0
- flexlock/mlflow.py +192 -0
- flexlock/parallel.py +378 -0
- flexlock/query.py +739 -0
- flexlock/report.py +78 -0
- flexlock/report_template.html +246 -0
- flexlock/resolvers.py +248 -0
- flexlock/run_cli.py +44 -0
- flexlock/run_record.py +173 -0
- flexlock/runner.py +571 -0
- flexlock/save_dir.py +232 -0
- flexlock/skills/flexlock-new-stage/SKILL.md +57 -0
- flexlock/skills/flexlock-report/SKILL.md +50 -0
- flexlock/skills/flexlock-run-and-watch/SKILL.md +60 -0
- flexlock/skills/flexlock-survey/SKILL.md +61 -0
- flexlock/snapshot.py +245 -0
- flexlock/status_cli.py +299 -0
- flexlock/taskdb.py +685 -0
- flexlock/utils.py +710 -0
- flexlock/worker.py +228 -0
- flexlock/worker_cli.py +143 -0
- flexlock-0.8.2.dist-info/METADATA +357 -0
- flexlock-0.8.2.dist-info/RECORD +45 -0
- flexlock-0.8.2.dist-info/WHEEL +4 -0
- flexlock-0.8.2.dist-info/entry_points.txt +7 -0
flexlock/query.py
ADDED
|
@@ -0,0 +1,739 @@
|
|
|
1
|
+
"""Read-only queries over FlexLock run directories — the agent-facing contract.
|
|
2
|
+
|
|
3
|
+
Everything here is side-effect free: run locks are read with ``yaml.safe_load``
|
|
4
|
+
(NOT ``RunRecord.load``, which resolves interpolations and can fire ``${vinc:}``
|
|
5
|
+
resolvers that create directories). The JSON shapes returned by these functions
|
|
6
|
+
are the stable contract consumed by the CLI (``show``/``graph``/``why``/
|
|
7
|
+
``stages``), the shipped skills, and the HTML report.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from .run_record import (
|
|
17
|
+
RunRecord,
|
|
18
|
+
LOCK_NAME,
|
|
19
|
+
COMPLETE_MARKER,
|
|
20
|
+
ERROR_NAME,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
SCHEMA_VERSION = 1
|
|
24
|
+
|
|
25
|
+
MARKER_NAME = ".flexlock_marker"
|
|
26
|
+
TASKS_DB_NAME = "run.lock.tasks.db"
|
|
27
|
+
|
|
28
|
+
# Map task-DB statuses onto the run-dir status vocabulary.
|
|
29
|
+
_TASK_STATUS_MAP = {
|
|
30
|
+
"done": "complete",
|
|
31
|
+
"failed": "failed",
|
|
32
|
+
"interrupted": "interrupted",
|
|
33
|
+
"running": "running",
|
|
34
|
+
"pending": "pending",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_lock(run_dir: Path) -> Optional[dict]:
|
|
39
|
+
"""Parse ``run.lock`` with a side-effect-free loader, or ``None``."""
|
|
40
|
+
lock = Path(run_dir) / LOCK_NAME
|
|
41
|
+
if not lock.exists():
|
|
42
|
+
return None
|
|
43
|
+
try:
|
|
44
|
+
with open(lock) as f:
|
|
45
|
+
return yaml.safe_load(f)
|
|
46
|
+
except Exception:
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _read_marker(run_dir: Path) -> Optional[dict]:
|
|
51
|
+
marker = Path(run_dir) / MARKER_NAME
|
|
52
|
+
if not marker.exists():
|
|
53
|
+
return None
|
|
54
|
+
try:
|
|
55
|
+
return json.loads(marker.read_text())
|
|
56
|
+
except Exception:
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _resolve_marker_db(run_dir: Path, marker: dict) -> Path:
|
|
61
|
+
"""Resolve the task DB path recorded in a ``.flexlock_marker``.
|
|
62
|
+
|
|
63
|
+
The worker stores ``db`` relative to the task dir's *parent* (worker.py),
|
|
64
|
+
so a relative path is resolved against ``run_dir.parent``.
|
|
65
|
+
"""
|
|
66
|
+
db = Path(marker.get("db", ""))
|
|
67
|
+
if not db.is_absolute():
|
|
68
|
+
db = (Path(run_dir).parent / db)
|
|
69
|
+
return db
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def run_status(run_dir) -> dict:
|
|
73
|
+
"""Classify a run directory from what's on disk.
|
|
74
|
+
|
|
75
|
+
Returns ``{"status", "detail", "kind", "tasks"|None, "error"|None}``.
|
|
76
|
+
|
|
77
|
+
Precedence (see plan Phase B):
|
|
78
|
+
1. ``.flexlock_marker`` → sweep-task dir (query the task DB row).
|
|
79
|
+
2. ``run.complete`` → ``complete``.
|
|
80
|
+
3. ``run.error`` → ``failed`` (payload attached).
|
|
81
|
+
4. ``run.lock`` + task DB in dir → sweep master (aggregate counts).
|
|
82
|
+
5. bare ``run.lock`` → ``interrupted`` (may still be running in-process).
|
|
83
|
+
6. otherwise → ``unknown``.
|
|
84
|
+
"""
|
|
85
|
+
run_dir = Path(run_dir)
|
|
86
|
+
rec = RunRecord(run_dir)
|
|
87
|
+
|
|
88
|
+
# 1. Sweep-task dir.
|
|
89
|
+
marker = _read_marker(run_dir)
|
|
90
|
+
if marker is not None:
|
|
91
|
+
task_id = marker.get("task_id")
|
|
92
|
+
db_path = _resolve_marker_db(run_dir, marker)
|
|
93
|
+
if db_path.exists() and task_id:
|
|
94
|
+
from .taskdb import get_task_status
|
|
95
|
+
|
|
96
|
+
row = get_task_status(db_path, task_id)
|
|
97
|
+
if row is not None:
|
|
98
|
+
status = _TASK_STATUS_MAP.get(row["status"], "unknown")
|
|
99
|
+
out = {
|
|
100
|
+
"status": status,
|
|
101
|
+
"detail": None,
|
|
102
|
+
"kind": "sweep_task",
|
|
103
|
+
"tasks": None,
|
|
104
|
+
"error": None,
|
|
105
|
+
}
|
|
106
|
+
if status == "failed" and row.get("error"):
|
|
107
|
+
out["error"] = {"traceback": row["error"], "node": row.get("node")}
|
|
108
|
+
return out
|
|
109
|
+
# DB missing/unreadable — fall through with a hint, but still note kind.
|
|
110
|
+
detail = f"sweep-task dir but task DB unavailable at {db_path}"
|
|
111
|
+
# Prefer local completion/failure markers if present.
|
|
112
|
+
if rec.complete_path.exists():
|
|
113
|
+
return {"status": "complete", "detail": detail, "kind": "sweep_task",
|
|
114
|
+
"tasks": None, "error": None}
|
|
115
|
+
err = rec.load_error()
|
|
116
|
+
if err is not None:
|
|
117
|
+
return {"status": "failed", "detail": detail, "kind": "sweep_task",
|
|
118
|
+
"tasks": None, "error": err}
|
|
119
|
+
return {"status": "unknown", "detail": detail, "kind": "sweep_task",
|
|
120
|
+
"tasks": None, "error": None}
|
|
121
|
+
|
|
122
|
+
# 2. Explicit success marker.
|
|
123
|
+
if rec.complete_path.exists():
|
|
124
|
+
return {"status": "complete", "detail": None, "kind": _plain_kind(run_dir),
|
|
125
|
+
"tasks": _maybe_tasks(run_dir), "error": None}
|
|
126
|
+
|
|
127
|
+
# 3. Failure sidecar.
|
|
128
|
+
err = rec.load_error()
|
|
129
|
+
if err is not None:
|
|
130
|
+
return {"status": "failed", "detail": None, "kind": _plain_kind(run_dir),
|
|
131
|
+
"tasks": None, "error": err}
|
|
132
|
+
|
|
133
|
+
# 4. Sweep master (run.lock + task DB).
|
|
134
|
+
lock = Path(run_dir) / LOCK_NAME
|
|
135
|
+
db_path = Path(run_dir) / TASKS_DB_NAME
|
|
136
|
+
if lock.exists() and db_path.exists():
|
|
137
|
+
from .taskdb import get_status_counts
|
|
138
|
+
|
|
139
|
+
counts = get_status_counts(db_path)
|
|
140
|
+
status = _aggregate_sweep_status(counts)
|
|
141
|
+
return {"status": status, "detail": None, "kind": "sweep_master",
|
|
142
|
+
"tasks": counts, "error": None}
|
|
143
|
+
|
|
144
|
+
# 5. Bare run.lock.
|
|
145
|
+
if lock.exists():
|
|
146
|
+
return {
|
|
147
|
+
"status": "interrupted",
|
|
148
|
+
"detail": "run.lock present without run.complete — interrupted, "
|
|
149
|
+
"or still running in-process (no liveness probe).",
|
|
150
|
+
"kind": "run",
|
|
151
|
+
"tasks": None,
|
|
152
|
+
"error": None,
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
# 6. Nothing recognisable.
|
|
156
|
+
return {"status": "unknown", "detail": "no run.lock/run.complete/marker found",
|
|
157
|
+
"kind": "unknown", "tasks": None, "error": None}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _plain_kind(run_dir: Path) -> str:
|
|
161
|
+
return "sweep_master" if (Path(run_dir) / TASKS_DB_NAME).exists() else "run"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _maybe_tasks(run_dir: Path) -> Optional[dict]:
|
|
165
|
+
db_path = Path(run_dir) / TASKS_DB_NAME
|
|
166
|
+
if not db_path.exists():
|
|
167
|
+
return None
|
|
168
|
+
from .taskdb import get_status_counts
|
|
169
|
+
|
|
170
|
+
return get_status_counts(db_path)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _aggregate_sweep_status(counts: dict) -> str:
|
|
174
|
+
"""Roll per-task counts into one master status."""
|
|
175
|
+
if counts.get("pending", 0) or counts.get("running", 0):
|
|
176
|
+
return "running"
|
|
177
|
+
if counts.get("failed", 0):
|
|
178
|
+
return "failed"
|
|
179
|
+
if counts.get("interrupted", 0):
|
|
180
|
+
return "interrupted"
|
|
181
|
+
return "complete"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _lineage_from_lock(lock: dict) -> dict:
|
|
185
|
+
"""Return the lineage mapping, tolerating the ``lineage``/``prevs`` spellings."""
|
|
186
|
+
return lock.get("lineage") or lock.get("prevs") or {}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _upstream_list(lock: dict) -> list:
|
|
190
|
+
out = []
|
|
191
|
+
for name, entry in _lineage_from_lock(lock).items():
|
|
192
|
+
if not isinstance(entry, dict):
|
|
193
|
+
continue
|
|
194
|
+
path = entry.get("path") or entry.get("config", {}).get("save_dir")
|
|
195
|
+
info = entry.get("info", {})
|
|
196
|
+
ts = entry.get("timestamp") or info.get("timestamp")
|
|
197
|
+
out.append({"name": name, "path": str(path) if path else None, "timestamp": ts})
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _tag_for_path(run_dir: Path, scan_root: Path) -> Optional[str]:
|
|
202
|
+
"""Find a flexlock tag whose ``Path:`` message resolves to ``run_dir``."""
|
|
203
|
+
from .cli import find_git_repo, get_flexlock_tags, get_tag_details
|
|
204
|
+
|
|
205
|
+
repo = find_git_repo(str(scan_root))
|
|
206
|
+
if not repo:
|
|
207
|
+
return None
|
|
208
|
+
target = str(Path(run_dir).resolve())
|
|
209
|
+
try:
|
|
210
|
+
for tag_name, tag_commit in get_flexlock_tags(repo).items():
|
|
211
|
+
details = get_tag_details(repo, tag_commit)
|
|
212
|
+
for line in details["message"].splitlines():
|
|
213
|
+
if line.startswith("Path: "):
|
|
214
|
+
tagged = line[6:].strip()
|
|
215
|
+
if str(Path(tagged).resolve()) == target:
|
|
216
|
+
return tag_name
|
|
217
|
+
except Exception:
|
|
218
|
+
return None
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _downstream_list(run_dir: Path, scan_root: Path) -> list:
|
|
223
|
+
"""Scan ``scan_root`` for runs whose lineage resolves to ``run_dir``."""
|
|
224
|
+
from .cli import find_results_dirs
|
|
225
|
+
|
|
226
|
+
target = str(Path(run_dir).resolve())
|
|
227
|
+
out = []
|
|
228
|
+
for run in find_results_dirs(str(scan_root)):
|
|
229
|
+
if str(Path(run["path"]).resolve()) == target:
|
|
230
|
+
continue
|
|
231
|
+
for entry in (run.get("lineage") or {}).values():
|
|
232
|
+
if not isinstance(entry, dict):
|
|
233
|
+
continue
|
|
234
|
+
path = entry.get("path") or entry.get("config", {}).get("save_dir")
|
|
235
|
+
if path and str(Path(path).resolve()) == target:
|
|
236
|
+
lock = _load_lock(Path(run["path"])) or {}
|
|
237
|
+
out.append({
|
|
238
|
+
"path": str(Path(run["path"]).resolve()),
|
|
239
|
+
"timestamp": run.get("timestamp"),
|
|
240
|
+
"note": lock.get("note"),
|
|
241
|
+
})
|
|
242
|
+
break
|
|
243
|
+
return out
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def load_run_summary(run_dir, scan_root=None, downstream=True) -> dict:
|
|
247
|
+
"""Full JSON summary for ``flexlock show`` — the agent-facing run contract."""
|
|
248
|
+
run_dir = Path(run_dir).resolve()
|
|
249
|
+
scan_root = Path(scan_root).resolve() if scan_root else run_dir.parent
|
|
250
|
+
|
|
251
|
+
st = run_status(run_dir)
|
|
252
|
+
kind = st["kind"]
|
|
253
|
+
lock = _load_lock(run_dir)
|
|
254
|
+
|
|
255
|
+
# Sweep-task dirs carry no run.lock; reconstruct from the task DB snapshot
|
|
256
|
+
# and inherit note/repos from the master lock.
|
|
257
|
+
marker = _read_marker(run_dir)
|
|
258
|
+
master_lock = None
|
|
259
|
+
if lock is None and marker is not None:
|
|
260
|
+
db_path = _resolve_marker_db(run_dir, marker)
|
|
261
|
+
task_id = marker.get("task_id")
|
|
262
|
+
if db_path.exists() and task_id:
|
|
263
|
+
from .taskdb import get_task_snapshot
|
|
264
|
+
|
|
265
|
+
snap = get_task_snapshot(db_path, task_id)
|
|
266
|
+
if snap:
|
|
267
|
+
lock = snap
|
|
268
|
+
master_lock = _load_lock(db_path.parent) if db_path.exists() else None
|
|
269
|
+
|
|
270
|
+
lock = lock or {}
|
|
271
|
+
config = lock.get("config", {}) or {}
|
|
272
|
+
repos = lock.get("repos") or (master_lock or {}).get("repos") or {}
|
|
273
|
+
note = lock.get("note")
|
|
274
|
+
if note is None and master_lock:
|
|
275
|
+
note = master_lock.get("note")
|
|
276
|
+
|
|
277
|
+
summary = {
|
|
278
|
+
"schema_version": SCHEMA_VERSION,
|
|
279
|
+
"path": str(run_dir),
|
|
280
|
+
"kind": kind,
|
|
281
|
+
"status": st["status"],
|
|
282
|
+
"status_detail": st.get("detail"),
|
|
283
|
+
"timestamp": lock.get("timestamp"),
|
|
284
|
+
"note": note,
|
|
285
|
+
"target": config.get("_target_") if isinstance(config, dict) else None,
|
|
286
|
+
"fingerprint": lock.get("fingerprint"),
|
|
287
|
+
"tag": _tag_for_path(run_dir, scan_root),
|
|
288
|
+
"error": st.get("error"),
|
|
289
|
+
"config": config,
|
|
290
|
+
"results": RunRecord(run_dir).load_results() or {},
|
|
291
|
+
"repos": repos,
|
|
292
|
+
"data": lock.get("data") or {},
|
|
293
|
+
"tasks": st.get("tasks"),
|
|
294
|
+
"lineage": {
|
|
295
|
+
"upstream": _upstream_list(lock),
|
|
296
|
+
"downstream": _downstream_list(run_dir, scan_root) if downstream else [],
|
|
297
|
+
},
|
|
298
|
+
}
|
|
299
|
+
return summary
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _git_log_between(repo_path, a_commit, b_commit) -> list:
|
|
303
|
+
"""``git log a..b`` between two shadow commits → list of commit dicts.
|
|
304
|
+
|
|
305
|
+
Raises on git errors (caller records them per repo and continues).
|
|
306
|
+
"""
|
|
307
|
+
from git.repo import Repo as GitRepo
|
|
308
|
+
|
|
309
|
+
repo = GitRepo(repo_path, search_parent_directories=True)
|
|
310
|
+
fmt = "%H%x00%an%x00%aI%x00%s"
|
|
311
|
+
out = repo.git.log(f"--format={fmt}", f"{a_commit}..{b_commit}")
|
|
312
|
+
commits = []
|
|
313
|
+
for line in out.splitlines():
|
|
314
|
+
if not line.strip():
|
|
315
|
+
continue
|
|
316
|
+
sha, author, date, subject = line.split("\x00")
|
|
317
|
+
commits.append({"sha": sha, "author": author, "date": date, "subject": subject})
|
|
318
|
+
return commits
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def why(run_a, run_b) -> dict:
|
|
322
|
+
"""Explain the difference between two runs — config/git/data + real commits.
|
|
323
|
+
|
|
324
|
+
For each repo present in both locks with differing tree hashes, walk the
|
|
325
|
+
recorded shadow commits (``git log a..b`` / ``b..a``). Git failures (e.g.
|
|
326
|
+
gc'd refs) are captured per-repo so the command always succeeds (exit 0).
|
|
327
|
+
"""
|
|
328
|
+
from .diff_cli import run_comparison
|
|
329
|
+
|
|
330
|
+
run_a = Path(run_a).resolve()
|
|
331
|
+
run_b = Path(run_b).resolve()
|
|
332
|
+
lock_a = _load_lock(run_a) or {}
|
|
333
|
+
lock_b = _load_lock(run_b) or {}
|
|
334
|
+
|
|
335
|
+
is_match, diffs = run_comparison(lock_a, lock_b)
|
|
336
|
+
|
|
337
|
+
repos_a = lock_a.get("repos") or {}
|
|
338
|
+
repos_b = lock_b.get("repos") or {}
|
|
339
|
+
commits = {}
|
|
340
|
+
for name in sorted(set(repos_a) & set(repos_b)):
|
|
341
|
+
info_a = repos_a[name] or {}
|
|
342
|
+
info_b = repos_b[name] or {}
|
|
343
|
+
entry = {"trees_identical": False, "a_to_b": [], "b_to_a": [], "error": None}
|
|
344
|
+
if info_a.get("tree") and info_a.get("tree") == info_b.get("tree"):
|
|
345
|
+
entry["trees_identical"] = True
|
|
346
|
+
commits[name] = entry
|
|
347
|
+
continue
|
|
348
|
+
ca = info_a.get("commit")
|
|
349
|
+
cb = info_b.get("commit")
|
|
350
|
+
repo_path = info_a.get("path") or info_b.get("path")
|
|
351
|
+
if not (ca and cb and repo_path):
|
|
352
|
+
entry["error"] = "missing shadow commit or repo path in run.lock"
|
|
353
|
+
commits[name] = entry
|
|
354
|
+
continue
|
|
355
|
+
try:
|
|
356
|
+
entry["a_to_b"] = _git_log_between(repo_path, ca, cb)
|
|
357
|
+
entry["b_to_a"] = _git_log_between(repo_path, cb, ca)
|
|
358
|
+
except Exception as exc:
|
|
359
|
+
entry["error"] = f"{exc} — shadow commits may have been gc'd"
|
|
360
|
+
commits[name] = entry
|
|
361
|
+
|
|
362
|
+
return {
|
|
363
|
+
"schema_version": SCHEMA_VERSION,
|
|
364
|
+
"run_a": str(run_a),
|
|
365
|
+
"run_b": str(run_b),
|
|
366
|
+
"match": is_match,
|
|
367
|
+
"diffs": diffs,
|
|
368
|
+
"commits": commits,
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def format_why_text(result: dict) -> str:
|
|
373
|
+
"""Human-readable rendering of a ``why`` result."""
|
|
374
|
+
lines = []
|
|
375
|
+
mark = "✓ match" if result["match"] else "✗ differ"
|
|
376
|
+
lines.append(f"{mark}: {Path(result['run_a']).name} vs {Path(result['run_b']).name}")
|
|
377
|
+
lines.append("")
|
|
378
|
+
|
|
379
|
+
for name, entry in result.get("commits", {}).items():
|
|
380
|
+
lines.append(f"## repo: {name}")
|
|
381
|
+
if entry.get("error"):
|
|
382
|
+
lines.append(f" (git unavailable: {entry['error']})")
|
|
383
|
+
elif entry.get("trees_identical"):
|
|
384
|
+
lines.append(" trees identical")
|
|
385
|
+
else:
|
|
386
|
+
if entry["a_to_b"]:
|
|
387
|
+
lines.append(f" commits in {Path(result['run_b']).name} not in "
|
|
388
|
+
f"{Path(result['run_a']).name}:")
|
|
389
|
+
for c in entry["a_to_b"]:
|
|
390
|
+
lines.append(f" + {c['sha'][:10]} {c['subject']}")
|
|
391
|
+
if entry["b_to_a"]:
|
|
392
|
+
lines.append(f" commits in {Path(result['run_a']).name} not in "
|
|
393
|
+
f"{Path(result['run_b']).name}:")
|
|
394
|
+
for c in entry["b_to_a"]:
|
|
395
|
+
lines.append(f" - {c['sha'][:10]} {c['subject']}")
|
|
396
|
+
lines.append("")
|
|
397
|
+
|
|
398
|
+
for key in ("config", "data", "git"):
|
|
399
|
+
vals = result.get("diffs", {}).get(key)
|
|
400
|
+
if vals:
|
|
401
|
+
lines.append(f"## {key} diffs")
|
|
402
|
+
for d in vals:
|
|
403
|
+
lines.append(f" - {d}")
|
|
404
|
+
lines.append("")
|
|
405
|
+
|
|
406
|
+
return "\n".join(lines)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def list_stage_nodes(root_cfg) -> list:
|
|
410
|
+
"""Enumerate every stage (a mapping carrying ``_target_``) in a config tree.
|
|
411
|
+
|
|
412
|
+
The config is walked as a *plain* container with ``resolve=False`` — resolving
|
|
413
|
+
would fire resolvers like ``${vinc:}`` and create directories on disk. Each
|
|
414
|
+
stage is reported by its dot-path key, its ``_target_``, its declared
|
|
415
|
+
``save_dir`` (unresolved), and its nesting depth. ``_snapshot_`` subtrees are
|
|
416
|
+
skipped (tracking declarations, not runnable stages).
|
|
417
|
+
"""
|
|
418
|
+
from omegaconf import OmegaConf, DictConfig
|
|
419
|
+
|
|
420
|
+
if isinstance(root_cfg, DictConfig):
|
|
421
|
+
container = OmegaConf.to_container(root_cfg, resolve=False)
|
|
422
|
+
elif OmegaConf.is_config(root_cfg):
|
|
423
|
+
container = OmegaConf.to_container(root_cfg, resolve=False)
|
|
424
|
+
else:
|
|
425
|
+
container = root_cfg
|
|
426
|
+
|
|
427
|
+
stages = []
|
|
428
|
+
|
|
429
|
+
def _walk(node, path, depth):
|
|
430
|
+
if not isinstance(node, dict):
|
|
431
|
+
return
|
|
432
|
+
if "_target_" in node:
|
|
433
|
+
stages.append({
|
|
434
|
+
"key": path,
|
|
435
|
+
"target": node.get("_target_"),
|
|
436
|
+
"save_dir": node.get("save_dir"),
|
|
437
|
+
"depth": depth,
|
|
438
|
+
})
|
|
439
|
+
for key, value in node.items():
|
|
440
|
+
if key == "_snapshot_":
|
|
441
|
+
continue
|
|
442
|
+
if isinstance(value, dict):
|
|
443
|
+
child_path = f"{path}.{key}" if path else key
|
|
444
|
+
_walk(value, child_path, depth + 1)
|
|
445
|
+
|
|
446
|
+
_walk(container, "", 0)
|
|
447
|
+
return stages
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _all_tags_by_path(scan_root) -> dict:
|
|
451
|
+
"""One-shot ``{resolved_path: tag_name}`` map for a whole tree."""
|
|
452
|
+
from .cli import find_git_repo, get_flexlock_tags, get_tag_details
|
|
453
|
+
|
|
454
|
+
out = {}
|
|
455
|
+
repo = find_git_repo(str(scan_root))
|
|
456
|
+
if not repo:
|
|
457
|
+
return out
|
|
458
|
+
try:
|
|
459
|
+
for tag_name, tag_commit in get_flexlock_tags(repo).items():
|
|
460
|
+
details = get_tag_details(repo, tag_commit)
|
|
461
|
+
for line in details["message"].splitlines():
|
|
462
|
+
if line.startswith("Path: "):
|
|
463
|
+
out[str(Path(line[6:].strip()).resolve())] = tag_name
|
|
464
|
+
except Exception:
|
|
465
|
+
pass
|
|
466
|
+
return out
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _numeric_metrics(results) -> dict:
|
|
470
|
+
"""Top-level numeric keys of a results payload."""
|
|
471
|
+
if not isinstance(results, dict):
|
|
472
|
+
return {}
|
|
473
|
+
return {
|
|
474
|
+
k: v for k, v in results.items()
|
|
475
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def build_graph(results_root, include_groups=False) -> dict:
|
|
480
|
+
"""Build the experiment DAG JSON — backend for ``graph`` and the report.
|
|
481
|
+
|
|
482
|
+
Nodes come from every ``run.lock`` under ``results_root`` plus every sweep
|
|
483
|
+
task dir (identified by ``.flexlock_marker``, which has no run.lock). Edges
|
|
484
|
+
capture lineage and sweep-item containment; edges pointing outside the scan
|
|
485
|
+
materialise ``kind="external"`` stub nodes so the graph stays closed.
|
|
486
|
+
"""
|
|
487
|
+
from datetime import datetime as _dt
|
|
488
|
+
|
|
489
|
+
from .cli import find_results_dirs
|
|
490
|
+
|
|
491
|
+
root = Path(results_root).resolve()
|
|
492
|
+
tag_map = _all_tags_by_path(root)
|
|
493
|
+
|
|
494
|
+
nodes: dict[str, dict] = {}
|
|
495
|
+
edges: list[dict] = []
|
|
496
|
+
|
|
497
|
+
def _node(node_id, **fields):
|
|
498
|
+
nodes[node_id] = fields
|
|
499
|
+
|
|
500
|
+
# 1. run.lock-backed nodes (runs + sweep masters).
|
|
501
|
+
for run in find_results_dirs(str(root)):
|
|
502
|
+
run_dir = Path(run["path"]).resolve()
|
|
503
|
+
node_id = str(run_dir)
|
|
504
|
+
lock = _load_lock(run_dir) or {}
|
|
505
|
+
st = run_status(run_dir)
|
|
506
|
+
config = lock.get("config", {}) or {}
|
|
507
|
+
repos = lock.get("repos") or {}
|
|
508
|
+
results = RunRecord(run_dir).load_results()
|
|
509
|
+
_node(
|
|
510
|
+
node_id,
|
|
511
|
+
id=node_id,
|
|
512
|
+
label=run_dir.name,
|
|
513
|
+
kind=st["kind"] if st["kind"] != "unknown" else "run",
|
|
514
|
+
status=st["status"],
|
|
515
|
+
timestamp=lock.get("timestamp"),
|
|
516
|
+
note=lock.get("note"),
|
|
517
|
+
target=config.get("_target_") if isinstance(config, dict) else None,
|
|
518
|
+
fingerprint=lock.get("fingerprint"),
|
|
519
|
+
tag=tag_map.get(node_id),
|
|
520
|
+
tree_hashes={n: (i or {}).get("tree") for n, i in repos.items()},
|
|
521
|
+
data_hashes=lock.get("data") or {},
|
|
522
|
+
metrics=_numeric_metrics(results),
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
# 2. sweep-task nodes (marker dirs, no run.lock).
|
|
526
|
+
for marker_file in root.rglob(MARKER_NAME):
|
|
527
|
+
task_dir = marker_file.parent.resolve()
|
|
528
|
+
node_id = str(task_dir)
|
|
529
|
+
if node_id in nodes:
|
|
530
|
+
continue
|
|
531
|
+
marker = _read_marker(task_dir) or {}
|
|
532
|
+
st = run_status(task_dir)
|
|
533
|
+
db_path = _resolve_marker_db(task_dir, marker)
|
|
534
|
+
results = RunRecord(task_dir).load_results()
|
|
535
|
+
_node(
|
|
536
|
+
node_id,
|
|
537
|
+
id=node_id,
|
|
538
|
+
label=task_dir.name,
|
|
539
|
+
kind="sweep_task",
|
|
540
|
+
status=st["status"],
|
|
541
|
+
timestamp=None,
|
|
542
|
+
note=None,
|
|
543
|
+
target=None,
|
|
544
|
+
fingerprint=None,
|
|
545
|
+
tag=tag_map.get(node_id),
|
|
546
|
+
tree_hashes={},
|
|
547
|
+
data_hashes={},
|
|
548
|
+
metrics=_numeric_metrics(results),
|
|
549
|
+
)
|
|
550
|
+
# sweep-item edge from the master (task DB's parent dir).
|
|
551
|
+
master_id = str(db_path.parent.resolve())
|
|
552
|
+
edges.append({"source": master_id, "target": node_id, "type": "sweep_item"})
|
|
553
|
+
|
|
554
|
+
# 3. lineage edges (materialise external stubs for out-of-scan sources).
|
|
555
|
+
def _ensure_external(node_id):
|
|
556
|
+
if node_id not in nodes:
|
|
557
|
+
_node(node_id, id=node_id, label=Path(node_id).name, kind="external",
|
|
558
|
+
status="unknown", timestamp=None, note=None, target=None,
|
|
559
|
+
fingerprint=None, tag=tag_map.get(node_id), tree_hashes={},
|
|
560
|
+
data_hashes={}, metrics={})
|
|
561
|
+
|
|
562
|
+
for run in find_results_dirs(str(root)):
|
|
563
|
+
run_dir = Path(run["path"]).resolve()
|
|
564
|
+
lock = _load_lock(run_dir) or {}
|
|
565
|
+
for entry in _lineage_from_lock(lock).values():
|
|
566
|
+
if not isinstance(entry, dict):
|
|
567
|
+
continue
|
|
568
|
+
path = entry.get("path") or entry.get("config", {}).get("save_dir")
|
|
569
|
+
if not path:
|
|
570
|
+
continue
|
|
571
|
+
src = str(Path(path).resolve())
|
|
572
|
+
_ensure_external(src)
|
|
573
|
+
edges.append({"source": src, "target": str(run_dir), "type": "lineage"})
|
|
574
|
+
|
|
575
|
+
# Drop sweep_item edges whose master somehow isn't a node.
|
|
576
|
+
for e in edges:
|
|
577
|
+
if e["type"] == "sweep_item":
|
|
578
|
+
_ensure_external(e["source"])
|
|
579
|
+
|
|
580
|
+
graph = {
|
|
581
|
+
"schema_version": SCHEMA_VERSION,
|
|
582
|
+
"root": str(root),
|
|
583
|
+
"generated_at": _dt.now().isoformat(),
|
|
584
|
+
"nodes": list(nodes.values()),
|
|
585
|
+
"edges": edges,
|
|
586
|
+
}
|
|
587
|
+
if include_groups:
|
|
588
|
+
graph["groups"] = _compute_groups(nodes)
|
|
589
|
+
return graph
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _compute_groups(nodes: dict) -> dict:
|
|
593
|
+
"""Group node ids that share an identical tree-hash tuple / data dict."""
|
|
594
|
+
same_tree: dict = {}
|
|
595
|
+
same_data: dict = {}
|
|
596
|
+
for node_id, n in nodes.items():
|
|
597
|
+
trees = n.get("tree_hashes") or {}
|
|
598
|
+
if trees and all(v for v in trees.values()):
|
|
599
|
+
key = tuple(sorted(trees.items()))
|
|
600
|
+
same_tree.setdefault(key, []).append(node_id)
|
|
601
|
+
data = n.get("data_hashes") or {}
|
|
602
|
+
if data:
|
|
603
|
+
dkey = tuple(sorted(data.items()))
|
|
604
|
+
same_data.setdefault(dkey, []).append(node_id)
|
|
605
|
+
return {
|
|
606
|
+
"same_tree": [sorted(v) for v in same_tree.values() if len(v) >= 2],
|
|
607
|
+
"same_data": [sorted(v) for v in same_data.values() if len(v) >= 2],
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
# Status → display colour, shared by mermaid/dot/HTML.
|
|
612
|
+
STATUS_COLORS = {
|
|
613
|
+
"complete": "#4caf50",
|
|
614
|
+
"failed": "#e53935",
|
|
615
|
+
"interrupted": "#fb8c00",
|
|
616
|
+
"running": "#1e88e5",
|
|
617
|
+
"pending": "#9e9e9e",
|
|
618
|
+
"unknown": "#bdbdbd",
|
|
619
|
+
"external": "#e0e0e0",
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def graph_to_mermaid(graph: dict) -> str:
|
|
624
|
+
"""Render the graph JSON as a Mermaid ``graph TD`` diagram."""
|
|
625
|
+
lines = ["graph TD"]
|
|
626
|
+
id_map = {n["id"]: f"n{i}" for i, n in enumerate(graph["nodes"])}
|
|
627
|
+
statuses_seen = set()
|
|
628
|
+
for n in graph["nodes"]:
|
|
629
|
+
nid = id_map[n["id"]]
|
|
630
|
+
label = f"{n['label']}<br/>{n['status']}"
|
|
631
|
+
lines.append(f' {nid}["{label}"]')
|
|
632
|
+
statuses_seen.add(n["status"])
|
|
633
|
+
for e in graph["edges"]:
|
|
634
|
+
s = id_map.get(e["source"])
|
|
635
|
+
t = id_map.get(e["target"])
|
|
636
|
+
if s and t:
|
|
637
|
+
arrow = "-.->" if e["type"] == "sweep_item" else "-->"
|
|
638
|
+
lines.append(f" {s} {arrow} {t}")
|
|
639
|
+
# classDef per status + assignments.
|
|
640
|
+
for status in sorted(statuses_seen):
|
|
641
|
+
color = STATUS_COLORS.get(status, "#bdbdbd")
|
|
642
|
+
lines.append(f" classDef {status} fill:{color},color:#fff;")
|
|
643
|
+
for n in graph["nodes"]:
|
|
644
|
+
lines.append(f" class {id_map[n['id']]} {n['status']};")
|
|
645
|
+
return "\n".join(lines)
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def graph_to_dot(graph: dict) -> str:
|
|
649
|
+
"""Render the graph JSON as Graphviz DOT."""
|
|
650
|
+
lines = ["digraph flexlock {", " rankdir=TB;", ' node [style=filled, shape=box];']
|
|
651
|
+
id_map = {n["id"]: f"n{i}" for i, n in enumerate(graph["nodes"])}
|
|
652
|
+
for n in graph["nodes"]:
|
|
653
|
+
color = STATUS_COLORS.get(n["status"], "#bdbdbd")
|
|
654
|
+
label = f"{n['label']}\\n{n['status']}"
|
|
655
|
+
lines.append(f' {id_map[n["id"]]} [label="{label}", fillcolor="{color}"];')
|
|
656
|
+
for e in graph["edges"]:
|
|
657
|
+
s = id_map.get(e["source"])
|
|
658
|
+
t = id_map.get(e["target"])
|
|
659
|
+
if s and t:
|
|
660
|
+
style = "dashed" if e["type"] == "sweep_item" else "solid"
|
|
661
|
+
lines.append(f' {s} -> {t} [style={style}];')
|
|
662
|
+
lines.append("}")
|
|
663
|
+
return "\n".join(lines)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def format_summary_md(summary: dict) -> str:
|
|
667
|
+
"""Render a human-readable Markdown view of a ``load_run_summary`` dict."""
|
|
668
|
+
name = Path(summary["path"]).name
|
|
669
|
+
lines = [f"# {name} — {summary['status'].upper()}", ""]
|
|
670
|
+
|
|
671
|
+
meta = []
|
|
672
|
+
if summary.get("tag"):
|
|
673
|
+
meta.append(f"- **tag:** {summary['tag']}")
|
|
674
|
+
if summary.get("note"):
|
|
675
|
+
meta.append(f"- **note:** {summary['note']}")
|
|
676
|
+
if summary.get("target"):
|
|
677
|
+
meta.append(f"- **target:** `{summary['target']}`")
|
|
678
|
+
if summary.get("timestamp"):
|
|
679
|
+
meta.append(f"- **timestamp:** {summary['timestamp']}")
|
|
680
|
+
meta.append(f"- **kind:** {summary['kind']}")
|
|
681
|
+
meta.append(f"- **path:** `{summary['path']}`")
|
|
682
|
+
if summary.get("status_detail"):
|
|
683
|
+
meta.append(f"- **note on status:** {summary['status_detail']}")
|
|
684
|
+
lines += meta + [""]
|
|
685
|
+
|
|
686
|
+
err = summary.get("error")
|
|
687
|
+
if err:
|
|
688
|
+
lines += ["## Error", ""]
|
|
689
|
+
if err.get("exc_type"):
|
|
690
|
+
lines.append(f"`{err.get('exc_type')}`: {err.get('exc_message', '')}")
|
|
691
|
+
lines.append("")
|
|
692
|
+
tb = err.get("traceback")
|
|
693
|
+
if tb:
|
|
694
|
+
lines += ["```", tb.rstrip(), "```", ""]
|
|
695
|
+
|
|
696
|
+
tasks = summary.get("tasks")
|
|
697
|
+
if tasks:
|
|
698
|
+
lines += ["## Tasks", ""]
|
|
699
|
+
lines.append("| pending | running | done | failed | interrupted |")
|
|
700
|
+
lines.append("|---|---|---|---|---|")
|
|
701
|
+
lines.append(
|
|
702
|
+
f"| {tasks.get('pending', 0)} | {tasks.get('running', 0)} | "
|
|
703
|
+
f"{tasks.get('done', 0)} | {tasks.get('failed', 0)} | "
|
|
704
|
+
f"{tasks.get('interrupted', 0)} |"
|
|
705
|
+
)
|
|
706
|
+
lines.append("")
|
|
707
|
+
|
|
708
|
+
results = summary.get("results") or {}
|
|
709
|
+
if isinstance(results, dict) and results:
|
|
710
|
+
lines += ["## Results", "", "| key | value |", "|---|---|"]
|
|
711
|
+
for k, v in results.items():
|
|
712
|
+
lines.append(f"| {k} | {v} |")
|
|
713
|
+
lines.append("")
|
|
714
|
+
|
|
715
|
+
lineage = summary.get("lineage", {})
|
|
716
|
+
up = lineage.get("upstream") or []
|
|
717
|
+
down = lineage.get("downstream") or []
|
|
718
|
+
if up or down:
|
|
719
|
+
lines += ["## Lineage", ""]
|
|
720
|
+
if up:
|
|
721
|
+
lines.append("**Upstream:**")
|
|
722
|
+
for u in up:
|
|
723
|
+
lines.append(f"- {u.get('name')}: `{u.get('path')}`")
|
|
724
|
+
if down:
|
|
725
|
+
lines.append("**Downstream:**")
|
|
726
|
+
for d in down:
|
|
727
|
+
note = f" — {d['note']}" if d.get("note") else ""
|
|
728
|
+
lines.append(f"- `{d.get('path')}`{note}")
|
|
729
|
+
lines.append("")
|
|
730
|
+
|
|
731
|
+
config = summary.get("config") or {}
|
|
732
|
+
if config:
|
|
733
|
+
import yaml as _yaml
|
|
734
|
+
|
|
735
|
+
lines += ["## Config", "", "```yaml",
|
|
736
|
+
_yaml.safe_dump(config, default_flow_style=False, sort_keys=False).rstrip(),
|
|
737
|
+
"```", ""]
|
|
738
|
+
|
|
739
|
+
return "\n".join(lines)
|