runprint 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qr/__init__.py +17 -0
- qr/cli.py +643 -0
- qr/manifest.py +108 -0
- qr/project.py +86 -0
- qr/rerun.py +226 -0
- qr/sdk.py +182 -0
- qr/snapshot/__init__.py +15 -0
- qr/snapshot/git.py +93 -0
- qr/snapshot/hardware.py +109 -0
- qr/snapshot/runtime.py +87 -0
- qr/storage/__init__.py +14 -0
- qr/storage/index.py +120 -0
- qr/storage/local.py +90 -0
- qr/subprocess_runner.py +132 -0
- runprint/__init__.py +17 -0
- runprint-0.1.0.dist-info/METADATA +278 -0
- runprint-0.1.0.dist-info/RECORD +21 -0
- runprint-0.1.0.dist-info/WHEEL +5 -0
- runprint-0.1.0.dist-info/entry_points.txt +3 -0
- runprint-0.1.0.dist-info/licenses/LICENSE +176 -0
- runprint-0.1.0.dist-info/top_level.txt +2 -0
qr/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
QR — Reproducible Research Execution Wrapper
|
|
3
|
+
=============================================
|
|
4
|
+
Run anything. Capture everything needed to reproduce it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from qr.sdk import artifact, input_dataset, log, note
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"__version__",
|
|
13
|
+
"log",
|
|
14
|
+
"artifact",
|
|
15
|
+
"input_dataset",
|
|
16
|
+
"note",
|
|
17
|
+
]
|
qr/cli.py
ADDED
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""
|
|
2
|
+
QR CLI Interface
|
|
3
|
+
================
|
|
4
|
+
Command-line interface for the QR reproducible research execution wrapper.
|
|
5
|
+
Commands: init, run, list, show, rerun, doctor, diff.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import difflib
|
|
11
|
+
import json
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
import os
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
import shutil
|
|
16
|
+
import sys
|
|
17
|
+
from typing import List, Optional
|
|
18
|
+
|
|
19
|
+
# Ensure UTF-8 output on Windows consoles
|
|
20
|
+
if sys.platform == "win32":
|
|
21
|
+
try:
|
|
22
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
23
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
24
|
+
if hasattr(sys.stderr, "reconfigure"):
|
|
25
|
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
26
|
+
except Exception:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
import click
|
|
30
|
+
from rich.console import Console
|
|
31
|
+
from rich.panel import Panel
|
|
32
|
+
from rich.table import Table
|
|
33
|
+
from rich.text import Text
|
|
34
|
+
|
|
35
|
+
from qr import __version__
|
|
36
|
+
from qr.manifest import (
|
|
37
|
+
ArtifactItem,
|
|
38
|
+
DatasetInput,
|
|
39
|
+
RunInputs,
|
|
40
|
+
RunManifest,
|
|
41
|
+
Timestamps,
|
|
42
|
+
)
|
|
43
|
+
from qr.project import find_project_root, get_qr_dir, init_project, is_project_initialized
|
|
44
|
+
from qr.rerun import (
|
|
45
|
+
cleanup_isolated_worktree,
|
|
46
|
+
create_isolated_worktree,
|
|
47
|
+
run_rerun_preflight,
|
|
48
|
+
)
|
|
49
|
+
from qr.snapshot.git import capture_git_snapshot
|
|
50
|
+
from qr.snapshot.hardware import capture_hardware_snapshot
|
|
51
|
+
from qr.snapshot.runtime import capture_runtime_snapshot
|
|
52
|
+
from qr.storage.index import RunIndex
|
|
53
|
+
from qr.storage.local import RunStorage, generate_run_id
|
|
54
|
+
from qr.subprocess_runner import run_command_with_tee
|
|
55
|
+
|
|
56
|
+
console = Console()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
|
|
60
|
+
@click.version_option(version=__version__, prog_name="qr")
|
|
61
|
+
def main():
|
|
62
|
+
"""QR -- Reproducible Research Execution Wrapper.
|
|
63
|
+
Run anything. Capture everything needed to reproduce it.
|
|
64
|
+
"""
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@main.command()
|
|
69
|
+
@click.option("--name", default=None, help="Custom project name")
|
|
70
|
+
def init(name: Optional[str]):
|
|
71
|
+
"""Initialize a QR project in the current directory."""
|
|
72
|
+
cwd = Path.cwd()
|
|
73
|
+
qr_dir = init_project(cwd, project_name=name)
|
|
74
|
+
console.print(
|
|
75
|
+
Panel(
|
|
76
|
+
f"[bold green]Initialized QR project in[/bold green] [cyan]{qr_dir}[/cyan]\n"
|
|
77
|
+
f"[dim]Run your first experiment with:[/dim]\n"
|
|
78
|
+
f" [bold white]qr run -- python train.py[/bold white]",
|
|
79
|
+
title="[bold cyan]Runprint (QR)[/bold cyan]",
|
|
80
|
+
border_style="green",
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _coerce_param_value(val: str):
|
|
86
|
+
"""Convert numeric string to float/int if possible."""
|
|
87
|
+
try:
|
|
88
|
+
if "." in val:
|
|
89
|
+
return float(val)
|
|
90
|
+
return int(val)
|
|
91
|
+
except ValueError:
|
|
92
|
+
return val
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def extract_cli_parameters(cmd_list: List[str]):
|
|
96
|
+
"""Extract config files and CLI flags/parameters from command."""
|
|
97
|
+
configs: List[str] = []
|
|
98
|
+
params = {}
|
|
99
|
+
skip_next = False
|
|
100
|
+
|
|
101
|
+
for i, arg in enumerate(cmd_list):
|
|
102
|
+
if skip_next:
|
|
103
|
+
skip_next = False
|
|
104
|
+
continue
|
|
105
|
+
|
|
106
|
+
if arg in ("--config", "-c", "--cfg", "--config-file") and i + 1 < len(cmd_list):
|
|
107
|
+
configs.append(cmd_list[i + 1])
|
|
108
|
+
skip_next = True
|
|
109
|
+
elif arg.startswith("--config=") or arg.startswith("--cfg="):
|
|
110
|
+
configs.append(arg.split("=", 1)[1])
|
|
111
|
+
elif arg.startswith("--"):
|
|
112
|
+
key_part = arg[2:]
|
|
113
|
+
if "=" in key_part:
|
|
114
|
+
k, v = key_part.split("=", 1)
|
|
115
|
+
params[k] = _coerce_param_value(v)
|
|
116
|
+
elif i + 1 < len(cmd_list) and not cmd_list[i + 1].startswith("-"):
|
|
117
|
+
params[key_part] = _coerce_param_value(cmd_list[i + 1])
|
|
118
|
+
skip_next = True
|
|
119
|
+
else:
|
|
120
|
+
params[key_part] = True
|
|
121
|
+
return configs, params
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@main.command(context_settings={"ignore_unknown_options": True})
|
|
125
|
+
@click.argument("command", nargs=-1, required=True, type=click.UNPROCESSED)
|
|
126
|
+
@click.option("--tag", "-t", multiple=True, help="Tag for labeling the run (can specify multiple)")
|
|
127
|
+
@click.option("--parent", default=None, help="Parent run ID (used for lineage tracking)")
|
|
128
|
+
@click.option("--cwd", "custom_cwd", default=None, help="Custom working directory")
|
|
129
|
+
def run(command: tuple, tag: tuple, parent: Optional[str], custom_cwd: Optional[str]):
|
|
130
|
+
"""Execute a command, stream logs, and capture reproducible provenance.
|
|
131
|
+
|
|
132
|
+
Example:
|
|
133
|
+
qr run --tag baseline -- python train.py --config configs/baseline.yaml
|
|
134
|
+
"""
|
|
135
|
+
cmd_list: List[str] = list(command)
|
|
136
|
+
# Strip leading '--' if present
|
|
137
|
+
if cmd_list and cmd_list[0] == "--":
|
|
138
|
+
cmd_list = cmd_list[1:]
|
|
139
|
+
|
|
140
|
+
if not cmd_list:
|
|
141
|
+
console.print("[bold red]Error:[/bold red] No command specified to run.")
|
|
142
|
+
sys.exit(1)
|
|
143
|
+
|
|
144
|
+
work_dir = Path(custom_cwd).resolve() if custom_cwd else Path.cwd()
|
|
145
|
+
project_root = find_project_root(work_dir)
|
|
146
|
+
if project_root is None or not is_project_initialized(project_root):
|
|
147
|
+
# Auto-init if not already initialized
|
|
148
|
+
project_root = work_dir
|
|
149
|
+
init_project(project_root)
|
|
150
|
+
|
|
151
|
+
qr_dir = get_qr_dir(project_root)
|
|
152
|
+
run_id = generate_run_id()
|
|
153
|
+
storage = RunStorage(qr_dir, run_id)
|
|
154
|
+
run_dir = storage.init_run_dir()
|
|
155
|
+
|
|
156
|
+
# Pre-run Snapshots
|
|
157
|
+
git_snap = capture_git_snapshot(project_root, run_dir)
|
|
158
|
+
if git_snap and git_snap.dirty:
|
|
159
|
+
warning_text = Text("[!] Working tree contains uncommitted changes. Snapshotting diff:\n", style="bold yellow")
|
|
160
|
+
for f in git_snap.modified_files[:10]:
|
|
161
|
+
warning_text.append(f" - {f}\n", style="yellow")
|
|
162
|
+
if len(git_snap.modified_files) > 10:
|
|
163
|
+
warning_text.append(f" ... and {len(git_snap.modified_files) - 10} more\n", style="dim yellow")
|
|
164
|
+
console.print(warning_text)
|
|
165
|
+
|
|
166
|
+
runtime_snap = capture_runtime_snapshot(run_dir)
|
|
167
|
+
hardware_snap = capture_hardware_snapshot()
|
|
168
|
+
|
|
169
|
+
# Detect explicit config file flags and CLI parameters in argv
|
|
170
|
+
detected_configs, detected_params = extract_cli_parameters(cmd_list)
|
|
171
|
+
|
|
172
|
+
# Initial minimal manifest (persisted before process starts for crash-safety)
|
|
173
|
+
manifest = RunManifest(
|
|
174
|
+
run_id=run_id,
|
|
175
|
+
parent_run_id=parent,
|
|
176
|
+
tags=list(tag),
|
|
177
|
+
command=cmd_list,
|
|
178
|
+
cwd=str(work_dir),
|
|
179
|
+
timestamps=Timestamps(),
|
|
180
|
+
git=git_snap,
|
|
181
|
+
runtime=runtime_snap,
|
|
182
|
+
hardware=hardware_snap,
|
|
183
|
+
inputs=RunInputs(configs=detected_configs, parameters=detected_params),
|
|
184
|
+
status="running",
|
|
185
|
+
)
|
|
186
|
+
storage.save_manifest(manifest)
|
|
187
|
+
|
|
188
|
+
tag_str = f" [magenta][{', '.join(tag)}][/magenta]" if tag else ""
|
|
189
|
+
console.print(f"[bold cyan]Run {run_id}[/bold cyan]{tag_str} started [dim]({datetime.now().strftime('%H:%M:%S')})[/dim]")
|
|
190
|
+
|
|
191
|
+
# Execute command with live tee logging
|
|
192
|
+
env_overrides = {
|
|
193
|
+
"QR_RUN_ID": run_id,
|
|
194
|
+
"QR_RUN_DIR": str(run_dir),
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
exec_result = run_command_with_tee(
|
|
198
|
+
command=cmd_list,
|
|
199
|
+
cwd=work_dir,
|
|
200
|
+
stdout_log_path=storage.stdout_path,
|
|
201
|
+
stderr_log_path=storage.stderr_path,
|
|
202
|
+
env_overrides=env_overrides,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
# Collect SDK outputs (artifacts & datasets if registered)
|
|
206
|
+
datasets_file = run_dir / "inputs_datasets.json"
|
|
207
|
+
if datasets_file.exists():
|
|
208
|
+
try:
|
|
209
|
+
ds_data = json.loads(datasets_file.read_text(encoding="utf-8"))
|
|
210
|
+
manifest.inputs.datasets = [DatasetInput(**item) for item in ds_data]
|
|
211
|
+
except Exception:
|
|
212
|
+
pass
|
|
213
|
+
|
|
214
|
+
# Finalize manifest
|
|
215
|
+
manifest.timestamps.finished_at = datetime.now(timezone.utc).isoformat()
|
|
216
|
+
manifest.duration_seconds = exec_result.duration_seconds
|
|
217
|
+
manifest.exit_code = exec_result.exit_code
|
|
218
|
+
|
|
219
|
+
if exec_result.interrupted:
|
|
220
|
+
manifest.status = "interrupted"
|
|
221
|
+
elif exec_result.exit_code == 0:
|
|
222
|
+
manifest.status = "completed"
|
|
223
|
+
else:
|
|
224
|
+
manifest.status = "failed"
|
|
225
|
+
|
|
226
|
+
storage.save_manifest(manifest)
|
|
227
|
+
|
|
228
|
+
# Update SQLite index
|
|
229
|
+
db_path = qr_dir / "index.sqlite"
|
|
230
|
+
index = RunIndex(db_path)
|
|
231
|
+
index.upsert_run(manifest)
|
|
232
|
+
|
|
233
|
+
status_color = "green" if manifest.status == "completed" else "red"
|
|
234
|
+
console.print(
|
|
235
|
+
f"\n[bold {status_color}]Run {manifest.status}:[/bold {status_color}] "
|
|
236
|
+
f"exit={exec_result.exit_code}, duration={exec_result.duration_seconds}s "
|
|
237
|
+
f"[dim]({run_id})[/dim]"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
sys.exit(exec_result.exit_code)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@main.command(name="list")
|
|
244
|
+
@click.option("--limit", default=20, help="Number of runs to display")
|
|
245
|
+
@click.option("--tag", "-t", default=None, help="Filter runs by tag")
|
|
246
|
+
def list_runs(limit: int, tag: Optional[str]):
|
|
247
|
+
"""List recorded experiments, optionally filtered by tag."""
|
|
248
|
+
project_root = find_project_root()
|
|
249
|
+
if not project_root:
|
|
250
|
+
console.print("[yellow]No QR project found. Run 'qr init' first.[/yellow]")
|
|
251
|
+
return
|
|
252
|
+
|
|
253
|
+
qr_dir = get_qr_dir(project_root)
|
|
254
|
+
db_path = qr_dir / "index.sqlite"
|
|
255
|
+
index = RunIndex(db_path)
|
|
256
|
+
|
|
257
|
+
# If DB is empty, sync from storage
|
|
258
|
+
rows = index.list_runs(limit=limit, tag=tag)
|
|
259
|
+
if not rows and not tag:
|
|
260
|
+
manifests = RunStorage.list_all_runs(qr_dir)
|
|
261
|
+
if manifests:
|
|
262
|
+
index.rebuild_from_manifests(manifests)
|
|
263
|
+
rows = index.list_runs(limit=limit)
|
|
264
|
+
|
|
265
|
+
if not rows:
|
|
266
|
+
msg = f"[dim]No runs found{' matching tag: ' + tag if tag else ''}. Start with: qr run -- <command>[/dim]"
|
|
267
|
+
console.print(msg)
|
|
268
|
+
return
|
|
269
|
+
|
|
270
|
+
table = Table(title="[bold cyan]QR Experiment Runs[/bold cyan]", border_style="dim")
|
|
271
|
+
table.add_column("Run ID", style="bold cyan", no_wrap=True)
|
|
272
|
+
table.add_column("Status", no_wrap=True)
|
|
273
|
+
table.add_column("Tags", style="magenta")
|
|
274
|
+
table.add_column("Command", style="white")
|
|
275
|
+
table.add_column("Duration", justify="right")
|
|
276
|
+
table.add_column("Commit", style="dim")
|
|
277
|
+
table.add_column("Dirty", justify="center")
|
|
278
|
+
table.add_column("Started", style="dim")
|
|
279
|
+
|
|
280
|
+
for row in rows:
|
|
281
|
+
st = row["status"]
|
|
282
|
+
st_color = "green" if st == "completed" else ("red" if st == "failed" else "yellow")
|
|
283
|
+
dur_str = f"{row['duration_seconds']:.1f}s" if row["duration_seconds"] is not None else "-"
|
|
284
|
+
commit_str = (row["git_commit"][:7]) if row.get("git_commit") else "-"
|
|
285
|
+
dirty_str = "[yellow]yes[/yellow]" if row.get("git_dirty") else "[green]no[/green]"
|
|
286
|
+
started_str = row["started_at"][:19].replace("T", " ") if row.get("started_at") else "-"
|
|
287
|
+
tags_str = row.get("tags") or "-"
|
|
288
|
+
|
|
289
|
+
table.add_row(
|
|
290
|
+
row["run_id"],
|
|
291
|
+
f"[{st_color}]{st}[/{st_color}]",
|
|
292
|
+
tags_str,
|
|
293
|
+
row["command"],
|
|
294
|
+
dur_str,
|
|
295
|
+
commit_str,
|
|
296
|
+
dirty_str,
|
|
297
|
+
started_str,
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
console.print(table)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@main.command()
|
|
304
|
+
@click.argument("run_id")
|
|
305
|
+
def show(run_id: str):
|
|
306
|
+
"""Inspect full provenance and artifacts for a specific run."""
|
|
307
|
+
project_root = find_project_root()
|
|
308
|
+
if not project_root:
|
|
309
|
+
console.print("[yellow]No QR project found.[/yellow]")
|
|
310
|
+
return
|
|
311
|
+
|
|
312
|
+
qr_dir = get_qr_dir(project_root)
|
|
313
|
+
storage = RunStorage(qr_dir, run_id)
|
|
314
|
+
try:
|
|
315
|
+
manifest = storage.load_manifest()
|
|
316
|
+
except FileNotFoundError:
|
|
317
|
+
console.print(f"[bold red]Error:[/bold red] Run '{run_id}' not found.")
|
|
318
|
+
return
|
|
319
|
+
|
|
320
|
+
# Build Rich Display matching Phụ lục A
|
|
321
|
+
st = manifest.status.upper()
|
|
322
|
+
st_color = "green" if st == "COMPLETED" else "red"
|
|
323
|
+
|
|
324
|
+
content = []
|
|
325
|
+
content.append(f"[bold]Status:[/bold] [{st_color}]{st}[/{st_color}] (exit={manifest.exit_code})")
|
|
326
|
+
content.append(f"[bold]Started:[/bold] {manifest.timestamps.started_at}")
|
|
327
|
+
content.append(f"[bold]Duration:[/bold] {manifest.duration_seconds}s")
|
|
328
|
+
if manifest.tags:
|
|
329
|
+
content.append(f"[bold]Tags:[/bold] [magenta]{', '.join(manifest.tags)}[/magenta]")
|
|
330
|
+
content.append(f"[bold]Command:[/bold] [white]{' '.join(manifest.command)}[/white]")
|
|
331
|
+
content.append(f"[bold]Directory:[/bold] [dim]{manifest.cwd}[/dim]")
|
|
332
|
+
|
|
333
|
+
content.append("\n[bold cyan]SOURCE[/bold cyan]")
|
|
334
|
+
if manifest.git:
|
|
335
|
+
content.append(f" Commit: {manifest.git.commit or 'N/A'}")
|
|
336
|
+
content.append(f" Branch: {manifest.git.branch or 'N/A'}")
|
|
337
|
+
content.append(f" Dirty diff: {'yes (patch saved to git.diff)' if manifest.git.dirty else 'clean'}")
|
|
338
|
+
else:
|
|
339
|
+
content.append(" Git: Not a git repository")
|
|
340
|
+
|
|
341
|
+
content.append("\n[bold cyan]RUNTIME[/bold cyan]")
|
|
342
|
+
content.append(f" OS: {manifest.runtime.os}")
|
|
343
|
+
content.append(f" Python: {manifest.runtime.python_version}")
|
|
344
|
+
content.append(f" Lockfile: {manifest.runtime.packages_lock or 'N/A'}")
|
|
345
|
+
|
|
346
|
+
content.append("\n[bold cyan]HARDWARE[/bold cyan]")
|
|
347
|
+
content.append(f" CPU: {manifest.hardware.cpu or 'N/A'} ({manifest.hardware.cpu_count or '?'} cores)")
|
|
348
|
+
content.append(f" RAM: {manifest.hardware.ram_gb or '?'} GB")
|
|
349
|
+
gpu_desc = ", ".join(manifest.hardware.gpu) if manifest.hardware.gpu else "No GPU detected"
|
|
350
|
+
content.append(f" GPU: {gpu_desc}")
|
|
351
|
+
|
|
352
|
+
# Display declared inputs
|
|
353
|
+
if manifest.inputs.datasets or manifest.inputs.configs or manifest.inputs.parameters:
|
|
354
|
+
content.append("\n[bold cyan]INPUTS[/bold cyan]")
|
|
355
|
+
for cfg in manifest.inputs.configs:
|
|
356
|
+
content.append(f" Config: {cfg}")
|
|
357
|
+
if manifest.inputs.parameters:
|
|
358
|
+
params_str = ", ".join(f"--{k} {v}" for k, v in manifest.inputs.parameters.items())
|
|
359
|
+
content.append(f" Parameters: {params_str}")
|
|
360
|
+
for ds in manifest.inputs.datasets:
|
|
361
|
+
fp_str = f" ({ds.fingerprint[:16]}...)" if ds.fingerprint else ""
|
|
362
|
+
content.append(f" Dataset: {ds.name} [uri={ds.uri}{', ver=' + ds.version if ds.version else ''}{fp_str}]")
|
|
363
|
+
|
|
364
|
+
# Check metrics
|
|
365
|
+
if storage.metrics_path.exists():
|
|
366
|
+
content.append("\n[bold cyan]METRICS LOGGED[/bold cyan]")
|
|
367
|
+
lines = storage.metrics_path.read_text(encoding="utf-8").strip().splitlines()
|
|
368
|
+
content.append(f" Total logged steps: {len(lines)}")
|
|
369
|
+
if lines:
|
|
370
|
+
try:
|
|
371
|
+
last_metric = json.loads(lines[-1])
|
|
372
|
+
summary = ", ".join(f"{k}={v}" for k, v in last_metric.items() if k != "timestamp")
|
|
373
|
+
content.append(f" Latest step: {summary}")
|
|
374
|
+
except Exception:
|
|
375
|
+
pass
|
|
376
|
+
|
|
377
|
+
# Check artifacts
|
|
378
|
+
if storage.artifacts_path.exists():
|
|
379
|
+
content.append("\n[bold cyan]ARTIFACTS REGISTERED[/bold cyan]")
|
|
380
|
+
try:
|
|
381
|
+
art_list = json.loads(storage.artifacts_path.read_text(encoding="utf-8"))
|
|
382
|
+
for art in art_list:
|
|
383
|
+
size_str = f"{round(art['size_bytes']/1024, 1)} KB" if art.get("size_bytes") else "unknown size"
|
|
384
|
+
content.append(f" - {art['path']} [{art.get('kind', 'generic')}] ({size_str})")
|
|
385
|
+
except Exception:
|
|
386
|
+
pass
|
|
387
|
+
|
|
388
|
+
content.append("\n[bold cyan]REPRODUCIBILITY CONTRACT[/bold cyan]")
|
|
389
|
+
content.append(" Traceable: [bold green]YES[/bold green] (Full snapshot & log preserved)")
|
|
390
|
+
has_lock = bool(manifest.runtime.packages_lock)
|
|
391
|
+
content.append(f" Restorable: {'[green]LIKELY[/green]' if has_lock else '[yellow]PARTIAL[/yellow]'}")
|
|
392
|
+
content.append(" Lineage: Parent=" + (manifest.parent_run_id or "Root run"))
|
|
393
|
+
|
|
394
|
+
console.print(
|
|
395
|
+
Panel(
|
|
396
|
+
"\n".join(content),
|
|
397
|
+
title=f"[bold cyan]Experiment Run Details -- {run_id}[/bold cyan]",
|
|
398
|
+
border_style="cyan",
|
|
399
|
+
)
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
@main.command()
|
|
404
|
+
@click.argument("run_id")
|
|
405
|
+
@click.option("--allow-hardware-change", is_flag=True, default=False, help="Bypass hardware warnings")
|
|
406
|
+
@click.option("--allow-dataset-mismatch", is_flag=True, default=False, help="Bypass dataset fingerprint mismatch")
|
|
407
|
+
@click.option("--isolated", is_flag=True, default=False, help="Run in an isolated git worktree")
|
|
408
|
+
def rerun(run_id: str, allow_hardware_change: bool, allow_dataset_mismatch: bool, isolated: bool):
|
|
409
|
+
"""Re-execute a previous run with reproducibility preflight and lineage tracking."""
|
|
410
|
+
project_root = find_project_root()
|
|
411
|
+
if not project_root:
|
|
412
|
+
console.print("[yellow]No QR project found.[/yellow]")
|
|
413
|
+
return
|
|
414
|
+
|
|
415
|
+
qr_dir = get_qr_dir(project_root)
|
|
416
|
+
storage = RunStorage(qr_dir, run_id)
|
|
417
|
+
try:
|
|
418
|
+
manifest = storage.load_manifest()
|
|
419
|
+
except FileNotFoundError:
|
|
420
|
+
console.print(f"[bold red]Error:[/bold red] Run '{run_id}' not found.")
|
|
421
|
+
return
|
|
422
|
+
|
|
423
|
+
console.print(f"[bold cyan]RERUN PREFLIGHT -- {run_id}[/bold cyan]")
|
|
424
|
+
report = run_rerun_preflight(
|
|
425
|
+
manifest,
|
|
426
|
+
project_root,
|
|
427
|
+
storage.run_dir,
|
|
428
|
+
allow_dataset_mismatch=allow_dataset_mismatch,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
for item in report.checks:
|
|
432
|
+
if item.passed:
|
|
433
|
+
console.print(f" [bold green][OK][/bold green] {item.name}: {item.message}")
|
|
434
|
+
elif item.is_warning:
|
|
435
|
+
console.print(f" [bold yellow][!][/bold yellow] {item.name}: {item.message}")
|
|
436
|
+
else:
|
|
437
|
+
console.print(f" [bold red][FAIL][/bold red] {item.name}: {item.message}")
|
|
438
|
+
|
|
439
|
+
if not report.can_proceed:
|
|
440
|
+
console.print("\n[bold red]Preflight failed:[/bold red] Cannot guarantee faithful rerun.")
|
|
441
|
+
sys.exit(1)
|
|
442
|
+
|
|
443
|
+
work_dir = project_root
|
|
444
|
+
isolated_wt_path: Optional[Path] = None
|
|
445
|
+
|
|
446
|
+
if isolated:
|
|
447
|
+
if not manifest.git or not manifest.git.commit:
|
|
448
|
+
console.print("[bold red]Error:[/bold red] Cannot create isolated worktree: no git commit recorded.")
|
|
449
|
+
sys.exit(1)
|
|
450
|
+
|
|
451
|
+
wt_dir = qr_dir / "worktrees" / f"wt_{run_id}"
|
|
452
|
+
console.print(f"\n[cyan]Setting up isolated git worktree at:[/cyan] {wt_dir}")
|
|
453
|
+
try:
|
|
454
|
+
diff_file = storage.git_diff_path if storage.git_diff_path.exists() else None
|
|
455
|
+
work_dir = create_isolated_worktree(project_root, manifest.git.commit, diff_file, wt_dir)
|
|
456
|
+
isolated_wt_path = work_dir
|
|
457
|
+
console.print("[bold green][OK][/bold green] Isolated worktree ready.")
|
|
458
|
+
except Exception as e:
|
|
459
|
+
console.print(f"[bold red]Worktree creation failed:[/bold red] {e}")
|
|
460
|
+
sys.exit(1)
|
|
461
|
+
|
|
462
|
+
console.print("\n[bold green]Preflight passed.[/bold green] Spawning rerun process...")
|
|
463
|
+
|
|
464
|
+
# Invoke run with --parent run_id
|
|
465
|
+
ctx = click.get_current_context()
|
|
466
|
+
try:
|
|
467
|
+
ctx.invoke(
|
|
468
|
+
run,
|
|
469
|
+
command=tuple(manifest.command),
|
|
470
|
+
tag=tuple(manifest.tags),
|
|
471
|
+
parent=run_id,
|
|
472
|
+
custom_cwd=str(work_dir),
|
|
473
|
+
)
|
|
474
|
+
finally:
|
|
475
|
+
if isolated_wt_path and isolated_wt_path.exists():
|
|
476
|
+
console.print(f"[dim]Cleaning up isolated worktree at {isolated_wt_path}...[/dim]")
|
|
477
|
+
cleanup_isolated_worktree(project_root, isolated_wt_path)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
@main.command()
|
|
481
|
+
@click.option("--fix", is_flag=True, default=False, help="Auto-repair orphaned or unindexed runs")
|
|
482
|
+
def doctor(fix: bool):
|
|
483
|
+
"""Diagnose repository health, detect orphaned/interrupted runs, and sync SQLite cache."""
|
|
484
|
+
project_root = find_project_root()
|
|
485
|
+
if not project_root:
|
|
486
|
+
console.print("[yellow]No QR project found. Run 'qr init' first.[/yellow]")
|
|
487
|
+
return
|
|
488
|
+
|
|
489
|
+
qr_dir = get_qr_dir(project_root)
|
|
490
|
+
runs = RunStorage.list_all_runs(qr_dir)
|
|
491
|
+
db_path = qr_dir / "index.sqlite"
|
|
492
|
+
index = RunIndex(db_path)
|
|
493
|
+
|
|
494
|
+
table = Table(title="[bold cyan]QR Health & Integrity Report[/bold cyan]", border_style="dim")
|
|
495
|
+
table.add_column("Category", style="bold white")
|
|
496
|
+
table.add_column("Status")
|
|
497
|
+
table.add_column("Details")
|
|
498
|
+
|
|
499
|
+
# 1. Check Project Config
|
|
500
|
+
proj_file = qr_dir / "project.json"
|
|
501
|
+
if proj_file.is_file():
|
|
502
|
+
table.add_row("Project Config", "[green][OK][/green]", str(proj_file))
|
|
503
|
+
else:
|
|
504
|
+
table.add_row("Project Config", "[red][MISSING][/red]", "project.json missing")
|
|
505
|
+
|
|
506
|
+
# 2. Check for orphaned runs
|
|
507
|
+
orphaned_runs = []
|
|
508
|
+
for r in runs:
|
|
509
|
+
if r.status == "running":
|
|
510
|
+
orphaned_runs.append(r)
|
|
511
|
+
|
|
512
|
+
if orphaned_runs:
|
|
513
|
+
st_color = "[yellow]REPAIRED[/yellow]" if fix else "[red][ORPHANED][/red]"
|
|
514
|
+
table.add_row(
|
|
515
|
+
"Active Runs",
|
|
516
|
+
st_color,
|
|
517
|
+
f"Found {len(orphaned_runs)} run(s) stuck in 'running' status",
|
|
518
|
+
)
|
|
519
|
+
if fix:
|
|
520
|
+
for r in orphaned_runs:
|
|
521
|
+
r.status = "interrupted"
|
|
522
|
+
r.timestamps.finished_at = datetime.now(timezone.utc).isoformat()
|
|
523
|
+
storage = RunStorage(qr_dir, r.run_id)
|
|
524
|
+
storage.save_manifest(r)
|
|
525
|
+
index.upsert_run(r)
|
|
526
|
+
else:
|
|
527
|
+
table.add_row("Active Runs", "[green][OK][/green]", "All recorded runs properly finalized")
|
|
528
|
+
|
|
529
|
+
# 3. Check SQLite Index consistency
|
|
530
|
+
indexed_rows = index.list_runs(limit=1000)
|
|
531
|
+
if len(indexed_rows) != len(runs):
|
|
532
|
+
table.add_row(
|
|
533
|
+
"SQLite Index",
|
|
534
|
+
"[yellow]REBUILT[/yellow]" if fix else "[yellow]OUT OF SYNC[/yellow]",
|
|
535
|
+
f"Index has {len(indexed_rows)} rows, but filesystem has {len(runs)} manifests",
|
|
536
|
+
)
|
|
537
|
+
if fix:
|
|
538
|
+
index.rebuild_from_manifests(runs)
|
|
539
|
+
else:
|
|
540
|
+
table.add_row("SQLite Index", "[green][OK][/green]", f"{len(runs)} runs indexed accurately")
|
|
541
|
+
|
|
542
|
+
console.print(table)
|
|
543
|
+
if not fix and (orphaned_runs or len(indexed_rows) != len(runs)):
|
|
544
|
+
console.print("\n[yellow]Run [bold]qr doctor --fix[/bold] to auto-repair these issues.[/yellow]")
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
@main.command()
|
|
548
|
+
@click.argument("run_id_1")
|
|
549
|
+
@click.argument("run_id_2")
|
|
550
|
+
def diff(run_id_1: str, run_id_2: str):
|
|
551
|
+
"""Compare configuration, source, metrics, and outcomes between two runs."""
|
|
552
|
+
project_root = find_project_root()
|
|
553
|
+
if not project_root:
|
|
554
|
+
console.print("[yellow]No QR project found.[/yellow]")
|
|
555
|
+
return
|
|
556
|
+
|
|
557
|
+
qr_dir = get_qr_dir(project_root)
|
|
558
|
+
storage1 = RunStorage(qr_dir, run_id_1)
|
|
559
|
+
storage2 = RunStorage(qr_dir, run_id_2)
|
|
560
|
+
|
|
561
|
+
try:
|
|
562
|
+
m1 = storage1.load_manifest()
|
|
563
|
+
except FileNotFoundError:
|
|
564
|
+
console.print(f"[bold red]Error:[/bold red] Run '{run_id_1}' not found.")
|
|
565
|
+
return
|
|
566
|
+
|
|
567
|
+
try:
|
|
568
|
+
m2 = storage2.load_manifest()
|
|
569
|
+
except FileNotFoundError:
|
|
570
|
+
console.print(f"[bold red]Error:[/bold red] Run '{run_id_2}' not found.")
|
|
571
|
+
return
|
|
572
|
+
|
|
573
|
+
table = Table(title=f"[bold cyan]Run Comparison: {run_id_1} vs {run_id_2}[/bold cyan]", border_style="dim")
|
|
574
|
+
table.add_column("Property", style="bold white")
|
|
575
|
+
table.add_column(run_id_1, style="cyan")
|
|
576
|
+
table.add_column(run_id_2, style="cyan")
|
|
577
|
+
|
|
578
|
+
# Status & Exit
|
|
579
|
+
table.add_row(
|
|
580
|
+
"Status",
|
|
581
|
+
f"[{'green' if m1.status == 'completed' else 'red'}]{m1.status}[/] (exit={m1.exit_code})",
|
|
582
|
+
f"[{'green' if m2.status == 'completed' else 'red'}]{m2.status}[/] (exit={m2.exit_code})",
|
|
583
|
+
)
|
|
584
|
+
table.add_row("Duration", f"{m1.duration_seconds}s", f"{m2.duration_seconds}s")
|
|
585
|
+
table.add_row("Tags", ", ".join(m1.tags) or "-", ", ".join(m2.tags) or "-")
|
|
586
|
+
table.add_row("Command", " ".join(m1.command), " ".join(m2.command))
|
|
587
|
+
|
|
588
|
+
# Git
|
|
589
|
+
c1 = (m1.git.commit[:7] if m1.git and m1.git.commit else "-") + (" (dirty)" if m1.git and m1.git.dirty else "")
|
|
590
|
+
c2 = (m2.git.commit[:7] if m2.git and m2.git.commit else "-") + (" (dirty)" if m2.git and m2.git.dirty else "")
|
|
591
|
+
table.add_row("Git Commit", c1, c2)
|
|
592
|
+
|
|
593
|
+
# Runtime & Hardware
|
|
594
|
+
table.add_row("Python", m1.runtime.python_version, m2.runtime.python_version)
|
|
595
|
+
table.add_row("OS", m1.runtime.os, m2.runtime.os)
|
|
596
|
+
g1 = ", ".join(m1.hardware.gpu) if m1.hardware.gpu else "None"
|
|
597
|
+
g2 = ", ".join(m2.hardware.gpu) if m2.hardware.gpu else "None"
|
|
598
|
+
table.add_row("GPU", g1, g2)
|
|
599
|
+
|
|
600
|
+
# Compare CLI parameters if present
|
|
601
|
+
all_params = sorted(set(list(m1.inputs.parameters.keys()) + list(m2.inputs.parameters.keys())))
|
|
602
|
+
for p in all_params:
|
|
603
|
+
v1 = m1.inputs.parameters.get(p, "-")
|
|
604
|
+
v2 = m2.inputs.parameters.get(p, "-")
|
|
605
|
+
diff_str = ""
|
|
606
|
+
if isinstance(v1, (int, float)) and isinstance(v2, (int, float)):
|
|
607
|
+
delta = v2 - v1
|
|
608
|
+
sign = "+" if delta > 0 else ""
|
|
609
|
+
diff_str = f" [dim]({sign}{delta})[/dim]"
|
|
610
|
+
table.add_row(f"Param: --{p}", str(v1), f"{v2}{diff_str}")
|
|
611
|
+
|
|
612
|
+
# Compare latest metrics
|
|
613
|
+
def _get_latest_metrics(storage: RunStorage):
|
|
614
|
+
if storage.metrics_path.exists():
|
|
615
|
+
lines = storage.metrics_path.read_text(encoding="utf-8").strip().splitlines()
|
|
616
|
+
if lines:
|
|
617
|
+
try:
|
|
618
|
+
return json.loads(lines[-1])
|
|
619
|
+
except Exception:
|
|
620
|
+
pass
|
|
621
|
+
return {}
|
|
622
|
+
|
|
623
|
+
met1 = _get_latest_metrics(storage1)
|
|
624
|
+
met2 = _get_latest_metrics(storage2)
|
|
625
|
+
all_keys = sorted(set(list(met1.keys()) + list(met2.keys())))
|
|
626
|
+
|
|
627
|
+
for k in all_keys:
|
|
628
|
+
if k == "timestamp":
|
|
629
|
+
continue
|
|
630
|
+
v1 = met1.get(k, "-")
|
|
631
|
+
v2 = met2.get(k, "-")
|
|
632
|
+
diff_str = ""
|
|
633
|
+
if isinstance(v1, (int, float)) and isinstance(v2, (int, float)):
|
|
634
|
+
delta = v2 - v1
|
|
635
|
+
sign = "+" if delta > 0 else ""
|
|
636
|
+
diff_str = f" [dim]({sign}{delta:.4f})[/dim]"
|
|
637
|
+
table.add_row(f"Metric: {k}", str(v1), f"{v2}{diff_str}")
|
|
638
|
+
|
|
639
|
+
console.print(table)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
if __name__ == "__main__":
|
|
643
|
+
main()
|