researchloop 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. researchloop/__init__.py +1 -0
  2. researchloop/__main__.py +3 -0
  3. researchloop/cli.py +1138 -0
  4. researchloop/clusters/__init__.py +4 -0
  5. researchloop/clusters/monitor.py +199 -0
  6. researchloop/clusters/ssh.py +183 -0
  7. researchloop/comms/__init__.py +0 -0
  8. researchloop/comms/base.py +34 -0
  9. researchloop/comms/conversation.py +465 -0
  10. researchloop/comms/ntfy.py +95 -0
  11. researchloop/comms/router.py +71 -0
  12. researchloop/comms/slack.py +188 -0
  13. researchloop/core/__init__.py +0 -0
  14. researchloop/core/auth.py +78 -0
  15. researchloop/core/config.py +328 -0
  16. researchloop/core/credentials.py +38 -0
  17. researchloop/core/models.py +119 -0
  18. researchloop/core/orchestrator.py +910 -0
  19. researchloop/dashboard/__init__.py +0 -0
  20. researchloop/dashboard/app.py +15 -0
  21. researchloop/dashboard/auth.py +60 -0
  22. researchloop/dashboard/routes.py +912 -0
  23. researchloop/dashboard/templates/base.html +84 -0
  24. researchloop/dashboard/templates/login.html +12 -0
  25. researchloop/dashboard/templates/loop_detail.html +58 -0
  26. researchloop/dashboard/templates/loops.html +61 -0
  27. researchloop/dashboard/templates/setup.html +14 -0
  28. researchloop/dashboard/templates/sprint_detail.html +109 -0
  29. researchloop/dashboard/templates/sprints.html +48 -0
  30. researchloop/dashboard/templates/studies.html +18 -0
  31. researchloop/dashboard/templates/study_detail.html +64 -0
  32. researchloop/db/__init__.py +5 -0
  33. researchloop/db/database.py +86 -0
  34. researchloop/db/migrations.py +172 -0
  35. researchloop/db/queries.py +351 -0
  36. researchloop/runner/__init__.py +1 -0
  37. researchloop/runner/claude.py +169 -0
  38. researchloop/runner/job_templates/sge.sh.j2 +319 -0
  39. researchloop/runner/job_templates/slurm.sh.j2 +336 -0
  40. researchloop/runner/main.py +156 -0
  41. researchloop/runner/pipeline.py +272 -0
  42. researchloop/runner/templates/fix_issues.md.j2 +11 -0
  43. researchloop/runner/templates/idea_generator.md.j2 +16 -0
  44. researchloop/runner/templates/red_team.md.j2 +15 -0
  45. researchloop/runner/templates/report.md.j2 +31 -0
  46. researchloop/runner/templates/research_sprint.md.j2 +51 -0
  47. researchloop/runner/templates/summarizer.md.j2 +7 -0
  48. researchloop/runner/upload.py +153 -0
  49. researchloop/schedulers/__init__.py +11 -0
  50. researchloop/schedulers/base.py +43 -0
  51. researchloop/schedulers/local.py +188 -0
  52. researchloop/schedulers/sge.py +163 -0
  53. researchloop/schedulers/slurm.py +179 -0
  54. researchloop/sprints/__init__.py +0 -0
  55. researchloop/sprints/auto_loop.py +458 -0
  56. researchloop/sprints/manager.py +750 -0
  57. researchloop/studies/__init__.py +0 -0
  58. researchloop/studies/manager.py +102 -0
  59. researchloop-0.1.0.dist-info/METADATA +596 -0
  60. researchloop-0.1.0.dist-info/RECORD +63 -0
  61. researchloop-0.1.0.dist-info/WHEEL +4 -0
  62. researchloop-0.1.0.dist-info/entry_points.txt +3 -0
  63. researchloop-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,912 @@
1
+ """Dashboard HTML routes for the web UI."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import logging
7
+ from pathlib import Path
8
+ from typing import TYPE_CHECKING, Any
9
+
10
+ import markdown as _md
11
+ from fastapi import FastAPI, HTTPException, Request
12
+ from fastapi.responses import (
13
+ FileResponse,
14
+ JSONResponse,
15
+ RedirectResponse,
16
+ )
17
+ from starlette.templating import Jinja2Templates
18
+
19
+ from researchloop.dashboard.auth import (
20
+ SESSION_COOKIE,
21
+ SessionManager,
22
+ check_password,
23
+ generate_csrf_token,
24
+ hash_password,
25
+ verify_csrf_token,
26
+ )
27
+ from researchloop.db import queries
28
+
29
+ if TYPE_CHECKING:
30
+ from researchloop.core.orchestrator import Orchestrator
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ _TEMPLATES_DIR = Path(__file__).resolve().parent / "templates"
35
+ templates = Jinja2Templates(directory=str(_TEMPLATES_DIR))
36
+
37
+ # Add a markdown filter for rendering reports.
38
+ templates.env.filters["markdown"] = lambda text: _md.markdown(
39
+ text,
40
+ extensions=["fenced_code", "tables", "codehilite"],
41
+ )
42
+
43
+
44
+ def add_dashboard_routes(
45
+ app: FastAPI,
46
+ orchestrator: Orchestrator,
47
+ ) -> None:
48
+ """Register all dashboard HTML routes on *app*."""
49
+
50
+ # Session signing key — loaded lazily from DB.
51
+ _session_mgr: SessionManager | None = None
52
+
53
+ async def _get_session_mgr() -> SessionManager:
54
+ nonlocal _session_mgr
55
+ if _session_mgr is not None:
56
+ return _session_mgr
57
+ key: str | None = None
58
+ if orchestrator.db is not None:
59
+ row = await orchestrator.db.fetch_one(
60
+ "SELECT value FROM settings WHERE key = ?",
61
+ ("signing_key",),
62
+ )
63
+ if row:
64
+ key = row["value"]
65
+ _session_mgr = SessionManager(secret_key=key)
66
+ return _session_mgr
67
+
68
+ # ----------------------------------------------------------
69
+ # Password resolution — config, env, or DB
70
+ # ----------------------------------------------------------
71
+
72
+ async def _get_password_hash() -> str | None:
73
+ """Get password hash from config or DB settings."""
74
+ # Config / env var takes priority
75
+ cfg_hash = orchestrator.config.dashboard.password_hash
76
+ if cfg_hash:
77
+ return cfg_hash
78
+ # Fall back to DB
79
+ if orchestrator.db is not None:
80
+ row = await orchestrator.db.fetch_one(
81
+ "SELECT value FROM settings WHERE key = ?",
82
+ ("dashboard_password_hash",),
83
+ )
84
+ if row:
85
+ return row["value"]
86
+ return None
87
+
88
+ async def _set_password_hash(pw_hash: str) -> None:
89
+ """Store password hash in the DB settings table."""
90
+ if orchestrator.db is None:
91
+ return
92
+ await orchestrator.db.execute(
93
+ "INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)",
94
+ ("dashboard_password_hash", pw_hash),
95
+ )
96
+
97
+ # ----------------------------------------------------------
98
+ # Auth helpers
99
+ # ----------------------------------------------------------
100
+
101
+ async def _is_authenticated(request: Request) -> bool:
102
+ pw_hash = await _get_password_hash()
103
+ if not pw_hash:
104
+ return False # no password = needs setup
105
+ token = request.cookies.get(SESSION_COOKIE)
106
+ if not token:
107
+ return False
108
+ mgr = await _get_session_mgr()
109
+ return mgr.verify_token(token)
110
+
111
+ async def _needs_setup() -> bool:
112
+ return await _get_password_hash() is None
113
+
114
+ def _parse_job_options(form: object) -> dict[str, str]:
115
+ """Extract GPU/memory/CPU overrides from form data."""
116
+ opts: dict[str, str] = {}
117
+ gpu = str(getattr(form, "get", lambda k, d: d)("gpu", "")).strip()
118
+ mem = str(getattr(form, "get", lambda k, d: d)("mem", "")).strip()
119
+ cpus = str(getattr(form, "get", lambda k, d: d)("cpus", "")).strip()
120
+ if gpu:
121
+ opts["gres"] = gpu
122
+ if mem:
123
+ opts["mem"] = mem
124
+ if cpus:
125
+ opts["cpus-per-task"] = cpus
126
+ return opts
127
+
128
+ def _csrf_token(request: Request) -> str:
129
+ """Return a CSRF token for the current session, or empty string."""
130
+ token = request.cookies.get(SESSION_COOKIE, "")
131
+ if not token or _session_mgr is None:
132
+ return ""
133
+ return generate_csrf_token(token, _session_mgr.secret_key)
134
+
135
+ async def _check_csrf(request: Request) -> None:
136
+ """Validate the CSRF token from form data or header.
137
+
138
+ Checks ``X-CSRF-Token`` header first, then falls back to the
139
+ ``csrf_token`` form field. Raises 403 on failure.
140
+ """
141
+ csrf_tok = request.headers.get("X-CSRF-Token", "")
142
+ if not csrf_tok:
143
+ form = await request.form()
144
+ csrf_tok = str(form.get("csrf_token", ""))
145
+ session_tok = request.cookies.get(SESSION_COOKIE, "")
146
+ mgr = await _get_session_mgr()
147
+ if not session_tok or not verify_csrf_token(
148
+ session_tok, mgr.secret_key, csrf_tok
149
+ ):
150
+ raise HTTPException(status_code=403, detail="CSRF token invalid")
151
+
152
+ def _ctx(request: Request, authenticated: bool = False, **kwargs: object) -> dict:
153
+ return {
154
+ "request": request,
155
+ "authenticated": authenticated,
156
+ "csrf_token": _csrf_token(request),
157
+ **kwargs,
158
+ }
159
+
160
+ # ----------------------------------------------------------
161
+ # Setup (first run)
162
+ # ----------------------------------------------------------
163
+
164
+ @app.get("/dashboard/setup")
165
+ async def dashboard_setup(request: Request): # type: ignore[no-untyped-def]
166
+ if not await _needs_setup():
167
+ return RedirectResponse("/dashboard/", status_code=303)
168
+ return templates.TemplateResponse("setup.html", _ctx(request, error=None))
169
+
170
+ @app.post("/dashboard/setup")
171
+ async def dashboard_setup_post(request: Request): # type: ignore[no-untyped-def]
172
+ if not await _needs_setup():
173
+ return RedirectResponse("/dashboard/", status_code=303)
174
+
175
+ form = await request.form()
176
+ password = str(form.get("password", ""))
177
+ confirm = str(form.get("confirm", ""))
178
+
179
+ if len(password) < 8:
180
+ return templates.TemplateResponse(
181
+ "setup.html",
182
+ _ctx(
183
+ request,
184
+ error="Password must be at least 8 characters",
185
+ ),
186
+ status_code=400,
187
+ )
188
+
189
+ if password != confirm:
190
+ return templates.TemplateResponse(
191
+ "setup.html",
192
+ _ctx(request, error="Passwords do not match"),
193
+ status_code=400,
194
+ )
195
+
196
+ pw_hash = hash_password(password)
197
+ await _set_password_hash(pw_hash)
198
+
199
+ logger.info("Dashboard password set via first-run setup")
200
+
201
+ # Auto-login after setup
202
+ mgr = await _get_session_mgr()
203
+ token = mgr.create_token()
204
+ response = RedirectResponse("/dashboard/", status_code=303)
205
+ response.set_cookie(
206
+ SESSION_COOKIE,
207
+ token,
208
+ httponly=True,
209
+ samesite="lax",
210
+ )
211
+ return response
212
+
213
+ # ----------------------------------------------------------
214
+ # Login / Logout
215
+ # ----------------------------------------------------------
216
+
217
+ @app.get("/dashboard/login")
218
+ async def dashboard_login(request: Request): # type: ignore[no-untyped-def]
219
+ if await _needs_setup():
220
+ return RedirectResponse("/dashboard/setup", status_code=303)
221
+ return templates.TemplateResponse("login.html", _ctx(request, error=None))
222
+
223
+ @app.post("/dashboard/login")
224
+ async def dashboard_login_post(request: Request): # type: ignore[no-untyped-def]
225
+ if await _needs_setup():
226
+ return RedirectResponse("/dashboard/setup", status_code=303)
227
+
228
+ form = await request.form()
229
+ pwd = str(form.get("password", ""))
230
+ pw_hash = await _get_password_hash()
231
+
232
+ if pw_hash and check_password(pwd, pw_hash):
233
+ mgr = await _get_session_mgr()
234
+ token = mgr.create_token()
235
+ response = RedirectResponse("/dashboard/", status_code=303)
236
+ response.set_cookie(
237
+ SESSION_COOKIE,
238
+ token,
239
+ httponly=True,
240
+ samesite="lax",
241
+ )
242
+ return response
243
+
244
+ return templates.TemplateResponse(
245
+ "login.html",
246
+ _ctx(request, error="Invalid password"),
247
+ status_code=401,
248
+ )
249
+
250
+ @app.get("/dashboard/logout")
251
+ async def dashboard_logout(): # type: ignore[no-untyped-def]
252
+ response = RedirectResponse("/dashboard/login", status_code=303)
253
+ response.delete_cookie(SESSION_COOKIE)
254
+ return response
255
+
256
+ # ----------------------------------------------------------
257
+ # Auth gate for all pages below
258
+ # ----------------------------------------------------------
259
+
260
+ async def _gate(request: Request): # type: ignore[no-untyped-def]
261
+ """Redirect to setup or login if needed."""
262
+ if await _needs_setup():
263
+ return RedirectResponse("/dashboard/setup", status_code=303)
264
+ if not await _is_authenticated(request):
265
+ return RedirectResponse("/dashboard/login", status_code=303)
266
+ return None
267
+
268
+ # ----------------------------------------------------------
269
+ # Studies
270
+ # ----------------------------------------------------------
271
+
272
+ @app.get("/dashboard/")
273
+ async def dashboard_studies(request: Request): # type: ignore[no-untyped-def]
274
+ if redir := await _gate(request):
275
+ return redir
276
+ assert orchestrator.db is not None
277
+
278
+ rows = await queries.list_studies(orchestrator.db)
279
+ study_list = []
280
+ for s in rows:
281
+ sprints = await queries.list_sprints(
282
+ orchestrator.db,
283
+ study_name=s["name"],
284
+ limit=10000,
285
+ )
286
+ study_list.append(
287
+ {
288
+ "name": s["name"],
289
+ "cluster": s.get("cluster", ""),
290
+ "description": s.get("description", ""),
291
+ "sprint_count": len(sprints),
292
+ }
293
+ )
294
+ return templates.TemplateResponse(
295
+ "studies.html",
296
+ _ctx(request, authenticated=True, studies=study_list),
297
+ )
298
+
299
+ @app.get("/dashboard/studies/{name}")
300
+ async def dashboard_study_detail(name: str, request: Request): # type: ignore[no-untyped-def]
301
+ if redir := await _gate(request):
302
+ return redir
303
+ assert orchestrator.db is not None
304
+
305
+ study = await queries.get_study(orchestrator.db, name)
306
+ if study is None:
307
+ raise HTTPException(status_code=404, detail="Study not found")
308
+
309
+ sprints = await queries.list_sprints(orchestrator.db, study_name=name, limit=50)
310
+ prefill_idea = request.query_params.get("idea", "")
311
+ return templates.TemplateResponse(
312
+ "study_detail.html",
313
+ _ctx(
314
+ request,
315
+ authenticated=True,
316
+ study=study,
317
+ sprints=sprints,
318
+ prefill_idea=prefill_idea,
319
+ ),
320
+ )
321
+
322
+ # ----------------------------------------------------------
323
+ # Sprints
324
+ # ----------------------------------------------------------
325
+
326
+ @app.get("/dashboard/sprints")
327
+ async def dashboard_sprints(request: Request): # type: ignore[no-untyped-def]
328
+ if redir := await _gate(request):
329
+ return redir
330
+ assert orchestrator.db is not None
331
+
332
+ sprints = await queries.list_sprints(orchestrator.db, limit=100)
333
+ study_rows = await queries.list_studies(orchestrator.db)
334
+ study_names = [s["name"] for s in study_rows]
335
+ return templates.TemplateResponse(
336
+ "sprints.html",
337
+ _ctx(
338
+ request,
339
+ authenticated=True,
340
+ sprints=sprints,
341
+ studies=study_names,
342
+ ),
343
+ )
344
+
345
+ @app.get("/dashboard/sprints/{sprint_id}")
346
+ async def dashboard_sprint_detail(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
347
+ if redir := await _gate(request):
348
+ return redir
349
+ assert orchestrator.db is not None
350
+
351
+ sprint = await queries.get_sprint(orchestrator.db, sprint_id)
352
+ if sprint is None:
353
+ raise HTTPException(status_code=404, detail="Sprint not found")
354
+
355
+ artifacts = await queries.list_artifacts(orchestrator.db, sprint_id)
356
+
357
+ # Extract report and has_pdf from metadata_json.
358
+ report = None
359
+ has_pdf = False
360
+ meta = sprint.get("metadata_json")
361
+ if meta:
362
+ try:
363
+ md = json.loads(meta)
364
+ report = md.get("report")
365
+ has_pdf = md.get("has_pdf", False)
366
+ except (json.JSONDecodeError, TypeError):
367
+ pass
368
+
369
+ return templates.TemplateResponse(
370
+ "sprint_detail.html",
371
+ _ctx(
372
+ request,
373
+ authenticated=True,
374
+ sprint=sprint,
375
+ artifacts=artifacts,
376
+ report=report,
377
+ has_pdf=has_pdf,
378
+ ),
379
+ )
380
+
381
+ @app.get("/dashboard/sprints/{sprint_id}/report.pdf")
382
+ async def dashboard_sprint_pdf(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
383
+ """Download the sprint's PDF report."""
384
+ if redir := await _gate(request):
385
+ return redir
386
+ artifact_dir = Path(orchestrator.config.artifact_dir).resolve()
387
+ pdf_path = (artifact_dir / sprint_id / "report.pdf").resolve()
388
+ if not str(pdf_path).startswith(str(artifact_dir) + "/"):
389
+ raise HTTPException(
390
+ status_code=403,
391
+ detail="Access denied: path traversal detected",
392
+ )
393
+ if not pdf_path.exists():
394
+ raise HTTPException(
395
+ status_code=404,
396
+ detail="PDF report not found. Try Refresh first.",
397
+ )
398
+ return FileResponse(
399
+ path=str(pdf_path),
400
+ media_type="application/pdf",
401
+ headers={"Content-Disposition": "inline"},
402
+ )
403
+
404
+ # ----------------------------------------------------------
405
+ # Sprint actions
406
+ # ----------------------------------------------------------
407
+
408
+ @app.api_route(
409
+ "/dashboard/sprints/{sprint_id}/refresh",
410
+ methods=["GET", "POST"],
411
+ )
412
+ async def dashboard_sprint_refresh(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
413
+ """Check real job status on the cluster and update."""
414
+ if redir := await _gate(request):
415
+ return redir
416
+ if request.method == "POST":
417
+ await _check_csrf(request)
418
+ assert orchestrator.db is not None
419
+ assert orchestrator.sprint_manager is not None
420
+
421
+ sprint = await queries.get_sprint(orchestrator.db, sprint_id)
422
+ if sprint and sprint.get("job_id"):
423
+ try:
424
+ # Resolve cluster config
425
+ study_name = sprint["study_name"]
426
+ cluster_cfg = None
427
+ if orchestrator.study_manager:
428
+ cluster_cfg = await orchestrator.study_manager.get_cluster_config(
429
+ study_name
430
+ )
431
+
432
+ if cluster_cfg:
433
+ scheduler = orchestrator.sprint_manager.schedulers.get(
434
+ cluster_cfg.name
435
+ ) or orchestrator.sprint_manager.schedulers.get(
436
+ cluster_cfg.scheduler_type
437
+ )
438
+ if scheduler:
439
+ mgr = orchestrator.sprint_manager
440
+ conn = {
441
+ "host": cluster_cfg.host,
442
+ "port": cluster_cfg.port,
443
+ "user": cluster_cfg.user,
444
+ "key_path": cluster_cfg.key_path,
445
+ }
446
+ ssh = await mgr.ssh_manager.get_connection(conn)
447
+ real_status = await scheduler.status(ssh, sprint["job_id"])
448
+
449
+ terminal = {
450
+ "completed",
451
+ "failed",
452
+ "cancelled",
453
+ }
454
+ cur = sprint["status"]
455
+ if real_status in terminal and cur not in terminal:
456
+ from datetime import (
457
+ datetime,
458
+ timezone,
459
+ )
460
+
461
+ now = datetime.now(timezone.utc).isoformat()
462
+ await queries.update_sprint(
463
+ orchestrator.db,
464
+ sprint_id,
465
+ status=real_status,
466
+ completed_at=now,
467
+ )
468
+
469
+ # Resolve sprints_base the same way
470
+ # sprint manager does.
471
+ study_cfg = None
472
+ for s in orchestrator.config.studies:
473
+ if s.name == study_name:
474
+ study_cfg = s
475
+ break
476
+ if study_cfg and study_cfg.sprints_dir:
477
+ sbase = study_cfg.sprints_dir
478
+ else:
479
+ sbase = f"{cluster_cfg.working_dir}/{study_name}"
480
+ sp_dir = sprint.get("directory", "")
481
+ log_pat = f"{sbase}/{sp_dir}/slurm-*.out"
482
+ sprint_path = f"{sbase}/{sp_dir}"
483
+
484
+ # Read SLURM log.
485
+ stdout, _, _ = await ssh.run(
486
+ f"tail -50 {log_pat} 2>/dev/null || echo '(no log found)'"
487
+ )
488
+ log_text = stdout.strip()
489
+
490
+ # Read sprint log for detailed progress.
491
+ sprint_log_out, _, _ = await ssh.run(
492
+ f"tail -100 {sprint_path}/sprint_log.txt"
493
+ f" 2>/dev/null || true"
494
+ )
495
+
496
+ # Read summary and report from cluster.
497
+ summary_out, _, _ = await ssh.run(
498
+ f"cat {sprint_path}/summary.txt 2>/dev/null || true"
499
+ )
500
+ report_out, _, _ = await ssh.run(
501
+ f"cat {sprint_path}/report.md 2>/dev/null || true"
502
+ )
503
+
504
+ # Check if PDF exists.
505
+ pdf_path = f"{sprint_path}/report.pdf"
506
+ _, _, pdf_rc = await ssh.run(f"test -f {pdf_path}")
507
+ has_pdf = pdf_rc == 0
508
+
509
+ # If PDF exists, download it locally.
510
+ if has_pdf:
511
+ art_dir = Path(orchestrator.config.artifact_dir) / sprint_id
512
+ art_dir.mkdir(parents=True, exist_ok=True)
513
+ local_pdf = art_dir / "report.pdf"
514
+ if not local_pdf.exists():
515
+ try:
516
+ await ssh.download_file(
517
+ pdf_path,
518
+ str(local_pdf),
519
+ )
520
+ except Exception:
521
+ logger.warning("PDF download failed")
522
+ has_pdf = False
523
+
524
+ # Detect current pipeline step from log.
525
+ current_step = None
526
+ if log_text:
527
+ for line in reversed(log_text.split("\n")):
528
+ line = line.strip()
529
+ if line.startswith(">>> Step:"):
530
+ current_step = line.split(">>> Step:")[1].strip()
531
+ break
532
+ if line.startswith("<<<"):
533
+ # Last step finished
534
+ break
535
+
536
+ # Read idea.txt from cluster.
537
+ idea_out, _, _ = await ssh.run(
538
+ f"cat {sprint_path}/idea.txt 2>/dev/null || true"
539
+ )
540
+
541
+ # Read findings.md, progress.md, and output.log.
542
+ findings_out, _, _ = await ssh.run(
543
+ f"cat {sprint_path}/findings.md 2>/dev/null || true"
544
+ )
545
+ progress_out, _, _ = await ssh.run(
546
+ f"cat {sprint_path}/progress.md 2>/dev/null || true"
547
+ )
548
+ output_log_out, _, _ = await ssh.run(
549
+ f"tail -50 {sprint_path}/output.log 2>/dev/null || true"
550
+ )
551
+
552
+ # Build update dict.
553
+ update_kw: dict[str, Any] = {}
554
+
555
+ # Update idea from idea.txt if it differs.
556
+ idea_text = idea_out.strip()
557
+ cur_idea = sprint.get("idea", "")
558
+ if idea_text and idea_text != cur_idea:
559
+ update_kw["idea"] = idea_text[:500]
560
+
561
+ # Update status: running with step, or terminal.
562
+ if real_status == "running":
563
+ step_label = (
564
+ f"running ({current_step})"
565
+ if current_step
566
+ else "running"
567
+ )
568
+ update_kw["status"] = step_label
569
+ elif real_status in terminal and cur not in terminal:
570
+ update_kw["status"] = real_status
571
+
572
+ if summary_out.strip():
573
+ update_kw["summary"] = summary_out.strip()
574
+
575
+ # Build log display: progress + output + tool log.
576
+ parts: list[str] = []
577
+ progress_text = progress_out.strip()
578
+ if progress_text:
579
+ parts.append(progress_text)
580
+ output_text = output_log_out.strip()
581
+ if output_text:
582
+ parts.append(
583
+ f"--- Script output (last 50 lines) ---\n{output_text}"
584
+ )
585
+ sprint_log = sprint_log_out.strip()
586
+ display_log = sprint_log or log_text
587
+ if display_log:
588
+ if parts:
589
+ parts.append(f"--- Tool log ---\n{display_log}")
590
+ else:
591
+ parts.append(f"[{real_status}] Log:\n{display_log}")
592
+ if parts:
593
+ update_kw["error"] = "\n\n".join(parts)
594
+
595
+ meta_dict: dict[str, Any] = {}
596
+ if report_out.strip():
597
+ meta_dict["report"] = report_out.strip()
598
+ elif findings_out.strip():
599
+ meta_dict["report"] = findings_out.strip()
600
+ if has_pdf:
601
+ meta_dict["has_pdf"] = True
602
+ if meta_dict:
603
+ update_kw["metadata_json"] = json.dumps(meta_dict)
604
+ if update_kw:
605
+ await queries.update_sprint(
606
+ orchestrator.db,
607
+ sprint_id,
608
+ **update_kw,
609
+ )
610
+ except Exception as exc:
611
+ logger.warning("Refresh status failed: %s", exc)
612
+
613
+ # Return JSON if requested (JS refresh), otherwise redirect.
614
+ if request.headers.get("accept", "").startswith("application/json"):
615
+ updated = await queries.get_sprint(orchestrator.db, sprint_id)
616
+ return JSONResponse(
617
+ {
618
+ "status": updated["status"] if updated else None,
619
+ "idea": updated.get("idea") if updated else None,
620
+ "summary": updated.get("summary") if updated else None,
621
+ "completed_at": updated.get("completed_at") if updated else None,
622
+ }
623
+ )
624
+
625
+ return RedirectResponse(
626
+ f"/dashboard/sprints/{sprint_id}",
627
+ status_code=303,
628
+ )
629
+
630
+ @app.post("/dashboard/sprints/{sprint_id}/cancel")
631
+ async def dashboard_sprint_cancel(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
632
+ if redir := await _gate(request):
633
+ return redir
634
+ await _check_csrf(request)
635
+ assert orchestrator.sprint_manager is not None
636
+ try:
637
+ await orchestrator.sprint_manager.cancel_sprint(sprint_id)
638
+ except Exception as exc:
639
+ logger.warning("Cancel failed: %s", exc)
640
+ return RedirectResponse(
641
+ f"/dashboard/sprints/{sprint_id}",
642
+ status_code=303,
643
+ )
644
+
645
+ @app.post("/dashboard/sprints/{sprint_id}/delete")
646
+ async def dashboard_sprint_delete(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
647
+ if redir := await _gate(request):
648
+ return redir
649
+ await _check_csrf(request)
650
+ assert orchestrator.db is not None
651
+ await queries.delete_sprint(orchestrator.db, sprint_id)
652
+ return RedirectResponse("/dashboard/sprints", status_code=303)
653
+
654
+ @app.post("/dashboard/sprints/{sprint_id}/resubmit")
655
+ async def dashboard_sprint_resubmit(sprint_id: str, request: Request): # type: ignore[no-untyped-def]
656
+ """Resubmit a failed/cancelled sprint with the same idea."""
657
+ if redir := await _gate(request):
658
+ return redir
659
+ await _check_csrf(request)
660
+ assert orchestrator.db is not None
661
+ assert orchestrator.sprint_manager is not None
662
+
663
+ sprint = await queries.get_sprint(orchestrator.db, sprint_id)
664
+ if sprint is None:
665
+ raise HTTPException(status_code=404, detail="Sprint not found")
666
+
667
+ idea = sprint.get("idea") or sprint.get("summary") or "Retry"
668
+ study_name = sprint["study_name"]
669
+
670
+ try:
671
+ new_sprint = await orchestrator.sprint_manager.run_sprint(study_name, idea)
672
+ return RedirectResponse(
673
+ f"/dashboard/sprints/{new_sprint.id}",
674
+ status_code=303,
675
+ )
676
+ except Exception as exc:
677
+ logger.warning("Resubmit failed: %s", exc)
678
+ return RedirectResponse(
679
+ f"/dashboard/sprints/{sprint_id}",
680
+ status_code=303,
681
+ )
682
+
683
+ @app.post("/dashboard/sprints/new")
684
+ async def dashboard_sprint_new(request: Request): # type: ignore[no-untyped-def]
685
+ if redir := await _gate(request):
686
+ return redir
687
+ await _check_csrf(request)
688
+ assert orchestrator.sprint_manager is not None
689
+
690
+ form = await request.form()
691
+ study_name = str(form.get("study_name", ""))
692
+ idea = str(form.get("idea", "")).strip()
693
+
694
+ if not study_name or not idea:
695
+ return RedirectResponse("/dashboard/sprints", status_code=303)
696
+
697
+ job_opts = _parse_job_options(form)
698
+ try:
699
+ sprint = await orchestrator.sprint_manager.run_sprint(
700
+ study_name, idea, job_options=job_opts or None
701
+ )
702
+ return RedirectResponse(
703
+ f"/dashboard/sprints/{sprint.id}",
704
+ status_code=303,
705
+ )
706
+ except Exception as exc:
707
+ logger.warning("Sprint submission failed: %s", exc)
708
+ return RedirectResponse("/dashboard/sprints", status_code=303)
709
+
710
+ @app.post("/dashboard/studies/{name}/sprint")
711
+ async def dashboard_study_sprint(name: str, request: Request): # type: ignore[no-untyped-def]
712
+ if redir := await _gate(request):
713
+ return redir
714
+ await _check_csrf(request)
715
+ assert orchestrator.sprint_manager is not None
716
+
717
+ form = await request.form()
718
+ idea = str(form.get("idea", "")).strip()
719
+
720
+ if not idea:
721
+ return RedirectResponse(
722
+ f"/dashboard/studies/{name}",
723
+ status_code=303,
724
+ )
725
+
726
+ job_opts = _parse_job_options(form)
727
+ try:
728
+ sprint = await orchestrator.sprint_manager.run_sprint(
729
+ name, idea, job_options=job_opts or None
730
+ )
731
+ return RedirectResponse(
732
+ f"/dashboard/sprints/{sprint.id}",
733
+ status_code=303,
734
+ )
735
+ except Exception as exc:
736
+ logger.warning("Sprint submission failed: %s", exc)
737
+ return RedirectResponse(
738
+ f"/dashboard/studies/{name}",
739
+ status_code=303,
740
+ )
741
+
742
+ # ----------------------------------------------------------
743
+ # Auto-Loops
744
+ # ----------------------------------------------------------
745
+
746
+ @app.get("/dashboard/loops")
747
+ async def dashboard_loops(request: Request): # type: ignore[no-untyped-def]
748
+ if redir := await _gate(request):
749
+ return redir
750
+ assert orchestrator.db is not None
751
+
752
+ loops = await queries.list_auto_loops(orchestrator.db)
753
+ study_rows = await queries.list_studies(orchestrator.db)
754
+ # Only show studies that allow loops.
755
+ loopable = {s.name for s in orchestrator.config.studies if s.allow_loop}
756
+ study_names = [s["name"] for s in study_rows if s["name"] in loopable]
757
+ return templates.TemplateResponse(
758
+ "loops.html",
759
+ _ctx(
760
+ request,
761
+ authenticated=True,
762
+ loops=loops,
763
+ studies=study_names,
764
+ ),
765
+ )
766
+
767
+ @app.get("/dashboard/loops/{loop_id}")
768
+ async def dashboard_loop_detail(loop_id: str, request: Request): # type: ignore[no-untyped-def]
769
+ if redir := await _gate(request):
770
+ return redir
771
+ assert orchestrator.db is not None
772
+
773
+ loop = await queries.get_auto_loop(orchestrator.db, loop_id)
774
+ if loop is None:
775
+ raise HTTPException(status_code=404, detail="Loop not found")
776
+
777
+ # Get sprints belonging to this loop.
778
+ all_sprints = await queries.list_sprints(
779
+ orchestrator.db,
780
+ study_name=loop["study_name"],
781
+ limit=200,
782
+ )
783
+ loop_sprints = [
784
+ sp
785
+ for sp in all_sprints
786
+ if sp.get("loop_id") == loop_id or loop_id in (sp.get("idea") or "")
787
+ ]
788
+
789
+ # Extract context from metadata_json.
790
+ context = ""
791
+ meta = loop.get("metadata_json")
792
+ if meta:
793
+ try:
794
+ context = json.loads(meta).get("context", "")
795
+ except (json.JSONDecodeError, TypeError):
796
+ pass
797
+
798
+ return templates.TemplateResponse(
799
+ "loop_detail.html",
800
+ _ctx(
801
+ request,
802
+ authenticated=True,
803
+ loop=loop,
804
+ sprints=loop_sprints,
805
+ context=context,
806
+ ),
807
+ )
808
+
809
+ @app.post("/dashboard/loops/{loop_id}/stop")
810
+ async def dashboard_loop_stop(loop_id: str, request: Request): # type: ignore[no-untyped-def]
811
+ if redir := await _gate(request):
812
+ return redir
813
+ await _check_csrf(request)
814
+ assert orchestrator.auto_loop is not None
815
+ try:
816
+ await orchestrator.auto_loop.stop(loop_id)
817
+ except Exception as exc:
818
+ logger.warning("Loop stop failed: %s", exc)
819
+ return RedirectResponse(
820
+ f"/dashboard/loops/{loop_id}",
821
+ status_code=303,
822
+ )
823
+
824
+ @app.post("/dashboard/loops/{loop_id}/resume")
825
+ async def dashboard_loop_resume(loop_id: str, request: Request): # type: ignore[no-untyped-def]
826
+ if redir := await _gate(request):
827
+ return redir
828
+ await _check_csrf(request)
829
+ assert orchestrator.auto_loop is not None
830
+ try:
831
+ await orchestrator.auto_loop.resume(loop_id)
832
+ except Exception as exc:
833
+ logger.warning("Loop resume failed: %s", exc)
834
+ return RedirectResponse(
835
+ f"/dashboard/loops/{loop_id}",
836
+ status_code=303,
837
+ )
838
+
839
+ @app.post("/dashboard/loops/new")
840
+ async def dashboard_loop_new(request: Request): # type: ignore[no-untyped-def]
841
+ if redir := await _gate(request):
842
+ return redir
843
+ await _check_csrf(request)
844
+ assert orchestrator.auto_loop is not None
845
+
846
+ form = await request.form()
847
+ study_name = str(form.get("study_name", ""))
848
+ count_str = str(form.get("count", "5"))
849
+ context = str(form.get("context", "")).strip()
850
+
851
+ if not study_name:
852
+ return RedirectResponse("/dashboard/loops", status_code=303)
853
+
854
+ try:
855
+ count = int(count_str)
856
+ except ValueError:
857
+ count = 5
858
+
859
+ job_opts = _parse_job_options(form)
860
+ try:
861
+ loop_id = await orchestrator.auto_loop.start(
862
+ study_name,
863
+ count,
864
+ context,
865
+ job_options=job_opts or None,
866
+ )
867
+ return RedirectResponse(
868
+ f"/dashboard/loops/{loop_id}",
869
+ status_code=303,
870
+ )
871
+ except Exception as exc:
872
+ logger.warning("Loop creation failed: %s", exc)
873
+ return RedirectResponse(
874
+ "/dashboard/loops",
875
+ status_code=303,
876
+ )
877
+
878
+ # ----------------------------------------------------------
879
+ # Artifact download
880
+ # ----------------------------------------------------------
881
+
882
+ @app.get("/dashboard/artifacts/{artifact_id}/download")
883
+ async def dashboard_artifact_download(artifact_id: int, request: Request): # type: ignore[no-untyped-def]
884
+ if redir := await _gate(request):
885
+ return redir
886
+ assert orchestrator.db is not None
887
+
888
+ artifact = await queries.get_artifact(orchestrator.db, artifact_id)
889
+ if artifact is None:
890
+ raise HTTPException(status_code=404, detail="Artifact not found")
891
+
892
+ file_path = Path(artifact["path"]).resolve()
893
+ artifact_dir = Path(orchestrator.config.artifact_dir).resolve()
894
+ if (
895
+ not str(file_path).startswith(str(artifact_dir) + "/")
896
+ and file_path != artifact_dir
897
+ ):
898
+ raise HTTPException(
899
+ status_code=403,
900
+ detail="Access denied: path traversal detected",
901
+ )
902
+
903
+ if not file_path.exists():
904
+ raise HTTPException(
905
+ status_code=404,
906
+ detail="Artifact file not found on disk",
907
+ )
908
+
909
+ return FileResponse(
910
+ path=str(file_path),
911
+ filename=artifact["filename"],
912
+ )