lightcone-cli 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. lightcone/cli/__init__.py +16 -0
  2. lightcone/cli/claude/lightcone/agents/lc-extractor.md +114 -0
  3. lightcone/cli/claude/lightcone/guides/astra-reference.md +290 -0
  4. lightcone/cli/claude/lightcone/guides/lightcone-cli-reference.md +75 -0
  5. lightcone/cli/claude/lightcone/guides/ui-brand.md +86 -0
  6. lightcone/cli/claude/lightcone/hooks/langfuse_git_commit_hook.py +303 -0
  7. lightcone/cli/claude/lightcone/hooks/langfuse_hook.py +894 -0
  8. lightcone/cli/claude/lightcone/hooks/langfuse_prepare_commit_msg.py +142 -0
  9. lightcone/cli/claude/lightcone/hooks/langfuse_session_init_hook.py +83 -0
  10. lightcone/cli/claude/lightcone/hooks/langfuse_utils.py +457 -0
  11. lightcone/cli/claude/lightcone/scripts/activate-venv.sh +44 -0
  12. lightcone/cli/claude/lightcone/scripts/check-lc-run.sh +140 -0
  13. lightcone/cli/claude/lightcone/scripts/session-start.sh +140 -0
  14. lightcone/cli/claude/lightcone/scripts/validate-on-save.sh +77 -0
  15. lightcone/cli/claude/lightcone/skills/lc-build/SKILL.md +92 -0
  16. lightcone/cli/claude/lightcone/skills/lc-build/assets/loop-prompt.md +92 -0
  17. lightcone/cli/claude/lightcone/skills/lc-build/scripts/setup-lc-build.sh +240 -0
  18. lightcone/cli/claude/lightcone/skills/lc-feedback/SKILL.md +94 -0
  19. lightcone/cli/claude/lightcone/skills/lc-migrate/SKILL.md +98 -0
  20. lightcone/cli/claude/lightcone/skills/lc-new/SKILL.md +183 -0
  21. lightcone/cli/claude/lightcone/skills/lc-verify/SKILL.md +53 -0
  22. lightcone/cli/claude/lightcone/templates/CLAUDE.md +32 -0
  23. lightcone/cli/commands.py +2327 -0
  24. lightcone/cli/plugin.py +34 -0
  25. lightcone/engine/__init__.py +42 -0
  26. lightcone/engine/assets.py +418 -0
  27. lightcone/engine/container.py +370 -0
  28. lightcone/engine/io_manager.py +27 -0
  29. lightcone/engine/runner.py +1017 -0
  30. lightcone/engine/site_registry.py +142 -0
  31. lightcone/engine/status.py +135 -0
  32. lightcone/engine/targets.py +68 -0
  33. lightcone/engine/tree.py +245 -0
  34. lightcone/eval/__init__.py +25 -0
  35. lightcone/eval/build.py +148 -0
  36. lightcone/eval/cli.py +176 -0
  37. lightcone/eval/graders.py +192 -0
  38. lightcone/eval/harness.py +265 -0
  39. lightcone/eval/models.py +117 -0
  40. lightcone/eval/report.py +214 -0
  41. lightcone/eval/sandbox.py +394 -0
  42. lightcone_cli-0.2.0.dist-info/METADATA +16 -0
  43. lightcone_cli-0.2.0.dist-info/RECORD +46 -0
  44. lightcone_cli-0.2.0.dist-info/WHEEL +4 -0
  45. lightcone_cli-0.2.0.dist-info/entry_points.txt +2 -0
  46. lightcone_cli-0.2.0.dist-info/licenses/LICENSE +29 -0
@@ -0,0 +1,2327 @@
1
+ """Command-line interface for lightcone-cli — the ASTRA-compliant agentic layer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import shutil
8
+ import subprocess
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ import click
14
+ import yaml
15
+ from rich.console import Console
16
+
17
+ from lightcone.cli.plugin import get_plugin_source_dir
18
+
19
+ console = Console()
20
+
21
+ #: Permission tier definitions for Claude Code's ``.claude/settings.json``.
22
+ #:
23
+ #: Three tiers are available:
24
+ #:
25
+ #: * ``yolo`` — All tools allowed, including MCP servers. No guardrails.
26
+ #: Suitable for trusted, isolated development environments.
27
+ #: * ``recommended`` — Full read/write/bash access with deny rules for
28
+ #: sensitive dotfiles, HPC scratch filesystems, and destructive commands
29
+ #: (``sudo``, ``rm -rf /``, ``git push``). Default for new projects.
30
+ #: * ``minimal`` — Read-only. Every write or shell action requires explicit
31
+ #: human confirmation. Use when working on shared or production systems.
32
+ PERMISSION_TIERS: dict[str, dict[str, list[str]]] = {
33
+ "yolo": {
34
+ "allow": [
35
+ "Bash(*)",
36
+ "Edit",
37
+ "Read",
38
+ "Write",
39
+ "WebSearch",
40
+ "WebFetch",
41
+ "mcp__*",
42
+ ],
43
+ },
44
+ "recommended": {
45
+ "allow": [
46
+ "Read",
47
+ "Edit",
48
+ "Write",
49
+ "Bash(*)",
50
+ "WebSearch",
51
+ "WebFetch",
52
+ ],
53
+ "deny": [
54
+ # Sensitive dotfiles — don't silently modify credentials/keys
55
+ "Edit(~/.ssh/**)",
56
+ "Edit(~/.aws/**)",
57
+ "Edit(~/.gnupg/**)",
58
+ # Common HPC scratch filesystems
59
+ "Edit(//scratch/**)",
60
+ "Edit(//pscratch/**)",
61
+ # Dangerous bash — require explicit confirmation
62
+ "Bash(sudo *)",
63
+ "Bash(rm -rf /*)",
64
+ "Bash(git push *)",
65
+ "Bash(git push)",
66
+ ],
67
+ },
68
+ "minimal": {
69
+ "allow": ["Read"],
70
+ },
71
+ }
72
+
73
+
74
+ @click.group()
75
+ @click.version_option(package_name="lightcone-cli")
76
+ @click.pass_context
77
+ def main(ctx: click.Context) -> None:
78
+ """lightcone-cli — ASTRA-compliant Agentic Layer CLI."""
79
+ ctx.ensure_object(dict)
80
+ if ctx.invoked_subcommand in ("setup", "target", "update", "eval"):
81
+ return
82
+ from lightcone.engine.targets import get_config_path
83
+ if not get_config_path().exists():
84
+ console.print(
85
+ "\n[bold yellow]No execution environment configured.[/bold yellow]"
86
+ )
87
+ console.print(
88
+ " lightcone-cli needs a default target configured before you can use it.\n"
89
+ )
90
+ ctx.invoke(setup)
91
+ return
92
+
93
+
94
+
95
+ # =============================================================================
96
+ # Path helpers
97
+ # =============================================================================
98
+
99
+
100
+ def _find_lightcone_yaml(project_path: Path) -> Path | None:
101
+ """Find lightcone.yaml, checking .lightcone/ first then root for backwards compat."""
102
+ candidate = project_path / ".lightcone" / "lightcone.yaml"
103
+ if candidate.exists():
104
+ return candidate
105
+ candidate = project_path / "lightcone.yaml"
106
+ if candidate.exists():
107
+ return candidate
108
+ return None
109
+
110
+
111
+ def _find_dagster_yaml(project_path: Path) -> Path | None:
112
+ """Find dagster.yaml, checking .lightcone/ first then root for backwards compat."""
113
+ candidate = project_path / ".lightcone" / "dagster.yaml"
114
+ if candidate.exists():
115
+ return candidate
116
+ candidate = project_path / "dagster.yaml"
117
+ if candidate.exists():
118
+ return candidate
119
+ return None
120
+
121
+
122
+ def _load_lightcone_config(project_path: Path) -> dict:
123
+ """Load lightcone.yaml config, returning empty dict if not found."""
124
+ path = _find_lightcone_yaml(project_path)
125
+ if path is None:
126
+ return {}
127
+ with open(path) as f:
128
+ return yaml.safe_load(f) or {}
129
+
130
+
131
+ # =============================================================================
132
+ # Init command
133
+ # =============================================================================
134
+
135
+
136
+ @main.command()
137
+ @click.argument("directory", type=click.Path(path_type=Path), default=".")
138
+ @click.option("--no-git", is_flag=True, help="Don't initialize git repository")
139
+ @click.option("--no-venv", is_flag=True, help="Don't create Python virtual environment")
140
+ @click.option("--target", "-t", default=None, help="Execution target name")
141
+ @click.option(
142
+ "--permissions",
143
+ type=click.Choice(["yolo", "recommended", "minimal"]),
144
+ default=None,
145
+ help="Claude Code permission tier (default: prompt or saved default)",
146
+ )
147
+ @click.option(
148
+ "--existing-project", "existing_project",
149
+ type=click.Path(exists=True, path_type=Path),
150
+ default=None,
151
+ help=(
152
+ "Path to existing code to migrate "
153
+ "(copies into DIRECTORY, adds lightcone-cli infrastructure)"
154
+ ),
155
+ )
156
+ @click.option(
157
+ "--sub-analysis", "sub_analysis",
158
+ is_flag=True,
159
+ default=False,
160
+ help="Create a sub-analysis directory and wire it into the parent project",
161
+ )
162
+ def init(
163
+ directory: Path, no_git: bool, no_venv: bool,
164
+ target: str | None, permissions: str | None,
165
+ existing_project: Path | None,
166
+ sub_analysis: bool,
167
+ ) -> None:
168
+ """Create a new ASTRA analysis project with full agentic scaffolding.
169
+
170
+ Creates the project with ASTRA specification files, Claude Code plugin
171
+ configuration, skills, hooks, and a Python virtual environment.
172
+
173
+ Use --existing-project to migrate existing code into ASTRA. If the
174
+ source path differs from DIRECTORY, code is copied in. Then run
175
+ /lc-migrate in Claude Code to generate the spec.
176
+
177
+ Use --sub-analysis to scaffold a sub-analysis directory and wire it
178
+ into the parent project's astra.yaml and universe files.
179
+
180
+ DIRECTORY is the project folder to create (default: current directory).
181
+
182
+ Examples:
183
+ lc init my-analysis
184
+ lc init my-analysis --target perlmutter-gpu
185
+ lc init . --existing-project .
186
+ lc init my-analysis --existing-project ../old-code
187
+ lc init analyses/new_stage --sub-analysis
188
+ lc init --sub-analysis new_stage
189
+ """
190
+ if sub_analysis:
191
+ _init_sub_analysis(directory)
192
+ return
193
+
194
+ if existing_project is not None:
195
+ _init_existing_project(
196
+ directory, source=existing_project,
197
+ no_git=no_git, no_venv=no_venv,
198
+ target=target, permissions=permissions,
199
+ )
200
+ return
201
+
202
+ # Check if this is already an ASTRA project
203
+ if (directory / "astra.yaml").exists():
204
+ console.print(
205
+ f"[red]Error:[/red] [cyan]{directory}[/cyan] is already an ASTRA project "
206
+ f"(astra.yaml exists)."
207
+ )
208
+ console.print(
209
+ "Use [cyan]astra validate[/cyan] to check it, or delete astra.yaml to re-init."
210
+ )
211
+ raise SystemExit(1)
212
+
213
+ # Create project directory
214
+ if directory != Path("."):
215
+ if directory.exists() and any(directory.iterdir()):
216
+ if not click.confirm(
217
+ f"[yellow]{directory}[/yellow] already exists and is not empty. Continue?"
218
+ ):
219
+ raise SystemExit(0)
220
+ directory.mkdir(parents=True, exist_ok=True)
221
+
222
+ # Create directory structure
223
+ subdirs = [
224
+ "universes",
225
+ "scripts",
226
+ "results",
227
+ ".lightcone",
228
+ ]
229
+ for subdir in subdirs:
230
+ (directory / subdir).mkdir(parents=True, exist_ok=True)
231
+
232
+ # Create dagster.yaml inside .lightcone/
233
+ _create_dagster_yaml(directory)
234
+
235
+ # Create .gitignore
236
+ _create_or_append_gitignore(directory)
237
+
238
+ # Create boilerplate astra.yaml
239
+ _create_boilerplate_astra_yaml(directory)
240
+
241
+ # Create CLAUDE.md with project conventions
242
+ _create_claude_md(directory)
243
+
244
+ # Resolve target and permission tier, then create Claude Code settings
245
+ effective_target = target
246
+ if not effective_target:
247
+ from lightcone.engine.targets import load_user_config
248
+ effective_target = load_user_config().get("default_target", "local")
249
+
250
+ tier = _resolve_permission_tier(permissions)
251
+ _create_claude_settings(directory, tier, target=effective_target)
252
+
253
+ # Write lightcone.yaml project config
254
+ _create_lightcone_config(directory, effective_target)
255
+
256
+ # If user explicitly passed --target, ensure it's been configured
257
+ if target and target != "local":
258
+ from lightcone.engine.targets import load_target
259
+ if load_target(target) is None:
260
+ console.print(
261
+ f"\n[yellow]Target [cyan]{target}[/cyan] "
262
+ "is not configured yet.[/yellow]"
263
+ )
264
+ console.print(
265
+ " Run [cyan]lc setup[/cyan] to configure execution targets."
266
+ )
267
+
268
+ # Create virtual environment
269
+ _create_venv(directory, no_venv)
270
+
271
+ # Initialize git repository
272
+ _init_git_repo(directory, no_git)
273
+
274
+ # Print success message
275
+ console.print(f"[green]✓[/green] Created ASTRA analysis project: [cyan]{directory}[/cyan]")
276
+ if target:
277
+ console.print(f" Target: [cyan]{target}[/cyan]")
278
+
279
+ # Container runtime detection and guidance
280
+ from lightcone.engine.container import detect_container_runtime
281
+ rt = detect_container_runtime()
282
+ if rt:
283
+ console.print(f" Container runtime: [cyan]{rt}[/cyan]")
284
+ else:
285
+ console.print(
286
+ "\n[yellow]Note:[/yellow] No container runtime (Docker or Podman) detected.\n"
287
+ " Recipes will run in the project venv "
288
+ "(dependencies from requirements.txt).\n"
289
+ " For full container isolation, install one of:\n"
290
+ " Podman: [cyan]https://podman.io/docs/installation[/cyan]\n"
291
+ " (recommended — rootless, no daemon)\n"
292
+ " Docker: [cyan]https://docs.docker.com/engine/install/[/cyan]"
293
+ )
294
+
295
+ console.print(
296
+ "\n[bold yellow]Note:[/bold yellow] Telemetry is enabled by default. "
297
+ "Claude Code sessions in this project will be traced to Langfuse.\n"
298
+ " To disable, set [cyan]TRACE_TO_LANGFUSE=false[/cyan] "
299
+ "in [cyan].claude/settings.local.json[/cyan]."
300
+ )
301
+
302
+ # Detect SLURM environment and suggest interactive allocation
303
+ if shutil.which("salloc") and not os.environ.get("SLURM_JOB_ID"):
304
+ target_name = target
305
+ if not target_name:
306
+ from lightcone.engine.targets import load_user_config
307
+ target_name = load_user_config().get("default_target")
308
+ if target_name and target_name != "local":
309
+ from lightcone.engine.targets import load_target
310
+ target_config = load_target(target_name)
311
+ if target_config and target_config.get("backend") == "slurm":
312
+ console.print(
313
+ "\n[bold]Tip:[/bold] For fast execution, start an interactive "
314
+ "allocation ([cyan]salloc[/cyan]) before launching Claude Code. "
315
+ "This lets [cyan]lc run[/cyan] execute instantly via srun "
316
+ "instead of waiting in the batch queue."
317
+ )
318
+
319
+ console.print(f"\n[bold]cd {directory}[/bold] && [bold]claude[/bold]")
320
+ console.print("Then run [cyan]/lc-new[/cyan] to scope your research question.")
321
+
322
+
323
+ _GITIGNORE_LINES = [
324
+ "results/",
325
+ "results/.dagster/",
326
+ "__pycache__/",
327
+ "*.py[cod]",
328
+ ".venv/",
329
+ ".ipynb_checkpoints/",
330
+ ".DS_Store",
331
+ ".langfuse/",
332
+ ]
333
+
334
+
335
+ def _create_dagster_yaml(directory: Path) -> None:
336
+ """Create .lightcone/dagster.yaml for Dagster instance configuration."""
337
+ dagster_yaml_content = {
338
+ "storage": {
339
+ "sqlite": {
340
+ "base_dir": "results/.dagster",
341
+ },
342
+ },
343
+ }
344
+ lightcone_dir = directory / ".lightcone"
345
+ lightcone_dir.mkdir(parents=True, exist_ok=True)
346
+ (lightcone_dir / "dagster.yaml").write_text(
347
+ yaml.dump(dagster_yaml_content, default_flow_style=False, sort_keys=False)
348
+ )
349
+
350
+
351
+ def _create_or_append_gitignore(directory: Path) -> None:
352
+ """Create .gitignore or append missing lightcone-cli entries to an existing one."""
353
+ gitignore_path = directory / ".gitignore"
354
+ if gitignore_path.exists():
355
+ existing = gitignore_path.read_text()
356
+ existing_lines = {line.strip() for line in existing.splitlines()}
357
+ missing = [line for line in _GITIGNORE_LINES if line not in existing_lines]
358
+ if missing:
359
+ addition = "\n# lightcone-cli / ASTRA\n" + "\n".join(missing) + "\n"
360
+ with open(gitignore_path, "a") as f:
361
+ f.write(addition)
362
+ else:
363
+ content = "# ASTRA Analysis\n" + "\n".join(_GITIGNORE_LINES) + "\n"
364
+ gitignore_path.write_text(content)
365
+
366
+
367
+ def _init_existing_project(
368
+ directory: Path,
369
+ *,
370
+ source: Path,
371
+ no_git: bool,
372
+ no_venv: bool,
373
+ target: str | None,
374
+ permissions: str | None,
375
+ ) -> None:
376
+ """Add lightcone-cli infrastructure to an existing project.
377
+
378
+ If source != directory, copies source contents into directory first.
379
+ Adds .lightcone/, .claude/, universes/, CLAUDE.md, and .gitignore entries
380
+ without creating boilerplate astra.yaml or overwriting existing files.
381
+ The user then runs /lc-migrate in Claude Code to generate the spec.
382
+ """
383
+ source = source.resolve()
384
+ directory = directory if directory == Path(".") else directory
385
+
386
+ # Copy source into directory if they differ
387
+ if source.resolve() != directory.resolve():
388
+ directory.mkdir(parents=True, exist_ok=True)
389
+ # Copy all files from source, skipping hidden dirs and __pycache__
390
+ for item in source.iterdir():
391
+ if item.name.startswith(".") or item.name == "__pycache__":
392
+ continue
393
+ dest = directory / item.name
394
+ if dest.exists():
395
+ continue
396
+ if item.is_dir():
397
+ shutil.copytree(item, dest, ignore=shutil.ignore_patterns(
398
+ "__pycache__", "*.pyc", ".git",
399
+ ))
400
+ else:
401
+ shutil.copy2(item, dest)
402
+ console.print(
403
+ f"[green]✓[/green] Copied project from [cyan]{source}[/cyan] "
404
+ f"to [cyan]{directory}[/cyan]"
405
+ )
406
+
407
+ # Check if this is already an ASTRA project
408
+ if (directory / "astra.yaml").exists():
409
+ console.print(
410
+ f"[red]Error:[/red] [cyan]{directory}[/cyan] already has an astra.yaml."
411
+ )
412
+ console.print(
413
+ "Use [cyan]astra validate[/cyan] to check it, "
414
+ "or delete astra.yaml and re-run."
415
+ )
416
+ raise SystemExit(1)
417
+
418
+ console.print(
419
+ f"[bold]Adding lightcone-cli infrastructure to: [cyan]{directory}[/cyan][/bold]\n"
420
+ )
421
+
422
+ # Create directories that don't exist yet
423
+ for subdir in ["universes", "results", ".lightcone"]:
424
+ d = directory / subdir
425
+ if not d.exists():
426
+ d.mkdir(parents=True, exist_ok=True)
427
+
428
+ # .lightcone/ internals
429
+ _create_dagster_yaml(directory)
430
+
431
+ # .gitignore — append if exists, create if not
432
+ _create_or_append_gitignore(directory)
433
+
434
+ # CLAUDE.md — only if it doesn't exist
435
+ if not (directory / "CLAUDE.md").exists():
436
+ _create_claude_md(directory)
437
+ else:
438
+ console.print(" [dim]CLAUDE.md already exists, skipping[/dim]")
439
+
440
+ # Containerfile — only if it doesn't exist
441
+ if not (directory / "Containerfile").exists():
442
+ containerfile = """\
443
+ FROM python:3.12-slim
444
+
445
+ WORKDIR /app
446
+
447
+ COPY requirements.txt .
448
+ RUN pip install --no-cache-dir -r requirements.txt
449
+
450
+ COPY . .
451
+ """
452
+ (directory / "Containerfile").write_text(containerfile)
453
+ else:
454
+ console.print(" [dim]Containerfile already exists, skipping[/dim]")
455
+
456
+ # requirements.txt — don't touch if it exists
457
+ if not (directory / "requirements.txt").exists():
458
+ (directory / "requirements.txt").write_text("")
459
+ else:
460
+ console.print(" [dim]requirements.txt already exists, skipping[/dim]")
461
+
462
+ # Claude Code settings
463
+ tier = _resolve_permission_tier(permissions)
464
+ _create_claude_settings(directory, tier)
465
+
466
+ # lightcone.yaml
467
+ effective_target = target
468
+ if not effective_target:
469
+ from lightcone.engine.targets import load_user_config
470
+ effective_target = load_user_config().get("default_target", "local")
471
+ _create_lightcone_config(directory, effective_target)
472
+
473
+ if target and target != "local":
474
+ from lightcone.engine.targets import load_target
475
+ if load_target(target) is None:
476
+ console.print(
477
+ f"\n[yellow]Target [cyan]{target}[/cyan] "
478
+ "is not configured yet.[/yellow]"
479
+ )
480
+ console.print(
481
+ " Run [cyan]lc setup[/cyan] to configure execution targets."
482
+ )
483
+
484
+ # Virtual environment
485
+ _create_venv(directory, no_venv)
486
+
487
+ # Git
488
+ _init_git_repo(directory, no_git)
489
+
490
+ # Success
491
+ console.print(
492
+ f"\n[green]✓[/green] Added lightcone-cli infrastructure to: [cyan]{directory}[/cyan]"
493
+ )
494
+
495
+ console.print(
496
+ "\n[bold]Next steps:[/bold]"
497
+ )
498
+ if directory != Path("."):
499
+ console.print(f" [bold]cd {directory}[/bold]")
500
+ console.print(" [bold]claude[/bold]")
501
+ console.print(" [cyan]/lc-migrate[/cyan]")
502
+
503
+
504
+ def _create_boilerplate_astra_yaml(directory: Path) -> None:
505
+ """Create boilerplate astra.yaml with TODOs."""
506
+
507
+ name = directory.name if directory != Path(".") else "My Analysis"
508
+
509
+ astra_yaml = f"""# ASTRA Analysis Specification
510
+ # Documentation: https://github.com/LightconeResearch/ASTRA
511
+
512
+ version: "1.0"
513
+ name: "{name}"
514
+ description: |
515
+ TODO: What research question are you trying to answer?
516
+
517
+ container: Containerfile
518
+
519
+ inputs:
520
+ - id: primary_data
521
+ type: data
522
+ description: "TODO: Describe your primary data source"
523
+
524
+ outputs:
525
+ - id: main_result
526
+ type: metric
527
+ description: "TODO: Describe your primary output metric"
528
+ recipe:
529
+ command: python scripts/compute.py
530
+
531
+ - id: conclusion
532
+ type: report
533
+ description: "Summary addressing the problem statement"
534
+ recipe:
535
+ command: python scripts/summarize.py
536
+ inputs: [main_result]
537
+
538
+ decisions:
539
+ example_method:
540
+ label: "Example Method Choice"
541
+ tags: [analysis]
542
+ rationale: "TODO: Explain why this decision matters"
543
+ default: option_a
544
+ options:
545
+ option_a:
546
+ label: "Option A"
547
+ description: "TODO: Describe option A"
548
+ option_b:
549
+ label: "Option B"
550
+ description: "TODO: Describe option B"
551
+ """
552
+ (directory / "astra.yaml").write_text(astra_yaml)
553
+
554
+ # Create Containerfile
555
+ containerfile = """\
556
+ FROM python:3.12-slim
557
+
558
+ WORKDIR /app
559
+
560
+ COPY requirements.txt .
561
+ RUN pip install --no-cache-dir -r requirements.txt
562
+
563
+ COPY . .
564
+ """
565
+ (directory / "Containerfile").write_text(containerfile)
566
+
567
+ # Create requirements.txt
568
+ requirements = """\
569
+ numpy
570
+ pandas
571
+ """
572
+ (directory / "requirements.txt").write_text(requirements)
573
+
574
+ # Create baseline universe
575
+ baseline_universe = """# Baseline Universe
576
+ # Default configuration using standard practices
577
+
578
+ id: baseline
579
+ description: "Default configuration using standard practices"
580
+
581
+ decisions:
582
+ example_method: option_a
583
+ """
584
+ (directory / "universes" / "baseline.yaml").write_text(baseline_universe)
585
+
586
+
587
+ def _init_sub_analysis(directory: Path) -> None:
588
+ """Scaffold a sub-analysis directory and wire it into the parent project."""
589
+ from astra.helpers import load_yaml, save_yaml
590
+
591
+ # Resolve the sub-analysis path.
592
+ # If directory has no path separator (e.g. "new_stage"), default to analyses/<name>
593
+ sub_path = directory
594
+ if sub_path == Path("."):
595
+ console.print("[red]Error:[/red] Please provide a name or path for the sub-analysis.")
596
+ raise SystemExit(1)
597
+
598
+ # If the user gave a bare name (no directory separators), put it under analyses/
599
+ if len(sub_path.parts) == 1:
600
+ sub_path = Path("analyses") / sub_path
601
+
602
+ name = sub_path.name
603
+
604
+ # Find the project root by looking for astra.yaml
605
+ project_root = Path.cwd()
606
+ if not (project_root / "astra.yaml").exists():
607
+ console.print(
608
+ "[red]Error:[/red] No astra.yaml found in current directory. "
609
+ "Run this from the project root."
610
+ )
611
+ raise SystemExit(1)
612
+
613
+ abs_sub_path = project_root / sub_path
614
+
615
+ if abs_sub_path.exists() and (abs_sub_path / "astra.yaml").exists():
616
+ console.print(
617
+ f"[red]Error:[/red] Sub-analysis already exists at "
618
+ f"[cyan]{sub_path}[/cyan] (astra.yaml found)."
619
+ )
620
+ raise SystemExit(1)
621
+
622
+ # 1. Create the sub-analysis directory structure
623
+ abs_sub_path.mkdir(parents=True, exist_ok=True)
624
+ (abs_sub_path / "scripts").mkdir(exist_ok=True)
625
+ (abs_sub_path / "scripts" / ".gitkeep").touch()
626
+ (abs_sub_path / "universes").mkdir(exist_ok=True)
627
+ (abs_sub_path / "results").mkdir(exist_ok=True)
628
+
629
+ # Write the sub-analysis astra.yaml
630
+ label = name.replace("_", " ").replace("-", " ").title()
631
+ sub_spec = {
632
+ "name": label,
633
+ "description": "",
634
+ "inputs": [],
635
+ "outputs": [],
636
+ "decisions": {},
637
+ }
638
+ save_yaml(sub_spec, abs_sub_path / "astra.yaml")
639
+
640
+ # Write the sub-analysis baseline universe
641
+ sub_universe = {
642
+ "id": "baseline",
643
+ "description": "Default configuration",
644
+ "decisions": {},
645
+ }
646
+ save_yaml(sub_universe, abs_sub_path / "universes" / "baseline.yaml")
647
+
648
+ # Write CLAUDE.md
649
+ _create_claude_md(abs_sub_path)
650
+
651
+ # 2. Wire into the parent astra.yaml
652
+ root_spec = load_yaml(project_root / "astra.yaml")
653
+ if "analyses" not in root_spec or root_spec["analyses"] is None:
654
+ root_spec["analyses"] = {}
655
+ root_spec["analyses"][name] = {"path": f"./{sub_path}"}
656
+ save_yaml(root_spec, project_root / "astra.yaml")
657
+
658
+ # 3. Wire into all root universe files
659
+ universes_dir = project_root / "universes"
660
+ if universes_dir.is_dir():
661
+ for ufile in sorted(universes_dir.glob("*.yaml")):
662
+ udata = load_yaml(ufile)
663
+ if udata is None:
664
+ continue
665
+ if "analyses" not in udata or udata["analyses"] is None:
666
+ udata["analyses"] = {}
667
+ udata["analyses"][name] = {"universe": "baseline"}
668
+ save_yaml(udata, ufile)
669
+
670
+ console.print(
671
+ f"[green]\u2713[/green] Created sub-analysis "
672
+ f"[cyan]{name}[/cyan] at [cyan]{sub_path}[/cyan]"
673
+ )
674
+ console.print(f" - {sub_path}/astra.yaml")
675
+ console.print(f" - {sub_path}/CLAUDE.md")
676
+ console.print(f" - {sub_path}/scripts/")
677
+ console.print(f" - {sub_path}/results/")
678
+ console.print(f" - {sub_path}/universes/baseline.yaml")
679
+ console.print(" - Wired into root astra.yaml and universe files")
680
+
681
+
682
+ def _create_claude_md(directory: Path) -> None:
683
+ """Create CLAUDE.md from the template in the plugin source."""
684
+ name = directory.name if directory != Path(".") else "My Analysis"
685
+
686
+ # Find the template
687
+ plugin_source = get_plugin_source_dir()
688
+ template_path = plugin_source / "templates" / "CLAUDE.md" if plugin_source else None
689
+
690
+ if template_path and template_path.exists():
691
+ content = template_path.read_text()
692
+ content = content.replace("{{name}}", name)
693
+ else:
694
+ # Fallback: minimal CLAUDE.md if template not found
695
+ content = (
696
+ f"# CLAUDE.md\n\n## Project: {name}\n\n"
697
+ "This is an ASTRA analysis project. Read `astra.yaml` for the specification.\n\n"
698
+ "Run `/lc-new` to scope a research question.\n\n"
699
+ "---\n\n"
700
+ "<!-- AUTOGENERATED: /lc-new populates below during specification -->\n"
701
+ "## Analysis Context\n\n"
702
+ "_Run `/lc-new` to scope the research question and populate this section._\n"
703
+ )
704
+
705
+ (directory / "CLAUDE.md").write_text(content)
706
+
707
+
708
+ def _create_lightcone_config(directory: Path, target_name: str) -> None:
709
+ """Create .lightcone/lightcone.yaml with target reference."""
710
+ config = {
711
+ "target": target_name,
712
+ }
713
+ lightcone_dir = directory / ".lightcone"
714
+ lightcone_dir.mkdir(parents=True, exist_ok=True)
715
+ (lightcone_dir / "lightcone.yaml").write_text(
716
+ yaml.dump(config, default_flow_style=False, sort_keys=False)
717
+ )
718
+ console.print(f"[green]✓[/green] Created .lightcone/lightcone.yaml (target: {target_name})")
719
+
720
+
721
+
722
+ def _prompt_permission_tier() -> str:
723
+ """Interactively prompt the user to choose a permission tier.
724
+
725
+ Returns one of: 'yolo', 'recommended', 'minimal'.
726
+ Saves the choice as the default for future projects.
727
+ """
728
+ console.print("\n[bold]Claude Code permission level[/bold]")
729
+ console.print(" Controls what Claude can do without asking.\n")
730
+ console.print(" 1. yolo — Everything including MCP. No guardrails.")
731
+ console.print(" 2. recommended — Full access with guardrails (no sudo/push/scratch).")
732
+ console.print(" 3. minimal — Only file reading. Everything else prompts.")
733
+
734
+ choice_map = {"1": "yolo", "2": "recommended", "3": "minimal"}
735
+ raw = click.prompt(
736
+ "\n Select permission level",
737
+ type=click.Choice(["1", "2", "3"]),
738
+ default="2",
739
+ )
740
+ tier = choice_map.get(raw, "recommended")
741
+
742
+ from lightcone.engine.targets import load_user_config, save_user_config
743
+ global_config = load_user_config()
744
+ global_config["default_permission_tier"] = tier
745
+ save_user_config(global_config)
746
+ console.print(f" [green]✓[/green] Permissions: {tier}")
747
+
748
+ return tier
749
+
750
+
751
+ def _prompt_extraction_model() -> str:
752
+ """Interactively prompt the user to choose a model for literature extraction subagents.
753
+
754
+ Returns a model name (e.g. 'haiku', 'sonnet') or empty string for inherit.
755
+ Saves the choice to ~/.lightcone/config.yaml.
756
+ """
757
+ from lightcone.engine.targets import load_user_config, save_user_config
758
+
759
+ console.print("\n[bold]Literature extraction model[/bold]")
760
+ console.print(" Model used for paper-reading subagents during /lc-new.\n")
761
+ console.print(" 1. inherit — Use the same model as the main session (default)")
762
+ console.print(" 2. haiku — Fast and cheap, good for straightforward extraction")
763
+ console.print(" 3. sonnet — Balanced cost and capability")
764
+
765
+ choice_map = {"1": "", "2": "haiku", "3": "sonnet"}
766
+ raw = click.prompt(
767
+ "\n Select extraction model",
768
+ type=click.Choice(["1", "2", "3"]),
769
+ default="1",
770
+ )
771
+ model = choice_map.get(raw, "")
772
+
773
+ global_config = load_user_config()
774
+ global_config["extraction_model"] = model
775
+ save_user_config(global_config)
776
+
777
+ display = model if model else "inherit"
778
+ console.print(f" [green]✓[/green] Extraction model: {display}")
779
+ return model
780
+
781
+
782
+ def _resolve_permission_tier(flag_value: str | None) -> str:
783
+ """Resolve which permission tier to use.
784
+
785
+ Priority:
786
+ 1. --permissions flag (explicit override)
787
+ 2. Saved default in ~/.lightcone/config.yaml
788
+ 3. Interactive prompt (first time only)
789
+ """
790
+ # 1. Explicit flag
791
+ if flag_value is not None:
792
+ console.print(f" Permissions: [cyan]{flag_value}[/cyan] (--permissions flag)")
793
+ return flag_value
794
+
795
+ # 2. Saved default
796
+ from lightcone.engine.targets import load_user_config
797
+ global_config = load_user_config()
798
+ saved = global_config.get("default_permission_tier")
799
+ if saved:
800
+ console.print(f" Permissions: [cyan]{saved}[/cyan] (saved default)")
801
+ return saved
802
+
803
+ # 3. Interactive prompt
804
+ return _prompt_permission_tier()
805
+
806
+
807
+ def _update_extractor_agent_model(agents_dir: Path) -> None:
808
+ """Update the lc-extractor agent definition with the configured extraction model.
809
+
810
+ Reads extraction_model from ~/.lightcone/config.yaml and sets the model field
811
+ in the agent's YAML frontmatter. If empty/missing, removes the model field
812
+ so the agent inherits the parent model.
813
+ """
814
+ from lightcone.engine.targets import load_user_config
815
+
816
+ extractor_path = agents_dir / "lc-extractor.md"
817
+ if not extractor_path.exists():
818
+ return
819
+
820
+ user_config = load_user_config()
821
+ extraction_model = user_config.get("extraction_model", "sonnet")
822
+
823
+ content = extractor_path.read_text()
824
+
825
+ # Insert or remove model field in frontmatter
826
+ if extraction_model:
827
+ # Add model field after description line
828
+ if "model:" not in content:
829
+ content = content.replace(
830
+ "\ntools: Read, Bash",
831
+ f"\nmodel: {extraction_model}\ntools: Read, Bash",
832
+ )
833
+ else:
834
+ # Update existing model field
835
+ import re
836
+ content = re.sub(r"model: \w+", f"model: {extraction_model}", content)
837
+
838
+ extractor_path.write_text(content)
839
+
840
+
841
+ def _create_claude_settings(
842
+ directory: Path, tier: str = "recommended", target: str = "local",
843
+ ) -> None:
844
+ """Create Claude Code settings with lightcone-cli skills and agents.
845
+
846
+ If a non-local target maps to a known HPC site, site-specific deny rules
847
+ (e.g. scratch filesystem paths) are merged into the permissions.
848
+ """
849
+ claude_dir = directory / ".claude"
850
+ claude_dir.mkdir(parents=True, exist_ok=True)
851
+
852
+ # Find the plugin source directory
853
+ plugin_source = get_plugin_source_dir()
854
+ if plugin_source is None:
855
+ console.print(
856
+ "[yellow]Warning:[/yellow] Could not find lightcone-cli plugin source files. "
857
+ "Claude Code skills will not be available."
858
+ )
859
+ return
860
+
861
+ # Copy scripts
862
+ scripts_src = plugin_source / "scripts"
863
+ scripts_dst = claude_dir / "scripts"
864
+ if scripts_src.exists():
865
+ if scripts_dst.exists():
866
+ shutil.rmtree(scripts_dst)
867
+ shutil.copytree(scripts_src, scripts_dst)
868
+ # Make scripts executable
869
+ for script in scripts_dst.glob("*.sh"):
870
+ script.chmod(script.stat().st_mode | 0o111)
871
+
872
+ # Copy hooks
873
+ hooks_src = plugin_source / "hooks"
874
+ hooks_dst = claude_dir / "hooks"
875
+ if hooks_src.exists():
876
+ if hooks_dst.exists():
877
+ shutil.rmtree(hooks_dst)
878
+ shutil.copytree(hooks_src, hooks_dst)
879
+ # Make .py files executable
880
+ for hook in hooks_dst.glob("*.py"):
881
+ hook.chmod(hook.stat().st_mode | 0o111)
882
+
883
+ # Copy skills
884
+ skills_src = plugin_source / "skills"
885
+ skills_dst = claude_dir / "skills"
886
+ if skills_src.exists():
887
+ if skills_dst.exists():
888
+ shutil.rmtree(skills_dst)
889
+ shutil.copytree(skills_src, skills_dst)
890
+
891
+ # Copy agents and apply extraction model config
892
+ agents_src = plugin_source / "agents"
893
+ agents_dst = claude_dir / "agents"
894
+ if agents_src.exists():
895
+ if agents_dst.exists():
896
+ shutil.rmtree(agents_dst)
897
+ shutil.copytree(agents_src, agents_dst)
898
+ _update_extractor_agent_model(agents_dst)
899
+
900
+ # Copy guides
901
+ guides_src = plugin_source / "guides"
902
+ guides_dst = claude_dir / "guides"
903
+ if guides_src.exists():
904
+ if guides_dst.exists():
905
+ shutil.rmtree(guides_dst)
906
+ shutil.copytree(guides_src, guides_dst)
907
+
908
+ # Build permissions: start from tier, then merge site-specific deny rules
909
+ permissions: dict[str, list[str]] = {
910
+ k: list(v) for k, v in PERMISSION_TIERS[tier].items()
911
+ }
912
+ if target != "local" and "deny" in permissions:
913
+ from lightcone.engine.site_registry import detect_site, get_site_scratch_deny_rules
914
+ site_key = detect_site(target)
915
+ if site_key:
916
+ site_deny = get_site_scratch_deny_rules(site_key)
917
+ existing = set(permissions["deny"])
918
+ for rule in site_deny:
919
+ if rule not in existing:
920
+ permissions["deny"].append(rule)
921
+
922
+ # Build absolute paths for hook commands
923
+ abs_hooks = str(directory.resolve() / ".claude" / "hooks")
924
+
925
+ # Create settings.json with hooks configured directly
926
+ settings: dict[str, Any] = {
927
+ "permissions": permissions,
928
+ "hooks": {
929
+ "SessionStart": [
930
+ {
931
+ "hooks": [
932
+ {
933
+ "type": "command",
934
+ "command": ".claude/scripts/activate-venv.sh",
935
+ "timeout": 5,
936
+ },
937
+ {
938
+ "type": "command",
939
+ "command": ".claude/scripts/session-start.sh",
940
+ "timeout": 10,
941
+ },
942
+ ],
943
+ },
944
+ ],
945
+ "Stop": [
946
+ {
947
+ "matcher": "",
948
+ "hooks": [
949
+ {
950
+ "type": "command",
951
+ "command": f"python3 {abs_hooks}/langfuse_hook.py",
952
+ "timeout": 30,
953
+ },
954
+ ],
955
+ },
956
+ ],
957
+ "SessionEnd": [
958
+ {
959
+ "matcher": "",
960
+ "hooks": [
961
+ {
962
+ "type": "command",
963
+ "command": f"python3 {abs_hooks}/langfuse_hook.py",
964
+ "timeout": 30,
965
+ },
966
+ ],
967
+ },
968
+ ],
969
+ "PreToolUse": [
970
+ {
971
+ "matcher": "",
972
+ "hooks": [
973
+ {
974
+ "type": "command",
975
+ "command": f"python3 {abs_hooks}/langfuse_session_init_hook.py",
976
+ "timeout": 10,
977
+ },
978
+ ],
979
+ },
980
+ ],
981
+ "PostToolUse": [
982
+ {
983
+ "matcher": "Write|Edit",
984
+ "hooks": [
985
+ {
986
+ "type": "command",
987
+ "command": ".claude/scripts/validate-on-save.sh",
988
+ "timeout": 15,
989
+ },
990
+ ],
991
+ },
992
+ {
993
+ "matcher": "Bash",
994
+ "hooks": [
995
+ {
996
+ "type": "command",
997
+ "command": ".claude/scripts/check-lc-run.sh",
998
+ "timeout": 15,
999
+ },
1000
+ {
1001
+ "type": "command",
1002
+ "command": f"python3 {abs_hooks}/langfuse_git_commit_hook.py",
1003
+ "timeout": 15,
1004
+ },
1005
+ ],
1006
+ },
1007
+ ],
1008
+ },
1009
+ }
1010
+
1011
+ settings_file = claude_dir / "settings.json"
1012
+ settings_file.write_text(json.dumps(settings, indent=2) + "\n")
1013
+
1014
+ # Create settings.local.json with telemetry environment variables
1015
+ settings_local: dict[str, Any] = {
1016
+ "env": {
1017
+ "TRACE_TO_LANGFUSE": "true",
1018
+ "LANGFUSE_PUBLIC_KEY": (
1019
+ "ced0ca0cf048a05ac1f272cf1e70693233f6932722738eadd6a56fa361f213cf"
1020
+ ),
1021
+ "LANGFUSE_SECRET_KEY": "relay",
1022
+ "LANGFUSE_HOST": "https://prism-telemetry.lightconeresearch.workers.dev",
1023
+ },
1024
+ }
1025
+ settings_local_file = claude_dir / "settings.local.json"
1026
+ settings_local_file.write_text(json.dumps(settings_local, indent=2) + "\n")
1027
+
1028
+
1029
+ def _init_git_repo(directory: Path, no_git: bool) -> None:
1030
+ """Initialize git repository if requested."""
1031
+ if no_git or (directory / ".git").exists():
1032
+ return
1033
+
1034
+ try:
1035
+ subprocess.run(
1036
+ ["git", "init"],
1037
+ cwd=directory,
1038
+ capture_output=True,
1039
+ check=True,
1040
+ )
1041
+ console.print("[green]✓[/green] Initialized git repository")
1042
+ try:
1043
+ subprocess.run(["git", "add", "."], cwd=directory, capture_output=True, check=True)
1044
+ subprocess.run(
1045
+ ["git", "commit", "-m", "Initial ASTRA analysis structure"],
1046
+ cwd=directory,
1047
+ capture_output=True,
1048
+ check=True,
1049
+ )
1050
+ except subprocess.CalledProcessError:
1051
+ pass
1052
+ except (subprocess.CalledProcessError, FileNotFoundError):
1053
+ pass
1054
+
1055
+
1056
+ def _create_venv(directory: Path, no_venv: bool) -> bool:
1057
+ """Create a virtual environment with lightcone-cli installed from PyPI."""
1058
+ if no_venv:
1059
+ return False
1060
+
1061
+ venv_path = directory / ".venv"
1062
+
1063
+ try:
1064
+ subprocess.run(
1065
+ [sys.executable, "-m", "venv", str(venv_path)],
1066
+ capture_output=True,
1067
+ check=True,
1068
+ )
1069
+ except subprocess.CalledProcessError as e:
1070
+ console.print(f"[yellow]Warning:[/yellow] Failed to create virtual environment: {e}")
1071
+ return False
1072
+
1073
+ console.print("[green]✓[/green] Created virtual environment (.venv)")
1074
+
1075
+ pip_path = venv_path / ("Scripts" if sys.platform == "win32" else "bin") / "pip"
1076
+ try:
1077
+ subprocess.run(
1078
+ [str(pip_path), "install", "lightcone-cli"],
1079
+ capture_output=True,
1080
+ check=True,
1081
+ )
1082
+ console.print("[green]✓[/green] Installed lightcone-cli in virtual environment")
1083
+ except subprocess.CalledProcessError:
1084
+ console.print(
1085
+ "[yellow]Warning:[/yellow] Could not install lightcone-cli automatically. "
1086
+ "You can install manually with: .venv/bin/pip install lightcone-cli"
1087
+ )
1088
+
1089
+ return True
1090
+
1091
+
1092
+ # =============================================================================
1093
+ # Dagster execution commands
1094
+ # =============================================================================
1095
+
1096
+
1097
+ @main.command(context_settings={
1098
+ "ignore_unknown_options": True,
1099
+ "allow_extra_args": True,
1100
+ })
1101
+ @click.argument("outputs", nargs=-1, type=click.UNPROCESSED)
1102
+ @click.option("--universe", "-u", default=None, help="Universe to materialize for")
1103
+ @click.option("--target", "-t", default=None, help="Execution target name")
1104
+ @click.option("--no-build", is_flag=True, help="Skip automatic container image builds")
1105
+ @click.pass_context
1106
+ def run(
1107
+ ctx: click.Context,
1108
+ outputs: tuple[str, ...],
1109
+ universe: str | None,
1110
+ target: str | None,
1111
+ no_build: bool,
1112
+ ) -> None:
1113
+ """Materialize ASTRA outputs via Dagster.
1114
+
1115
+ Runs recipes to produce outputs. Without arguments, materializes all
1116
+ outputs for all universes. Container build specs are automatically
1117
+ built before execution unless --no-build is given.
1118
+
1119
+ Any unknown flags are passed through as SLURM scheduling directives
1120
+ (e.g. --partition, --qos, --constraint, --gres).
1121
+
1122
+ Examples:
1123
+ lc run # all outputs, all universes
1124
+ lc run accuracy # specific output
1125
+ lc run --universe baseline # specific universe
1126
+ lc run accuracy -u baseline # specific output + universe
1127
+ lc run --target perlmutter # run on SLURM
1128
+ lc run --qos shared --constraint gpu # SLURM scheduling flags
1129
+ lc run --partition gpu-a100 # works for any cluster
1130
+ lc run --no-build # skip container builds
1131
+ """
1132
+ from lightcone.engine.assets import build_definitions
1133
+ from lightcone.engine.targets import load_target
1134
+
1135
+ # Separate output names from SLURM flags in the combined args
1136
+ all_args = list(outputs) + ctx.args
1137
+ output_names = [a for a in all_args if not a.startswith("-")]
1138
+ slurm_args = [a for a in all_args if a.startswith("-")]
1139
+
1140
+ project_path = Path.cwd()
1141
+ if not (project_path / "astra.yaml").exists():
1142
+ console.print("[red]Error:[/red] No astra.yaml found in current directory.")
1143
+ raise SystemExit(1)
1144
+
1145
+ # Resolve target: --target flag > .lightcone/lightcone.yaml > default from user config
1146
+ target_name = target
1147
+ if not target_name:
1148
+ lightcone_data = _load_lightcone_config(project_path)
1149
+ target_name = lightcone_data.get("target")
1150
+ if not target_name:
1151
+ from lightcone.engine.targets import load_user_config
1152
+ target_name = load_user_config().get("default_target")
1153
+
1154
+ # Load target config directly — no merging
1155
+ target_config = None
1156
+ if target_name and target_name != "local":
1157
+ target_config = load_target(target_name)
1158
+
1159
+ # Pass through any extra SLURM flags
1160
+ if slurm_args and target_config:
1161
+ target_config["extra_slurm_args"] = slurm_args
1162
+
1163
+ universe_id = universe or "baseline"
1164
+ defs = build_definitions(
1165
+ project_path, target_config=target_config, universe_id=universe_id,
1166
+ no_build=no_build,
1167
+ )
1168
+
1169
+ console.print("[bold]Materializing outputs...[/bold]")
1170
+
1171
+ import dagster as dg
1172
+
1173
+ # Select assets to materialize (exclude external/input-only assets)
1174
+ all_assets = list(defs.resolve_all_asset_specs())
1175
+ if output_names:
1176
+ # Support dot-notation: hod_fitting.galaxy_mesh -> [universe, hod_fitting, galaxy_mesh]
1177
+ selection = [dg.AssetKey([universe_id] + o.split(".")) for o in output_names]
1178
+ else:
1179
+ selection = [
1180
+ spec.key for spec in all_assets
1181
+ if not (spec.metadata or {}).get('external', False)
1182
+ ]
1183
+
1184
+ # Ensure dagster.yaml exists so materialization events are persisted.
1185
+ # Without this, events are lost and lc status can't detect them.
1186
+ dagster_yaml_path = _find_dagster_yaml(project_path)
1187
+ if dagster_yaml_path is None:
1188
+ lightcone_dir = project_path / ".lightcone"
1189
+ lightcone_dir.mkdir(parents=True, exist_ok=True)
1190
+ dagster_yaml_path = lightcone_dir / "dagster.yaml"
1191
+ dagster_yaml_content = {
1192
+ "storage": {"sqlite": {"base_dir": "results/.dagster"}},
1193
+ }
1194
+ dagster_yaml_path.write_text(
1195
+ yaml.dump(dagster_yaml_content, default_flow_style=False, sort_keys=False)
1196
+ )
1197
+ instance = dg.DagsterInstance.from_config(str(dagster_yaml_path.parent))
1198
+
1199
+ # Execute
1200
+ try:
1201
+ result = dg.materialize(
1202
+ assets=list(defs.assets),
1203
+ selection=selection,
1204
+ instance=instance,
1205
+ )
1206
+ if result.success:
1207
+ console.print("[green]✓[/green] Materialization complete")
1208
+ else:
1209
+ console.print("[red]✗[/red] Materialization failed")
1210
+ raise SystemExit(1)
1211
+ except Exception as e:
1212
+ console.print(f"[red]Error:[/red] {e}")
1213
+ raise SystemExit(1)
1214
+
1215
+
1216
+ @main.command()
1217
+ @click.option("--force", is_flag=True, help="Rebuild images even if they already exist")
1218
+ @click.option(
1219
+ "--runtime", "-r",
1220
+ type=click.Choice(["docker", "podman", "podman-hpc"]),
1221
+ default=None,
1222
+ help="Container runtime to build with (auto-detected from target config)",
1223
+ )
1224
+ def build(force: bool, runtime: str | None) -> None:
1225
+ """Build container images from Containerfile specs in astra.yaml.
1226
+
1227
+ Scans the analysis specification for container build specs (both
1228
+ analysis-level and per-recipe) and builds any missing images.
1229
+ Images are content-addressed — rebuilds only happen when the
1230
+ Containerfile or dependency files change.
1231
+
1232
+ The container runtime is auto-detected from the project's target
1233
+ config (.lightcone/lightcone.yaml → ~/.lightcone/targets/). Use --runtime to override.
1234
+
1235
+ Examples:
1236
+ lc build # auto-detect runtime from target
1237
+ lc build --runtime podman-hpc # force podman-hpc
1238
+ lc build --runtime docker # force docker
1239
+ lc build --force # rebuild all images
1240
+ """
1241
+ from astra.helpers import get_outputs, load_yaml, resolve_analysis_tree
1242
+
1243
+ from lightcone.engine.container import (
1244
+ ContainerBuildError,
1245
+ resolve_container_for_slurm,
1246
+ resolve_container_spec,
1247
+ )
1248
+
1249
+ project_path = Path.cwd()
1250
+ if not (project_path / "astra.yaml").exists():
1251
+ console.print("[red]Error:[/red] No astra.yaml found in current directory.")
1252
+ raise SystemExit(1)
1253
+
1254
+ # Resolve runtime from target config if not explicitly provided
1255
+ if runtime is None:
1256
+ from lightcone.engine.targets import load_target, load_user_config
1257
+ lightcone_data = _load_lightcone_config(project_path)
1258
+ target_name = lightcone_data.get("target")
1259
+ if not target_name:
1260
+ target_name = load_user_config().get("default_target")
1261
+ if target_name and target_name != "local":
1262
+ target_config = load_target(target_name)
1263
+ if target_config:
1264
+ runtime = target_config.get("container_runtime", "docker")
1265
+ if runtime is None:
1266
+ from lightcone.engine.container import detect_container_runtime
1267
+ runtime = detect_container_runtime()
1268
+ if runtime is None:
1269
+ console.print(
1270
+ "[red]Error:[/red] No container runtime found (Docker or Podman).\n"
1271
+ " Install Docker or Podman to build container images."
1272
+ )
1273
+ raise SystemExit(1)
1274
+
1275
+ spec = load_yaml(project_path / "astra.yaml")
1276
+ spec = resolve_analysis_tree(spec, project_path)
1277
+ project_name = spec.get("name") or project_path.name
1278
+
1279
+ # Collect all unique container specs that need building or migrating.
1280
+ from lightcone.engine.container import is_containerfile
1281
+
1282
+ build_specs: list[tuple[str, str]] = [] # (label, spec)
1283
+ raw_default = spec.get("container")
1284
+ if raw_default is not None:
1285
+ if is_containerfile(raw_default, project_path):
1286
+ build_specs.append(("analysis-level", raw_default))
1287
+ elif runtime != "docker":
1288
+ # Pre-built images need pull/migrate for HPC runtimes
1289
+ build_specs.append(("analysis-level", raw_default))
1290
+
1291
+ for output_def in get_outputs(spec):
1292
+ recipe = output_def.get("recipe")
1293
+ if not recipe:
1294
+ continue
1295
+ raw = recipe.get("container")
1296
+ if raw is not None:
1297
+ label = f"recipe:{output_def.get('id', '?')}"
1298
+ if is_containerfile(raw, project_path):
1299
+ build_specs.append((label, raw))
1300
+ elif runtime != "docker":
1301
+ build_specs.append((label, raw))
1302
+
1303
+ if not build_specs:
1304
+ console.print("[dim]No container build specs found in astra.yaml.[/dim]")
1305
+ return
1306
+
1307
+ console.print(
1308
+ f"[bold]Found {len(build_specs)} container spec(s) "
1309
+ f"(runtime: {runtime})[/bold]\n"
1310
+ )
1311
+
1312
+ for label, bspec in build_specs:
1313
+ try:
1314
+ if runtime == "podman-hpc":
1315
+ tag = resolve_container_for_slurm(
1316
+ bspec, project_path, project_name, runtime, force=force,
1317
+ )
1318
+ else:
1319
+ tag = resolve_container_spec(
1320
+ bspec, project_path, project_name, force=force, runtime=runtime,
1321
+ )
1322
+ console.print(f" [green]ready[/green] {label} -> {tag}")
1323
+ except ContainerBuildError as e:
1324
+ console.print(f" [red]fail[/red] {label}: {e}")
1325
+
1326
+
1327
+ def _status_label(s: str) -> str:
1328
+ """Format a status string for rich display."""
1329
+ if s == "materialized":
1330
+ return "[green]ok[/green]"
1331
+ elif s == "pending":
1332
+ return "[dim]pending[/dim]"
1333
+ elif s == "alias":
1334
+ return "[cyan]alias[/cyan]"
1335
+ return "[yellow]no recipe[/yellow]"
1336
+
1337
+
1338
+ def _display_tree_status(
1339
+ name: str,
1340
+ groups: dict,
1341
+ all_status: dict[str, dict[str, str]],
1342
+ ) -> None:
1343
+ """Display status grouped by sub-analysis as a tree."""
1344
+ from rich.tree import Tree
1345
+
1346
+ for uid, universe_status in all_status.items():
1347
+ tree = Tree(f"[bold]{name}[/bold] universe: {uid}")
1348
+
1349
+ for analysis_id, outputs in groups.items():
1350
+ if analysis_id is None:
1351
+ # Root-level outputs
1352
+ for out_id, out_def in outputs:
1353
+ s = universe_status.get(out_id, "no_recipe")
1354
+ tree.add(f"{out_id:40s} {_status_label(s)}")
1355
+ else:
1356
+ branch = tree.add(f"[bold cyan]{analysis_id}/[/bold cyan]")
1357
+ for out_id, out_def in outputs:
1358
+ qualified = f"{analysis_id}/{out_id}"
1359
+ s = universe_status.get(qualified, "no_recipe")
1360
+ branch.add(f"{out_id:40s} {_status_label(s)}")
1361
+
1362
+ console.print(tree)
1363
+
1364
+
1365
+ def _display_flat_status(
1366
+ name: str,
1367
+ outputs: list[tuple[str, dict]],
1368
+ all_status: dict[str, dict[str, str]],
1369
+ ) -> None:
1370
+ """Display status as a flat table (original behavior)."""
1371
+ from rich.table import Table
1372
+
1373
+ table = Table(title=f"{name} -- Output Status")
1374
+ table.add_column("Output", style="cyan")
1375
+ for uid in all_status:
1376
+ table.add_column(uid)
1377
+
1378
+ for out_id, out_def in outputs:
1379
+ if not out_id:
1380
+ continue
1381
+ row = [out_id]
1382
+ for uid, universe_status in all_status.items():
1383
+ s = universe_status.get(out_id, "no_recipe")
1384
+ row.append(_status_label(s))
1385
+ table.add_row(*row)
1386
+
1387
+ console.print(table)
1388
+
1389
+
1390
+ @main.command()
1391
+ @click.option("--universe", "-u", default=None, help="Show status for specific universe")
1392
+ def status(universe: str | None) -> None:
1393
+ """Show materialization status of all outputs.
1394
+
1395
+ Displays a table of outputs vs universes with materialization state.
1396
+
1397
+ Examples:
1398
+ lc status
1399
+ lc status --universe baseline
1400
+ """
1401
+ from astra.helpers import load_yaml, resolve_analysis_tree
1402
+
1403
+ from lightcone.engine.status import get_all_universe_status, get_output_status
1404
+
1405
+ project_path = Path.cwd()
1406
+ if not (project_path / "astra.yaml").exists():
1407
+ console.print("[red]Error:[/red] No astra.yaml found in current directory.")
1408
+ raise SystemExit(1)
1409
+
1410
+ spec = load_yaml(project_path / "astra.yaml")
1411
+ spec = resolve_analysis_tree(spec, project_path)
1412
+ name = spec.get("name", "Unknown")
1413
+
1414
+ if universe:
1415
+ all_status = {universe: get_output_status(project_path, universe)}
1416
+ else:
1417
+ all_status = get_all_universe_status(project_path)
1418
+
1419
+ if not all_status:
1420
+ console.print("[yellow]No universes found.[/yellow]")
1421
+ return
1422
+
1423
+ # Collect all qualified output IDs grouped by sub-analysis
1424
+ from lightcone.engine.tree import collect_tree_outputs
1425
+
1426
+ tree_outputs = collect_tree_outputs(spec)
1427
+
1428
+ # Group outputs by analysis_id (None for root)
1429
+ from collections import OrderedDict
1430
+
1431
+ groups: OrderedDict[str | None, list[tuple[str, dict]]] = OrderedDict()
1432
+ for tree_out in tree_outputs:
1433
+ gid = tree_out.analysis_id
1434
+ if gid not in groups:
1435
+ groups[gid] = []
1436
+ groups[gid].append((tree_out.output_id, tree_out.output_def))
1437
+
1438
+ # Display as tree when sub-analyses exist
1439
+ has_sub = any(k is not None for k in groups)
1440
+
1441
+ if has_sub:
1442
+ _display_tree_status(name, groups, all_status)
1443
+ else:
1444
+ _display_flat_status(name, groups.get(None, []), all_status)
1445
+
1446
+ # Count totals across all groups
1447
+ recipe_count = 0
1448
+ total_outputs = 0
1449
+ materialized_count = 0
1450
+ total_cells = 0
1451
+ for tree_out in tree_outputs:
1452
+ out_id = tree_out.output_id
1453
+ if not out_id:
1454
+ continue
1455
+ total_outputs += 1
1456
+ has_recipe = bool(tree_out.output_def.get("recipe"))
1457
+ if has_recipe:
1458
+ recipe_count += 1
1459
+ if tree_out.analysis_id:
1460
+ qualified = f"{tree_out.analysis_id}/{out_id}"
1461
+ else:
1462
+ qualified = out_id
1463
+ for uid, universe_status in all_status.items():
1464
+ if has_recipe:
1465
+ total_cells += 1
1466
+ if universe_status.get(qualified) == "materialized":
1467
+ materialized_count += 1
1468
+
1469
+ console.print(f"\n Recipes: {recipe_count}/{total_outputs} outputs integrated")
1470
+ console.print(f" Materialized: {materialized_count}/{total_cells} runs")
1471
+
1472
+ # Show container status
1473
+ from lightcone.engine.container import detect_container_runtime, get_container_status
1474
+
1475
+ raw_container = spec.get("container")
1476
+ rt = detect_container_runtime() or "docker"
1477
+ cstatus = get_container_status(raw_container, project_path, name, runtime=rt)
1478
+ if cstatus.type == "prebuilt":
1479
+ console.print(f" Container: prebuilt [cyan]{cstatus.image}[/cyan]")
1480
+ elif cstatus.type == "build":
1481
+ if cstatus.exists:
1482
+ console.print(
1483
+ f" Container: build {cstatus.containerfile} "
1484
+ f"[green]{cstatus.image} (built)[/green]"
1485
+ )
1486
+ else:
1487
+ console.print(
1488
+ f" Container: build {cstatus.containerfile} "
1489
+ f"[yellow]{cstatus.image} (not built)[/yellow]"
1490
+ )
1491
+
1492
+
1493
+ @main.command()
1494
+ @click.option("--port", "-p", default=3000, type=int, help="Port for Dagster webserver")
1495
+ @click.option("--universe", "-u", default="baseline", help="Universe to load definitions for")
1496
+ def dev(port: int, universe: str) -> None:
1497
+ """Launch Dagster webserver UI for the current project.
1498
+
1499
+ Opens a web UI showing the asset graph, run history, and
1500
+ materialization status.
1501
+
1502
+ Examples:
1503
+ lc dev
1504
+ lc dev --port 8080
1505
+ lc dev --universe experiment1
1506
+ """
1507
+ import tempfile
1508
+
1509
+ project_path = Path.cwd()
1510
+ if not (project_path / "astra.yaml").exists():
1511
+ console.print("[red]Error:[/red] No astra.yaml found in current directory.")
1512
+ raise SystemExit(1)
1513
+
1514
+ console.print(f"[bold]Starting Dagster webserver on port {port}...[/bold]")
1515
+ console.print(f" Open [cyan]http://localhost:{port}[/cyan] in your browser")
1516
+ console.print("[dim]Press Ctrl+C to stop[/dim]\n")
1517
+
1518
+ # Generate a temporary Python file that builds Dagster Definitions from
1519
+ # the current ASTRA project. dagster-webserver discovers assets via -f.
1520
+ defs_code = (
1521
+ "from pathlib import Path\n"
1522
+ "from lightcone.engine.assets import build_definitions\n"
1523
+ f"defs = build_definitions(Path({str(project_path)!r}), "
1524
+ f"universe_id={universe!r}, no_build=True)\n"
1525
+ )
1526
+
1527
+ try:
1528
+ with tempfile.NamedTemporaryFile(
1529
+ mode="w", suffix=".py", prefix="lightcone_defs_", delete=False,
1530
+ ) as f:
1531
+ f.write(defs_code)
1532
+ defs_file = f.name
1533
+
1534
+ dagster_yaml_path = _find_dagster_yaml(project_path)
1535
+ dagster_home = str(dagster_yaml_path.parent) if dagster_yaml_path else str(project_path)
1536
+ env = {**os.environ, "DAGSTER_HOME": dagster_home}
1537
+ subprocess.run(
1538
+ ["dagster-webserver", "-f", defs_file, "-h", "0.0.0.0", "-p", str(port)],
1539
+ check=True,
1540
+ env=env,
1541
+ )
1542
+ except KeyboardInterrupt:
1543
+ console.print("\n[dim]Dagster webserver stopped[/dim]")
1544
+ except FileNotFoundError:
1545
+ console.print("[red]Error:[/red] dagster-webserver not found.")
1546
+ console.print(" Install with: [cyan]pip install lightcone-cli[/cyan]")
1547
+ raise SystemExit(1)
1548
+ finally:
1549
+ # Clean up the temporary definitions file
1550
+ try:
1551
+ Path(defs_file).unlink(missing_ok=True)
1552
+ except NameError:
1553
+ pass
1554
+
1555
+
1556
+ # =============================================================================
1557
+ # Target command
1558
+ # =============================================================================
1559
+
1560
+
1561
+ @main.group(invoke_without_command=True)
1562
+ @click.option("--set", "set_target", default=None, help="Set project target")
1563
+ @click.option("--list", "list_flag", is_flag=True, help="List available targets")
1564
+ @click.option("--show", "show_name", default=None, help="Show a target's config")
1565
+ @click.pass_context
1566
+ def target(
1567
+ ctx: click.Context,
1568
+ set_target: str | None,
1569
+ list_flag: bool,
1570
+ show_name: str | None,
1571
+ ) -> None:
1572
+ """Show or manage execution targets for this project."""
1573
+ if ctx.invoked_subcommand is not None:
1574
+ return
1575
+
1576
+ from lightcone.engine.targets import list_targets, load_target
1577
+
1578
+ if set_target:
1579
+ # Update target key in .lightcone/lightcone.yaml
1580
+ project_path = Path.cwd()
1581
+ lightcone_yaml = _find_lightcone_yaml(project_path)
1582
+ if lightcone_yaml is None:
1583
+ console.print("[red]Error:[/red] No lightcone.yaml found. Run 'lc init' first.")
1584
+ raise SystemExit(1)
1585
+
1586
+ # Verify target exists (or is "local")
1587
+ if set_target != "local" and load_target(set_target) is None:
1588
+ console.print(f"[red]Error:[/red] No configured target '{set_target}'.")
1589
+ console.print(
1590
+ f" Available: {', '.join(list_targets()) or 'none'}"
1591
+ )
1592
+ raise SystemExit(1)
1593
+
1594
+ with open(lightcone_yaml) as f:
1595
+ data = yaml.safe_load(f) or {}
1596
+ data["target"] = set_target
1597
+ with open(lightcone_yaml, "w") as f:
1598
+ yaml.dump(data, f, default_flow_style=False, sort_keys=False)
1599
+ console.print(f"[green]✓[/green] Project target set to '{set_target}'")
1600
+ return
1601
+
1602
+ if show_name:
1603
+ config = load_target(show_name)
1604
+ if config is None:
1605
+ console.print(f"[red]Error:[/red] No configured target '{show_name}'.")
1606
+ raise SystemExit(1)
1607
+ console.print(f"[bold]Target: {show_name}[/bold]\n")
1608
+ console.print(yaml.dump(config, default_flow_style=False, sort_keys=False))
1609
+ return
1610
+
1611
+ if list_flag:
1612
+ from lightcone.engine.targets import load_user_config
1613
+ saved = list_targets()
1614
+ user_config = load_user_config()
1615
+ default = user_config.get("default_target", "")
1616
+
1617
+ console.print("[bold]Available targets:[/bold]")
1618
+ local_marker = " [green](default)[/green]" if default == "local" else ""
1619
+ console.print(f" - local (built-in){local_marker}")
1620
+ if saved:
1621
+ for t in saved:
1622
+ marker = " [green](default)[/green]" if t == default else ""
1623
+ console.print(f" - {t}{marker}")
1624
+ else:
1625
+ console.print(" [dim](no additional targets configured)[/dim]")
1626
+ console.print(
1627
+ "\nRun [cyan]lc target add[/cyan] to create a new target."
1628
+ )
1629
+ return
1630
+
1631
+ # Default: show current project target
1632
+ project_path = Path.cwd()
1633
+ lightcone_yaml = _find_lightcone_yaml(project_path)
1634
+ if lightcone_yaml is None:
1635
+ console.print("No lightcone.yaml found. Run [cyan]lc init[/cyan] first.")
1636
+ return
1637
+
1638
+ with open(lightcone_yaml) as f:
1639
+ data = yaml.safe_load(f) or {}
1640
+ current = data.get("target", "not set")
1641
+ console.print(f" Current target: [cyan]{current}[/cyan]")
1642
+
1643
+ # Check if it's configured
1644
+ if current != "local" and current != "not set":
1645
+ config = load_target(current)
1646
+ if config:
1647
+ console.print(f" Backend: {config.get('backend', 'unknown')}")
1648
+ conn = config.get("connection", {})
1649
+ if conn.get("hostname"):
1650
+ console.print(f" Host: {conn['hostname']}")
1651
+ else:
1652
+ console.print(" [yellow]Warning: target not found in ~/.lightcone/targets/[/yellow]")
1653
+ console.print("\n Use [cyan]lc target --set <name>[/cyan] to change.")
1654
+
1655
+
1656
+ @target.command("add")
1657
+ @click.argument("name", required=False)
1658
+ def target_add(name: str | None) -> None:
1659
+ """Create a new execution target."""
1660
+ from lightcone.engine.site_registry import get_site_defaults, list_known_sites
1661
+ from lightcone.engine.targets import save_target
1662
+
1663
+ console.print("\n[bold]Create New Target[/bold]\n")
1664
+
1665
+ # --- Site selection ---
1666
+ known = list_known_sites()
1667
+ hpc_sites = [(k, d) for k, d in known if k != "local"]
1668
+
1669
+ console.print(" [bold]Site type:[/bold]")
1670
+ console.print(" 1. Local (Docker)")
1671
+ for i, (_key, display) in enumerate(hpc_sites, 2):
1672
+ console.print(f" {i}. {display}")
1673
+
1674
+ choices = [str(i) for i in range(1, len(hpc_sites) + 2)]
1675
+ choice = click.prompt(
1676
+ "\n Select site type",
1677
+ type=click.Choice(choices),
1678
+ default="1",
1679
+ )
1680
+
1681
+ if choice == "1":
1682
+ # Local target
1683
+ target_name = name or "local"
1684
+ config: dict[str, Any] = {
1685
+ "site": "local",
1686
+ "backend": "local",
1687
+ "connection": {},
1688
+ }
1689
+ else:
1690
+ site_key = hpc_sites[int(choice) - 2][0]
1691
+ site = get_site_defaults(site_key) or {}
1692
+ hostname = site.get("connection", {}).get("hostname", "")
1693
+
1694
+ # --- Connection ---
1695
+ username = click.prompt(
1696
+ " Username",
1697
+ default=os.environ.get("USER", ""),
1698
+ )
1699
+ account = click.prompt(" Account/allocation")
1700
+
1701
+ # --- Container runtime ---
1702
+ site_runtimes = site.get("container_runtimes", [])
1703
+ if len(site_runtimes) > 1:
1704
+ console.print("\n [bold]Container runtime:[/bold]")
1705
+ for i, rt in enumerate(site_runtimes, 1):
1706
+ console.print(f" {i}. {rt}")
1707
+ rt_choices = [str(i) for i in range(1, len(site_runtimes) + 1)]
1708
+ rt_idx = click.prompt(
1709
+ " Select runtime",
1710
+ type=click.Choice(rt_choices),
1711
+ default="1",
1712
+ )
1713
+ container_runtime = site_runtimes[int(rt_idx) - 1]
1714
+ elif site_runtimes:
1715
+ container_runtime = site_runtimes[0]
1716
+ else:
1717
+ container_runtime = site.get(
1718
+ "scheduler", {},
1719
+ ).get("container_runtime", "docker")
1720
+
1721
+ # --- Node type selection ---
1722
+ node_types = site.get("node_types", {})
1723
+ if node_types:
1724
+ console.print("\n [bold]Node type:[/bold]")
1725
+ nt_list = list(node_types.items())
1726
+ for i, (nt_key, nt_info) in enumerate(nt_list, 1):
1727
+ console.print(f" {i}. {nt_key} — {nt_info.get('description', '')}")
1728
+ nt_choices = [str(i) for i in range(1, len(nt_list) + 1)]
1729
+ nt_idx = click.prompt(
1730
+ " Select node type",
1731
+ type=click.Choice(nt_choices),
1732
+ default="1",
1733
+ )
1734
+ nt_key, nt_info = nt_list[int(nt_idx) - 1]
1735
+ constraint = nt_info.get("constraint", nt_key)
1736
+ else:
1737
+ nt_key = "default"
1738
+ constraint = ""
1739
+
1740
+ target_name = name or f"{site_key}-{nt_key}"
1741
+
1742
+ # QOS selection
1743
+ qos_options = site.get("qos_options", {})
1744
+ qos = "regular"
1745
+ if qos_options:
1746
+ console.print("\n [bold]QOS:[/bold]")
1747
+ qos_list = list(qos_options.items())
1748
+ for i, (qos_key, qos_info) in enumerate(qos_list, 1):
1749
+ console.print(f" {i}. {qos_key} — {qos_info.get('description', '')}")
1750
+ qos_choices = [str(i) for i in range(1, len(qos_list) + 1)]
1751
+ qos_idx = click.prompt(
1752
+ " Select QOS",
1753
+ type=click.Choice(qos_choices),
1754
+ default="1",
1755
+ )
1756
+ qos = qos_list[int(qos_idx) - 1][0]
1757
+
1758
+ resource_limits = site.get("resource_limits", {})
1759
+ config = {
1760
+ "site": site_key,
1761
+ "backend": site.get("backend", "slurm"),
1762
+ "connection": {
1763
+ "hostname": hostname,
1764
+ "username": username,
1765
+ },
1766
+ "account": account,
1767
+ "container_runtime": container_runtime,
1768
+ "constraint": constraint,
1769
+ "qos": qos,
1770
+ }
1771
+
1772
+ # --- Resource limits ---
1773
+ console.print("\n [bold]Resource limits[/bold]")
1774
+ console.print(" (these cap what Claude can request per job)\n")
1775
+
1776
+ config["max_nodes"] = click.prompt(
1777
+ " Max nodes per job",
1778
+ type=int,
1779
+ default=resource_limits.get("max_nodes", 4),
1780
+ )
1781
+ config["max_walltime_minutes"] = click.prompt(
1782
+ " Max walltime (minutes)",
1783
+ type=int,
1784
+ default=resource_limits.get("max_walltime_minutes", 360),
1785
+ )
1786
+ config["max_concurrent_jobs"] = click.prompt(
1787
+ " Max concurrent jobs",
1788
+ type=int,
1789
+ default=resource_limits.get("max_concurrent_jobs", 8),
1790
+ )
1791
+
1792
+ path = save_target(target_name, config)
1793
+ console.print(f"\n [green]✓[/green] Created target '{target_name}' at {path}")
1794
+
1795
+
1796
+ @target.command("edit")
1797
+ @click.argument("name")
1798
+ def target_edit(name: str) -> None:
1799
+ """Edit an existing execution target."""
1800
+ from lightcone.engine.targets import load_target, save_target
1801
+
1802
+ config = load_target(name)
1803
+ if config is None:
1804
+ console.print(f"[red]Error:[/red] No configured target '{name}'.")
1805
+ raise SystemExit(1)
1806
+
1807
+ console.print(f"\n[bold]Edit Target: {name}[/bold]")
1808
+ console.print(" Press Enter to keep current value.\n")
1809
+
1810
+ # Edit each field
1811
+ backend = click.prompt(" Backend", default=config.get("backend", "local"))
1812
+ config["backend"] = backend
1813
+
1814
+ conn = config.get("connection", {})
1815
+ if backend != "local":
1816
+ hostname = click.prompt(" Hostname", default=conn.get("hostname", ""))
1817
+ username = click.prompt(" Username", default=conn.get("username", ""))
1818
+ config["connection"] = {"hostname": hostname, "username": username}
1819
+
1820
+ account = click.prompt(" Account", default=config.get("account", ""))
1821
+ if account:
1822
+ config["account"] = account
1823
+
1824
+ runtime = click.prompt(
1825
+ " Container runtime",
1826
+ default=config.get("container_runtime", "docker"),
1827
+ )
1828
+ config["container_runtime"] = runtime
1829
+
1830
+ constraint = click.prompt(
1831
+ " Constraint",
1832
+ default=config.get("constraint", ""),
1833
+ )
1834
+ if constraint:
1835
+ config["constraint"] = constraint
1836
+
1837
+ qos = click.prompt(" QOS", default=config.get("qos", ""))
1838
+ if qos:
1839
+ config["qos"] = qos
1840
+
1841
+ # --- Resource limits ---
1842
+ console.print("\n [bold]Resource limits:[/bold]")
1843
+ config["max_nodes"] = click.prompt(
1844
+ " Max nodes per job",
1845
+ type=int,
1846
+ default=config.get("max_nodes", 4),
1847
+ )
1848
+ config["max_walltime_minutes"] = click.prompt(
1849
+ " Max walltime (minutes)",
1850
+ type=int,
1851
+ default=config.get("max_walltime_minutes", 360),
1852
+ )
1853
+ config["max_concurrent_jobs"] = click.prompt(
1854
+ " Max concurrent jobs",
1855
+ type=int,
1856
+ default=config.get("max_concurrent_jobs", 8),
1857
+ )
1858
+
1859
+ path = save_target(name, config)
1860
+ console.print(f"\n [green]✓[/green] Updated target '{name}' at {path}")
1861
+
1862
+
1863
+ # =============================================================================
1864
+ # Setup command
1865
+ # =============================================================================
1866
+
1867
+
1868
+ def _run_setup_menu() -> None:
1869
+ """Show the setup management menu when config already exists."""
1870
+ from lightcone.engine.targets import list_targets, load_user_config, save_user_config
1871
+
1872
+ user_config = load_user_config()
1873
+ default = user_config.get("default_target", "local")
1874
+ tier = user_config.get("default_permission_tier", "recommended")
1875
+ extraction_model = user_config.get("extraction_model", "sonnet")
1876
+ extraction_display = extraction_model if extraction_model else "inherit"
1877
+ targets = list_targets()
1878
+ target_names = ["local"] + [t for t in targets if t != "local"]
1879
+
1880
+ console.print("\n[bold]lightcone-cli Setup[/bold]")
1881
+ console.print(f" Default target: {default}")
1882
+ console.print(f" Permission level: {tier}")
1883
+ console.print(f" Extraction model: {extraction_display}")
1884
+ console.print(f" Targets: {', '.join(target_names)}")
1885
+
1886
+ console.print("\n 1. Change permission level")
1887
+ console.print(" 2. Change extraction model")
1888
+ console.print(" 3. Add a target")
1889
+ console.print(" 4. Edit a target")
1890
+ console.print(" 5. Change default target")
1891
+ console.print(" 6. Re-run setup wizard")
1892
+ console.print(" 7. Exit")
1893
+
1894
+ choice = click.prompt(
1895
+ "\n Select action",
1896
+ type=click.Choice(["1", "2", "3", "4", "5", "6", "7"]),
1897
+ default="7",
1898
+ )
1899
+
1900
+ if choice == "7":
1901
+ return
1902
+ elif choice == "1":
1903
+ _prompt_permission_tier()
1904
+ elif choice == "2":
1905
+ _prompt_extraction_model()
1906
+ elif choice == "3":
1907
+ ctx = click.get_current_context()
1908
+ ctx.invoke(target_add)
1909
+ elif choice == "4":
1910
+ console.print("\n [bold]Targets:[/bold]")
1911
+ for i, t in enumerate(target_names, 1):
1912
+ console.print(f" {i}. {t}")
1913
+ idx = click.prompt(
1914
+ " Select target to edit",
1915
+ type=click.Choice([str(i) for i in range(1, len(target_names) + 1)]),
1916
+ default="1",
1917
+ )
1918
+ chosen = target_names[int(idx) - 1]
1919
+ ctx = click.get_current_context()
1920
+ ctx.invoke(target_edit, name=chosen)
1921
+ elif choice == "5":
1922
+ console.print("\n [bold]Targets:[/bold]")
1923
+ for i, t in enumerate(target_names, 1):
1924
+ console.print(f" {i}. {t}")
1925
+ idx = click.prompt(
1926
+ " Select new default",
1927
+ type=click.Choice([str(i) for i in range(1, len(target_names) + 1)]),
1928
+ default="1",
1929
+ )
1930
+ chosen = target_names[int(idx) - 1]
1931
+ user_config["default_target"] = chosen
1932
+ save_user_config(user_config)
1933
+ console.print(f" [green]✓[/green] Default target: {chosen}")
1934
+ elif choice == "6":
1935
+ _run_setup_wizard()
1936
+
1937
+
1938
+ def _run_setup_wizard() -> list[Path]:
1939
+ """Run the interactive setup wizard.
1940
+
1941
+ Creates one target per node type for HPC sites, plus a local target.
1942
+ Returns the list of paths where target configs were saved.
1943
+ """
1944
+ from lightcone.engine.site_registry import get_site_defaults, list_known_sites
1945
+ from lightcone.engine.targets import load_user_config, save_target, save_user_config
1946
+
1947
+ console.print("\n[bold]lightcone-cli Setup — Target Configuration[/bold]")
1948
+ console.print(
1949
+ " These settings are stored in [cyan]~/.lightcone/targets/[/cyan] and "
1950
+ "referenced by projects via .lightcone/lightcone.yaml.\n"
1951
+ )
1952
+
1953
+ saved_paths: list[Path] = []
1954
+ default_target = "local"
1955
+
1956
+ # --- Configure HPC? ---
1957
+ configure_hpc = click.confirm(
1958
+ " Configure a remote execution site (HPC)?",
1959
+ default=False,
1960
+ )
1961
+
1962
+ if configure_hpc:
1963
+ # --- HPC site selection ---
1964
+ known = list_known_sites()
1965
+ hpc_sites = [(k, d) for k, d in known if k != "local"]
1966
+ console.print("\n [bold]HPC sites:[/bold]")
1967
+ for i, (_key, display) in enumerate(hpc_sites, 1):
1968
+ console.print(f" {i}. {display}")
1969
+ console.print(f" {len(hpc_sites) + 1}. Other SLURM cluster")
1970
+
1971
+ site_choices = [str(i) for i in range(1, len(hpc_sites) + 2)]
1972
+ site_idx = click.prompt(
1973
+ "\n Select site",
1974
+ type=click.Choice(site_choices),
1975
+ default="1",
1976
+ )
1977
+ selected_idx = int(site_idx) - 1
1978
+
1979
+ if selected_idx < len(hpc_sites):
1980
+ # --- Known site ---
1981
+ site_key = hpc_sites[selected_idx][0]
1982
+ site = get_site_defaults(site_key) or {}
1983
+
1984
+ display = site.get("display_name", site_key)
1985
+ hostname = site.get("connection", {}).get("hostname", "")
1986
+ console.print(
1987
+ f" Detected: [cyan]{display}[/cyan] ({hostname})\n"
1988
+ )
1989
+
1990
+ username = click.prompt(
1991
+ " Username",
1992
+ default=os.environ.get("USER", ""),
1993
+ )
1994
+ account = click.prompt(" Account/allocation")
1995
+
1996
+ # Container runtime — auto-select if only one
1997
+ site_runtimes = site.get("container_runtimes", [])
1998
+ if len(site_runtimes) > 1:
1999
+ console.print("\n [bold]Container runtime:[/bold]")
2000
+ for i, rt in enumerate(site_runtimes, 1):
2001
+ console.print(f" {i}. {rt}")
2002
+ rt_choices = [
2003
+ str(i) for i in range(1, len(site_runtimes) + 1)
2004
+ ]
2005
+ rt_idx = click.prompt(
2006
+ " Select runtime",
2007
+ type=click.Choice(rt_choices),
2008
+ default="1",
2009
+ )
2010
+ container_runtime = site_runtimes[int(rt_idx) - 1]
2011
+ elif site_runtimes:
2012
+ container_runtime = site_runtimes[0]
2013
+ else:
2014
+ container_runtime = None
2015
+
2016
+ default_name = f"{site_key}-{account}"
2017
+ target_name = click.prompt(" Target name", default=default_name)
2018
+
2019
+ target_config: dict[str, Any] = {
2020
+ "site": site_key,
2021
+ "backend": site.get("backend", "slurm"),
2022
+ "connection": {
2023
+ "hostname": hostname,
2024
+ "username": username,
2025
+ },
2026
+ "account": account,
2027
+ }
2028
+ if container_runtime:
2029
+ target_config["container_runtime"] = container_runtime
2030
+
2031
+ else:
2032
+ # --- Custom SLURM cluster ---
2033
+ console.print("\n [bold]Custom SLURM cluster[/bold]\n")
2034
+
2035
+ cluster_name = click.prompt(" Cluster name (e.g. frontier, summit)")
2036
+ hostname = click.prompt(" Hostname", default=cluster_name)
2037
+ username = click.prompt(
2038
+ " Username",
2039
+ default=os.environ.get("USER", ""),
2040
+ )
2041
+ account = click.prompt(" Account/allocation")
2042
+
2043
+ use_containers = click.confirm(
2044
+ " Use a container runtime?",
2045
+ default=False,
2046
+ )
2047
+ container_runtime = None
2048
+ if use_containers:
2049
+ container_runtime = click.prompt(
2050
+ " Container runtime",
2051
+ default="singularity",
2052
+ )
2053
+
2054
+ default_name = f"{cluster_name}-{account}"
2055
+ target_name = click.prompt(" Target name", default=default_name)
2056
+
2057
+ target_config = {
2058
+ "backend": "slurm",
2059
+ "connection": {
2060
+ "hostname": hostname,
2061
+ "username": username,
2062
+ },
2063
+ "account": account,
2064
+ }
2065
+ if container_runtime:
2066
+ target_config["container_runtime"] = container_runtime
2067
+
2068
+ path = save_target(target_name, target_config)
2069
+ saved_paths.append(path)
2070
+ console.print(f" [green]✓[/green] Created target: {target_name}")
2071
+
2072
+ default_target = target_name
2073
+
2074
+ # --- Always create local target ---
2075
+ local_config: dict[str, Any] = {
2076
+ "site": "local",
2077
+ "backend": "local",
2078
+ "connection": {},
2079
+ }
2080
+ path = save_target("local", local_config)
2081
+ saved_paths.append(path)
2082
+ console.print(" [green]✓[/green] Created target: local")
2083
+
2084
+ # --- Set default ---
2085
+ user_config = load_user_config()
2086
+ user_config["default_target"] = default_target
2087
+ save_user_config(user_config)
2088
+ console.print(f"\n [green]✓[/green] Default target: {default_target}")
2089
+
2090
+ # --- Extraction model (default to sonnet) ---
2091
+ if "extraction_model" not in user_config:
2092
+ user_config["extraction_model"] = "sonnet"
2093
+ save_user_config(user_config)
2094
+
2095
+ console.print(
2096
+ "\n To list configured targets: [cyan]lc target --list[/cyan]"
2097
+ "\n To add more targets: [cyan]lc target add[/cyan]"
2098
+ "\n To edit a target: [cyan]lc target edit <name>[/cyan]"
2099
+ )
2100
+
2101
+ return saved_paths
2102
+
2103
+
2104
+ @main.command()
2105
+ @click.option("--list", "list_flag", is_flag=True, help="List configured targets")
2106
+ @click.option("--show", "show_name", default=None, help="Show a target's config")
2107
+ @click.option("--default", "set_default", default=None, help="Set default target")
2108
+ def setup(
2109
+ list_flag: bool,
2110
+ show_name: str | None, set_default: str | None,
2111
+ ) -> None:
2112
+ """Set up execution targets (first-time experience).
2113
+
2114
+ Configures connection details and container runtime for remote
2115
+ execution backends (SLURM). Creates one target per node type.
2116
+
2117
+ Settings are stored at the user level (~/.lightcone/targets/) and
2118
+ referenced by projects via .lightcone/lightcone.yaml.
2119
+
2120
+ Examples:
2121
+ lc setup # interactive wizard
2122
+ lc setup --list # list configured targets
2123
+ lc setup --show perlmutter-gpu # show a target's config
2124
+ lc setup --default local # change default target
2125
+ """
2126
+ if set_default:
2127
+ from lightcone.engine.targets import load_target, load_user_config, save_user_config
2128
+ # Validate target exists (or is "local")
2129
+ if set_default != "local":
2130
+ target_config = load_target(set_default)
2131
+ if target_config is None:
2132
+ console.print(f"[red]Error:[/red] No configured target '{set_default}'.")
2133
+ raise SystemExit(1)
2134
+ user_config = load_user_config()
2135
+ user_config["default_target"] = set_default
2136
+ save_user_config(user_config)
2137
+ console.print(f"[green]✓[/green] Default target set to '{set_default}'")
2138
+ return
2139
+
2140
+ if show_name:
2141
+ from lightcone.engine.targets import load_target
2142
+ config = load_target(show_name)
2143
+ if config is None:
2144
+ console.print(f"[red]Error:[/red] No configured target '{show_name}'.")
2145
+ raise SystemExit(1)
2146
+ console.print(f"[bold]Target: {show_name}[/bold]\n")
2147
+ console.print(yaml.dump(config, default_flow_style=False, sort_keys=False))
2148
+ return
2149
+
2150
+ if list_flag:
2151
+ from lightcone.engine.targets import list_targets, load_user_config
2152
+ saved = list_targets()
2153
+ user_config = load_user_config()
2154
+ default = user_config.get("default_target", "")
2155
+
2156
+ console.print("[bold]Configured targets:[/bold]")
2157
+ # Always show local
2158
+ local_marker = " [green](default)[/green]" if default == "local" else ""
2159
+ console.print(f" - local (built-in){local_marker}")
2160
+ if saved:
2161
+ for t in saved:
2162
+ marker = " [green](default)[/green]" if t == default else ""
2163
+ console.print(f" - {t}{marker}")
2164
+ else:
2165
+ console.print(" [dim](no additional targets configured)[/dim]")
2166
+ console.print(
2167
+ "\nRun [cyan]lc target add[/cyan] to create a new target."
2168
+ )
2169
+ return
2170
+
2171
+ from lightcone.engine.targets import get_config_path
2172
+ if get_config_path().exists():
2173
+ _run_setup_menu()
2174
+ else:
2175
+ _run_setup_wizard()
2176
+
2177
+
2178
+ # =============================================================================
2179
+ # Update command
2180
+ # =============================================================================
2181
+
2182
+ # Marker that separates the lightcone-cli-managed portion of CLAUDE.md from user content.
2183
+ _CLAUDE_MD_SEPARATOR = "## Analysis Context"
2184
+
2185
+
2186
+ def _sync_project_plugins(project_dir: Path) -> bool:
2187
+ """Sync plugin files (skills, hooks, scripts, agents, CLAUDE.md) into a project.
2188
+
2189
+ Returns True if the sync succeeded.
2190
+ """
2191
+ if not (project_dir / "astra.yaml").exists():
2192
+ console.print(f" [red]✗[/red] {project_dir}: not an ASTRA project (no astra.yaml)")
2193
+ return False
2194
+
2195
+ plugin_source = get_plugin_source_dir()
2196
+ if plugin_source is None:
2197
+ console.print(" [red]✗[/red] Could not find lightcone-cli plugin source files.")
2198
+ return False
2199
+
2200
+ claude_dir = project_dir / ".claude"
2201
+ claude_dir.mkdir(parents=True, exist_ok=True)
2202
+
2203
+ # Sync directories: skills, hooks, scripts, agents, guides
2204
+ for subdir in ("scripts", "hooks", "skills", "agents", "guides"):
2205
+ src = plugin_source / subdir
2206
+ dst = claude_dir / subdir
2207
+ if not src.exists():
2208
+ continue
2209
+ if dst.exists():
2210
+ shutil.rmtree(dst)
2211
+ shutil.copytree(src, dst)
2212
+ # Make executable as needed
2213
+ if subdir == "scripts":
2214
+ for f in dst.glob("*.sh"):
2215
+ f.chmod(f.stat().st_mode | 0o111)
2216
+ elif subdir == "hooks":
2217
+ for f in dst.glob("*.py"):
2218
+ f.chmod(f.stat().st_mode | 0o111)
2219
+
2220
+ # Apply extraction model config to agents
2221
+ agents_dst = claude_dir / "agents"
2222
+ if agents_dst.exists():
2223
+ _update_extractor_agent_model(agents_dst)
2224
+
2225
+ # Update the managed portion of CLAUDE.md (everything above "## Analysis Context")
2226
+ claude_md = project_dir / "CLAUDE.md"
2227
+ if claude_md.exists():
2228
+ existing = claude_md.read_text()
2229
+ # Find the separator
2230
+ sep_idx = existing.find(_CLAUDE_MD_SEPARATOR)
2231
+ if sep_idx != -1:
2232
+ user_section = existing[sep_idx:]
2233
+ else:
2234
+ # No separator found — preserve everything as user content
2235
+ user_section = (
2236
+ f"{_CLAUDE_MD_SEPARATOR}\n\n"
2237
+ "_Run `/lc-new` to scope the research question and populate "
2238
+ "this section with domain context and implementation notes not "
2239
+ "captured in astra.yaml._\n"
2240
+ )
2241
+
2242
+ # Get fresh template
2243
+ name = project_dir.name
2244
+ template_path = plugin_source / "templates" / "CLAUDE.md"
2245
+ if template_path.exists():
2246
+ template = template_path.read_text().replace("{{name}}", name)
2247
+ template_sep_idx = template.find(_CLAUDE_MD_SEPARATOR)
2248
+ if template_sep_idx != -1:
2249
+ managed_section = template[:template_sep_idx]
2250
+ else:
2251
+ managed_section = template + "\n"
2252
+ else:
2253
+ managed_section = (
2254
+ f"# CLAUDE.md\n\n## Project: {name}\n\n"
2255
+ "ASTRA analysis project, built with lightcone-cli.\n\n---\n\n"
2256
+ "<!-- AUTOGENERATED: /lc-new populates below during specification -->\n"
2257
+ )
2258
+
2259
+ claude_md.write_text(managed_section + user_section)
2260
+
2261
+ console.print(f" [green]✓[/green] {project_dir}")
2262
+ return True
2263
+
2264
+
2265
+ def _prompt_sync_projects() -> None:
2266
+ """Prompt the user to sync plugin files into existing projects."""
2267
+ console.print(
2268
+ "\n[bold]Sync updated plugin files to your projects?[/bold]"
2269
+ )
2270
+ console.print(
2271
+ " This updates skills, hooks, scripts, and CLAUDE.md in each project's .claude/ directory."
2272
+ )
2273
+ raw = click.prompt(
2274
+ "\n Enter project paths (comma-separated), or skip",
2275
+ default="skip",
2276
+ )
2277
+ if raw.strip().lower() in ("skip", "s", ""):
2278
+ return
2279
+
2280
+ paths = [Path(p.strip()).expanduser().resolve() for p in raw.split(",") if p.strip()]
2281
+ if not paths:
2282
+ return
2283
+
2284
+ console.print()
2285
+ for p in paths:
2286
+ _sync_project_plugins(p)
2287
+
2288
+
2289
+ @main.command()
2290
+ @click.option("--sync", is_flag=True, help="Only sync plugin files to projects (skip upgrade)")
2291
+ def update(sync: bool) -> None:
2292
+ """Upgrade lightcone-cli and sync plugin files to projects.
2293
+
2294
+ Upgrades lightcone-cli from PyPI, then offers to sync
2295
+ updated skills, hooks, and scripts into your projects.
2296
+
2297
+ Examples:
2298
+ lc update # upgrade package & sync projects
2299
+ lc update --sync # just sync plugin files (no upgrade)
2300
+ """
2301
+ if not sync:
2302
+ console.print("[bold]Upgrading lightcone-cli...[/bold]\n")
2303
+ proc = subprocess.run(
2304
+ [sys.executable, "-m", "pip", "install", "--upgrade", "lightcone-cli"],
2305
+ capture_output=True,
2306
+ text=True,
2307
+ )
2308
+ if proc.returncode == 0:
2309
+ console.print(" [green]✓[/green] lightcone-cli upgraded")
2310
+ else:
2311
+ console.print(f" [red]✗[/red] upgrade failed: {proc.stderr.strip()[:200]}")
2312
+ raise SystemExit(1)
2313
+
2314
+ _prompt_sync_projects()
2315
+
2316
+
2317
+ # Register eval subgroup (requires optional 'eval' extra)
2318
+ try:
2319
+ from lightcone.eval.cli import eval_group
2320
+
2321
+ main.add_command(eval_group, "eval")
2322
+ except ImportError:
2323
+ pass
2324
+
2325
+
2326
+ if __name__ == "__main__":
2327
+ main()