lightcone-cli 0.3.3__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/PKG-INFO +1 -1
  2. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/agents/lc-extractor.md +1 -0
  3. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/guides/lightcone-cli-reference.md +0 -1
  4. lightcone_cli-0.3.4/claude/lightcone/hooks.json +30 -0
  5. lightcone_cli-0.3.4/claude/lightcone/scripts/activate-venv.sh +23 -0
  6. lightcone_cli-0.3.4/claude/lightcone/scripts/session-start.sh +72 -0
  7. lightcone_cli-0.3.4/claude/lightcone/scripts/validate-on-save.sh +44 -0
  8. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/skills/lc-new/SKILL.md +1 -1
  9. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/cli/commands.py +8 -48
  10. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/build.py +14 -51
  11. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/sandbox.py +73 -64
  12. lightcone_cli-0.3.3/claude/lightcone/scripts/activate-venv.sh +0 -44
  13. lightcone_cli-0.3.3/claude/lightcone/scripts/check-lc-run.sh +0 -107
  14. lightcone_cli-0.3.3/claude/lightcone/scripts/session-start.sh +0 -120
  15. lightcone_cli-0.3.3/claude/lightcone/scripts/validate-on-save.sh +0 -77
  16. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/.gitignore +0 -0
  17. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/LICENSE +0 -0
  18. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/guides/astra-reference.md +0 -0
  19. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/hooks/langfuse_git_commit_hook.py +0 -0
  20. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/hooks/langfuse_hook.py +0 -0
  21. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/hooks/langfuse_prepare_commit_msg.py +0 -0
  22. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/hooks/langfuse_session_init_hook.py +0 -0
  23. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/hooks/langfuse_utils.py +0 -0
  24. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/skills/lc-feedback/SKILL.md +0 -0
  25. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/skills/lc-migrate/SKILL.md +0 -0
  26. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/claude/lightcone/templates/CLAUDE.md +0 -0
  27. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/pyproject.toml +0 -0
  28. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/cli/__init__.py +0 -0
  29. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/cli/plugin.py +0 -0
  30. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/__init__.py +0 -0
  31. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/container.py +0 -0
  32. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/dask_cluster.py +0 -0
  33. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/manifest.py +0 -0
  34. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/runner.py +0 -0
  35. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/scratch.py +0 -0
  36. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/site_registry.py +0 -0
  37. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/snakefile.py +0 -0
  38. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/status.py +0 -0
  39. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/tree.py +0 -0
  40. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/validation.py +0 -0
  41. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/engine/verify.py +0 -0
  42. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/__init__.py +0 -0
  43. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/cli.py +0 -0
  44. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/graders.py +0 -0
  45. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/harness.py +0 -0
  46. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/models.py +0 -0
  47. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/lightcone/eval/report.py +0 -0
  48. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/snakemake_executor_plugin_dask/__init__.py +0 -0
  49. {lightcone_cli-0.3.3 → lightcone_cli-0.3.4}/src/snakemake_executor_plugin_dask/executor.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lightcone-cli
3
- Version: 0.3.3
3
+ Version: 0.3.4
4
4
  Summary: Lightcone command-line toolchain
5
5
  Author: Lightcone Research
6
6
  License: BSD-3-Clause
@@ -2,6 +2,7 @@
2
2
  name: lc-extractor
3
3
  description: Extract prior insights from scientific papers for ASTRA analyses. Reads PDFs, identifies claims relevant to target decisions, extracts verbatim quotes, and verifies them. Use for literature extraction during /lc-new.
4
4
  tools: Read, Bash
5
+ model: sonnet
5
6
  ---
6
7
 
7
8
  You are an ASTRA prior insight extraction agent with self-validation capability. Your task is to extract prior insights from a single paper and format them for an ASTRA analysis. Prior insights are knowledge from literature that informs analysis decisions — they go in the `prior_insights:` section of astra.yaml.
@@ -20,7 +20,6 @@ The first `lc` invocation auto-creates `~/.lightcone/config.yaml`:
20
20
  ```yaml
21
21
  container:
22
22
  runtime: auto # or: docker | podman | podman-hpc | none
23
- # extraction_model: sonnet # optional; controls the lc-extractor agent
24
23
  ```
25
24
 
26
25
  **Always run via `lc`.** Recipes must execute through `lc run` so that container builds, option resolution, resource limits, and result paths are applied. Treat the underlying execution engine as a black box — never invoke schedulers or container runtimes directly, that will bypass reproducibility guarantees.
@@ -0,0 +1,30 @@
1
+ {
2
+ "SessionStart": [
3
+ {
4
+ "hooks": [
5
+ {
6
+ "type": "command",
7
+ "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/activate-venv.sh",
8
+ "timeout": 5
9
+ },
10
+ {
11
+ "type": "command",
12
+ "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/session-start.sh",
13
+ "timeout": 15
14
+ }
15
+ ]
16
+ }
17
+ ],
18
+ "PostToolUse": [
19
+ {
20
+ "matcher": "Write|Edit",
21
+ "hooks": [
22
+ {
23
+ "type": "command",
24
+ "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/validate-on-save.sh",
25
+ "timeout": 15
26
+ }
27
+ ]
28
+ }
29
+ ]
30
+ }
@@ -0,0 +1,23 @@
1
+ #!/bin/bash
2
+ # SessionStart hook: prepend the project venv to PATH for every Bash command
3
+ # Claude Code spawns this session.
4
+ #
5
+ # We don't `source` activate -- we only need PATH and VIRTUAL_ENV. Sourcing
6
+ # activate also runs prompt mutations and defines a deactivate function we
7
+ # don't want, and any non-zero status under `set -e` would silently skip
8
+ # the writeback (the failure mode behind issue #103). Writing the two
9
+ # exports directly to CLAUDE_ENV_FILE is the documented mechanism and is
10
+ # what every downstream hook implicitly relies on for `astra` / `lc` to
11
+ # resolve to the project venv rather than whatever system install happens
12
+ # to be on PATH.
13
+
14
+ VENV="$CLAUDE_PROJECT_DIR/.venv"
15
+ [ -d "$VENV/bin" ] || exit 0
16
+ [ -n "$CLAUDE_ENV_FILE" ] || exit 0
17
+
18
+ {
19
+ echo "export VIRTUAL_ENV=$VENV"
20
+ echo "export PATH=$VENV/bin:\$PATH"
21
+ } >> "$CLAUDE_ENV_FILE"
22
+
23
+ exit 0
@@ -0,0 +1,72 @@
1
+ #!/bin/bash
2
+ # SessionStart hook: surface a terse project status to the agent.
3
+ #
4
+ # Reports validation status, materialization counts, and pointers to the
5
+ # canonical reference docs. Project name / decision count / universe count
6
+ # are intentionally omitted -- they are trivia the agent reads from
7
+ # astra.yaml and CLAUDE.md when needed, and they cost against the 10k
8
+ # additionalContext budget.
9
+
10
+ input=$(cat)
11
+ cwd=$(echo "$input" | jq -r '.cwd // empty')
12
+
13
+ [ -z "$cwd" ] && exit 0
14
+ cd "$cwd" 2>/dev/null || exit 0
15
+ [ -f "astra.yaml" ] || exit 0
16
+
17
+ # astra and lc both come from the project venv (prepended to PATH by
18
+ # activate-venv.sh). If neither resolved, the venv setup is broken and
19
+ # there is nothing useful we can report.
20
+ command -v astra &>/dev/null || exit 0
21
+ command -v lc &>/dev/null || exit 0
22
+
23
+ validation_output=$(astra validate astra.yaml 2>&1)
24
+ validation_ok=$?
25
+
26
+ status_json=$(lc status --json 2>/dev/null)
27
+ counts=$(echo "$status_json" | jq -r '
28
+ [.universes[].outputs[].status] as $s |
29
+ {
30
+ ok: ($s | map(select(. == "ok")) | length),
31
+ stale: ($s | map(select(. == "stale")) | length),
32
+ missing: ($s | map(select(. == "missing")) | length),
33
+ alias: ($s | map(select(. == "alias")) | length)
34
+ } | "\(.ok) \(.stale) \(.missing) \(.alias)"
35
+ ' 2>/dev/null)
36
+ read -r ok_count stale_count missing_count alias_count <<<"$counts"
37
+ ok_count=${ok_count:-0}
38
+ stale_count=${stale_count:-0}
39
+ missing_count=${missing_count:-0}
40
+ alias_count=${alias_count:-0}
41
+
42
+ if [ "$validation_ok" -eq 0 ]; then
43
+ summary="ASTRA project — validation: valid"
44
+ else
45
+ summary="ASTRA project — validation: has errors"
46
+ fi
47
+
48
+ summary="$summary
49
+ Materialization: ok=$ok_count stale=$stale_count missing=$missing_count alias=$alias_count
50
+
51
+ References: .claude/guides/astra-reference.md (spec) and .claude/guides/lightcone-cli-reference.md (CLI)."
52
+
53
+ if [ "$validation_ok" -ne 0 ]; then
54
+ # tail rather than head -- the leading lines are success markers
55
+ # ("✓ Schema validation passed" etc.) and the actual error block is
56
+ # at the bottom.
57
+ error_preview=$(echo "$validation_output" | tail -20)
58
+ summary="$summary
59
+
60
+ Validation errors (run 'astra validate astra.yaml' for full output):
61
+ $error_preview"
62
+ fi
63
+
64
+ needs_run=$((missing_count + stale_count))
65
+ if [ "$needs_run" -gt 0 ]; then
66
+ summary="$summary
67
+
68
+ ACTION REQUIRED: $needs_run output(s) need \`lc run\` ($missing_count missing, $stale_count stale)."
69
+ fi
70
+
71
+ jq -n --arg ctx "$summary" '{hookSpecificOutput: {hookEventName: "SessionStart", additionalContext: $ctx}}'
72
+ exit 0
@@ -0,0 +1,44 @@
1
+ #!/bin/bash
2
+ # PostToolUse(Write|Edit) hook: re-validate after writes to astra.yaml or a
3
+ # universe file, push errors back to the agent as additionalContext.
4
+ #
5
+ # astra is resolved from the project venv via PATH (prepended at
6
+ # SessionStart by activate-venv.sh). Issue #103 traced back to this hook
7
+ # running an older, globally-installed astra when activate-venv failed
8
+ # silently -- with the venv reliably on PATH that whole code path is
9
+ # unnecessary.
10
+
11
+ input=$(cat)
12
+ file_path=$(echo "$input" | jq -r '.tool_input.file_path // .tool_response.filePath // empty')
13
+ [ -z "$file_path" ] && exit 0
14
+
15
+ filename=$(basename "$file_path")
16
+ parent=$(basename "$(dirname "$file_path")")
17
+
18
+ # Filter to astra.yaml at any depth and universe files (universes/*.yaml)
19
+ if [ "$filename" = "astra.yaml" ]; then
20
+ project_root=$(dirname "$file_path")
21
+ elif [ "$parent" = "universes" ] && [[ "$filename" == *.yaml ]]; then
22
+ project_root=$(dirname "$(dirname "$file_path")")
23
+ else
24
+ exit 0
25
+ fi
26
+
27
+ command -v astra &>/dev/null || exit 0
28
+ cd "$project_root" 2>/dev/null || exit 0
29
+
30
+ if [ "$filename" = "astra.yaml" ]; then
31
+ result=$(astra validate astra.yaml 2>&1)
32
+ else
33
+ result=$(astra validate "$file_path" 2>&1)
34
+ fi
35
+ exit_code=$?
36
+
37
+ if [ $exit_code -eq 0 ]; then
38
+ msg="ASTRA validation passed for $filename"
39
+ else
40
+ msg=$(printf 'ASTRA validation FAILED for %s:\n%s' "$filename" "$result")
41
+ fi
42
+
43
+ jq -n --arg ctx "$msg" '{hookSpecificOutput: {hookEventName: "PostToolUse", additionalContext: $ctx}}'
44
+ exit 0
@@ -68,7 +68,7 @@ For each approved paper: `astra paper add <doi>`, `astra paper path <doi>`, then
68
68
  - **Target decisions**: each decision ID, label, and options with descriptions
69
69
  - **Timestamp**: current time in ISO 8601
70
70
 
71
- The agent type is pre-configured with the user's preferred extraction model (the `extraction_model` field in `~/.lightcone/config.yaml`). Spawn all in a single message (parallel). Show progress as results come in:
71
+ Spawn all in a single message (parallel). Show progress as results come in:
72
72
 
73
73
  ```
74
74
  ✓ Ba et al. 2016 -- 3 prior insights
@@ -293,7 +293,12 @@ def _install_claude_plugin(
293
293
  plugin_source: Path,
294
294
  permissions: str,
295
295
  ) -> None:
296
- """Copy the bundled Claude Code plugin into the project's ``.claude/``."""
296
+ """Copy the bundled Claude Code plugin into the project's ``.claude/``.
297
+
298
+ The hook configuration ships with the plugin as ``hooks.json`` so
299
+ that hook entries live next to the scripts they reference. The CLI
300
+ only owns the ``--permissions`` tier selection.
301
+ """
297
302
  claude_dir = project_dir / ".claude"
298
303
  claude_dir.mkdir(exist_ok=True)
299
304
  for sub in ("skills", "agents", "hooks", "scripts", "guides", "templates"):
@@ -303,59 +308,14 @@ def _install_claude_plugin(
303
308
  if dest.exists():
304
309
  shutil.rmtree(dest)
305
310
  shutil.copytree(src, dest)
311
+ hooks = json.loads((plugin_source / "hooks.json").read_text())
306
312
  settings = {
307
313
  "permissions": PERMISSION_TIERS[permissions],
308
- "hooks": _PROJECT_HOOKS,
314
+ "hooks": hooks,
309
315
  }
310
316
  (claude_dir / "settings.json").write_text(json.dumps(settings, indent=2))
311
317
 
312
318
 
313
- # Hook entries written into ``<project>/.claude/settings.json``. Scripts are
314
- # invoked via ``bash`` so a missing +x bit on a contributor's checkout
315
- # doesn't silently break the hook. Paths use ``${CLAUDE_PROJECT_DIR}`` so
316
- # the wiring is portable across machines and survives a project move.
317
- _PROJECT_HOOKS: dict[str, list[dict[str, object]]] = {
318
- "SessionStart": [
319
- {
320
- "hooks": [
321
- {
322
- "type": "command",
323
- "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/activate-venv.sh",
324
- "timeout": 5,
325
- },
326
- {
327
- "type": "command",
328
- "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/session-start.sh",
329
- "timeout": 15,
330
- },
331
- ],
332
- },
333
- ],
334
- "PostToolUse": [
335
- {
336
- "matcher": "Write|Edit",
337
- "hooks": [
338
- {
339
- "type": "command",
340
- "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/validate-on-save.sh",
341
- "timeout": 15,
342
- },
343
- ],
344
- },
345
- {
346
- "matcher": "Bash",
347
- "hooks": [
348
- {
349
- "type": "command",
350
- "command": "bash ${CLAUDE_PROJECT_DIR}/.claude/scripts/check-lc-run.sh",
351
- "timeout": 15,
352
- },
353
- ],
354
- },
355
- ],
356
- }
357
-
358
-
359
319
  # =============================================================================
360
320
  # lc run
361
321
  # =============================================================================
@@ -78,53 +78,22 @@ def _extract_version(wheel_path: Path) -> str:
78
78
  return match.group(1) if match else wheel_path.name
79
79
 
80
80
 
81
- def _build_astra_wheel(outdir: Path) -> Path:
82
- """Build an ASTRA wheel from the version installed in the current environment.
83
-
84
- Uses importlib.metadata to find the git URL and commit of the installed astra
85
- package, then builds a wheel pinned to that exact commit via pip wheel.
86
- """
87
- import importlib.metadata
88
- import json
89
-
90
- dist = importlib.metadata.distribution("astra")
91
- url_text = dist.read_text("direct_url.json")
92
- if not url_text:
93
- raise RuntimeError(
94
- "Cannot determine ASTRA source — installed package has no direct_url.json. "
95
- "Ensure astra is installed from its git repo (not a plain wheel)."
96
- )
97
- url_info = json.loads(url_text)
98
- git_url = url_info["url"]
99
- commit = url_info.get("vcs_info", {}).get("commit_id", "")
100
- spec = f"astra @ git+{git_url}@{commit}" if commit else f"astra @ git+{git_url}"
101
-
102
- logger.info("Building ASTRA wheel from %s (commit %s) ...", git_url, commit[:8] or "HEAD")
103
- result = subprocess.run(
104
- ["pip", "wheel", "--no-deps", "--wheel-dir", str(outdir), spec],
105
- capture_output=True, text=True,
106
- )
107
- if result.returncode != 0:
108
- raise RuntimeError(f"ASTRA wheel build failed:\n{result.stderr}")
109
-
110
- wheels = list(outdir.glob("astra-*.whl"))
111
- if not wheels:
112
- raise RuntimeError(f"No astra wheel found in {outdir} after build")
113
- return wheels[0]
114
-
115
-
116
81
  def build_eval_wheels(evals_dir: Path) -> tuple[VersionInfo, list[Path]]:
117
- """Build the lightcone-cli and ASTRA wheels for sandbox injection.
82
+ """Build the lightcone-cli wheel for sandbox injection.
83
+
84
+ Only lightcone-cli is built locally — that's the package under test
85
+ and must come from the branch's working tree, not from PyPI. ASTRA
86
+ (``astra-tools`` + ``astra-spec``) is pre-installed in the sandbox
87
+ image from PyPI alongside the other third-party deps; we record the
88
+ host's resolved version here for the run report.
118
89
 
119
90
  Returns (version_info, [wheel_paths]).
120
- Both wheels are built fresh from the current environment:
121
- - lightcone-cli: built from the local git working tree
122
- - ASTRA: built from the git URL/commit recorded in the installed package metadata
123
91
  """
92
+ import importlib.metadata
93
+
124
94
  repo_root = _get_repo_root()
125
95
  version_info = _get_git_info(repo_root)
126
96
 
127
- # Build lightcone-cli wheel into a temp dir
128
97
  tmpdir = Path(tempfile.mkdtemp(prefix="lightcone-eval-wheels-"))
129
98
  logger.info("Building lightcone-cli wheel from %s ...", repo_root)
130
99
  lightcone_wheel = _build_wheel(repo_root, tmpdir)
@@ -134,15 +103,9 @@ def build_eval_wheels(evals_dir: Path) -> tuple[VersionInfo, list[Path]]:
134
103
  version_info.lightcone_commit[:8],
135
104
  " dirty" if version_info.lightcone_dirty else "")
136
105
 
137
- wheels: list[Path] = [lightcone_wheel]
138
-
139
- # Build ASTRA wheel from the installed package's git source
140
106
  try:
141
- astra_wheel = _build_astra_wheel(tmpdir)
142
- version_info.astra_version = _extract_version(astra_wheel)
143
- wheels.append(astra_wheel)
144
- logger.info("Built %s", astra_wheel.name)
145
- except Exception as exc:
146
- logger.warning("Failed to build ASTRA wheel: %s — sandbox may not have astra CLI", exc)
147
-
148
- return version_info, wheels
107
+ version_info.astra_version = importlib.metadata.version("astra-tools")
108
+ except importlib.metadata.PackageNotFoundError:
109
+ pass
110
+
111
+ return version_info, [lightcone_wheel]
@@ -71,12 +71,18 @@ class EvalSandbox:
71
71
 
72
72
  # Build the sandbox image
73
73
  if self.sandbox_image is None:
74
- # Direct deps from astra + lightcone-cli pyproject.toml (pip resolves transitive).
75
- # Kept in sync with the runtime deps declared there — the local astra
76
- # and lightcone-cli wheels are uploaded with --no-deps, so anything not
77
- # listed here must come from the wheels themselves.
74
+ # Pre-install third-party Python deps so the only thing we
75
+ # upload at trial-setup time is the lightcone-cli wheel
76
+ # built from the branch under test. ``astra-tools`` (which
77
+ # ships the ``astra`` module) and ``astra-spec`` come from
78
+ # PyPI here — astra isn't the package under test, just a
79
+ # dependency, so we let pip resolve like a normal user
80
+ # install. The lightcone-cli wheel is installed below with
81
+ # ``--no-deps``, so any runtime dep it relies on must be
82
+ # listed here.
78
83
  deps = (
79
- "click httpx jinja2 jsonschema pydantic pypdf pyyaml rapidfuzz rich"
84
+ "astra-tools astra-spec"
85
+ " jinja2 jsonschema"
80
86
  " snakemake snakemake-interface-executor-plugins"
81
87
  " snakemake-interface-common dask distributed langfuse"
82
88
  )
@@ -156,39 +162,73 @@ class EvalSandbox:
156
162
  loop_prompt_template: str,
157
163
  wheels: list[Path] | None = None,
158
164
  ) -> None:
159
- """Upload seed project and template the loop prompt."""
165
+ """Scaffold the project via ``lc init`` and overlay task seed files.
166
+
167
+ The scaffold (``.claude/``, ``CLAUDE.md``, ``Containerfile``,
168
+ ``requirements.txt``, ``.lightcone/``, ``.gitignore``, default
169
+ ``universes/baseline.yaml``) is produced by the same ``lc init``
170
+ command users run — no separate uploaded copies of skills/hooks
171
+ to drift out of sync. The task seed dir contributes only the
172
+ bits that ``lc init`` cannot generate: ``astra.yaml``, ``data/``,
173
+ and ``task.yaml``.
174
+ """
160
175
  assert self._sandbox is not None, "Call create() first"
161
176
 
162
- # Upload and install dependency wheels
177
+ # Install dependency wheels system-wide so `lc` and `astra` are
178
+ # importable before we run them (lc init below, astra universe
179
+ # generate after the overlay).
163
180
  if wheels:
164
181
  self._install_wheels(wheels)
165
182
 
166
- # Upload seed project files
167
- self._upload_directory(seed_dir, self.WORK_DIR)
183
+ # Seed a minimal global config before invoking `lc`. `lc`'s main
184
+ # group would auto-create this, but writing it explicitly pins
185
+ # the runtime for reproducibility across eval runs.
186
+ self.exec(
187
+ "mkdir -p ~/.lightcone"
188
+ " && printf 'container:\\n runtime: auto\\n' > ~/.lightcone/config.yaml"
189
+ )
168
190
 
169
- # Template the loop prompt
170
- prompt = loop_prompt_template.replace("{{UNIVERSE}}", universe)
171
- self.upload_file("/tmp/loop-prompt.md", prompt.encode())
191
+ # Scaffold the project using the lc init from the wheel under
192
+ # test. --no-venv: deps are pre-installed system-wide on the
193
+ # sandbox image. --no-git: we run git init below, after the
194
+ # overlay, so the seed commit captures the task files too.
195
+ result = self.exec(
196
+ f"mkdir -p {self.WORK_DIR}"
197
+ f" && lc init {self.WORK_DIR} --no-git --no-venv",
198
+ timeout=120,
199
+ )
200
+ if result.exit_code != 0:
201
+ raise RuntimeError(
202
+ f"`lc init` failed (exit {result.exit_code}):\n"
203
+ f"{result.output[-2000:]}"
204
+ )
172
205
 
173
- # Upload lightcone-cli plugin files (.claude/ with skills, hooks, scripts, agents)
174
- # We can't use `lc init` because the seed project already has astra.yaml,
175
- # and lc init refuses to run in that case. Instead, upload directly.
176
- self._install_claude_plugins()
206
+ # Overlay task seed files (astra.yaml, data/, task.yaml). The
207
+ # task's astra.yaml replaces the one `astra init` boilerplated.
208
+ self._upload_directory(seed_dir, self.WORK_DIR)
177
209
 
178
- # Seed a minimal global config. The `lc` main group would auto-create
179
- # this on first invocation, but we write it explicitly so the runtime
180
- # is pinned for reproducibility across eval runs.
210
+ # Regenerate baseline.yaml from the task astra.yaml's defaults.
211
+ # The one lc init produced matches the boilerplate astra.yaml
212
+ # (different decisions), so it would no longer be coherent.
181
213
  self.exec(
182
- "mkdir -p ~/.lightcone"
183
- " && printf 'container:\\n runtime: auto\\n' > ~/.lightcone/config.yaml"
214
+ f"cd {self.WORK_DIR}"
215
+ f" && rm -f universes/baseline.yaml"
216
+ f" && astra universe generate -n baseline"
217
+ f" -d 'Default configuration using standard practices'",
218
+ timeout=60,
184
219
  )
185
220
 
186
- # Git init (exec runs as evaluser via Dockerfile USER directive)
221
+ # Template and stage the loop prompt for the agent.
222
+ prompt = loop_prompt_template.replace("{{UNIVERSE}}", universe)
223
+ self.upload_file("/tmp/loop-prompt.md", prompt.encode())
224
+
225
+ # Initial commit captures the full project state (scaffold +
226
+ # task overlay + regenerated universe).
187
227
  self.exec(
188
228
  f"cd {self.WORK_DIR}"
189
229
  " && git config --global user.name Eval"
190
230
  " && git config --global user.email eval@lightcone"
191
- " && git init && git add -A && git commit -m 'seed'"
231
+ " && git init -q && git add -A && git commit -q -m 'seed'"
192
232
  )
193
233
 
194
234
  def exec(self, cmd: str, timeout: int = 300, cwd: str | None = None) -> ExecuteResult:
@@ -290,48 +330,14 @@ class EvalSandbox:
290
330
  )
291
331
  self._sandbox = None
292
332
 
293
- def _install_claude_plugins(self) -> None:
294
- """Upload lightcone-cli's Claude Code plugin files (skills, hooks, scripts, agents)."""
295
- from lightcone.cli.plugin import get_plugin_source_dir
296
-
297
- plugin_source = get_plugin_source_dir()
298
- if plugin_source is None:
299
- logger.warning("Could not find lightcone-cli plugin source files — skipping")
300
- return
301
-
302
- claude_dir = f"{self.WORK_DIR}/.claude"
303
-
304
- # Upload plugin subdirectories
305
- for subdir in ("skills", "hooks", "scripts", "agents"):
306
- src = plugin_source / subdir
307
- if src.exists():
308
- self._upload_directory(src, f"{claude_dir}/{subdir}")
309
-
310
- # Upload CLAUDE.md template as project CLAUDE.md if not already present
311
- template = plugin_source / "templates" / "CLAUDE.md"
312
- if template.exists():
313
- self.upload_file(f"{self.WORK_DIR}/CLAUDE.md", template.read_bytes())
314
-
315
- # Make scripts executable
316
- self.exec(
317
- f"chmod -R +x {claude_dir}/scripts/ {claude_dir}/hooks/ 2>/dev/null || true"
318
- )
319
-
320
- # Create minimal settings.json (bypassPermissions, no hooks that need
321
- # absolute paths or venvs — the eval sandbox is ephemeral)
322
- settings = json.dumps({
323
- "permissions": {"allow": ["Bash", "Read", "Edit", "Write", "WebSearch", "WebFetch"]},
324
- })
325
- self.upload_file(
326
- f"{claude_dir}/settings.json",
327
- settings.encode(),
328
- )
329
-
330
333
  def _install_wheels(self, wheels: list[Path]) -> None:
331
- """Upload and install wheel files into the sandbox.
334
+ """Upload and install the lightcone-cli wheel into the sandbox.
332
335
 
333
- Third-party deps are pre-installed in the sandbox image.
334
- Only the local astra/lightcone-cli wheels need to be uploaded and installed here.
336
+ The wheel is built from the branch under test in
337
+ :func:`lightcone.eval.build.build_eval_wheels`. ``--no-deps``
338
+ because every runtime dep is already in the sandbox image;
339
+ ``--force-reinstall`` so the local wheel always overrides any
340
+ pre-existing install and we never silently run a PyPI version.
335
341
  """
336
342
  self.exec("mkdir -p /tmp/deps")
337
343
 
@@ -341,7 +347,10 @@ class EvalSandbox:
341
347
  self.upload_file(remote_path, whl.read_bytes())
342
348
  remote_paths.append(remote_path)
343
349
 
344
- whl_cmd = "pip install --no-deps " + " ".join(shlex.quote(p) for p in remote_paths)
350
+ whl_cmd = (
351
+ "pip install --no-deps --force-reinstall "
352
+ + " ".join(shlex.quote(p) for p in remote_paths)
353
+ )
345
354
  result = self.exec(whl_cmd, timeout=120)
346
355
  if result.exit_code != 0:
347
356
  logger.warning(
@@ -1,44 +0,0 @@
1
- #!/bin/bash
2
- # Activate Python virtual environment if it exists in the project directory
3
- # This hook runs at SessionStart to ensure the venv is active for Claude Code
4
-
5
- set -e
6
-
7
- # Check if we have a project directory
8
- if [ -z "$CLAUDE_PROJECT_DIR" ]; then
9
- exit 0
10
- fi
11
-
12
- # Check for .venv in project directory
13
- VENV_DIR="$CLAUDE_PROJECT_DIR/.venv"
14
- if [ ! -d "$VENV_DIR" ]; then
15
- exit 0
16
- fi
17
-
18
- # Determine activate script path based on OS
19
- if [ -f "$VENV_DIR/bin/activate" ]; then
20
- ACTIVATE_SCRIPT="$VENV_DIR/bin/activate"
21
- elif [ -f "$VENV_DIR/Scripts/activate" ]; then
22
- ACTIVATE_SCRIPT="$VENV_DIR/Scripts/activate"
23
- else
24
- exit 0
25
- fi
26
-
27
- # Capture environment before activation
28
- BEFORE_PATH="$PATH"
29
- BEFORE_VIRTUAL_ENV="${VIRTUAL_ENV:-}"
30
-
31
- # Source the activation script
32
- # shellcheck source=/dev/null
33
- source "$ACTIVATE_SCRIPT"
34
-
35
- # If CLAUDE_ENV_FILE is set, write environment changes to it
36
- if [ -n "$CLAUDE_ENV_FILE" ]; then
37
- # Write PATH update
38
- echo "PATH=$PATH" >> "$CLAUDE_ENV_FILE"
39
-
40
- # Write VIRTUAL_ENV
41
- echo "VIRTUAL_ENV=$VIRTUAL_ENV" >> "$CLAUDE_ENV_FILE"
42
- fi
43
-
44
- exit 0
@@ -1,107 +0,0 @@
1
- #!/bin/bash
2
- # PostToolUse hook: warn when the agent runs a Python script that has an
3
- # integrated recipe — they should use `lc run` so a manifest gets written.
4
- #
5
- # Pure bash + jq + the `lc` binary on PATH. No Python imports — the hook
6
- # previously imported `lightcone.engine.status` which silently failed in
7
- # the empty venv `lc init` creates, so the hook was a no-op for nearly
8
- # every project. Status data now comes from `lc status --json`, which
9
- # carries the recipe template per output as of the same change.
10
-
11
- input=$(cat)
12
- command=$(echo "$input" | jq -r '.tool_input.command // empty')
13
-
14
- [ -z "$command" ] && exit 0
15
-
16
- # Only inspect commands that contain "python" / "python3"
17
- echo "$command" | grep -qE 'python[23]?[[:space:]]' || exit 0
18
-
19
- # Skip lc/astra invocations (they're already going through the right path)
20
- echo "$command" | grep -qE '(^|[[:space:]/])(lc|astra)[[:space:]]' && exit 0
21
-
22
- # Must be in an ASTRA project
23
- [ -f "astra.yaml" ] || exit 0
24
-
25
- # Must have lc on PATH; otherwise we can't get status
26
- command -v lc &> /dev/null || exit 0
27
-
28
- # Parse --universe NAME or --universe=NAME from the agent's command;
29
- # default to baseline.
30
- universe=$(echo "$command" \
31
- | grep -oE -- '--universe[= ][[:space:]]*[^[:space:]]+' \
32
- | head -1 \
33
- | sed -E 's/--universe[= ][[:space:]]*//')
34
- [ -z "$universe" ] && universe="baseline"
35
-
36
- status_json=$(lc status --json --universe "$universe" 2>/dev/null) || exit 0
37
-
38
- # Collect script paths the agent invoked. Split on && || ; first so each
39
- # chained command is considered separately, then look for tokens ending
40
- # in .py and `python -m module.path`.
41
- agent_scripts=$(echo "$command" | awk '
42
- {
43
- # Split on shell separators
44
- gsub(/&&|\|\||;/, "\n")
45
- print
46
- }' | awk '
47
- {
48
- n = split($0, tok, /[[:space:]]+/)
49
- for (i = 1; i <= n; i++) {
50
- t = tok[i]
51
- if (t ~ /\.py$/) {
52
- sub(/^\.\//, "", t)
53
- print t
54
- n2 = split(t, pp, "/")
55
- print pp[n2]
56
- }
57
- if (t == "-m" && i + 1 <= n) {
58
- mod = tok[i+1]
59
- gsub(/\./, "/", mod)
60
- print mod ".py"
61
- n2 = split(mod, pp, "/")
62
- print pp[n2] ".py"
63
- }
64
- }
65
- }' | sort -u)
66
-
67
- [ -z "$agent_scripts" ] && exit 0
68
-
69
- # For each output that has a recipe, extract the script path from the
70
- # recipe template and check whether the agent's command invoked it.
71
- matched=$(echo "$status_json" | jq -r --arg uni "$universe" '
72
- .universes[]
73
- | select(.universe_id == $uni)
74
- | .outputs[]
75
- | select(.recipe_command != null and .recipe_command != "")
76
- | "\(.output_id)\t\(.status)\t\(.recipe_command)"
77
- ' | while IFS=$'\t' read -r out_id out_status recipe_cmd; do
78
- recipe_scripts=$(echo "$recipe_cmd" | tr ' \t\n' '\n' | grep '\.py$' | sed 's|^\./||')
79
- for r in $recipe_scripts; do
80
- rb=$(basename "$r")
81
- if echo "$agent_scripts" | grep -qFx "$r" || echo "$agent_scripts" | grep -qFx "$rb"; then
82
- printf '%s\t%s\n' "$out_id" "$out_status"
83
- break
84
- fi
85
- done
86
- done | head -1)
87
-
88
- [ -z "$matched" ] && exit 0
89
-
90
- matched_id=$(echo "$matched" | cut -f1)
91
- matched_status=$(echo "$matched" | cut -f2)
92
-
93
- case "$matched_status" in
94
- missing|stale)
95
- msg="WARNING: You ran the script for output \`$matched_id\` directly (status: $matched_status in $universe), which has an integrated recipe. Use \`lc run $matched_id --universe $universe\` instead so a manifest is written and the result is reproducible."
96
- ;;
97
- ok)
98
- msg="NOTE: Output \`$matched_id\` already has current results in $universe from \`lc run\`. Running the script directly bypasses the manifest — use \`lc run $matched_id --universe $universe\` to regenerate reproducibly."
99
- ;;
100
- *)
101
- exit 0
102
- ;;
103
- esac
104
-
105
- escaped_msg=$(echo "$msg" | jq -Rs .)
106
- echo "{\"hookSpecificOutput\": {\"hookEventName\": \"PostToolUse\", \"additionalContext\": $escaped_msg}}"
107
- exit 0
@@ -1,120 +0,0 @@
1
- #!/bin/bash
2
- # SessionStart hook: Show ASTRA analysis summary when entering a project
3
- # Provides context about the current analysis state
4
-
5
- # Read JSON input from stdin
6
- input=$(cat)
7
-
8
- # Get the current working directory
9
- cwd=$(echo "$input" | jq -r '.cwd // empty')
10
-
11
- if [ -z "$cwd" ]; then
12
- exit 0
13
- fi
14
-
15
- cd "$cwd" 2>/dev/null || exit 0
16
-
17
- # Sync extraction model from ~/.lightcone/config.yaml to .claude/agents/lc-extractor.md
18
- if [ -f ".claude/agents/lc-extractor.md" ] && [ -f "$HOME/.lightcone/config.yaml" ]; then
19
- ext_model=$(grep '^extraction_model:' "$HOME/.lightcone/config.yaml" 2>/dev/null | awk '{print $2}' | tr -d "'\"")
20
- # Default to sonnet if not configured
21
- [ -z "$ext_model" ] && ext_model="sonnet"
22
- if [ -n "$ext_model" ]; then
23
- if ! grep -q "^model:" .claude/agents/lc-extractor.md 2>/dev/null; then
24
- # Insert model field after description line
25
- sed -i.bak '/^tools:/i\
26
- model: '"$ext_model" .claude/agents/lc-extractor.md 2>/dev/null && rm -f .claude/agents/lc-extractor.md.bak
27
- else
28
- # Update existing model field
29
- sed -i.bak 's/^model: .*/model: '"$ext_model"'/' .claude/agents/lc-extractor.md 2>/dev/null && rm -f .claude/agents/lc-extractor.md.bak
30
- fi
31
- else
32
- # Empty model = inherit, remove model line if present
33
- sed -i.bak '/^model: /d' .claude/agents/lc-extractor.md 2>/dev/null && rm -f .claude/agents/lc-extractor.md.bak
34
- fi
35
- fi
36
-
37
- # Check if this is an ASTRA project (has astra.yaml)
38
- if [ ! -f "astra.yaml" ]; then
39
- exit 0
40
- fi
41
-
42
- # Check if astra command is available
43
- if ! command -v astra &> /dev/null; then
44
- # Provide minimal info without CLI
45
- echo "{\"hookSpecificOutput\": {\"hookEventName\": \"SessionStart\", \"additionalContext\": \"This is an ASTRA project. The astra CLI is not installed - run 'pip install astra' to enable validation and other commands.\"}}"
46
- exit 0
47
- fi
48
-
49
- # Gather analysis information. `astra init` writes `name:` at the top
50
- # level (no indent); previous indented form was pre-asset-centric.
51
- # Use -E (ERE) so `?` works on both BSD and GNU sed.
52
- analysis_name=$(grep -m1 "^name:" astra.yaml 2>/dev/null | sed -E 's/^name:[[:space:]]*"?([^"]*)"?[[:space:]]*$/\1/')
53
-
54
- # Count keys under 'decisions:' (block-form decisions only)
55
- decision_count=$(awk '/^decisions:/{found=1; next} found && /^ [a-z_]+:/{count++} found && /^[a-z]/{exit} END{print count}' astra.yaml 2>/dev/null)
56
-
57
- # Count universes
58
- universe_count=$(ls -1 universes/*.yaml 2>/dev/null | wc -l | tr -d ' ')
59
-
60
- # Check validation status
61
- validation_result=$(astra validate astra.yaml 2>&1)
62
- if [ $? -eq 0 ]; then
63
- validation_status="valid"
64
- else
65
- validation_status="has errors"
66
- fi
67
-
68
- # Build summary
69
- summary="ASTRA Project: ${analysis_name:-unnamed}
70
- - Decisions: ${decision_count:-0}
71
- - Universes: ${universe_count:-0}
72
- - Validation: ${validation_status}
73
- - Reference: For astra.yaml syntax and spec format, read .claude/guides/astra-reference.md; for CLI and execution, read .claude/guides/lightcone-cli-reference.md"
74
-
75
- # If validation failed, add error summary. We prefer the tail because the
76
- # leading lines are the success header (`✓ Schema validation passed` etc.)
77
- # and the actual error block is at the bottom. `head -5` historically hid
78
- # every real error.
79
- if [ "$validation_status" = "has errors" ]; then
80
- error_preview=$(echo "$validation_result" | tail -20)
81
- summary="$summary
82
-
83
- Validation errors (run 'astra validate astra.yaml' for full output):
84
- $error_preview"
85
- fi
86
-
87
- # Add lc status if lc CLI is available. States are 'ok' / 'stale' /
88
- # 'missing' / 'alias' (manifest-driven). The output uses Rich glyphs:
89
- # '✓ ok', '✸ stale', '✗ miss', '→ alias'.
90
- if command -v lc &> /dev/null; then
91
- lc_status=$(lc status 2>&1)
92
- lc_exit=$?
93
- if [ $lc_exit -eq 0 ]; then
94
- ok_count=$(echo "$lc_status" | grep -c "✓ ok")
95
- stale_count=$(echo "$lc_status" | grep -c "✸ stale")
96
- missing_count=$(echo "$lc_status" | grep -c "✗ miss")
97
- alias_count=$(echo "$lc_status" | grep -c "→ alias")
98
-
99
- summary="$summary
100
-
101
- Materialization status:
102
- - ok: ${ok_count}
103
- - stale: ${stale_count}
104
- - missing: ${missing_count}
105
- - alias: ${alias_count}"
106
-
107
- needs_run=$((missing_count + stale_count))
108
- if [ "$needs_run" -gt 0 ]; then
109
- summary="$summary
110
-
111
- ACTION REQUIRED: ${needs_run} output(s) need \`lc run\` (${missing_count} missing, ${stale_count} stale)."
112
- fi
113
- fi
114
- fi
115
-
116
- # Output as JSON
117
- escaped_summary=$(echo "$summary" | jq -Rs .)
118
- echo "{\"hookSpecificOutput\": {\"hookEventName\": \"SessionStart\", \"additionalContext\": $escaped_summary}}"
119
-
120
- exit 0
@@ -1,77 +0,0 @@
1
- #!/bin/bash
2
- # PostToolUse hook: Auto-validate ASTRA files after modification
3
- # Triggers on Write|Edit of astra.yaml or universes/*.yaml
4
-
5
- # Read JSON input from stdin
6
- input=$(cat)
7
-
8
- # Extract the file path that was modified
9
- file_path=$(echo "$input" | jq -r '.tool_input.file_path // .tool_response.filePath // empty')
10
-
11
- # Exit silently if no file path
12
- if [ -z "$file_path" ]; then
13
- exit 0
14
- fi
15
-
16
- # Get the filename
17
- filename=$(basename "$file_path")
18
- dirpath=$(dirname "$file_path")
19
- dirname=$(basename "$dirpath")
20
-
21
- # Check if this is an ASTRA-related file
22
- is_astra_file=false
23
-
24
- # Check for astra.yaml (main analysis file)
25
- if [ "$filename" = "astra.yaml" ]; then
26
- is_astra_file=true
27
- file_type="analysis"
28
- fi
29
-
30
- # Check for universe files (in universes/ directory)
31
- if [ "$dirname" = "universes" ] && [[ "$filename" == *.yaml ]]; then
32
- is_astra_file=true
33
- file_type="universe"
34
- fi
35
-
36
- # Exit if not an ASTRA file
37
- if [ "$is_astra_file" = false ]; then
38
- exit 0
39
- fi
40
-
41
- # Find the project root (where astra.yaml lives)
42
- if [ "$file_type" = "analysis" ]; then
43
- project_root="$dirpath"
44
- else
45
- # For universe files, go up one level
46
- project_root=$(dirname "$dirpath")
47
- fi
48
-
49
- # Check if astra command is available
50
- if ! command -v astra &> /dev/null; then
51
- # ASTRA CLI not installed, skip validation
52
- exit 0
53
- fi
54
-
55
- # Run validation
56
- cd "$project_root" 2>/dev/null || exit 0
57
-
58
- if [ "$file_type" = "analysis" ]; then
59
- result=$(astra validate astra.yaml 2>&1)
60
- exit_code=$?
61
- else
62
- result=$(astra validate "$file_path" 2>&1)
63
- exit_code=$?
64
- fi
65
-
66
- # Prepare response
67
- if [ $exit_code -eq 0 ]; then
68
- # Validation passed
69
- echo "{\"hookSpecificOutput\": {\"hookEventName\": \"PostToolUse\", \"additionalContext\": \"ASTRA validation passed for $filename\"}}"
70
- else
71
- # Validation failed - provide context to Claude
72
- # Escape the result for JSON (jq -Rs . adds quotes, so use it directly)
73
- escaped_result=$(echo "ASTRA validation FAILED for $filename:\n$result" | jq -Rs .)
74
- echo "{\"hookSpecificOutput\": {\"hookEventName\": \"PostToolUse\", \"additionalContext\": $escaped_result}}"
75
- fi
76
-
77
- exit 0
File without changes
File without changes