lightcone-cli 0.2.0__py3-none-any.whl

lightcone/engine/site_registry.py

@@ -0,0 +1,142 @@
+"""Known HPC site defaults for site configuration.
+
+When ``lc setup`` detects a known site, it auto-populates scheduler
+settings with site-specific defaults (node types, QOS options, container
+runtimes, etc.).  Users can override any value during the wizard.
+
+To add a new site, append an entry to ``SITE_DEFAULTS``.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+# Each entry maps a site key to its defaults.  The ``hostname_patterns``
+# list is used to auto-detect the site from user-provided hostnames.
+SITE_DEFAULTS: dict[str, dict[str, Any]] = {
+    "perlmutter": {
+        "hostname_patterns": ["perlmutter", "saul"],
+        "display_name": "NERSC Perlmutter",
+        "backend": "slurm",
+        "connection": {
+            "hostname": "perlmutter.nersc.gov",
+        },
+        "scheduler": {
+            "container_runtime": "podman-hpc",
+        },
+        "node_types": {
+            "gpu": {
+                "description": "GPU (A100 40GB) — 1,536 nodes, 4 GPUs/node",
+                "constraint": "gpu",
+                "container_flags": ["--gpu"],
+            },
+            "gpu_hbm80": {
+                "description": "GPU (A100 80GB) — 256 nodes, 4 GPUs/node",
+                "constraint": "gpu&hbm80g",
+                "container_flags": ["--gpu"],
+            },
+            "cpu": {
+                "description": "CPU only — 3,072 nodes, 128 cores/node",
+                "constraint": "cpu",
+                "container_flags": [],
+            },
+        },
+        "qos_options": {
+            "regular": {"description": "Standard priority, max 48h", "default": True},
+            "debug": {"description": "Quick tests, max 30min, 8 nodes max"},
+            "shared": {"description": "Fractional GPU (1-2 GPUs), max 48h"},
+            "preempt": {"description": "0.25x cost, can be preempted after 2h"},
+        },
+        "container_runtimes": ["podman-hpc"],
+        "resource_limits": {
+            "max_nodes": 4,
+            "max_walltime_minutes": 360,
+            "max_concurrent_jobs": 8,
+        },
+        "safe_defaults": {
+            "node_type": "gpu",
+            "constraint": "gpu",
+            "qos": "debug",
+            "nodes": 1,
+            "time_limit": "30m",
+        },
+        "account_suffixes": {
+            "gpu": "_g",
+            "gpu&hbm80g": "_g",
+        },
+        "scratch_paths": [
+            "//pscratch/**",
+            "//global/cscratch1/**",
+            "//global/cfs/cdirs/**",
+        ],
+    },
+    "local": {
+        "hostname_patterns": [],
+        "display_name": "Local",
+        "backend": "local",
+        "connection": {},
+        "scheduler": {},
+        "node_types": {},
+        "qos_options": {},
+        "container_runtimes": [],
+        "resource_limits": {},
+    },
+}
+
+
+def detect_site(hostname_or_name: str) -> str | None:
+    """Detect a known HPC site from a hostname or site name.
+
+    Returns the site key (e.g. ``"perlmutter"``) or ``None`` if no match.
+    """
+    normalized = hostname_or_name.lower()
+    for site_key, site in SITE_DEFAULTS.items():
+        if site.get("backend") == "local":
+            continue
+        if site_key in normalized:
+            return site_key
+        for pattern in site.get("hostname_patterns", []):
+            if pattern in normalized:
+                return site_key
+    return None
+
+
+def get_site_defaults(site_key: str) -> dict[str, Any] | None:
+    """Return defaults for a known site, or ``None``."""
+    return SITE_DEFAULTS.get(site_key)
+
+
+def list_known_sites() -> list[tuple[str, str]]:
+    """Return list of (site_key, display_name) for all known sites."""
+    return [
+        (key, site.get("display_name", key))
+        for key, site in SITE_DEFAULTS.items()
+    ]
+
+
+def resolve_account(site_key: str, account: str, constraint: str | None) -> str:
+    """Apply site-specific account suffix based on constraint.
+
+    For example, on Perlmutter GPU jobs require ``m4031_g`` instead of
+    ``m4031``.  If the account already has the suffix, it is not added again.
+    """
+    site = SITE_DEFAULTS.get(site_key)
+    if not site or not constraint:
+        return account
+    suffixes = site.get("account_suffixes", {})
+    suffix = suffixes.get(constraint)
+    if suffix and not account.endswith(suffix):
+        return account + suffix
+    return account
+
+
+def get_site_scratch_deny_rules(site_key: str) -> list[str]:
+    """Return Edit deny rules for a site's scratch/shared filesystem paths.
+
+    These are used in Claude Code permissions to prevent accidental writes
+    to shared HPC filesystems.
+    """
+    site = SITE_DEFAULTS.get(site_key)
+    if not site:
+        return []
+    scratch_paths = site.get("scratch_paths", [])
+    return [f"Edit({path})" for path in scratch_paths]

lightcone/engine/status.py

@@ -0,0 +1,135 @@
+"""Materialization status queries for ASTRA outputs."""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+import dagster as dg
+from astra.helpers import load_yaml, resolve_analysis_tree
+
+from lightcone.engine.tree import collect_tree_outputs
+
+logger = logging.getLogger(__name__)
+
+
+def _get_dagster_instance(project_path: Path) -> dg.DagsterInstance | None:
+    """Load a DagsterInstance from the project's dagster.yaml.
+
+    Returns None if dagster.yaml doesn't exist or the instance can't be loaded
+    (e.g. corrupted SQLite). Callers treat None as "no events recorded."
+
+    Temporarily changes to project_path so that relative paths in dagster.yaml
+    (e.g. ``base_dir: results/.dagster``) resolve correctly.
+    """
+    # Check .lightcone/ first, then root for backwards compat
+    dagster_yaml = project_path / ".lightcone" / "dagster.yaml"
+    if not dagster_yaml.exists():
+        dagster_yaml = project_path / "dagster.yaml"
+    if not dagster_yaml.exists():
+        return None
+    config_dir = dagster_yaml.parent
+    old_cwd = os.getcwd()
+    try:
+        os.chdir(project_path)
+        return dg.DagsterInstance.from_config(str(config_dir))
+    except Exception:
+        logger.warning("Failed to load Dagster instance from %s", project_path, exc_info=True)
+        return None
+    finally:
+        os.chdir(old_cwd)
+
+
+def get_output_status(
+    project_path: Path,
+    universe_id: str,
+    instance: dg.DagsterInstance | None = None,
+) -> dict[str, str]:
+    """Get materialization status for all outputs in a universe.
+
+    Returns dict mapping qualified output_id to status string:
+    - "no_recipe": output declared but has no recipe block
+    - "pending": has recipe, not yet materialized
+    - "materialized": has recipe and Dagster event log confirms materialization
+
+    For sub-analysis outputs, keys are qualified: "analysis_id/output_id".
+    Root-level outputs use just "output_id".
+    """
+    spec = load_yaml(project_path / "astra.yaml")
+    # Resolve sub-analysis tree
+    spec = resolve_analysis_tree(spec, project_path)
+
+    if instance is None:
+        instance = _get_dagster_instance(project_path)
+
+    # Collect all outputs from the tree
+    tree_outputs = collect_tree_outputs(spec)
+
+    # Build asset keys for outputs with recipes, then batch-query Dagster
+    recipe_keys: dict[str, dg.AssetKey] = {}  # qualified_id -> asset key
+    for tree_out in tree_outputs:
+        out_id = tree_out.output_id
+        if not out_id or not tree_out.output_def.get("recipe"):
+            continue
+        if tree_out.analysis_id:
+            qualified = f"{tree_out.analysis_id}/{out_id}"
+            key = dg.AssetKey([universe_id, tree_out.analysis_id, out_id])
+        else:
+            qualified = out_id
+            key = dg.AssetKey([universe_id, out_id])
+        recipe_keys[qualified] = key
+
+    materialized: set[str] = set()
+    if instance is not None and recipe_keys:
+        events = instance.get_latest_materialization_events(list(recipe_keys.values()))
+        materialized_asset_keys = {k for k, v in events.items() if v is not None}
+        for qualified, key in recipe_keys.items():
+            if key in materialized_asset_keys:
+                materialized.add(qualified)
+
+    status: dict[str, str] = {}
+    for tree_out in tree_outputs:
+        out_id = tree_out.output_id
+        if not out_id:
+            continue
+        if tree_out.analysis_id:
+            qualified = f"{tree_out.analysis_id}/{out_id}"
+        else:
+            qualified = out_id
+
+        if not tree_out.output_def.get("recipe"):
+            # Check for alias outputs (from: sub.output)
+            from_ref = tree_out.output_def.get("from")
+            if from_ref and tree_out.analysis_id is None:
+                status[qualified] = "alias"
+                continue
+            status[qualified] = "no_recipe"
+        elif qualified in materialized:
+            status[qualified] = "materialized"
+        else:
+            status[qualified] = "pending"
+
+    return status
+
+
+def get_all_universe_status(
+    project_path: Path,
+) -> dict[str, dict[str, str]]:
+    """Get status for all universes.
+
+    Returns dict mapping universe_id to output status dict.
+    """
+    universes_dir = project_path / "universes"
+    if not universes_dir.exists():
+        return {}
+
+    # Create instance once and share across all universe checks
+    instance = _get_dagster_instance(project_path)
+
+    result: dict[str, dict[str, str]] = {}
+    for universe_file in sorted(universes_dir.glob("*.yaml")):
+        universe_data = load_yaml(universe_file)
+        universe_id = universe_data.get("id", universe_file.stem)
+        result[universe_id] = get_output_status(project_path, universe_id, instance=instance)
+
+    return result

lightcone/engine/targets.py

@@ -0,0 +1,68 @@
+"""Target configuration management for Dagster execution backends."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def get_targets_dir() -> Path:
+    """Return the user-level targets directory (~/.lightcone/targets/)."""
+    return Path.home() / ".lightcone" / "targets"
+
+
+def list_targets() -> list[str]:
+    """Return names of saved target configurations."""
+    targets_dir = get_targets_dir()
+    if not targets_dir.exists():
+        return []
+    return sorted(p.stem for p in targets_dir.glob("*.yaml"))
+
+
+def load_target(name: str) -> dict[str, Any] | None:
+    """Load a saved target configuration by name. Returns None if missing."""
+    config_path = get_targets_dir() / f"{name}.yaml"
+    if not config_path.exists():
+        return None
+    with open(config_path) as f:
+        return yaml.safe_load(f)
+
+
+def save_target(name: str, config: dict[str, Any]) -> Path:
+    """Save a target configuration to ~/.lightcone/targets/{name}.yaml."""
+    targets_dir = get_targets_dir()
+    targets_dir.mkdir(parents=True, exist_ok=True)
+    config_path = targets_dir / f"{name}.yaml"
+    with open(config_path, "w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+    return config_path
+
+
+def get_config_path() -> Path:
+    """Return the user-level config file path (~/.lightcone/config.yaml)."""
+    return Path.home() / ".lightcone" / "config.yaml"
+
+
+def load_user_config() -> dict[str, Any]:
+    """Load the user-level lightcone-cli configuration.
+
+    Returns an empty dict if the config file doesn't exist.
+    """
+    config_path = get_config_path()
+    if not config_path.exists():
+        return {}
+    with open(config_path) as f:
+        return yaml.safe_load(f) or {}
+
+
+def save_user_config(config: dict[str, Any]) -> Path:
+    """Save user-level lightcone-cli configuration to ~/.lightcone/config.yaml.
+
+    Returns the path where it was saved.
+    """
+    config_path = get_config_path()
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(config_path, "w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+    return config_path

lightcone/engine/tree.py

@@ -0,0 +1,245 @@
+"""Analysis tree helpers — walk resolved sub-analysis trees.
+
+After ``resolve_analysis_tree()`` from astra.helpers expands ``path:``
+references, this module provides utilities to:
+
+- Collect all outputs across the tree (with their sub-analysis context)
+- Resolve ``from:`` references on inputs to concrete output paths
+- Resolve ``from:`` references on decisions to parent decision values
+- Build merged decision dicts from composable universe files
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from astra.helpers import get_inputs, get_outputs, load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TreeOutput:
+    """An output from the resolved analysis tree, with its sub-analysis context."""
+
+    output_id: str
+    output_def: dict[str, Any]
+    analysis_id: str | None  # None for root-level outputs
+    analysis_path: str | None  # relative path, e.g. "./analyses/hod_fitting"
+    analysis_spec: dict[str, Any]  # the sub-analysis spec dict
+
+
+def collect_tree_outputs(spec: dict[str, Any]) -> list[TreeOutput]:
+    """Walk the resolved tree and collect all outputs with context.
+
+    Returns outputs from root level (analysis_id=None) and from each
+    sub-analysis. Root-level outputs with ``from:`` pointing to a
+    sub-analysis output are included but flagged for alias handling.
+    """
+    results: list[TreeOutput] = []
+
+    # Root-level outputs
+    for out in get_outputs(spec):
+        results.append(TreeOutput(
+            output_id=out.get("id", ""),
+            output_def=out,
+            analysis_id=None,
+            analysis_path=None,
+            analysis_spec=spec,
+        ))
+
+    # Sub-analysis outputs
+    for analysis_id, analysis_node in (spec.get("analyses") or {}).items():
+        sub_path = analysis_node.get("path")
+        for out in get_outputs(analysis_node):
+            results.append(TreeOutput(
+                output_id=out.get("id", ""),
+                output_def=out,
+                analysis_id=analysis_id,
+                analysis_path=sub_path,
+                analysis_spec=analysis_node,
+            ))
+
+    return results
+
+
+def collect_tree_inputs(spec: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """Collect all inputs from root and sub-analyses.
+
+    Returns {qualified_id: input_def} where qualified_id is:
+    - "input_id" for root inputs
+    - "analysis_id.input_id" for sub-analysis inputs
+    """
+    result: dict[str, dict[str, Any]] = {}
+
+    for inp in get_inputs(spec):
+        inp_id = inp.get("id", "")
+        if inp_id:
+            result[inp_id] = inp
+
+    for analysis_id, analysis_node in (spec.get("analyses") or {}).items():
+        for inp in get_inputs(analysis_node):
+            inp_id = inp.get("id", "")
+            if inp_id:
+                result[f"{analysis_id}.{inp_id}"] = inp
+
+    return result
+
+
+def resolve_universe_decisions(
+    project_path: Path,
+    spec: dict[str, Any],
+    universe_id: str,
+) -> dict[str, Any]:
+    """Load and merge universe decisions from root and sub-analysis universes.
+
+    Returns a flat dict of all decisions for execution:
+    - Root decisions from ``universes/<universe_id>.yaml``
+    - Sub-analysis decisions from ``<sub_path>/universes/<sub_universe_id>.yaml``
+    - ``from:`` decisions in sub-analyses are resolved to parent values
+
+    The returned dict uses qualified keys for sub-analysis decisions:
+    ``{analysis_id}.{decision_id}`` to avoid collisions.
+    """
+    # Load root universe
+    root_universe_file = project_path / "universes" / f"{universe_id}.yaml"
+    root_decisions: dict[str, Any] = {}
+    sub_universe_refs: dict[str, str] = {}
+
+    if root_universe_file.exists():
+        root_data = load_yaml(root_universe_file)
+        root_decisions = root_data.get("decisions", {})
+        # Parse sub-analysis universe references
+        for analysis_id, ref in (root_data.get("analyses") or {}).items():
+            if isinstance(ref, dict) and ref.get("universe"):
+                sub_universe_refs[analysis_id] = ref["universe"]
+
+    merged: dict[str, Any] = dict(root_decisions)
+
+    # Load sub-analysis universes
+    for analysis_id, analysis_node in (spec.get("analyses") or {}).items():
+        sub_path = analysis_node.get("path")
+        if not sub_path:
+            continue
+
+        sub_universe_id = sub_universe_refs.get(analysis_id, universe_id)
+        sub_dir = (project_path / sub_path).resolve()
+        sub_universe_file = sub_dir / "universes" / f"{sub_universe_id}.yaml"
+
+        if sub_universe_file.exists():
+            sub_data = load_yaml(sub_universe_file)
+            sub_decisions = sub_data.get("decisions", {})
+        else:
+            sub_decisions = {}
+
+        # Resolve from: references in sub-analysis decisions
+        for decision_id, decision_def in (analysis_node.get("decisions") or {}).items():
+            if isinstance(decision_def, dict) and decision_def.get("from"):
+                from_ref = decision_def["from"]
+                # ../parent_decision -> look up in root decisions
+                if from_ref.startswith("../"):
+                    parent_key = from_ref[3:]
+                    if parent_key in root_decisions:
+                        merged[f"{analysis_id}.{decision_id}"] = root_decisions[parent_key]
+                    else:
+                        logger.warning(
+                            "Decision '%s' in '%s' references '%s' which is not in "
+                            "root universe decisions",
+                            decision_id, analysis_id, from_ref,
+                        )
+            elif decision_id in sub_decisions:
+                merged[f"{analysis_id}.{decision_id}"] = sub_decisions[decision_id]
+
+        # Also add un-referenced local decisions from the sub-universe
+        for decision_id, value in sub_decisions.items():
+            key = f"{analysis_id}.{decision_id}"
+            if key not in merged:
+                merged[key] = value
+
+    return merged
+
+
+def get_decisions_for_analysis(
+    merged_decisions: dict[str, Any],
+    analysis_id: str | None,
+) -> dict[str, Any]:
+    """Extract the decisions relevant to a specific analysis from merged dict.
+
+    For root (analysis_id=None): returns unqualified keys.
+    For sub-analysis: returns decisions with matching prefix, stripped to local names.
+    Also includes root-level decisions (for from: references).
+    """
+    if analysis_id is None:
+        # Root analysis: return all unqualified keys
+        return {k: v for k, v in merged_decisions.items() if "." not in k}
+
+    prefix = f"{analysis_id}."
+    result: dict[str, Any] = {}
+
+    # Add qualified decisions with prefix stripped
+    for k, v in merged_decisions.items():
+        if k.startswith(prefix):
+            local_key = k[len(prefix):]
+            result[local_key] = v
+
+    return result
+
+
+def resolve_output_path(
+    project_path: Path,
+    tree_output: TreeOutput,
+    universe_id: str,
+) -> Path:
+    """Resolve the results directory for an output.
+
+    Root outputs: ``results/<universe_id>/``
+    Sub-analysis outputs: ``<sub_path>/results/<universe_id>/``
+    """
+    if tree_output.analysis_path:
+        return (project_path / tree_output.analysis_path).resolve() / "results" / universe_id
+    return project_path / "results" / universe_id
+
+
+def resolve_input_path(
+    project_path: Path,
+    spec: dict[str, Any],
+    from_ref: str,
+    universe_id: str,
+) -> str | None:
+    """Resolve a ``from:`` reference on an input to a concrete filesystem path.
+
+    Handles:
+    - ``../parent_input`` -> root input's source
+    - ``../sibling.output_id`` -> sibling sub-analysis's results path
+    - ``sibling.output_id`` -> sibling sub-analysis's results path (no ../ needed at root)
+    """
+    # Strip leading ../ if present
+    ref = from_ref.removeprefix("../")
+    ref = ref.removeprefix("/")
+
+    # Check if it's a root input reference
+    for inp in get_inputs(spec):
+        if inp.get("id") == ref:
+            source = inp.get("source")
+            if source and isinstance(source, str) and source.startswith("/"):
+                return source
+            return None
+
+    # Check if it's a sibling.output_id reference
+    if "." in ref:
+        analysis_id, output_id = ref.split(".", 1)
+        analyses = spec.get("analyses") or {}
+        if analysis_id in analyses:
+            sub_node = analyses[analysis_id]
+            sub_path = sub_node.get("path")
+            if sub_path:
+                return str(
+                    (project_path / sub_path).resolve()
+                    / "results" / universe_id / output_id
+                )
+
+    return None
+
+

lightcone/eval/__init__.py

@@ -0,0 +1,25 @@
+"""Lightcone eval harness — quantitative evaluation of the agentic build loop."""
+
+from __future__ import annotations
+
+from lightcone.eval.models import (
+    EvalRun,
+    EvalRunConfig,
+    GraderResult,
+    GraderSpec,
+    IterationResult,
+    TaskSpec,
+    TrialResult,
+    VersionInfo,
+)
+
+__all__ = [
+    "EvalRun",
+    "EvalRunConfig",
+    "GraderResult",
+    "GraderSpec",
+    "IterationResult",
+    "TaskSpec",
+    "TrialResult",
+    "VersionInfo",
+]