varve 0.1.0__py3-none-any.whl

varve/__init__.py

@@ -0,0 +1,16 @@
+"""Public API for varve."""
+
+from varve.context import Ctx
+from varve.decorators import StageSpec, batch_stage, stage
+from varve.keyspec import JSON, KeySpec
+from varve.pipeline import Pipeline
+
+__all__ = [
+    "Ctx",
+    "Pipeline",
+    "JSON",
+    "KeySpec",
+    "StageSpec",
+    "batch_stage",
+    "stage",
+]

varve/branch.py

@@ -0,0 +1,92 @@
+"""Branch selection helpers for varve experiments."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import re
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+BRANCH_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+
+def validate_branch_name(name: str) -> str:
+    """Validate a branch name before it is interpolated into output paths."""
+    if not isinstance(name, str) or BRANCH_NAME_RE.fullmatch(name) is None:
+        raise ValueError(
+            f"Invalid varve branch name {name!r}; branch names must match "
+            "[A-Za-z0-9][A-Za-z0-9._-]* and stay within one path segment."
+        )
+    return name
+
+
+def load_branches(yaml_path: Path | None) -> dict[str, tuple[dict[str, Any], bool]]:
+    """Load all branch configs from a varve.yaml file."""
+    if yaml_path is None or not Path(yaml_path).exists():
+        return {}
+
+    raw = yaml.safe_load(Path(yaml_path).read_text(encoding="utf-8"))
+    if raw is None:
+        raw = {}
+    if not isinstance(raw, Mapping):
+        raise ValueError(f"varve.yaml must be a mapping of branch names to configs: {yaml_path}")
+    for name in raw:
+        validate_branch_name(name)
+
+    result: dict[str, tuple[dict[str, Any], bool]] = {}
+    for branch, section in raw.items():
+        if section is None:
+            section = {}
+        if not isinstance(section, Mapping):
+            raise ValueError(f"Varve branch {branch!r} must be a mapping in {yaml_path}")
+
+        config = dict(section)
+        is_temporary = config.pop("is_temporary", False)
+        if not isinstance(is_temporary, bool):
+            raise ValueError(f"Varve branch {branch!r} has non-boolean is_temporary in {yaml_path}")
+        result[branch] = (config, is_temporary)
+    return result
+
+
+def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
+    merged = copy.deepcopy(dict(base))
+    for key, value in override.items():
+        current = merged.get(key)
+        if isinstance(current, Mapping) and isinstance(value, Mapping):
+            merged[key] = _deep_merge(current, value)
+        else:
+            merged[key] = copy.deepcopy(value)
+    return merged
+
+
+def merge_override(base_config: Mapping[str, Any], override_json: str) -> dict[str, Any]:
+    """Apply an override JSON object to a raw config mapping."""
+    override = json.loads(override_json)
+    if not isinstance(override, Mapping):
+        raise ValueError("--override must be a JSON object")
+    return _deep_merge(base_config, override)
+
+
+def canonical_config_json(config: Mapping[str, Any]) -> str:
+    """Return stable JSON for a validated config snapshot."""
+    return json.dumps(config, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def override_branch_name(config: Mapping[str, Any]) -> str:
+    """Derive the hash override branch name from a complete config snapshot."""
+    digest = hashlib.sha256(canonical_config_json(config).encode("utf-8")).hexdigest()[:12]
+    return f"main_override_{digest}"
+
+
+def assert_same_config(left: Mapping[str, Any], right: Mapping[str, Any], *, branch: str) -> None:
+    """Raise when a named temporary branch is reused with a different config."""
+    if canonical_config_json(left) != canonical_config_json(right):
+        raise ValueError(
+            f"Temporary varve branch {branch!r} was created with a different config; "
+            "use a different --branch name or clean the existing temporary branch first."
+        )

varve/branch_config.py

@@ -0,0 +1,172 @@
+"""Resolve varve branches into Config objects and output roots."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, ValidationError, create_model
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from varve.branch import (
+    assert_same_config,
+    load_branches,
+    merge_override,
+    override_branch_name,
+    validate_branch_name,
+)
+from varve.pipeline import Pipeline
+from varve.store.store import Store
+
+ConfigT = TypeVar("ConfigT", bound=BaseModel)
+
+
+@dataclass(frozen=True)
+class ResolvedBranch:
+    config: Any
+    branch: str
+    is_temporary: bool
+    output_base: Path | None
+    temporary_config: dict[str, Any] | None = None
+
+
+def _settings_type(config_type: type[BaseModel]) -> type[BaseSettings]:
+    class VarveSettings(BaseSettings):
+        model_config = SettingsConfigDict(
+            env_nested_delimiter="__",
+            env_file=".env",
+        )
+
+        @classmethod
+        def settings_customise_sources(
+            cls,
+            settings_cls,
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            file_secret_settings,
+        ):
+            return (init_settings, env_settings, dotenv_settings, file_secret_settings)
+
+    fields: dict[str, Any] = {
+        name: (field.annotation, field) for name, field in config_type.model_fields.items()
+    }
+    return create_model(f"{config_type.__name__}VarveSettings", __base__=VarveSettings, **fields)
+
+
+def config_from_init(config_type: type[ConfigT], init_kwargs: dict[str, Any]) -> ConfigT:
+    settings_type = _settings_type(config_type)
+    settings = settings_type(**init_kwargs)
+    return config_type.model_validate(settings.model_dump())
+
+
+def _snapshot(config: Any) -> dict[str, Any]:
+    if not hasattr(config, "model_dump"):
+        raise TypeError("Temporary varve branches require a pydantic Config model")
+    return config.model_dump(mode="json")
+
+
+def _main_config(
+    experiment: type[Pipeline],
+    raw_main: dict[str, Any],
+    *,
+    cli_out: Path | None,
+    allow_bare_output_root: bool,
+) -> Any:
+    try:
+        return config_from_init(experiment.Config, raw_main)
+    except ValidationError:
+        if allow_bare_output_root and cli_out is not None:
+            return SimpleNamespace()
+        raise
+
+
+def _temporary_config_from_manifest(main_base: Path, branch: str) -> dict[str, Any]:
+    manifest = Store(main_base / ".tmp" / branch).read_manifest()
+    if manifest is None or manifest.temporary_config is None:
+        raise ValueError(f"Unknown varve branch {branch!r}")
+    return manifest.temporary_config
+
+
+def resolve_branch(
+    experiment: type[Pipeline],
+    *,
+    branch: str,
+    override_json: str | None,
+    cli_out: Path | None,
+    allow_bare_output_root: bool = False,
+) -> ResolvedBranch:
+    validate_branch_name(branch)
+    branches = load_branches(experiment.varve_config_path())
+    raw_main = branches.get("main", ({}, False))[0]
+
+    if override_json is not None:
+        if branch in branches and branch != "main":
+            raise ValueError("--override is only supported on main or temporary branches")
+
+        final_config = config_from_init(experiment.Config, merge_override(raw_main, override_json))
+        temporary_config = _snapshot(final_config)
+        if cli_out is not None:
+            main_base = Path(cli_out)
+        else:
+            main_config = config_from_init(experiment.Config, raw_main)
+            main_base = experiment.default_output_root(main_config)
+        resolved_branch = override_branch_name(temporary_config) if branch == "main" else branch
+        validate_branch_name(resolved_branch)
+
+        manifest = Store(main_base / ".tmp" / resolved_branch).read_manifest()
+        if manifest is not None:
+            if manifest.temporary_config is None:
+                raise ValueError(f"Unknown varve branch {resolved_branch!r}")
+            assert_same_config(manifest.temporary_config, temporary_config, branch=resolved_branch)
+
+        return ResolvedBranch(
+            config=final_config,
+            branch=resolved_branch,
+            is_temporary=True,
+            output_base=main_base,
+            temporary_config=temporary_config,
+        )
+
+    if branch in branches:
+        raw_config, is_temporary = branches[branch]
+        return ResolvedBranch(
+            config=config_from_init(experiment.Config, raw_config),
+            branch=branch,
+            is_temporary=is_temporary,
+            output_base=Path(cli_out) if cli_out is not None else None,
+        )
+    if branch == "main":
+        main_config = _main_config(
+            experiment,
+            raw_main,
+            cli_out=cli_out,
+            allow_bare_output_root=allow_bare_output_root,
+        )
+        return ResolvedBranch(
+            config=main_config,
+            branch="main",
+            is_temporary=False,
+            output_base=Path(cli_out) if cli_out is not None else None,
+        )
+
+    if cli_out is not None:
+        main_base = Path(cli_out)
+    else:
+        main_config = _main_config(
+            experiment,
+            raw_main,
+            cli_out=None,
+            allow_bare_output_root=False,
+        )
+        main_base = experiment.default_output_root(main_config)
+    temporary_config = _temporary_config_from_manifest(main_base, branch)
+    return ResolvedBranch(
+        config=experiment.Config.model_validate(temporary_config),
+        branch=branch,
+        is_temporary=True,
+        output_base=main_base,
+        temporary_config=temporary_config,
+    )

varve/cli/app.py

@@ -0,0 +1,297 @@
+"""Command-line interface for Pipeline subclasses."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+from pydantic import BaseModel
+
+from varve.branch_config import resolve_branch
+from varve.cli import argmap
+from varve.cli.clean import clean
+from varve.engine.runner import evaluate_state, run, selected_stages
+from varve.log import configure_cli_logging
+from varve.pipeline import Pipeline
+
+_CONFIG_COMMANDS = {"run", "status", "clean"}
+_NEGATIVE_NUMBER_RE = re.compile(r"^-\d+$|^-\d*\.\d+$")
+_COMMAND_OPTION_ARITIES = {
+    "run": {
+        "--branch": 1,
+        "--override": 1,
+        "--upto": 1,
+        "--downstream": 1,
+        "--force": 0,
+        "-f": 0,
+        "--out": 1,
+    },
+    "status": {"--branch": 1, "--upto": 1, "--downstream": 1, "--out": 1},
+    "clean": {
+        "--branch": 1,
+        "--downstream": 1,
+        "--out": 1,
+        "--yes": 0,
+        "-y": 0,
+    },
+    "plan": {"--upto": 1, "--downstream": 1},
+}
+
+
+def _args_from_namespace(
+    pipeline: type[Pipeline],
+    namespace: argparse.Namespace,
+) -> BaseModel:
+    init_kwargs = argmap.collect_cli_args_namespace(namespace, pipeline.Args)
+    return pipeline.Args.model_validate(init_kwargs)
+
+
+def _print_list(pipeline: type[Pipeline]) -> None:
+    for name in pipeline.topo_order():
+        spec = pipeline.stages()[name]
+        needs = ",".join(spec.needs) if spec.needs else "-"
+        kind = "batch" if spec.kind == "batch" else "stage"
+        print(f"{name}\t{kind}\tneeds={needs}")
+
+
+def _print_plan(
+    pipeline: type[Pipeline],
+    *,
+    upto: str | None,
+    downstream: str | None,
+) -> None:
+    selected = selected_stages(pipeline, upto=upto, downstream=downstream)
+    print(" -> ".join(name for name in pipeline.topo_order() if name in selected))
+
+
+def _print_status(
+    pipeline: type[Pipeline],
+    config,
+    args,
+    *,
+    upto: str | None,
+    downstream: str | None,
+    cli_out: Path | None,
+    branch: str,
+    is_temporary: bool,
+) -> None:
+    outcomes = evaluate_state(
+        pipeline,
+        config,
+        args=args,
+        upto=upto,
+        downstream=downstream,
+        cli_out=cli_out,
+        branch=branch,
+        is_temporary=is_temporary,
+    )
+    for outcome in outcomes:
+        print(f"{outcome.stage}\t{outcome.status}\t{outcome.reason}")
+
+
+def _default_confirm(message: str) -> bool:
+    try:
+        answer = input(f"{message} [y/N] ").strip().lower()
+    except EOFError:
+        return False
+    return answer in {"y", "yes"}
+
+
+def _selected_command_index(argv: list[str]) -> int | None:
+    index = 0
+    while index < len(argv):
+        token = argv[index]
+        if token in {"-v", "--verbose"}:
+            index += 1
+            continue
+        if token == "--":
+            next_index = index + 1
+            return next_index if next_index < len(argv) else None
+        if token.startswith("-"):
+            return None
+        return index
+    return None
+
+
+def _option_name(token: str) -> str:
+    if token.startswith("--"):
+        return token.split("=", 1)[0]
+    return token
+
+
+def _looks_like_option(token: str) -> bool:
+    return token.startswith("-") and token != "-" and _NEGATIVE_NUMBER_RE.match(token) is None
+
+
+def _has_unknown_option_before_config_registration(
+    *,
+    command: str,
+    command_args: list[str],
+    args_type: type[BaseModel],
+) -> bool:
+    option_arities = argmap.args_option_arities(args_type)
+    option_arities.update(_COMMAND_OPTION_ARITIES[command])
+    # Let argparse handle help instead of failing the strict precheck.
+    option_arities.setdefault("--help", 0)
+    option_arities.setdefault("-h", 0)
+
+    index = 0
+    while index < len(command_args):
+        token = command_args[index]
+        if token == "--":
+            return False
+        if not token.startswith("-") or token == "-":
+            index += 1
+            continue
+        option = _option_name(token)
+        arity = option_arities.get(option)
+        if arity is None:
+            return True
+        index += 1
+        if arity == 1 and "=" not in token:
+            if index >= len(command_args) or _looks_like_option(command_args[index]):
+                return True
+            index += 1
+    return False
+
+
+def main(pipeline: type[Pipeline], argv: list[str] | None = None) -> int:
+    raw_argv = list(argv) if argv is not None else sys.argv[1:]
+    selected_command_index = _selected_command_index(raw_argv)
+    selected_command = (
+        raw_argv[selected_command_index] if selected_command_index is not None else None
+    )
+    parser = argparse.ArgumentParser(prog=pipeline.__name__)
+    parser.add_argument("-v", "--verbose", action="store_true")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    out_help = (
+        "Override the output base. For named yaml branches this replaces that branch's "
+        "default base; for main and temporary branches it replaces the main base."
+    )
+
+    run_parser = subparsers.add_parser("run", help="run selected stages")
+    run_parser.add_argument("--branch", default="main", metavar="NAME", help="Select a branch.")
+    run_parser.add_argument(
+        "--override",
+        metavar="JSON",
+        help="Merge JSON over main Config and run a temporary branch.",
+    )
+    run_stage = run_parser.add_mutually_exclusive_group()
+    run_stage.add_argument("--upto", metavar="STAGE", help="Run STAGE and all upstream stages.")
+    run_stage.add_argument(
+        "--downstream", metavar="STAGE", help="Run STAGE and all downstream stages."
+    )
+    run_parser.add_argument(
+        "--force", "-f", action="store_true", help="Ignore cache for selected stages."
+    )
+    run_parser.add_argument("--out", type=Path, metavar="PATH", help=out_help)
+
+    status_parser = subparsers.add_parser("status", help="show read-only stage status")
+    status_parser.add_argument("--branch", default="main", metavar="NAME", help="Select a branch.")
+    status_stage = status_parser.add_mutually_exclusive_group()
+    status_stage.add_argument("--upto", metavar="STAGE", help="Show STAGE and all upstream stages.")
+    status_stage.add_argument(
+        "--downstream", metavar="STAGE", help="Show STAGE and all downstream stages."
+    )
+    status_parser.add_argument("--out", type=Path, metavar="PATH", help=out_help)
+
+    clean_parser = subparsers.add_parser("clean", help="delete selected store records and outputs")
+    clean_parser.add_argument("--branch", default="main", metavar="NAME", help="Select a branch.")
+    clean_parser.add_argument(
+        "--downstream",
+        metavar="STAGE",
+        help="Clean STAGE and all downstream stages.",
+    )
+    clean_parser.add_argument("--out", type=Path, metavar="PATH", help=out_help)
+    clean_parser.add_argument("--yes", "-y", action="store_true", help="Skip confirmation.")
+
+    plan_parser = subparsers.add_parser("plan", help="print selected stage order")
+    plan_stage = plan_parser.add_mutually_exclusive_group()
+    plan_stage.add_argument("--upto", metavar="STAGE", help="Print STAGE and all upstream stages.")
+    plan_stage.add_argument(
+        "--downstream", metavar="STAGE", help="Print STAGE and all downstream stages."
+    )
+
+    subparsers.add_parser("list")
+
+    if selected_command in _CONFIG_COMMANDS and selected_command_index is not None:
+        command_args = raw_argv[selected_command_index + 1 :]
+        if _has_unknown_option_before_config_registration(
+            command=selected_command,
+            command_args=command_args,
+            args_type=pipeline.Args,
+        ):
+            parser.error("unknown option or missing option value")
+
+    if selected_command == "run":
+        argmap.register_args(run_parser, pipeline.Args)
+    if selected_command == "status":
+        argmap.register_args(status_parser, pipeline.Args)
+    if selected_command == "clean":
+        argmap.register_args(clean_parser, pipeline.Args)
+
+    namespace = parser.parse_args(raw_argv)
+    configure_cli_logging(namespace.verbose)
+
+    if namespace.command == "list":
+        _print_list(pipeline)
+        return 0
+    if namespace.command == "plan":
+        _print_plan(pipeline, upto=namespace.upto, downstream=namespace.downstream)
+        return 0
+
+    resolved = resolve_branch(
+        pipeline,
+        branch=namespace.branch,
+        override_json=namespace.override if namespace.command == "run" else None,
+        cli_out=namespace.out,
+        allow_bare_output_root=namespace.command == "clean",
+    )
+    config = resolved.config
+    args = _args_from_namespace(pipeline, namespace)
+    if namespace.command == "status":
+        _print_status(
+            pipeline,
+            config,
+            args,
+            upto=namespace.upto,
+            downstream=namespace.downstream,
+            cli_out=resolved.output_base,
+            branch=resolved.branch,
+            is_temporary=resolved.is_temporary,
+        )
+    elif namespace.command == "clean":
+        allowed_roots = (
+            None if isinstance(config, SimpleNamespace) else pipeline.clean_roots(config)
+        )
+        clean(
+            pipeline,
+            config,
+            cli_out=resolved.output_base,
+            branch=resolved.branch,
+            is_temporary=resolved.is_temporary,
+            target=namespace.downstream,
+            yes=namespace.yes,
+            allowed_roots=allowed_roots,
+            confirm=_default_confirm,
+        )
+    elif namespace.command == "run":
+        outcomes = run(
+            pipeline,
+            config,
+            args=args,
+            upto=namespace.upto,
+            downstream=namespace.downstream,
+            force=namespace.force,
+            cli_out=resolved.output_base,
+            branch=resolved.branch,
+            is_temporary=resolved.is_temporary,
+            temporary_config=resolved.temporary_config,
+        )
+        for outcome in outcomes:
+            print(f"{outcome.stage}\t{outcome.status}\t{outcome.reason}")
+    return 0