Orcca 0.1.0__py3-none-any.whl

orcca/__init__.py

@@ -0,0 +1,7 @@
+"""Public Orcca package surface."""
+
+from orcca.workflow import Workflow
+from orcca.launchers import register_launcher
+
+__all__ = ["Workflow", "register_launcher"]
+

orcca/cli.py

@@ -0,0 +1,617 @@
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import inspect
+import json
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+
+from orcca.dagger import _DagBuildError
+from orcca.executor import _ExecutionError
+from orcca.resolver import _ManifestResolutionError
+from orcca.runtime_launchers import _register_custom_launcher
+from orcca.validator import (
+    _ManifestValidationError,
+    _format_validation_report,
+)
+from orcca.workflow import Workflow
+from orcca.workflow_runner import _WorkflowRunError, _run_workflow
+
+
+class CliBootstrapError(Exception):
+    """Raised when CLI bootstrap steps fail."""
+
+
+def _find_project_launchers_dir(start_dir: Path) -> Path | None:
+    cur = start_dir.resolve()
+    for candidate_root in [cur, *cur.parents]:
+        launchers_dir = candidate_root / "launchers"
+        if launchers_dir.exists() and launchers_dir.is_dir():
+            return launchers_dir
+    return None
+
+
+def _load_module_from_path(py_file: Path, module_name: str) -> Any:
+    spec = importlib.util.spec_from_file_location(module_name, py_file)
+    if spec is None or spec.loader is None:
+        raise CliBootstrapError(f"Unable to load launcher module from {py_file}")
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except Exception as exc:  # noqa: BLE001
+        raise CliBootstrapError(f"Failed to execute launcher module: {py_file}: {exc}") from exc
+    return module
+
+
+def _autoload_launcher_modules_from_dir(launchers_dir: Path) -> None:
+    launcher_files = sorted(
+        p for p in launchers_dir.glob("*.py")
+        if p.name != "register.py" and not p.name.startswith("_")
+    )
+    for launcher_file in launcher_files:
+        launcher_id = launcher_file.stem
+        module_name = f"orcca_project_launcher_{abs(hash(str(launcher_file.resolve())))}"
+        module = _load_module_from_path(launcher_file, module_name)
+        candidate = getattr(module, launcher_id, None)
+        if candidate is None or not inspect.isfunction(candidate):
+            raise CliBootstrapError(
+                f"Launcher file '{launcher_file.name}' must define a function named '{launcher_id}'"
+            )
+        _register_custom_launcher(launcher_id, candidate)
+
+
+def _autoload_project_launchers(args: argparse.Namespace) -> None:
+    start_dir: Path
+    project_root = getattr(args, "project_root", None)
+    workspace_root = getattr(args, "workspace_root", None)
+    manifest_path = getattr(args, "manifest", None)
+    if isinstance(project_root, str) and project_root:
+        start_dir = Path(project_root).resolve()
+    elif isinstance(workspace_root, str) and workspace_root:
+        start_dir = Path(workspace_root).resolve()
+    elif isinstance(manifest_path, str) and manifest_path:
+        start_dir = Path(manifest_path).resolve().parent
+    else:
+        start_dir = Path.cwd()
+
+    launchers_dir = _find_project_launchers_dir(start_dir)
+    if launchers_dir is None:
+        return
+
+    register_file = launchers_dir / "register.py"
+    if register_file.exists() and register_file.is_file():
+        module_name = f"orcca_project_launchers_{abs(hash(str(register_file.resolve())))}"
+        _load_module_from_path(register_file, module_name)
+
+    _autoload_launcher_modules_from_dir(launchers_dir)
+
+
+def _log(message: str) -> None:
+    print(f"[orcca] {message}")
+
+
+def validate(
+    manifest: str | Path,
+    report_path: str | Path = ".orcca/validation/manifest_validation_report.html",
+) -> dict[str, Any]:
+    """Resolve, validate, and write a manifest validation report.
+
+    This is the Python counterpart to `orcca validate`.
+
+    Parameters
+    ----------
+    manifest : str or pathlib.Path
+        Path to the root workflow manifest YAML file.
+    report_path : str or pathlib.Path, default=".orcca/validation/manifest_validation_report.html"
+        Destination for the HTML validation report.
+    Returns
+    -------
+    dict[str, Any]
+        Payload containing the resolved validation result and report path.
+    """
+    workflow = Workflow(manifest)
+    return workflow.validate(report_path=report_path)
+
+
+def resolve(
+    manifest: str | Path,
+    output_path: str | Path = ".orcca/resolved_manifest.yaml",
+) -> dict[str, Any]:
+    """Resolve a manifest and write the resolved YAML to disk.
+
+    This is the Python counterpart to `orcca resolve`.
+
+    Parameters
+    ----------
+    manifest : str or pathlib.Path
+        Path to the root workflow manifest YAML file.
+    output_path : str or pathlib.Path, default=".orcca/resolved_manifest.yaml"
+        Destination for the resolved manifest YAML file.
+    Returns
+    -------
+    dict[str, Any]
+        Payload containing the resolved manifest path and resolved manifest.
+    """
+    workflow = Workflow(manifest)
+    return workflow.write_resolved_manifest(output_path=output_path)
+
+
+def plan(
+    manifest: str | Path,
+) -> dict[str, Any]:
+    """Resolve, validate, and compute the workflow plan.
+
+    This is the Python counterpart to `orcca plan`.
+
+    Parameters
+    ----------
+    manifest : str or pathlib.Path
+        Path to the root workflow manifest YAML file.
+    Returns
+    -------
+    dict[str, Any]
+        Planning payload, including validity and dependency information.
+    """
+    workflow = Workflow(manifest)
+    return workflow.plan()
+
+
+def run(
+    manifest: str | Path,
+    project_root: str | Path = ".",
+    run_dir: str | Path | None = None,
+    workers: int | None = None,
+    task_ids: list[str] | None = None,
+    artifact_ids: list[str] | None = None,
+    run_validations: bool = False,
+    validation_ids: list[str] | None = None,
+    no_execute: bool = False,
+    no_env_cache: bool = False,
+    refresh_env: bool = False,
+) -> Any:
+    """Run the full workflow pipeline.
+
+    This is the Python counterpart to `orcca run`.
+    """
+    workflow = Workflow(manifest, workspace_root=project_root, runs_root=Path(project_root) / ".orcca" / "runs")
+    if validation_ids and not run_validations:
+        raise _WorkflowRunError("--validation-id requires --run-validations")
+    return workflow.run(
+        run_dir=run_dir,
+        execute=not no_execute,
+        workers=workers,
+        task_ids=task_ids,
+        artifact_ids=artifact_ids,
+        run_validations=run_validations,
+        validation_ids=validation_ids,
+        use_env_cache=not no_env_cache,
+        refresh_env=refresh_env,
+        progress=_log,
+    )
+
+
+def dag(
+    manifest: str | Path,
+    project_root: str | Path = ".",
+    output_path: str | Path = ".orcca/graphs/artifact-dag.html",
+    artifact_id: str | None = None,
+) -> dict[str, Any]:
+    """Export an artifact dependency graph as an HTML file.
+
+    This is the Python counterpart to `orcca dag`.
+
+    Parameters
+    ----------
+    manifest
+        Path to the root workflow manifest YAML file.
+    project_root
+        Project workspace root used to resolve the manifest.
+    output_path
+        Destination HTML file for the graph export.
+    artifact_id
+        Optional generated artifact id to focus on.
+
+    Returns
+    -------
+    dict[str, Any]
+        HTML path plus graph summary metadata.
+    """
+    workflow = Workflow(manifest, workspace_root=project_root, runs_root=Path(project_root) / ".orcca" / "runs")
+    return workflow.export_artifact_dag(output_path=output_path, artifact_id=artifact_id)
+
+
+def env_cache(
+    action: str,
+    prune_all: bool = False,
+    output_format: str = "pretty",
+) -> dict[str, Any]:
+    """List or prune the package-manager environment cache.
+
+    This is the Python counterpart to `orcca env-cache`.
+    """
+    root = _env_cache_root()
+    root.mkdir(parents=True, exist_ok=True)
+    if action == "list":
+        entries: list[dict[str, Any]] = []
+        for d in sorted(p for p in root.iterdir() if p.is_dir()):
+            meta_path = d / "env_meta.json"
+            meta: dict[str, Any] = {}
+            if meta_path.exists():
+                try:
+                    meta = json.loads(meta_path.read_text(encoding="utf-8"))
+                except Exception:
+                    meta = {"error": "invalid_meta_json"}
+            entries.append({"path": str(d), "meta": meta})
+        return {"cache_root": str(root), "entries": entries}
+
+    if action == "prune":
+        removed = 0
+        for d in sorted(p for p in root.iterdir() if p.is_dir()):
+            if prune_all or not (d / "env_meta.json").exists():
+                shutil.rmtree(d, ignore_errors=True)
+                removed += 1
+        return {"cache_root": str(root), "removed": removed}
+
+    raise ValueError(f"unsupported env-cache action '{action}'")
+
+
+def init(
+    path: str | Path = ".",
+) -> dict[str, Any]:
+    """Scaffold a language-agnostic starter project.
+
+    This is the Python counterpart to `orcca init`.
+    """
+    project_dir = Path(path).resolve()
+    project_dir.mkdir(parents=True, exist_ok=True)
+    (project_dir / "launchers").mkdir(parents=True, exist_ok=True)
+    (project_dir / "workflows").mkdir(parents=True, exist_ok=True)
+    (project_dir / "manifests").mkdir(parents=True, exist_ok=True)
+
+    (project_dir / "launchers" / "default.py").write_text(
+        "def default(*, task, runtime, root):\n"
+        "    executable = runtime.get('executable', 'python')\n"
+        "    options = runtime.get('options', [])\n"
+        "    script = task['script']\n"
+        "    return [executable, *options, script]\n",
+        encoding="utf-8",
+    )
+    (project_dir / "workflows" / "main.py").write_text(
+        "from pathlib import Path\nPath('output.txt').write_text('ok\\n', encoding='utf-8')\n",
+        encoding="utf-8",
+    )
+    (project_dir / "requirements.txt").write_text("pandas==2.3.2\n", encoding="utf-8")
+    manifest = (
+        "id: starter\nname: Starter Workflow\nstudy:\n  id: ST-001\n"
+        "artifacts:\n  - id: out\n    name: Out\n    type: report\n    origin: generated\n    path: output.txt\n"
+        "runtimes:\n  - id: runtime\n    programming_language: python\n    version: '3.11'\n    launcher: default\n"
+        "    executable: python\n    environment:\n      type: package_manager\n      manager: pip\n      lockfile: requirements.txt\n"
+        "tasks:\n  - id: t1\n    runtimeId: runtime\n    script: workflows/main.py\n    outputs:\n      artifactIds: [out]\n"
+        "execution:\n  mode: serial\n"
+    )
+
+    (project_dir / "manifests" / "workflow.yaml").write_text(manifest, encoding="utf-8")
+    return {"initialized": str(project_dir)}
+
+
+def cmd_validate(args: argparse.Namespace) -> int:
+    """Resolve a manifest, validate it, and write a report.
+
+    This command checks the manifest structure and workflow rules without
+    running any workflow tasks. It is the CLI entry point for preflight
+    validation and is useful in both local development and CI pipelines.
+
+    Parameters
+    ----------
+    args : argparse.Namespace
+        Parsed CLI arguments for the `validate` command.
+
+    Returns
+    -------
+    int
+        Exit code (`0` when valid, `1` when invalid).
+    """
+    result = Workflow(args.manifest).validate(report_path=args.report_path)
+
+    print(f"validation_report_path: {result['validation_report_path']}")
+    print(f"valid: {result['valid']}")
+    return 0 if result["valid"] else 1
+
+
+def cmd_resolve(args: argparse.Namespace) -> int:
+    """Resolve a manifest and write the expanded YAML to disk.
+
+    The resolved manifest includes the final values after includes and config
+    files have been applied. Use this when you want to inspect or archive the
+    exact manifest that downstream tools will consume.
+    """
+    result = resolve(args.manifest, output_path=args.output_path)
+    print(f"resolved_manifest_path: {result['resolved_manifest_path']}")
+    return 0
+
+
+def cmd_plan(args: argparse.Namespace) -> int:
+    """Resolve a manifest, validate it, and print planning metadata.
+
+    The plan output is intended for inspection rather than execution. It helps
+    you understand task ordering, generated-artifact dependencies, and whether
+    the manifest is valid before any runtime work begins.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    isinstance(cmd_plan, object)
+    ```
+    True
+    """
+    result = Workflow(args.manifest).plan()
+
+    if not result["valid"]:
+        validation = result["validation"]
+        print(_format_validation_report(validation))
+        return 1
+
+    print(result)
+
+    return 0
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    """Run the workflow from manifest resolution through task execution.
+
+    The run command performs the complete workflow lifecycle: it resolves the
+    manifest, validates it, prepares runtimes, executes selected tasks, and
+    writes an HTML run report into the run directory.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    isinstance(cmd_run, object)
+    ```
+    True
+    """
+    _log("Starting workflow run")
+    result = Workflow(args.manifest, workspace_root=args.project_root, runs_root=Path(args.project_root) / ".orcca" / "runs").run(
+        run_dir=args.run_dir,
+        execute=not args.no_execute,
+        workers=args.workers,
+        task_ids=args.task_ids,
+        artifact_ids=args.artifact_ids,
+        run_validations=args.run_validations,
+        validation_ids=args.validation_ids,
+        use_env_cache=not args.no_env_cache,
+        refresh_env=args.refresh_env,
+        progress=_log,
+    )
+
+    status = "SUCCESS" if result.succeeded else "FAILED"
+    print(f"Run status: {status}")
+    print(f"Run ID: {result.run_id}")
+    print(f"Run directory: {result.run_dir}")
+    print(f"Run report: {result.run_report_path}")
+    if args.run_validations:
+        if result.validation_execution is None:
+            print("Validation task run: no validation tasks executed")
+        else:
+            print(f"Validation task run: {'SUCCESS' if result.validation_execution.succeeded else 'FAILED'}")
+
+    return 0 if result.succeeded else 1
+
+
+def cmd_dag(args: argparse.Namespace) -> int:
+    """Export the workflow artifact dependency graph as HTML.
+
+    Use this command to visualize how generated artifacts depend on one
+    another. You can export the full graph or focus on the upstream lineage of
+    a single artifact, which is especially helpful when debugging complex
+    workflows.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_dag, build_parser
+    isinstance(cmd_dag, object)
+    ```
+    True
+    """
+    result = dag(
+        args.manifest,
+        project_root=args.project_root,
+        output_path=args.output_path,
+        artifact_id=args.artifact_id,
+    )
+    print(f"html_path: {result['html_path']}")
+    print(f"artifact_id: {result['artifact_id']}")
+    print(f"node_count: {result['node_count']}")
+    print(f"edge_count: {result['edge_count']}")
+    return 0
+
+
+def _env_cache_root() -> Path:
+    return (Path(".") / ".orcca" / "cache" / "environments").resolve()
+
+
+def cmd_env_cache(args: argparse.Namespace) -> int:
+    """Inspect or prune the cached package-manager environments.
+
+    This command is useful when you want to see which runtime environments are
+    already prepared, or when you need to clean out stale environments before
+    a rebuild or CI run.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    isinstance(cmd_env_cache, object)
+    ```
+    True
+    """
+    try:
+        result = env_cache(args.action, prune_all=args.all)
+    except ValueError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    print(result)
+    return 0
+
+
+def cmd_init(args: argparse.Namespace) -> int:
+    """Create a starter project template.
+
+    This command sets up the minimum project structure needed to get started
+    with Orcca, including a manifest, workflow directory, launcher file, and
+    default runtime wiring.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    isinstance(cmd_init, object)
+    ```
+    True
+    """
+    result = init(path=args.path)
+    print(result)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the Orcca top-level argument parser and subcommands.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    parser = build_parser()
+    parser.prog is not None
+    ```
+    True
+    """
+    parser = argparse.ArgumentParser(
+        description="Orcca CLI for validating, planning, and running workflow manifests.",
+        epilog="Example: orcca run manifests/workflow.yaml --project-root . --run-dir .orcca/runs/demo",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    resolve_parser = subparsers.add_parser("resolve", help="Resolve a manifest and write the resolved YAML")
+    resolve_parser.add_argument("manifest", help="Path to root workflow manifest YAML")
+    resolve_parser.add_argument(
+        "--output-path",
+        default=".orcca/resolved_manifest.yaml",
+        help="YAML output path for the resolved manifest",
+    )
+    resolve_parser.set_defaults(func=cmd_resolve)
+
+    validate_parser = subparsers.add_parser("validate", help="Resolve and validate a workflow manifest")
+    validate_parser.add_argument("manifest", help="Path to root workflow manifest YAML")
+    validate_parser.add_argument(
+        "--report-path",
+        default=".orcca/validation/manifest_validation_report.html",
+        help="Path to write the manifest validation report HTML",
+    )
+    validate_parser.set_defaults(func=cmd_validate)
+
+    plan_parser = subparsers.add_parser("plan", help="Resolve, validate, and print execution planning details")
+    plan_parser.add_argument("manifest", help="Path to root workflow manifest YAML")
+    plan_parser.set_defaults(func=cmd_plan)
+
+    run_parser = subparsers.add_parser("run", help="Run end-to-end workflow pipeline")
+    run_parser.add_argument("manifest", help="Path to root workflow manifest YAML")
+    run_parser.add_argument(
+        "--project-root",
+        default=".",
+        help="Project root used for runtime execution and run artifacts (stored under .orcca/runs)",
+    )
+    run_parser.add_argument(
+        "--run-dir",
+        default=None,
+        help="Explicit run directory to write artifacts into instead of creating a timestamped subdirectory",
+    )
+    run_parser.add_argument("--workers", type=int, default=None, help="Override the number of concurrent workers")
+    run_parser.add_argument("--task", dest="task_ids", action="append", default=[], help="Target task id to execute; repeat to select multiple tasks")
+    run_parser.add_argument("--artifact", dest="artifact_ids", action="append", default=[], help="Target artifact id to generate; repeat to select multiple artifacts")
+    run_parser.add_argument(
+        "--run-validations",
+        action="store_true",
+        help="Execute validation tasks declared in validations[].validationTaskId",
+    )
+    run_parser.add_argument(
+        "--validation-id",
+        dest="validation_ids",
+        action="append",
+        default=[],
+        help="Validation id to execute (can be repeated). Requires --run-validations.",
+    )
+    run_parser.add_argument("--no-execute", action="store_true", help="Resolve and validate only; skip task execution")
+    run_parser.add_argument("--no-env-cache", action="store_true", help="Disable environment cache reuse for package-manager runtimes")
+    run_parser.add_argument("--refresh-env", action="store_true", help="Force rebuild or restore of package-manager environments")
+    run_parser.set_defaults(func=cmd_run)
+
+    dag_parser = subparsers.add_parser("dag", help="Export the artifact dependency graph as HTML")
+    dag_parser.add_argument("manifest", help="Path to root workflow manifest YAML")
+    dag_parser.add_argument(
+        "--project-root",
+        default=".",
+        help="Project root used to resolve the manifest and default graph output location",
+    )
+    dag_parser.add_argument(
+        "--artifact-id",
+        default=None,
+        help="Optional generated artifact id to export a focused upstream dependency graph",
+    )
+    dag_parser.add_argument(
+        "--output-path",
+        default=".orcca/graphs/artifact-dag.html",
+        help="HTML output path for the graph export",
+    )
+    dag_parser.set_defaults(func=cmd_dag)
+
+    env_cache_parser = subparsers.add_parser("env-cache", help="List or prune package-manager environment cache")
+    env_cache_parser.add_argument("action", choices=["list", "prune"], help="Cache action to perform")
+    env_cache_parser.add_argument("--all", action="store_true", help="When pruning, remove every cached environment")
+    env_cache_parser.set_defaults(func=cmd_env_cache)
+
+    init_parser = subparsers.add_parser("init", help="Scaffold a starter project")
+    init_parser.add_argument("path", nargs="?", default=".", help="Target project directory")
+    init_parser.set_defaults(func=cmd_init)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entrypoint for the `orcca` command.
+
+    Examples
+    --------
+    ```{python}
+from orcca.cli import cmd_plan, cmd_run, cmd_env_cache, cmd_init, cmd_inspect_run, build_parser, main
+    # Run help:
+    # main(["--help"])
+    isinstance(main, object)
+    ```
+    True
+    """
+    parser = build_parser()
+    effective_argv = argv if argv is not None else sys.argv[1:]
+    if not effective_argv:
+        parser.print_help()
+        return 0
+    args = parser.parse_args(effective_argv)
+
+    try:
+        _autoload_project_launchers(args)
+        return int(args.func(args))
+    except (CliBootstrapError, _ManifestResolutionError, _ManifestValidationError, _WorkflowRunError, _ExecutionError, _DagBuildError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+