roar-cli 0.4.1__tar.gz → 0.4.2__tar.gz

{roar_cli-0.4.1 → roar_cli-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roar-cli
-Version: 0.4.1
+Version: 0.4.2
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{roar_cli-0.4.1 → roar_cli-0.4.2}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "roar-cli"
-version = "0.4.1"
+version = "0.4.2"
 description = "Reproducibility and provenance tracker for ML training pipelines"
 authors = [
     { name="TReqs Team", email="info@treqs.ai" }
@@ -124,6 +124,7 @@ markers = [
     "diagnostic: Opt-in diagnostics or aspirational performance budgets outside the default gate",
     "large_pipeline: Stress-style pipeline coverage with larger DAG fixtures",
     "osmo_e2e: OSMO end-to-end tests requiring a Docker Compose managed KIND harness",
+    "k8s_e2e: Kubernetes end-to-end tests requiring the KIND harness (tests/backends/k8s)",
     "ray_e2e: Ray end-to-end tests requiring a running Docker cluster",
     "ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`",
     "ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details",

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/publish/put_execution.py

@@ -9,6 +9,7 @@ roar put ALWAYS registers lineage with GLaaS. This is not optional.
 
 from __future__ import annotations
 
+import shlex
 import time
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -52,6 +53,47 @@ from .put_composites import (
 from .results import PutCompositeRegistration, PutDryRunItem, PutUploadedFile
 
 
+def build_put_command(
+    sources: list[str],
+    destination: str | None,
+    *,
+    message: str,
+    public: bool = True,
+    anonymous: bool = False,
+    no_tag: bool = False,
+    as_dataset: bool = False,
+    step_name: str | None = None,
+) -> str:
+    """Reconstruct a faithful ``roar put`` command string for the job record.
+
+    The recorded command drives ``roar reproduce``, so it must include the
+    destination and every provenance-relevant flag — not just the sources and
+    message. Interaction/mode flags (``--yes``, ``--dry-run``) are intentionally
+    omitted: they control confirmation/preview, not the published result.
+    """
+    parts = ["roar", "put"]
+    if sources:
+        parts.extend(shlex.quote(s) for s in sources)
+    else:
+        parts.append("(session outputs)")
+    if destination:
+        parts.append(shlex.quote(destination))
+    if anonymous:
+        parts.append("--anonymous")
+    elif public is False:
+        parts.append("--private")
+    elif public:
+        parts.append("--public")
+    if no_tag:
+        parts.append("--no-tag")
+    if as_dataset:
+        parts.append("--as-dataset")
+    if step_name:
+        parts.extend(["--step-name", shlex.quote(step_name)])
+    parts.extend(["-m", shlex.quote(message)])
+    return " ".join(parts)
+
+
 def _upload_progress_message(done: int, total: int, uploaded_bytes: int) -> str:
     """Status line for the upload spinner, e.g. ``Uploading 12/100 files · 1.10 GB``."""
     return f"Uploading {done}/{total} files · {uploaded_bytes / 1e9:.2f} GB"
@@ -145,6 +187,7 @@ class PutService:
         git_commit: str | None = None,
         git_tag: str | None = None,
         declared: bool = False,
+        command: str | None = None,
     ) -> PutResult:
         """
         Execute a put operation.
@@ -294,6 +337,7 @@ class PutService:
                 git_context=git_context,
                 sources=sources,
                 message=message,
+                command=command,
                 uploads=uploads,
                 uploaded_files=uploaded_files,
                 artifacts_info=artifacts_info,
@@ -416,9 +460,10 @@ class PutService:
             for item in composite_registrations
         ]
 
-        # Build command string
-        source_str = " ".join(sources) if sources else "(session outputs)"
-        command = f'roar put {source_str} -m "{message}"'
+        # Build command string. Prefer the faithful command (destination + flags)
+        # supplied by the caller; fall back to a destination-aware reconstruction.
+        if command is None:
+            command = build_put_command(sources, self._destination, message=message)
 
         # Build metadata
         metadata_json = build_put_operation_metadata_json(
@@ -551,6 +596,7 @@ class PutService:
         git_context: GitContext,
         sources: list[str],
         message: str,
+        command: str | None = None,
         uploads: list[_UploadedArtifact],
         uploaded_files: list[PutUploadedFile],
         artifacts_info: list[tuple[str, str]],
@@ -571,8 +617,8 @@ class PutService:
         registration_errors: list[str] = []
         lineage_composite_registrations: list[dict[str, Any]] = []
         composite_registrations: list[dict[str, Any]] = []
-        source_str = " ".join(sources) if sources else "(session outputs)"
-        command = f'roar put {source_str} -m "{message}"'
+        if command is None:
+            command = build_put_command(sources, self._destination, message=message)
         source_type = normalize_registration_source_type(destination_type)
 
         provisional_metadata_json = build_put_operation_metadata_json(

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/publish/registration_package.py

@@ -206,6 +206,8 @@ def build_registration_package(
     if composites:
         package["composites"] = composites
 
+    if labels:
+        labels = _drop_labels_for_absent_jobs(labels, job_records)
     if labels:
         filtered_labels, label_detections = _filter_label_secrets(labels, omit_filter)
         package["labels"] = filtered_labels
@@ -292,6 +294,38 @@ def _apply_redaction(
     )
 
 
+def _drop_labels_for_absent_jobs(
+    labels: list[dict[str, Any]],
+    job_records: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """Drop job-scoped labels whose target job is not in the package.
+
+    Labels are collected from the raw lineage, but the package's jobs are
+    noise-filtered by ``normalize_jobs_for_registration``. A job label left
+    pointing at a dropped job makes the package internally inconsistent — GLaaS
+    rejects it on publish with "Package label references a job not in the
+    package". The job label's target reaches the publish side through the job
+    record's ``job_uid`` (GLaaS's ``sourceUid``), so match on that. Non-job
+    labels (dag, artifact) are unaffected.
+    """
+    package_job_uids = {str(job.get("job_uid")) for job in job_records if job.get("job_uid")}
+    kept: list[dict[str, Any]] = []
+    dropped = 0
+    for label in labels:
+        if str(label.get("entity_type")) == "job":
+            job_uid = label.get("job_uid")
+            if not isinstance(job_uid, str) or job_uid not in package_job_uids:
+                dropped += 1
+                continue
+        kept.append(label)
+    if dropped:
+        get_logger().debug(
+            "Dropped %d job label(s) referencing jobs not in the registration package",
+            dropped,
+        )
+    return kept
+
+
 def _filter_label_secrets(
     labels: list[dict[str, Any]],
     omit_filter: OmitFilter | None,

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/publish/service.py

@@ -894,6 +894,8 @@ def put_artifacts(request: PutRequest) -> PutResponse:
             )
             put_commit_on_remote = is_shareable_remote(prepared.git_context.repo)
 
+            from .put_execution import build_put_command
+
             result = service.put_prepared(
                 prepared=prepared,
                 sources=request.sources,
@@ -902,6 +904,16 @@ def put_artifacts(request: PutRequest) -> PutResponse:
                 git_commit=git_commit,
                 git_tag=expected_tag,
                 declared=request.as_dataset,
+                command=build_put_command(
+                    request.sources,
+                    request.destination,
+                    message=request.message,
+                    public=request.public,
+                    anonymous=request.anonymous,
+                    no_tag=request.no_tag,
+                    as_dataset=request.as_dataset,
+                    step_name=request.step_name,
+                ),
             )
 
             # Apply step name label if provided.

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/reproduce/requests.py

@@ -20,3 +20,8 @@ class ReproduceRequest:
     package_sync: bool = False
     list_requirements: bool = False
     out_path: str | None = None
+    # Skip publish (`roar put`) steps — for third-party reproduction that
+    # rebuilds the artifact without re-publishing to the owner's destination.
+    no_puts: bool = False
+    # Emit an editable reproduction shell script instead of previewing/running.
+    emit_script: bool = False

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/reproduce/service.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import re
+import shlex
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
@@ -42,7 +43,7 @@ def reproduce_artifact(
     server_url = _load_server_url(request.cwd)
     glaas_client = _build_glaas_client(server_url)
 
-    if request.out_path:
+    if request.out_path and not request.emit_script:
         _write_reproduction_output(
             request.hash_prefix,
             request.target_kind,
@@ -51,6 +52,9 @@ def reproduce_artifact(
             output,
         )
 
+    if request.emit_script and request.run_pipeline:
+        raise ValueError("choose either --script or --run, not both")
+
     pipeline, _pipeline_source = _resolve_pipeline(
         hash_prefix=request.hash_prefix,
         server_url=server_url,
@@ -58,6 +62,30 @@ def reproduce_artifact(
         glaas_client=glaas_client,
         target_kind=request.target_kind,
     )
+
+    # --no-puts: drop publish (`roar put`) steps so a third-party reproduction
+    # rebuilds the artifact without re-publishing to the owner's destination.
+    # Applied to the pipeline itself so preview, --script, and --run all agree
+    # (and the steps_run/steps_total accounting stays consistent).
+    if request.no_puts:
+        dropped = _drop_put_steps(pipeline)
+        # In --script mode stdout is the script itself; the omission is documented
+        # by a comment inside it, so don't pollute stdout with a notice here.
+        if dropped and not request.emit_script:
+            output.print(f"Skipping {dropped} publish (roar put) step(s) [--no-puts]")
+
+    # --script: emit an editable reproduction script instead of previewing/running.
+    if request.emit_script:
+        output.print(
+            build_reproduction_script(
+                pipeline,
+                request.hash_prefix,
+                target_kind=request.target_kind,
+                no_puts=request.no_puts,
+            )
+        )
+        return
+
     preview = build_preview_summary(
         pipeline,
         hash_prefix=request.hash_prefix,
@@ -357,6 +385,104 @@ def _resolve_pipeline(
     return pipeline, lookup.source
 
 
+def _is_put_step(step: dict) -> bool:
+    """A publish step: recorded as job_type 'put' or a `roar put …` command."""
+    kind = step.get("job_type") or step.get("jobType")
+    if isinstance(kind, str) and kind.strip().lower() == "put":
+        return True
+    cmd = str(step.get("command") or "").strip()
+    return cmd.split()[:2] == ["roar", "put"]
+
+
+def _drop_put_steps(pipeline: PipelineInfo) -> int:
+    """Remove publish steps from the pipeline in place; return how many were dropped."""
+    before = len(pipeline.run_steps)
+    pipeline.run_steps = [s for s in pipeline.run_steps if not _is_put_step(s)]
+    pipeline.total_steps = len(pipeline.build_steps) + len(pipeline.run_steps)
+    return before - len(pipeline.run_steps)
+
+
+def build_reproduction_script(
+    pipeline: PipelineInfo,
+    hash_prefix: str,
+    *,
+    target_kind: str = "artifact",
+    no_puts: bool = False,
+) -> str:
+    """Render an editable bash reproduction script (clone → env → pipeline).
+
+    Dry by design: nothing runs until the user executes it. Mirrors what
+    `--run` would do, so it composes with `--no-puts` (which has already
+    filtered publish steps out of *pipeline*).
+    """
+    from ..tags import run_modifier_flags
+
+    lines = [
+        "#!/usr/bin/env bash",
+        f"# Reproduction script for {target_kind} {hash_prefix}",
+        "# Generated by `roar reproduce --script`. Dry & editable: review the steps",
+        "# (e.g. change --device, or run on your own infra) before executing.",
+    ]
+    if no_puts:
+        lines.append("# Publish (roar put) steps are omitted (--no-puts).")
+    lines += ["set -euo pipefail", ""]
+
+    repo = pipeline.git_repo or "<unknown-git-repo>"
+    lines += [
+        "# 1. Clone the recorded code at its exact commit",
+        f"git clone {shlex.quote(repo)} reproduction",
+        "cd reproduction",
+    ]
+    if pipeline.git_commit:
+        lines.append(f"git checkout {shlex.quote(pipeline.git_commit)}")
+    if not pipeline.single_commit:
+        lines.append("# NOTE: the recorded session spanned multiple commits; results may differ.")
+    lines.append("")
+
+    lines += [
+        "# 2. Recreate the Python environment",
+        "python3 -m venv .venv",
+        "source .venv/bin/activate",
+    ]
+    for block in _build_requirement_blocks(pipeline):
+        if not block.values:
+            continue
+        if block.label in ("Pip packages", "Build tool pip packages"):
+            lines.append("pip install " + " ".join(shlex.quote(v) for v in block.values))
+        else:  # system / build-tool dpkg packages need sudo — emit as a comment
+            lines.append(f"# {block.label} (apt; requires sudo): " + " ".join(block.values))
+    lines.append("")
+
+    lines.append("# 3. Run the pipeline (recorded order)")
+    for step in pipeline.build_steps:
+        cmd = str(step.get("command") or "").strip()
+        if cmd:
+            lines.append(f"roar build {cmd}")
+    for step in pipeline.run_steps:
+        cmd = str(step.get("command") or "").strip()
+        if not cmd:
+            continue
+        # A step whose command is already a `roar …` invocation (e.g. `roar put`)
+        # is emitted as-is; other commands are wrapped in `roar run` (replaying
+        # any recorded run-modifiers) exactly as the executor would.
+        if cmd.split()[:1] == ["roar"]:
+            lines.append(cmd)
+            continue
+        metadata = step.get("metadata")
+        if isinstance(metadata, str):
+            try:
+                metadata = json.loads(metadata)
+            except (ValueError, TypeError):
+                metadata = {}
+        mods = (
+            run_modifier_flags(metadata.get("run_modifiers")) if isinstance(metadata, dict) else ""
+        )
+        prefix = f"roar run {mods}".strip()
+        lines.append(f"{prefix} {cmd}")
+    lines.append("")
+    return "\n".join(lines)
+
+
 def build_preview_summary(
     pipeline: PipelineInfo,
     *,

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/run/execution.py

@@ -28,6 +28,32 @@ class ExecutionReport:
     setup_error: bool = False
 
 
+RUN_REPORT_FILE_ENV = "ROAR_RUN_REPORT_FILE"
+
+
+def write_run_report_file(report: ExecutionReport) -> None:
+    """Write a machine-readable run outcome when ROAR_RUN_REPORT_FILE is set.
+
+    Wrappers that cannot see past the exit code (the k8s pod entrypoint) use
+    ``setup_error`` to distinguish "roar failed before launching the
+    workload" from "the workload ran and failed" — only the former is safe
+    to rerun uninstrumented. Advisory only: never fails the run.
+    """
+    target = str(os.environ.get(RUN_REPORT_FILE_ENV) or "").strip()
+    if not target:
+        return
+    import contextlib
+    import json
+
+    payload = {
+        "exit_code": report.exit_code,
+        "setup_error": report.setup_error,
+        "tracer_backend": report.tracer_backend,
+    }
+    with contextlib.suppress(OSError):
+        Path(target).write_text(json.dumps(payload), encoding="utf-8")
+
+
 def validate_git_clean(
     *,
     verb: str = "run",

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/run/service.py

@@ -17,6 +17,7 @@ from .execution import (
     execute_and_report,
     get_hash_algorithms,
     validate_git_clean,
+    write_run_report_file,
 )
 from .requests import BuildRequest, RunRequest
 from .verbosity import resolve_verbosity
@@ -150,6 +151,10 @@ def _execute_tracked_command(
         )
         raise
 
+    # Written before finalizers so a late finalizer crash cannot leave a
+    # wrapper believing the workload never launched.
+    write_run_report_file(report)
+
     try:
         if planned.finalize_run:
             planned.finalize_run(cast(Any, _FinalizerContext(roar_dir=roar_dir)))

{roar_cli-0.4.1 → roar_cli-0.4.2}/roar/application/system_labels.py

@@ -176,6 +176,10 @@ def _populate_run_build_common_labels(roar: dict[str, Any], metadata: dict[str,
                         row.get(key),
                     )
 
+        # Per-run GPU-usage fingerprint from NVIDIA accounting mode. Only
+        # present when accounting was enabled; land it alongside cuda/gpu.
+        _copy_scalar_map(runtime.get("gpu_accounting"), roar, ["runtime", "gpu_accounting"])
+
     packages = metadata.get("packages")
     if isinstance(packages, dict):
         for manager, package_map in packages.items():