roar-cli 0.4.0__tar.gz → 0.4.1__tar.gz

{roar_cli-0.4.0 → roar_cli-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roar-cli
-Version: 0.4.0
+Version: 0.4.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -34,6 +34,7 @@ Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
 Requires-Dist: mypy>=1.13.0 ; extra == 'dev'
 Requires-Dist: boto3>=1.28.0 ; extra == 'dev'
 Requires-Dist: google-cloud-storage>=2.10.0 ; extra == 'dev'
+Requires-Dist: huggingface-hub>=0.20.0 ; extra == 'dev'
 Provides-Extra: dev
 License-File: LICENSE
 Summary: Reproducibility and provenance tracker for ML training pipelines

{roar_cli-0.4.0 → roar_cli-0.4.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "roar-cli"
-version = "0.4.0"
+version = "0.4.1"
 description = "Reproducibility and provenance tracker for ML training pipelines"
 authors = [
     { name="TReqs Team", email="info@treqs.ai" }
@@ -68,6 +68,7 @@ dev = [
     "mypy>=1.13.0",
     "boto3>=1.28.0",
     "google-cloud-storage>=2.10.0",
+    "huggingface_hub>=0.20.0",
 ]
 
 [project.scripts]

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/analyzers/experiment_trackers.py

@@ -2,6 +2,7 @@
 
 import json
 import re
+import sqlite3
 from pathlib import Path
 from typing import Any, ClassVar
 
@@ -18,6 +19,7 @@ class ExperimentTrackerAnalyzer(Analyzer):
     TRACKER_PATTERNS: ClassVar[dict[str, list[str]]] = {
         "wandb": ["wandb/", ".wandb"],
         "mlflow": ["mlruns/", "mlartifacts/"],
+        "trackio": ["huggingface/trackio", "/trackio/"],
         "neptune": [".neptune/"],
         "tensorboard": ["/runs/", "events.out.tfevents"],
     }
@@ -72,7 +74,12 @@ class ExperimentTrackerAnalyzer(Analyzer):
         for tracker in trackers_found:
             run_info = self._extract_run_info(tracker, written, env)
             if run_info:
-                results["runs"].append(run_info)
+                # trackio reports one run per project DB (a run may touch
+                # several); the other trackers return a single run dict.
+                if isinstance(run_info, list):
+                    results["runs"].extend(run_info)
+                else:
+                    results["runs"].append(run_info)
 
             # Add ignore patterns for this tracker
             for pattern in self.IGNORE_PATTERNS:
@@ -88,18 +95,26 @@ class ExperimentTrackerAnalyzer(Analyzer):
             results["ignore_patterns"].extend(["mlruns/*", "mlartifacts/*"])
         if "neptune" in trackers_found:
             results["ignore_patterns"].append(".neptune/*")
+        if "trackio" in trackers_found:
+            results["ignore_patterns"].append("*/trackio/*")
 
         # Dedupe
         results["ignore_patterns"] = sorted(set(results["ignore_patterns"]))
 
         return results if results["trackers_detected"] else None
 
-    def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | None:
-        """Extract run URL and metadata for a specific tracker."""
+    def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | list | None:
+        """Extract run URL and metadata for a specific tracker.
+
+        Most trackers return a single run dict; ``trackio`` returns a list of
+        per-project run dicts (or ``None`` when nothing usable was found).
+        """
         if tracker == "wandb":
             return self._extract_wandb_info(written_files, env)
         elif tracker == "mlflow":
             return self._extract_mlflow_info(written_files, env)
+        elif tracker == "trackio":
+            return self._extract_trackio_info(written_files, env)
         elif tracker == "neptune":
             return self._extract_neptune_info(written_files, env)
         return None
@@ -198,6 +213,91 @@ class ExperimentTrackerAnalyzer(Analyzer):
 
         return info if len(info) > 1 else None
 
+    def _extract_trackio_info(self, written_files: list, env: dict) -> list | None:
+        """Extract trackio run info + the hosted-dashboard URL — scrape only.
+
+        trackio stores one SQLite DB per project at ``.../huggingface/trackio/
+        <project>.db`` and, when a run syncs to a HF Space, records the Space in
+        ``project_metadata`` (``key='space_id'``). We read that link + the run
+        identity, exactly as the W&B extractor reads wandb's on-disk run URL — no
+        roar-side config, no env, and **never the metrics**: TReqs stores no
+        customer data, so the DAG carries a link to the externally-hosted
+        experiment, and the metrics live on the dashboard.
+
+        Returns ONE record per project DB (never splicing one project's identity
+        onto another's Space), or ``None`` if no DB yielded anything usable. Each
+        record's URL is ``https://huggingface.co/spaces/<space_id>?project=<project>``.
+        (``env`` is unused — the space comes from what trackio itself persisted.)
+        """
+        del env
+        db_paths = sorted({p for p in written_files if "trackio/" in p and p.endswith(".db")})
+
+        runs: list[dict[str, Any]] = []
+        for db_path in db_paths:
+            path = Path(db_path)
+            if not path.exists():
+                continue
+            info: dict[str, Any] = {"tracker": "trackio", "project": path.stem}
+            try:
+                con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+            except sqlite3.Error:
+                continue
+            try:
+                # Two INDEPENDENT lookups: an older/imported trackio schema may
+                # lack a ``run_id`` column or the ``project_metadata`` table, so
+                # a failure of one must not discard the other.
+                run = self._trackio_run_identity(con)
+                space = self._trackio_space_id(con)
+            finally:
+                con.close()
+
+            if run:
+                info["run_id"], info["run_name"] = run
+            if space:
+                info["space_id"] = space
+                info["url"] = f"https://huggingface.co/spaces/{space}?project={info['project']}"
+
+            # Keep only DBs that yielded a run identity and/or a hosted Space
+            # (more than the always-present tracker+project keys).
+            if len(info) > 2:
+                runs.append(info)
+
+        return runs or None
+
+    @staticmethod
+    def _trackio_run_identity(con: "sqlite3.Connection") -> tuple | None:
+        """``(run_id, run_name)`` of the most recent metric row, tolerating
+        older schemas whose ``metrics`` table lacks a ``run_id`` column (trackio
+        produces these when importing TensorBoard/older runs)."""
+        try:
+            cols = {row[1] for row in con.execute("PRAGMA table_info(metrics)").fetchall()}
+        except sqlite3.Error:
+            return None
+        if not cols or not ({"run_id", "run_name"} & cols):
+            return None
+        run_id_expr = "run_id" if "run_id" in cols else "NULL"
+        run_name_expr = "run_name" if "run_name" in cols else "NULL"
+        try:
+            row = con.execute(
+                f"SELECT {run_id_expr}, {run_name_expr} FROM metrics ORDER BY rowid DESC LIMIT 1"
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+        if not row or (row[0] is None and row[1] is None):
+            return None
+        return row[0], row[1]
+
+    @staticmethod
+    def _trackio_space_id(con: "sqlite3.Connection") -> str | None:
+        """The hosted HF Space id trackio persisted once a run synced, or None."""
+        try:
+            row = con.execute(
+                "SELECT value FROM project_metadata WHERE key = 'space_id' LIMIT 1"
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+        return row[0] if row and row[0] else None
+
     def _extract_mlflow_info(self, written_files: list, env: dict) -> dict | None:
         """Extract MLflow run info from local files."""
         info = {"tracker": "mlflow"}

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/application/get/requests.py

@@ -23,3 +23,4 @@ class GetRequest:
     step_name: str | None = None
     limit: int | None = None  # download only the first N data files (subset get)
     full_anchor: bool = False  # re-hash non-LFS identity files past the budget
+    cache: str | None = None  # fetch bytes from this mirror; record `source` as canonical

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/application/get/service.py

@@ -12,6 +12,7 @@ from ...core.operation_metadata import build_operation_metadata_json
 from ...db.context import create_database_context
 from ...execution.recording import LocalJobRecorder, LocalRecordedArtifact
 from ...integrations.download import parse_source, resolve_download_backend
+from ...integrations.download.base import DownloadError
 from .requests import GetRequest
 from .results import GetDownloadedFile, GetResponse
 from .transfer import GetService, GetTransferResult
@@ -22,10 +23,27 @@ def get_artifacts(request: GetRequest) -> GetResponse:
     bootstrap(request.roar_dir)
     logger = get_logger()
 
-    parsed_source = parse_source(request.source)
-    backend = resolve_download_backend(request.source)
+    parsed_source = parse_source(request.source)  # canonical — always what we RECORD
+    # Fetch location: the mirror (`--cache`) when given, else the canonical source.
+    # Recording below always uses the canonical `parsed_source`, so the artifact's
+    # source_url / the AI-BOM downloadLocation stay the public URL, not the mirror.
+    fetch_url = request.cache or request.source
+    backend = resolve_download_backend(fetch_url)
+    fetch_parsed = parse_source(fetch_url)
     repo_root = request.repo_root or request.cwd
-    is_prefix = request.source.rstrip("/") != request.source or parsed_source.is_prefix
+    is_prefix = fetch_url.rstrip("/") != fetch_url or fetch_parsed.is_prefix
+
+    # A `--cache` mirror is only integrity-bound to the canonical identity when a
+    # `--hash` is given. Without it, the mirror's bytes are recorded under the
+    # canonical source_url unverified — warn so provenance isn't silently trusted.
+    if request.cache and not request.expected_hash:
+        logger.warning(
+            "--cache %s used without --hash: the mirror's bytes are not integrity-checked "
+            "against the canonical source %s, yet are recorded under it. Pass --hash "
+            "<digest> to bind the download to a known hash.",
+            request.cache,
+            request.source,
+        )
 
     git_commit = None
     git_branch = None
@@ -88,20 +106,57 @@ def get_artifacts(request: GetRequest) -> GetResponse:
             pass
 
     with create_database_context(request.roar_dir) as db_ctx:
-        service = GetService(
-            backend=backend,
-            source=parsed_source,
-            repo_root=repo_root,
-        )
         t0 = time.time()
-        transfer_result = service.get(
-            destination=request.destination,
-            expected_hash=request.expected_hash,
-            dry_run=request.dry_run,
-            force=request.force,
-            is_prefix=is_prefix,
-            limit=request.limit,
-        )
+        fetched_from_cache = False
+        try:
+            transfer_result = GetService(
+                backend=backend,
+                source=fetch_parsed,
+                repo_root=repo_root,
+            ).get(
+                destination=request.destination,
+                expected_hash=request.expected_hash,
+                dry_run=request.dry_run,
+                force=request.force,
+                is_prefix=is_prefix,
+                limit=request.limit,
+            )
+            fetched_from_cache = bool(request.cache)
+            # The transfer layer REPORTS a hash mismatch (and other soft failures)
+            # as success=False rather than raising, so trigger the same cache
+            # fallback here — otherwise a corrupt mirror hard-fails the whole get
+            # even when the canonical source would succeed (the documented behavior).
+            if request.cache and not transfer_result.dry_run and not transfer_result.success:
+                raise DownloadError(transfer_result.error or "cache fetch was unsuccessful")
+        except (DownloadError, FileNotFoundError, ValueError, OSError) as exc:
+            # A `--cache` mirror is best-effort: on miss / unreachable / hash
+            # mismatch, fall back to the canonical source (still hash-verified via
+            # --hash if given, and by content-addressing at reproduce time). A get
+            # without a cache re-raises unchanged.
+            if not request.cache:
+                raise
+            logger.warning(
+                "cache fetch from %s failed (%s); falling back to canonical source %s",
+                request.cache,
+                exc,
+                request.source,
+            )
+            fetched_from_cache = False
+            canonical_is_prefix = (
+                request.source.rstrip("/") != request.source or parsed_source.is_prefix
+            )
+            transfer_result = GetService(
+                backend=resolve_download_backend(request.source),
+                source=parsed_source,
+                repo_root=repo_root,
+            ).get(
+                destination=request.destination,
+                expected_hash=request.expected_hash,
+                dry_run=request.dry_run,
+                force=request.force,
+                is_prefix=canonical_is_prefix,
+                limit=request.limit,
+            )
         download_duration = time.time() - t0
 
         result = _materialize_get_result(
@@ -114,6 +169,8 @@ def get_artifacts(request: GetRequest) -> GetResponse:
             git_repo=git_repo_url,
             duration_seconds=download_duration,
             backend=backend,
+            fetched_from_cache=fetched_from_cache,
+            fetch_url=fetch_url,
         )
 
     git_tag_name = None
@@ -145,6 +202,26 @@ def get_artifacts(request: GetRequest) -> GetResponse:
     )
 
 
+def _canonical_source_url(file_info, request: GetRequest, fetch_url, fetched_from_cache: bool):
+    """The canonical (recorded) URL for one downloaded file.
+
+    When the bytes came from the canonical source (no ``--cache``, or a cache
+    miss/mismatch that fell back), each file's own fetch URL is already canonical.
+    When they came from a ``--cache`` mirror, remap each file's mirror URL back
+    onto the canonical prefix, so a PREFIX get records each file's own canonical
+    object (``s3://canon/data/a.pt``) rather than the bare prefix for every file.
+    """
+    remote_url = file_info.remote_url or ""
+    if not fetched_from_cache:
+        return remote_url or request.source
+    # Fetched from the mirror: swap the mirror prefix for the canonical one.
+    if fetch_url and remote_url.startswith(fetch_url) and remote_url != fetch_url:
+        suffix = remote_url[len(fetch_url) :]
+        return request.source.rstrip("/") + "/" + suffix.lstrip("/")
+    # Single-file cache (or an unmappable URL): the file itself IS the canonical source.
+    return request.source
+
+
 def _materialize_get_result(
     *,
     db_ctx,
@@ -156,6 +233,8 @@ def _materialize_get_result(
     git_repo: str | None = None,
     duration_seconds: float = 0.0,
     backend=None,
+    fetched_from_cache: bool = False,
+    fetch_url: str | None = None,
 ) -> GetResponse:
     if transfer_result.dry_run or not transfer_result.success:
         return GetResponse(
@@ -204,11 +283,18 @@ def _materialize_get_result(
         # downloaded files are the composite's components.
         output_artifacts: list[LocalRecordedArtifact] = []
     else:
+        # Record the canonical source on each downloaded artifact so it is a
+        # SOURCE node with a known origin (source_type/source_url), not a bare
+        # local file. This is what a downstream AI-BOM derives `downloadLocation`
+        # from — so it must be each file's OWN canonical URL, not a shared prefix.
+        source_type = parsed_source.scheme or None
         output_artifacts = [
             LocalRecordedArtifact(
                 path=file_info.local_path,
                 hashes={"blake3": str(file_info.hash)},
                 size=int(file_info.size or 0),
+                source_type=source_type,
+                source_url=_canonical_source_url(file_info, request, fetch_url, fetched_from_cache),
             )
             for file_info in transfer_result.downloaded_files
         ]
@@ -427,6 +513,9 @@ def _build_get_command(request: GetRequest) -> str:
     user typed.
     """
     parts = ["roar get", request.source]
+    if request.cache:
+        # Record the mirror so a reproduce replays cache-first-then-canonical.
+        parts.append(f"--cache {request.cache}")
     if request.limit is not None:
         parts.append(f"--limit {request.limit}")
     if request.step_name:

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/application/publish/put_preparation.py

@@ -100,7 +100,9 @@ def prepare_put_execution(
     )
 
     destination_type = _destination_type(destination)
-    composite_source_type = destination_type if destination_type in {"s3", "gs", "https"} else None
+    composite_source_type = (
+        destination_type if destination_type in {"s3", "gs", "https", "hf"} else None
+    )
 
     return PreparedPutExecution(
         glaas_client=runtime.glaas_client,

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/application/publish/registration.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import time
 from dataclasses import dataclass
 from typing import Any
@@ -329,14 +330,33 @@ def prepare_batch_registration_artifacts(
         except (TypeError, ValueError):
             size = 0
 
-        prepared.append(
-            {
-                "hashes": hashes,
-                "size": size,
-                "source_type": normalize_registration_source_type(artifact.get("source_type")),
-                "session_hash": session_hash,
-            }
-        )
+        entry = {
+            "hashes": hashes,
+            "size": size,
+            "source_type": normalize_registration_source_type(artifact.get("source_type")),
+            "session_hash": session_hash,
+        }
+
+        # Carry the artifact's source URL into its registration metadata so GLaaS
+        # can emit the AI-BOM `downloadLocation` external reference. The batch
+        # payload previously dropped `source_url`, so source-tracked inputs
+        # (e.g. s3://…) reached GLaaS with a sourceType but no distribution URL.
+        source_url = artifact.get("source_url")
+        if source_url and entry["source_type"]:
+            existing = artifact.get("metadata")
+            if isinstance(existing, str):
+                try:
+                    meta = json.loads(existing)
+                except (ValueError, TypeError):
+                    meta = {}
+            elif isinstance(existing, dict):
+                meta = dict(existing)
+            else:
+                meta = {}
+            meta.setdefault("source_url", source_url)
+            entry["metadata"] = json.dumps(meta)
+
+        prepared.append(entry)
 
     return prepared
 

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/application/reproducibility/report.py

@@ -75,6 +75,8 @@ def build_report(
     on_glaas: bool | None = None,
     single_commit: bool = True,
     untracked_paths: list[str] | None = None,
+    secrets_detected: int | None = None,
+    secrets_scanned: bool = True,
     notes: dict[str, str] | None = None,
     na: dict[str, str] | None = None,
 ) -> ReproducibilityReport:
@@ -147,6 +149,22 @@ def build_report(
                 "only on this machine — run `roar register` to publish it",
             )
         )
+    # Receipt-only, register/put: when the secret scan runs, any hits are redacted
+    # so the published lineage is clean (the note shows none/N redacted) and this
+    # passes. But it must NOT pass — and must not claim "none detected" — when the
+    # scan was disabled (registration.omit.enabled=false): that's "not scanned", not
+    # "scanned clean", and reporting it green would be false assurance. Doesn't bear
+    # on reproducibility, so `reproduce` omits it (secrets_detected None).
+    if secrets_detected is not None:
+        checks.append(
+            ReproCheck(
+                "secrets",
+                "no secrets in published lineage",
+                secrets_scanned,
+                "secret scan disabled (registration.omit.enabled=false) — the published "
+                "lineage was NOT scanned for secrets",
+            )
+        )
     report = ReproducibilityReport(checks=checks)
     for check in report.checks:
         if notes and check.key in notes:

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/cli/commands/get.py

@@ -78,6 +78,15 @@ from ..decorators import require_init
     "even past the 64 MB budget. Downloads them; without this they are excluded from "
     "the anchor's identity (LFS-only) when over budget.",
 )
+@click.option(
+    "--cache",
+    "cache",
+    default=None,
+    help="Fetch bytes from this mirror URL instead of SOURCE, while recording "
+    "SOURCE as the canonical origin (so the AI-BOM downloadLocation stays the public "
+    "URL, not the private mirror). The mirror is hash-verified against --hash when "
+    "given; roar falls back to SOURCE on miss/unreachable/mismatch.",
+)
 @click.pass_obj
 @require_init
 def get(
@@ -93,6 +102,7 @@ def get(
     step_name: str | None,
     limit: int | None,
     full_anchor: bool,
+    cache: str | None,
 ) -> None:
     """Download artifacts from cloud storage and record in the local DAG.
 
@@ -159,6 +169,7 @@ def get(
                 step_name=step_name,
                 limit=limit,
                 full_anchor=full_anchor,
+                cache=cache,
             )
         )
     except FileExistsError as e:

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/cli/commands/proxy.py

@@ -82,8 +82,9 @@ def proxy_disable() -> None:
 def proxy_start() -> None:
     """Start a standalone S3 proxy daemon.
 
-    The daemon runs in the background. Use its port with
-    AWS_ENDPOINT_URL to route S3 traffic through the proxy.
+    The daemon runs in the background. Point S3 clients at its port with
+    AWS_ENDPOINT_URL_S3 (SDKs) / S3_ENDPOINT_URL (s5cmd) to route S3 traffic
+    through the proxy.
     """
     from ...execution.cluster.proxy import ProxyService
     from ...integrations.config import get_roar_dir
@@ -104,7 +105,10 @@ def proxy_start() -> None:
         click.echo(f"Proxy daemon started (pid={info['pid']}, port={info['port']}).")
         click.echo("")
         click.echo("To use it:")
-        click.echo(f"  export AWS_ENDPOINT_URL=http://127.0.0.1:{info['port']}")
+        click.echo(
+            f"  export AWS_ENDPOINT_URL_S3=http://127.0.0.1:{info['port']}  # boto3, aws cli"
+        )
+        click.echo(f"  export S3_ENDPOINT_URL=http://127.0.0.1:{info['port']}     # s5cmd")
     except RuntimeError as e:
         raise click.ClickException(str(e)) from e
 

{roar_cli-0.4.0 → roar_cli-0.4.1}/roar/cli/commands/put.py

@@ -14,6 +14,10 @@ import click
 from ...application.publish.requests import PutRequest
 from ...application.publish.results import PutResponse
 from ...application.publish.service import put_artifacts
+from ...publish_auth import (
+    reset_requested_publish_visibility,
+    set_requested_publish_visibility,
+)
 from ..context import RoarContext
 from ..decorators import require_init
 from ..publish_intent import (
@@ -169,8 +173,11 @@ def put(
 
     \b
     Destination formats:
-        s3://bucket/prefix     AWS S3
-        gs://bucket/prefix     Google Cloud Storage
+        s3://bucket/prefix          AWS S3
+        gs://bucket/prefix          Google Cloud Storage
+        hf://owner/repo[/path]      Hugging Face Hub (model repo; recorded as the
+                                    public downloadLocation)
+        hf://datasets/owner/repo    Hugging Face Hub dataset repo
 
     \b
     Examples:
@@ -221,6 +228,9 @@ def put(
         click.echo("Publication aborted.")
         raise SystemExit(1)
 
+    # Record the explicit --public/--private choice so the publish resolves a
+    # flag-reconciled scope_request visibility (see publish_auth._scope_visibility).
+    _vis_token = set_requested_publish_visibility(public)
     try:
         response = put_artifacts(
             PutRequest(
@@ -242,6 +252,8 @@ def put(
         raise click.ClickException(str(e)) from e
     except Exception as e:  # pragma: no cover - defensive CLI boundary
         raise click.ClickException(f"Unexpected error during put: {e}") from e
+    finally:
+        reset_requested_publish_visibility(_vis_token)
 
     for warning in response.warnings:
         click.echo(f"Warning: {warning}", err=True)