roar-cli 0.3.7__tar.gz → 0.4.1__tar.gz

{roar_cli-0.3.7 → roar_cli-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roar-cli
-Version: 0.3.7
+Version: 0.4.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -34,6 +34,7 @@ Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
 Requires-Dist: mypy>=1.13.0 ; extra == 'dev'
 Requires-Dist: boto3>=1.28.0 ; extra == 'dev'
 Requires-Dist: google-cloud-storage>=2.10.0 ; extra == 'dev'
+Requires-Dist: huggingface-hub>=0.20.0 ; extra == 'dev'
 Provides-Extra: dev
 License-File: LICENSE
 Summary: Reproducibility and provenance tracker for ML training pipelines

{roar_cli-0.3.7 → roar_cli-0.4.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "roar-cli"
-version = "0.3.7"
+version = "0.4.1"
 description = "Reproducibility and provenance tracker for ML training pipelines"
 authors = [
     { name="TReqs Team", email="info@treqs.ai" }
@@ -68,6 +68,7 @@ dev = [
     "mypy>=1.13.0",
     "boto3>=1.28.0",
     "google-cloud-storage>=2.10.0",
+    "huggingface_hub>=0.20.0",
 ]
 
 [project.scripts]

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/analyzers/experiment_trackers.py

@@ -2,6 +2,7 @@
 
 import json
 import re
+import sqlite3
 from pathlib import Path
 from typing import Any, ClassVar
 
@@ -18,6 +19,7 @@ class ExperimentTrackerAnalyzer(Analyzer):
     TRACKER_PATTERNS: ClassVar[dict[str, list[str]]] = {
         "wandb": ["wandb/", ".wandb"],
         "mlflow": ["mlruns/", "mlartifacts/"],
+        "trackio": ["huggingface/trackio", "/trackio/"],
         "neptune": [".neptune/"],
         "tensorboard": ["/runs/", "events.out.tfevents"],
     }
@@ -72,7 +74,12 @@ class ExperimentTrackerAnalyzer(Analyzer):
         for tracker in trackers_found:
             run_info = self._extract_run_info(tracker, written, env)
             if run_info:
-                results["runs"].append(run_info)
+                # trackio reports one run per project DB (a run may touch
+                # several); the other trackers return a single run dict.
+                if isinstance(run_info, list):
+                    results["runs"].extend(run_info)
+                else:
+                    results["runs"].append(run_info)
 
             # Add ignore patterns for this tracker
             for pattern in self.IGNORE_PATTERNS:
@@ -88,18 +95,26 @@ class ExperimentTrackerAnalyzer(Analyzer):
             results["ignore_patterns"].extend(["mlruns/*", "mlartifacts/*"])
         if "neptune" in trackers_found:
             results["ignore_patterns"].append(".neptune/*")
+        if "trackio" in trackers_found:
+            results["ignore_patterns"].append("*/trackio/*")
 
         # Dedupe
         results["ignore_patterns"] = sorted(set(results["ignore_patterns"]))
 
         return results if results["trackers_detected"] else None
 
-    def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | None:
-        """Extract run URL and metadata for a specific tracker."""
+    def _extract_run_info(self, tracker: str, written_files: list, env: dict) -> dict | list | None:
+        """Extract run URL and metadata for a specific tracker.
+
+        Most trackers return a single run dict; ``trackio`` returns a list of
+        per-project run dicts (or ``None`` when nothing usable was found).
+        """
         if tracker == "wandb":
             return self._extract_wandb_info(written_files, env)
         elif tracker == "mlflow":
             return self._extract_mlflow_info(written_files, env)
+        elif tracker == "trackio":
+            return self._extract_trackio_info(written_files, env)
         elif tracker == "neptune":
             return self._extract_neptune_info(written_files, env)
         return None
@@ -198,6 +213,91 @@ class ExperimentTrackerAnalyzer(Analyzer):
 
         return info if len(info) > 1 else None
 
+    def _extract_trackio_info(self, written_files: list, env: dict) -> list | None:
+        """Extract trackio run info + the hosted-dashboard URL — scrape only.
+
+        trackio stores one SQLite DB per project at ``.../huggingface/trackio/
+        <project>.db`` and, when a run syncs to a HF Space, records the Space in
+        ``project_metadata`` (``key='space_id'``). We read that link + the run
+        identity, exactly as the W&B extractor reads wandb's on-disk run URL — no
+        roar-side config, no env, and **never the metrics**: TReqs stores no
+        customer data, so the DAG carries a link to the externally-hosted
+        experiment, and the metrics live on the dashboard.
+
+        Returns ONE record per project DB (never splicing one project's identity
+        onto another's Space), or ``None`` if no DB yielded anything usable. Each
+        record's URL is ``https://huggingface.co/spaces/<space_id>?project=<project>``.
+        (``env`` is unused — the space comes from what trackio itself persisted.)
+        """
+        del env
+        db_paths = sorted({p for p in written_files if "trackio/" in p and p.endswith(".db")})
+
+        runs: list[dict[str, Any]] = []
+        for db_path in db_paths:
+            path = Path(db_path)
+            if not path.exists():
+                continue
+            info: dict[str, Any] = {"tracker": "trackio", "project": path.stem}
+            try:
+                con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+            except sqlite3.Error:
+                continue
+            try:
+                # Two INDEPENDENT lookups: an older/imported trackio schema may
+                # lack a ``run_id`` column or the ``project_metadata`` table, so
+                # a failure of one must not discard the other.
+                run = self._trackio_run_identity(con)
+                space = self._trackio_space_id(con)
+            finally:
+                con.close()
+
+            if run:
+                info["run_id"], info["run_name"] = run
+            if space:
+                info["space_id"] = space
+                info["url"] = f"https://huggingface.co/spaces/{space}?project={info['project']}"
+
+            # Keep only DBs that yielded a run identity and/or a hosted Space
+            # (more than the always-present tracker+project keys).
+            if len(info) > 2:
+                runs.append(info)
+
+        return runs or None
+
+    @staticmethod
+    def _trackio_run_identity(con: "sqlite3.Connection") -> tuple | None:
+        """``(run_id, run_name)`` of the most recent metric row, tolerating
+        older schemas whose ``metrics`` table lacks a ``run_id`` column (trackio
+        produces these when importing TensorBoard/older runs)."""
+        try:
+            cols = {row[1] for row in con.execute("PRAGMA table_info(metrics)").fetchall()}
+        except sqlite3.Error:
+            return None
+        if not cols or not ({"run_id", "run_name"} & cols):
+            return None
+        run_id_expr = "run_id" if "run_id" in cols else "NULL"
+        run_name_expr = "run_name" if "run_name" in cols else "NULL"
+        try:
+            row = con.execute(
+                f"SELECT {run_id_expr}, {run_name_expr} FROM metrics ORDER BY rowid DESC LIMIT 1"
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+        if not row or (row[0] is None and row[1] is None):
+            return None
+        return row[0], row[1]
+
+    @staticmethod
+    def _trackio_space_id(con: "sqlite3.Connection") -> str | None:
+        """The hosted HF Space id trackio persisted once a run synced, or None."""
+        try:
+            row = con.execute(
+                "SELECT value FROM project_metadata WHERE key = 'space_id' LIMIT 1"
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+        return row[0] if row and row[0] else None
+
     def _extract_mlflow_info(self, written_files: list, env: dict) -> dict | None:
         """Extract MLflow run info from local files."""
         info = {"tracker": "mlflow"}

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/get/requests.py

@@ -23,3 +23,4 @@ class GetRequest:
     step_name: str | None = None
     limit: int | None = None  # download only the first N data files (subset get)
     full_anchor: bool = False  # re-hash non-LFS identity files past the budget
+    cache: str | None = None  # fetch bytes from this mirror; record `source` as canonical

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/get/service.py

@@ -12,6 +12,7 @@ from ...core.operation_metadata import build_operation_metadata_json
 from ...db.context import create_database_context
 from ...execution.recording import LocalJobRecorder, LocalRecordedArtifact
 from ...integrations.download import parse_source, resolve_download_backend
+from ...integrations.download.base import DownloadError
 from .requests import GetRequest
 from .results import GetDownloadedFile, GetResponse
 from .transfer import GetService, GetTransferResult
@@ -22,10 +23,27 @@ def get_artifacts(request: GetRequest) -> GetResponse:
     bootstrap(request.roar_dir)
     logger = get_logger()
 
-    parsed_source = parse_source(request.source)
-    backend = resolve_download_backend(request.source)
+    parsed_source = parse_source(request.source)  # canonical — always what we RECORD
+    # Fetch location: the mirror (`--cache`) when given, else the canonical source.
+    # Recording below always uses the canonical `parsed_source`, so the artifact's
+    # source_url / the AI-BOM downloadLocation stay the public URL, not the mirror.
+    fetch_url = request.cache or request.source
+    backend = resolve_download_backend(fetch_url)
+    fetch_parsed = parse_source(fetch_url)
     repo_root = request.repo_root or request.cwd
-    is_prefix = request.source.rstrip("/") != request.source or parsed_source.is_prefix
+    is_prefix = fetch_url.rstrip("/") != fetch_url or fetch_parsed.is_prefix
+
+    # A `--cache` mirror is only integrity-bound to the canonical identity when a
+    # `--hash` is given. Without it, the mirror's bytes are recorded under the
+    # canonical source_url unverified — warn so provenance isn't silently trusted.
+    if request.cache and not request.expected_hash:
+        logger.warning(
+            "--cache %s used without --hash: the mirror's bytes are not integrity-checked "
+            "against the canonical source %s, yet are recorded under it. Pass --hash "
+            "<digest> to bind the download to a known hash.",
+            request.cache,
+            request.source,
+        )
 
     git_commit = None
     git_branch = None
@@ -88,20 +106,57 @@ def get_artifacts(request: GetRequest) -> GetResponse:
             pass
 
     with create_database_context(request.roar_dir) as db_ctx:
-        service = GetService(
-            backend=backend,
-            source=parsed_source,
-            repo_root=repo_root,
-        )
         t0 = time.time()
-        transfer_result = service.get(
-            destination=request.destination,
-            expected_hash=request.expected_hash,
-            dry_run=request.dry_run,
-            force=request.force,
-            is_prefix=is_prefix,
-            limit=request.limit,
-        )
+        fetched_from_cache = False
+        try:
+            transfer_result = GetService(
+                backend=backend,
+                source=fetch_parsed,
+                repo_root=repo_root,
+            ).get(
+                destination=request.destination,
+                expected_hash=request.expected_hash,
+                dry_run=request.dry_run,
+                force=request.force,
+                is_prefix=is_prefix,
+                limit=request.limit,
+            )
+            fetched_from_cache = bool(request.cache)
+            # The transfer layer REPORTS a hash mismatch (and other soft failures)
+            # as success=False rather than raising, so trigger the same cache
+            # fallback here — otherwise a corrupt mirror hard-fails the whole get
+            # even when the canonical source would succeed (the documented behavior).
+            if request.cache and not transfer_result.dry_run and not transfer_result.success:
+                raise DownloadError(transfer_result.error or "cache fetch was unsuccessful")
+        except (DownloadError, FileNotFoundError, ValueError, OSError) as exc:
+            # A `--cache` mirror is best-effort: on miss / unreachable / hash
+            # mismatch, fall back to the canonical source (still hash-verified via
+            # --hash if given, and by content-addressing at reproduce time). A get
+            # without a cache re-raises unchanged.
+            if not request.cache:
+                raise
+            logger.warning(
+                "cache fetch from %s failed (%s); falling back to canonical source %s",
+                request.cache,
+                exc,
+                request.source,
+            )
+            fetched_from_cache = False
+            canonical_is_prefix = (
+                request.source.rstrip("/") != request.source or parsed_source.is_prefix
+            )
+            transfer_result = GetService(
+                backend=resolve_download_backend(request.source),
+                source=parsed_source,
+                repo_root=repo_root,
+            ).get(
+                destination=request.destination,
+                expected_hash=request.expected_hash,
+                dry_run=request.dry_run,
+                force=request.force,
+                is_prefix=canonical_is_prefix,
+                limit=request.limit,
+            )
         download_duration = time.time() - t0
 
         result = _materialize_get_result(
@@ -114,6 +169,8 @@ def get_artifacts(request: GetRequest) -> GetResponse:
             git_repo=git_repo_url,
             duration_seconds=download_duration,
             backend=backend,
+            fetched_from_cache=fetched_from_cache,
+            fetch_url=fetch_url,
         )
 
     git_tag_name = None
@@ -145,6 +202,26 @@ def get_artifacts(request: GetRequest) -> GetResponse:
     )
 
 
+def _canonical_source_url(file_info, request: GetRequest, fetch_url, fetched_from_cache: bool):
+    """The canonical (recorded) URL for one downloaded file.
+
+    When the bytes came from the canonical source (no ``--cache``, or a cache
+    miss/mismatch that fell back), each file's own fetch URL is already canonical.
+    When they came from a ``--cache`` mirror, remap each file's mirror URL back
+    onto the canonical prefix, so a PREFIX get records each file's own canonical
+    object (``s3://canon/data/a.pt``) rather than the bare prefix for every file.
+    """
+    remote_url = file_info.remote_url or ""
+    if not fetched_from_cache:
+        return remote_url or request.source
+    # Fetched from the mirror: swap the mirror prefix for the canonical one.
+    if fetch_url and remote_url.startswith(fetch_url) and remote_url != fetch_url:
+        suffix = remote_url[len(fetch_url) :]
+        return request.source.rstrip("/") + "/" + suffix.lstrip("/")
+    # Single-file cache (or an unmappable URL): the file itself IS the canonical source.
+    return request.source
+
+
 def _materialize_get_result(
     *,
     db_ctx,
@@ -156,6 +233,8 @@ def _materialize_get_result(
     git_repo: str | None = None,
     duration_seconds: float = 0.0,
     backend=None,
+    fetched_from_cache: bool = False,
+    fetch_url: str | None = None,
 ) -> GetResponse:
     if transfer_result.dry_run or not transfer_result.success:
         return GetResponse(
@@ -204,11 +283,18 @@ def _materialize_get_result(
         # downloaded files are the composite's components.
         output_artifacts: list[LocalRecordedArtifact] = []
     else:
+        # Record the canonical source on each downloaded artifact so it is a
+        # SOURCE node with a known origin (source_type/source_url), not a bare
+        # local file. This is what a downstream AI-BOM derives `downloadLocation`
+        # from — so it must be each file's OWN canonical URL, not a shared prefix.
+        source_type = parsed_source.scheme or None
         output_artifacts = [
             LocalRecordedArtifact(
                 path=file_info.local_path,
                 hashes={"blake3": str(file_info.hash)},
                 size=int(file_info.size or 0),
+                source_type=source_type,
+                source_url=_canonical_source_url(file_info, request, fetch_url, fetched_from_cache),
             )
             for file_info in transfer_result.downloaded_files
         ]
@@ -427,6 +513,9 @@ def _build_get_command(request: GetRequest) -> str:
     user typed.
     """
     parts = ["roar get", request.source]
+    if request.cache:
+        # Record the mirror so a reproduce replays cache-first-then-canonical.
+        parts.append(f"--cache {request.cache}")
     if request.limit is not None:
         parts.append(f"--limit {request.limit}")
     if request.step_name:

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/publish/put_preparation.py

@@ -100,7 +100,9 @@ def prepare_put_execution(
     )
 
     destination_type = _destination_type(destination)
-    composite_source_type = destination_type if destination_type in {"s3", "gs", "https"} else None
+    composite_source_type = (
+        destination_type if destination_type in {"s3", "gs", "https", "hf"} else None
+    )
 
     return PreparedPutExecution(
         glaas_client=runtime.glaas_client,

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/publish/register_execution.py

@@ -222,8 +222,15 @@ class RegisterService:
         skip_confirmation: bool,
         confirm_callback: Callable[[list[str]], bool] | None,
         prepared: PreparedRegisterExecution,
+        composite_leaf_hashes: frozenset[str] = frozenset(),
     ) -> RegisterResult:
-        """Register already-collected local lineage with GLaaS."""
+        """Register already-collected local lineage with GLaaS.
+
+        ``composite_leaf_hashes`` are true composite-member hashes subsumed into a view
+        edge (see ``_prepare_view_edges_for_lineage``) — they have no session-scoped edge
+        on GLaaS, so they're excluded from label sync even though they still appear in
+        ``lineage.artifacts`` for ordinary artifact registration.
+        """
         self._logger.debug(
             "Collected lineage: %d jobs, %d artifacts",
             len(lineage.jobs),
@@ -276,6 +283,12 @@ class RegisterService:
                 omit_filter=omit_filter,
             )
 
+        label_artifacts = (
+            [a for a in lineage.artifacts if a.get("hash") not in composite_leaf_hashes]
+            if composite_leaf_hashes
+            else lineage.artifacts
+        )
+
         registration_jobs = order_jobs_for_registration(
             normalize_jobs_for_registration(lineage.jobs)
         )
@@ -344,7 +357,7 @@ class RegisterService:
                                 session_id=session_id,
                                 session_hash=finalized_session_hash,
                                 jobs=remote_registration_jobs,
-                                artifacts=lineage.artifacts,
+                                artifacts=label_artifacts,
                                 errors=registration_errors,
                             )
                 elif batch_result.jobs_failed == 0 and batch_result.links_failed == 0:
@@ -391,7 +404,7 @@ class RegisterService:
                                             session_id=session_id,
                                             session_hash=finalized_session_hash,
                                             jobs=remote_registration_jobs,
-                                            artifacts=lineage.artifacts,
+                                            artifacts=label_artifacts,
                                             errors=registration_errors,
                                         )
                             except Exception as e:
@@ -411,7 +424,7 @@ class RegisterService:
                                     session_id=session_id,
                                     session_hash=finalized_session_hash,
                                     jobs=remote_registration_jobs,
-                                    artifacts=lineage.artifacts,
+                                    artifacts=label_artifacts,
                                     errors=registration_errors,
                                 )
             else:
@@ -456,7 +469,7 @@ class RegisterService:
                             ),
                             db_ctx=db_ctx,
                             session_id=session_id,
-                            label_artifacts=lineage.artifacts,
+                            label_artifacts=label_artifacts,
                         )
                 else:
                     batch_result = register_publish_lineage(
@@ -473,7 +486,7 @@ class RegisterService:
                         ),
                         db_ctx=None,
                         session_id=None,
-                        label_artifacts=lineage.artifacts,
+                        label_artifacts=label_artifacts,
                     )
                 registration_errors.extend(batch_result.errors)
 

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/publish/registration.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import time
 from dataclasses import dataclass
 from typing import Any
@@ -329,14 +330,33 @@ def prepare_batch_registration_artifacts(
         except (TypeError, ValueError):
             size = 0
 
-        prepared.append(
-            {
-                "hashes": hashes,
-                "size": size,
-                "source_type": normalize_registration_source_type(artifact.get("source_type")),
-                "session_hash": session_hash,
-            }
-        )
+        entry = {
+            "hashes": hashes,
+            "size": size,
+            "source_type": normalize_registration_source_type(artifact.get("source_type")),
+            "session_hash": session_hash,
+        }
+
+        # Carry the artifact's source URL into its registration metadata so GLaaS
+        # can emit the AI-BOM `downloadLocation` external reference. The batch
+        # payload previously dropped `source_url`, so source-tracked inputs
+        # (e.g. s3://…) reached GLaaS with a sourceType but no distribution URL.
+        source_url = artifact.get("source_url")
+        if source_url and entry["source_type"]:
+            existing = artifact.get("metadata")
+            if isinstance(existing, str):
+                try:
+                    meta = json.loads(existing)
+                except (ValueError, TypeError):
+                    meta = {}
+            elif isinstance(existing, dict):
+                meta = dict(existing)
+            else:
+                meta = {}
+            meta.setdefault("source_url", source_url)
+            entry["metadata"] = json.dumps(meta)
+
+        prepared.append(entry)
 
     return prepared
 

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/publish/service.py

@@ -496,7 +496,7 @@ def _load_remote_job_uid_mapping(*, roar_dir: Path, session_id: Any) -> dict[str
 
 def _prepare_view_edges_for_lineage(
     *, roar_dir: Path, lineage: Any, logger: Any
-) -> dict[str, list[Any]]:
+) -> tuple[dict[str, list[Any]], set[str]]:
     """Compute consumes/produces view edges per job and prune the subsumed leaves in-place.
 
     A job that read part of a dataset gets a ``consumes`` view edge over its inputs; a job
@@ -505,15 +505,21 @@ def _prepare_view_edges_for_lineage(
     and any anchor linked as a plain input/output — are removed from the bundle so the job
     links the dataset *view* (job -> view -> anchor -> leaves), not the loose leaves or the
     anchor directly. The anchor composite stays in ``lineage.artifacts`` (already
-    collected) so it is still registered. Returns ``{job_uid: [view_edge, ...]}`` to push
-    after the main registration.
+    collected) so it is still registered.
+
+    Returns ``({job_uid: [view_edge, ...]}, composite_leaf_hashes)``: the view edges to
+    push after the main registration, and the union of true component/leaf hashes (never
+    the anchor's own hash) subsumed across all jobs — a leaf has no session-scoped edge on
+    GLaaS, so the caller uses this set to keep leaf label/tag documents out of publish-time
+    label sync too.
     """
     from .view_edges import resolve_view_edges_for_job
 
     jobs = getattr(lineage, "jobs", []) or []
     if not jobs:
-        return {}
+        return {}, set()
     view_edges_by_job: dict[str, list[Any]] = {}
+    composite_leaf_hashes: set[str] = set()
     with create_database_context(roar_dir) as db:
         for job in jobs:
             if not isinstance(job, dict):
@@ -530,7 +536,7 @@ def _prepare_view_edges_for_lineage(
                 hashes = list(job.get(hashes_key) or [])
                 if not hashes:
                     continue
-                resolved, prune = resolve_view_edges_for_job(
+                resolved, prune, leaf_hashes = resolve_view_edges_for_job(
                     db_ctx=db, input_hashes=hashes, relation=relation
                 )
                 if not resolved:
@@ -539,9 +545,10 @@ def _prepare_view_edges_for_lineage(
                 job[hashes_key] = [h for h in hashes if h not in prune]
                 if isinstance(job.get(items_key), list):
                     job[items_key] = [i for i in job[items_key] if i.get("hash") not in prune]
+                composite_leaf_hashes |= leaf_hashes
             if edges:
                 view_edges_by_job[str(job_uid)] = edges
-    return view_edges_by_job
+    return view_edges_by_job, composite_leaf_hashes
 
 
 def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageResponse:
@@ -628,13 +635,15 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR
         # bloom over the touched leaves) and prune the subsumed leaves from the bundle.
         # Pushed after the main registration. Best-effort.
         view_edges_by_job: dict[str, list[Any]] = {}
+        composite_leaf_hashes: frozenset[str] = frozenset()
         if not request.dry_run:
             try:
-                view_edges_by_job = _prepare_view_edges_for_lineage(
+                view_edges_by_job, leaf_hashes = _prepare_view_edges_for_lineage(
                     roar_dir=request.roar_dir,
                     lineage=collected_lineage.lineage,
                     logger=logger,
                 )
+                composite_leaf_hashes = frozenset(leaf_hashes)
             except Exception as exc:  # view-edge prep is best-effort
                 logger.debug("view-edge preparation skipped: %s", exc)
 
@@ -728,6 +737,7 @@ def register_lineage_target(request: RegisterLineageRequest) -> RegisterLineageR
             skip_confirmation=request.skip_confirmation,
             confirm_callback=request.confirm_callback,
             prepared=prepared,
+            composite_leaf_hashes=composite_leaf_hashes,
         )
 
         # Push the consumes view edges now that the jobs + the anchor composite are

{roar_cli-0.3.7 → roar_cli-0.4.1}/roar/application/publish/view_edges.py

@@ -115,16 +115,23 @@ def resolve_view_edges_for_job(
     db_ctx: Any,
     input_hashes: list[str],
     relation: Literal["consumes", "produces"] = "consumes",
-) -> tuple[list[dict[str, Any]], set[str]]:
+) -> tuple[list[dict[str, Any]], set[str], set[str]]:
     """Resolve a job's leaf *hashes* into view edges + hashes to prune.
 
     Works from the collected-lineage hashes (not artifact ids). ``relation`` is the side
     being resolved: ``consumes`` over a job's input hashes (a run that read part of a
     dataset) or ``produces`` over its output hashes (a run that wrote one). The resolution
     is identical either way — each leaf is matched to the anchor composite(s) that carry
-    it as a component. Returns ``(view_edges, prune_hashes)`` — the second being the leaf
-    hashes that collapse into a view edge plus any anchor composite hash linked as a plain
-    input/output (the view edge replaces it).
+    it as a component. Returns ``(view_edges, prune_hashes, leaf_hashes)``:
+
+    - ``prune_hashes``: the leaf hashes that collapse into a view edge plus any anchor
+      composite hash linked as a plain input/output (the view edge replaces it). Used to
+      drop these from a job's plain input/output edges.
+    - ``leaf_hashes``: strictly the true component/leaf hashes subsumed into a view edge
+      (a subset of ``prune_hashes`` that excludes a directly-linked anchor's own hash).
+      A composite member never has its own session-scoped edge on GLaaS — it has no
+      standing to carry its own labels/tags either, so callers use this set to keep
+      per-leaf label/tag documents out of publish-time label sync.
 
     BOUNDARY: matching uses ``find_by_component_digest``, which only knows the anchor's
     *stored* components (capped at ``_MAX_STORED_COMPONENTS`` = 1000). For a dataset with
@@ -138,7 +145,7 @@ def resolve_view_edges_for_job(
     artifacts_repo: Any = optional_repo(db_ctx, "artifacts")
     composites_repo: Any = optional_repo(db_ctx, "composites")
     if artifacts_repo is None or composites_repo is None:
-        return [], set()
+        return [], set(), set()
 
     # anchor_id -> {"algorithm": str, "leaves": set, "shards": set}
     by_anchor: dict[str, dict[str, Any]] = {}
@@ -174,6 +181,7 @@ def resolve_view_edges_for_job(
                 _attribute(blake3_digest, "blake3", digest)
 
     view_edges: list[dict[str, Any]] = []
+    leaf_hashes: set[str] = set()
     for anchor_id, bucket in by_anchor.items():
         anchor = artifacts_repo.get(anchor_id)
         anchor_digest = _primary_composite_digest(anchor)
@@ -189,8 +197,9 @@ def resolve_view_edges_for_job(
             )
         )
         prune |= bucket["shards"]
+        leaf_hashes |= bucket["shards"]
 
-    return view_edges, prune
+    return view_edges, prune, leaf_hashes
 
 
 def _primary_composite_digest(artifact: dict[str, Any] | None) -> str | None: