roar-cli 0.4.2__tar.gz → 0.4.4__tar.gz

{roar_cli-0.4.2 → roar_cli-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roar-cli
-Version: 0.4.2
+Version: 0.4.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -25,6 +25,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pydantic-settings>=2.0.0
 Requires-Dist: textual>=0.80
 Requires-Dist: tomli>=2.0.0 ; python_full_version < '3.11'
+Requires-Dist: huggingface-hub>=0.20.0
 Requires-Dist: pytest>=7.0.0 ; extra == 'dev'
 Requires-Dist: pytest-timeout>=2.0.0 ; extra == 'dev'
 Requires-Dist: pytest-mock>=3.10.0 ; extra == 'dev'
@@ -34,7 +35,6 @@ Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
 Requires-Dist: mypy>=1.13.0 ; extra == 'dev'
 Requires-Dist: boto3>=1.28.0 ; extra == 'dev'
 Requires-Dist: google-cloud-storage>=2.10.0 ; extra == 'dev'
-Requires-Dist: huggingface-hub>=0.20.0 ; extra == 'dev'
 Provides-Extra: dev
 License-File: LICENSE
 Summary: Reproducibility and provenance tracker for ML training pipelines

{roar_cli-0.4.2 → roar_cli-0.4.4}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 
 [project]
 name = "roar-cli"
-version = "0.4.2"
+version = "0.4.4"
 description = "Reproducibility and provenance tracker for ML training pipelines"
 authors = [
     { name="TReqs Team", email="info@treqs.ai" }
@@ -49,6 +49,7 @@ dependencies = [
     "pydantic-settings>=2.0.0",
     "textual>=0.80",
     "tomli>=2.0.0; python_version < '3.11'",
+    "huggingface_hub>=0.20.0",  # `roar put hf://` is a shipped path, not dev-only (P0-19)
 ]
 
 [project.urls]
@@ -68,7 +69,6 @@ dev = [
     "mypy>=1.13.0",
     "boto3>=1.28.0",
     "google-cloud-storage>=2.10.0",
-    "huggingface_hub>=0.20.0",
 ]
 
 [project.scripts]
@@ -129,7 +129,7 @@ markers = [
     "ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`",
     "ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details",
 ]
-addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live"
+addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/backends/osmo --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live"
 timeout = 60
 filterwarnings = [
     "ignore::DeprecationWarning",

{roar_cli-0.4.2 → roar_cli-0.4.4}/roar/application/publish/collection.py

@@ -44,6 +44,11 @@ def collect_register_lineage(
     dry_run: bool = False,
 ) -> tuple[CollectedRegisterLineage | None, str | None]:
     """Collect local lineage for a resolved register target."""
+    if target.kind == "active_session":
+        return _collect_active_session_lineage(
+            roar_dir=roar_dir,
+            lineage_collector=lineage_collector,
+        )
     if target.kind == "step_reference":
         return _collect_step_lineage(
             step_reference=target.value,
@@ -81,6 +86,29 @@ def collect_register_lineage(
     return None, f"Unsupported register target type: {target.kind}"
 
 
+def _collect_active_session_lineage(
+    *,
+    roar_dir: Path,
+    lineage_collector: LineageCollector,
+) -> tuple[CollectedRegisterLineage | None, str | None]:
+    with create_database_context(roar_dir) as db_ctx:
+        session = db_ctx.sessions.get_active()
+        if not session:
+            return None, "No active session. Run 'roar run' to create a session first."
+        session_id = int(session["id"])
+        lineage = lineage_collector.collect_session(session_id, roar_dir)
+
+    return (
+        CollectedRegisterLineage(
+            lineage=lineage,
+            session_id=session_id,
+            artifact_hash="",
+            session_hash_override=None,
+        ),
+        None,
+    )
+
+
 def _collect_step_lineage(
     *,
     step_reference: str,

{roar_cli-0.4.2 → roar_cli-0.4.4}/roar/application/publish/lineage_composites.py

@@ -54,6 +54,7 @@ def preregister_lineage_composites_with_glaas(
     registration_errors: list[str],
     composite_builder: Any,
     logger: ILogger,
+    registration_session_id: str | None = None,
 ) -> list[dict[str, Any]]:
     """Prepare and preregister lineage composites before batch link resolution."""
     payloads = build_lineage_composite_payloads(
@@ -68,6 +69,7 @@ def preregister_lineage_composites_with_glaas(
         payloads=payloads,
         registration_errors=registration_errors,
         logger=logger,
+        registration_session_id=registration_session_id,
     )
 
 

{roar_cli-0.4.2 → roar_cli-0.4.4}/roar/application/publish/put_composites.py

@@ -39,6 +39,7 @@ def preregister_put_lineage_composites_with_glaas(
     dataset_identifiers: list[dict[str, Any]] | None,
     composite_builder: Any,
     logger: ILogger,
+    registration_session_id: str | None = None,
 ) -> list[dict[str, Any]]:
     """Prepare and preregister lineage composites for the put workflow."""
     payloads = build_put_lineage_composite_payloads(
@@ -55,6 +56,7 @@ def preregister_put_lineage_composites_with_glaas(
         payloads=payloads,
         registration_errors=registration_errors,
         logger=logger,
+        registration_session_id=registration_session_id,
     )
 
 
@@ -164,6 +166,7 @@ def register_put_composites_with_glaas(
     registration_errors: list[str],
     dataset_identifiers: list[dict[str, Any]] | None,
     logger: ILogger,
+    registration_session_id: str | None = None,
 ) -> list[dict[str, Any]]:
     """Register generated composite artifacts with GLaaS and persist local state."""
     composite_registrations: list[dict[str, Any]] = []
@@ -186,7 +189,14 @@ def register_put_composites_with_glaas(
         if metadata_json is not None:
             payload["metadata"] = metadata_json
 
-        response = resolved_remote_registry.register_composite_artifact(payload)
+        response = (
+            resolved_remote_registry.client.register_composite_artifact_under_registration_session(
+                registration_session_id,
+                payload,
+            )
+            if registration_session_id
+            else resolved_remote_registry.register_composite_artifact(payload)
+        )
         result, error = parse_composite_registration_response(response)
 
         composite_registration: dict[str, Any] = {

{roar_cli-0.4.2 → roar_cli-0.4.4}/roar/application/publish/put_execution.py

@@ -19,7 +19,7 @@ from ...application.publish.composite_builder import CompositeArtifactBuilder
 from ...application.publish.composites import build_publish_composite_results
 from ...application.publish.lineage import LineageCollector
 from ...application.publish.metadata import build_put_operation_metadata_json
-from ...application.publish.put_preparation import PreparedPutExecution
+from ...application.publish.put_preparation import DelegatedPutOperation, PreparedPutExecution
 from ...application.publish.registration import (
     normalize_registration_hashes,
     normalize_registration_source_type,
@@ -34,6 +34,7 @@ from ...application.publish.remote_job_uids import (
 from ...application.publish.session import build_staged_lineage_counts
 from ...application.system_labels import refresh_job_system_labels
 from ...core.interfaces.registration import GitContext
+from ...core.label_origins import LABEL_ORIGIN_SYSTEM
 from ...core.logging import get_logger
 from ...db.context import DatabaseContext
 from ...db.hashing import hash_files_blake3
@@ -219,6 +220,7 @@ class PutService:
         session_hash = prepared.session_hash
         registration_session_id = prepared.registration_session_id
         registration_session_mode = prepared.registration_session_mode
+        registration_session_status = prepared.registration_session_status
         git_context = prepared.git_context
         resolved = prepared.resolved_sources
         destination_type = prepared.destination_type
@@ -245,12 +247,31 @@ class PutService:
                 would_upload=[PutDryRunItem(path=str(r.path), exists=r.exists) for r in resolved],
             )
 
+        # Delegated broker sessions use a deterministic client-session id. If a
+        # previous attempt reached finalize but the caller lost the response,
+        # create/resume returns the closed session and its authoritative receipt.
+        # Treat that as a completed retry before hashing or uploading anything;
+        # closed registration-session capabilities cannot accept more staging and
+        # re-uploading large artifacts would be both wasteful and misleading.
+        if registration_session_id and registration_session_status == "closed":
+            self._logger.debug(
+                "Put publication already finalized for registration session %s",
+                registration_session_id,
+            )
+            return PutResult(
+                success=True,
+                session_hash=session_hash,
+                session_url=prepared.session_url,
+            )
+
         # Process each file: hash, create artifact, upload
         uploads: list[_UploadedArtifact] = []
         composite_registrations: list[dict[str, Any]] = []
         lineage_composite_registrations: list[dict[str, Any]] = []
-        with Spinner(f"Hashing {len(resolved)} file(s)..."):
-            hashes_by_path = self._hash_files_batch([source.path for source in resolved])
+        hashes_by_path = prepared.source_hashes
+        if not hashes_by_path:
+            with Spinner(f"Hashing {len(resolved)} file(s)..."):
+                hashes_by_path = self._hash_files_batch([source.path for source in resolved])
 
         # Uploads are the long pole of a put (multi-GB artifacts to S3/GCS); show a
         # live N/M + cumulative-bytes counter rather than a dead terminal.
@@ -309,6 +330,13 @@ class PutService:
         artifact_hashes = [u.hash for u in uploads]
         artifacts_info = [(u.artifact_id, u.local_path) for u in uploads]
 
+        # Record each uploaded artifact's distribution URL as a reserved system
+        # label, scoped to this publication, so GLaaS can surface it as the
+        # AI-BOM downloadLocation (see _set_publish_distribution_labels).
+        self._set_publish_distribution_labels(
+            uploads, normalize_registration_source_type(destination_type)
+        )
+
         self._logger.debug(
             "Upload complete: %d file(s), collecting lineage for hashes: %s",
             len(uploaded_files),
@@ -316,8 +344,11 @@ class PutService:
         )
 
         # Collect lineage for all uploaded artifacts (merged)
-        collector = self._lineage_collector or LineageCollector()
-        lineage = collector.collect(artifact_hashes, self._roar_dir)
+        if prepared.lineage is not None:
+            lineage = prepared.lineage
+        else:
+            collector = self._lineage_collector or LineageCollector()
+            lineage = collector.collect(artifact_hashes, self._roar_dir)
         self._logger.debug(
             "Lineage collected: %d job(s), %d artifact(s)",
             len(lineage.jobs),
@@ -332,6 +363,7 @@ class PutService:
                 coordinator=coordinator,
                 registration_session_id=registration_session_id,
                 registration_session_mode=registration_session_mode,
+                delegated_put_operation=prepared.delegated_put_operation,
                 session_id=session_id,
                 fallback_session_hash=session_hash or "",
                 git_context=git_context,
@@ -538,6 +570,7 @@ class PutService:
                     job_uid=job_uid,
                     remote_job_uid=job_uid,
                     registration_errors=registration_result.errors,
+                    uploads=uploads,
                 )
 
         registration_error = (
@@ -591,6 +624,7 @@ class PutService:
         coordinator: RegistrationCoordinator,
         registration_session_id: str,
         registration_session_mode: str | None,
+        delegated_put_operation: DelegatedPutOperation | None,
         session_id: int,
         fallback_session_hash: str,
         git_context: GitContext,
@@ -634,18 +668,30 @@ class PutService:
             timestamp=time.time(),
         )
 
-        step_number = self._db.sessions.get_next_step_number(session_id)
-        job_id, job_uid = self._db.jobs.create(
-            command=command,
-            timestamp=time.time(),
-            session_id=session_id,
-            step_number=step_number,
-            metadata=provisional_metadata_json,
-            execution_backend="local",
-            execution_role="host",
-            job_type="put",
-            exit_code=0,
+        stable_put_job_uid = (
+            delegated_put_operation.put_job_uid if delegated_put_operation is not None else None
         )
+        existing_put_job = (
+            self._db.jobs.get_by_uid(stable_put_job_uid) if stable_put_job_uid else None
+        )
+        if existing_put_job is not None:
+            job_id = int(existing_put_job["id"])
+            job_uid = str(existing_put_job["job_uid"])
+            step_number = int(existing_put_job.get("step_number") or 0)
+        else:
+            step_number = self._db.sessions.get_next_step_number(session_id)
+            job_id, job_uid = self._db.jobs.create(
+                command=command,
+                timestamp=time.time(),
+                job_uid=stable_put_job_uid,
+                session_id=session_id,
+                step_number=step_number,
+                metadata=provisional_metadata_json,
+                execution_backend="local",
+                execution_role="host",
+                job_type="put",
+                exit_code=0,
+            )
         self._logger.debug(
             "Put job created before registration-session finalize: id=%s, uid=%s, step=%d",
             job_id,
@@ -666,19 +712,56 @@ class PutService:
             composite_builder=self._composite_builder,
             declared=declared,
         )
+        uploaded_artifacts = self._build_uploaded_artifacts_for_registration(
+            uploads,
+            source_type,
+        )
+        staged_artifacts = prepare_batch_registration_artifacts(
+            uploaded_artifacts + lineage.artifacts,
+            registration_session_id,
+            fallback_to_hash=True,
+            prefer_blake3_first=True,
+        )
 
         put_job_registered = False
         put_job_links_succeeded = False
         with Spinner("Publishing lineage to GLaaS...") as spin:
+            spin.update("Staging lineage composites...")
+            lineage_composite_registrations = preregister_put_lineage_composites_with_glaas(
+                db_ctx=self._db,
+                glaas_client=client,
+                lineage_artifacts=lineage.artifacts,
+                session_hash=fallback_session_hash,
+                registration_errors=registration_errors,
+                dataset_identifiers=dataset_identifiers,
+                composite_builder=self._composite_builder,
+                logger=self._logger,
+                registration_session_id=registration_session_id,
+            )
+            spin.update("Staging output composites...")
+            composite_registrations = register_put_composites_with_glaas(
+                db_ctx=self._db,
+                glaas_client=client,
+                composite_results=composite_results_for_linking,
+                registration_errors=registration_errors,
+                dataset_identifiers=dataset_identifiers,
+                logger=self._logger,
+                registration_session_id=registration_session_id,
+            )
             spin.update("Staging lineage jobs and artifacts...")
             registration_result = coordinator.register_lineage_under_registration_session(
                 registration_session_id=registration_session_id,
                 git_context=git_context,
                 jobs=remote_lineage_jobs,
+                artifacts=staged_artifacts,
             )
             registration_errors.extend(registration_result.errors)
 
-            if registration_result.jobs_failed == 0 and registration_result.links_failed == 0:
+            if (
+                registration_result.jobs_failed == 0
+                and registration_result.links_failed == 0
+                and not registration_errors
+            ):
                 spin.update("Staging put job...")
                 put_job_result = coordinator.job_service.create_job_under_registration_session(
                     command=command,
@@ -748,39 +831,6 @@ class PutService:
                         session_hash = finalize_result.session_hash
                         session_url = finalize_result.session_url
 
-                        spin.update("Registering lineage composites...")
-                        lineage_composite_registrations = (
-                            preregister_put_lineage_composites_with_glaas(
-                                db_ctx=self._db,
-                                glaas_client=client,
-                                lineage_artifacts=lineage.artifacts,
-                                session_hash=session_hash,
-                                registration_errors=registration_errors,
-                                dataset_identifiers=dataset_identifiers,
-                                composite_builder=self._composite_builder,
-                                logger=self._logger,
-                            )
-                        )
-
-                        composite_results = build_publish_composite_results(
-                            resolved_sources=resolved,
-                            hashes_by_path=hashes_by_path,
-                            session_hash=session_hash,
-                            source_type=composite_source_type,
-                            additional_composite_roots=additional_composite_roots,
-                            composite_builder=self._composite_builder,
-                            declared=declared,
-                        )
-                        spin.update("Registering output composites...")
-                        composite_registrations = register_put_composites_with_glaas(
-                            db_ctx=self._db,
-                            glaas_client=client,
-                            composite_results=composite_results,
-                            registration_errors=registration_errors,
-                            dataset_identifiers=dataset_identifiers,
-                            logger=self._logger,
-                        )
-
         metadata_json = build_put_operation_metadata_json(
             message=message,
             destination=self._destination,
@@ -810,6 +860,8 @@ class PutService:
                 job_uid=job_uid,
                 remote_job_uid=remote_put_job_uid,
                 registration_errors=registration_errors,
+                uploads=uploads,
+                registration_session_id=registration_session_id,
             )
 
         composite_result_items = [
@@ -958,8 +1010,12 @@ class PutService:
                 "size": size,
                 "source_type": source_type,
             }
-            if uploaded.remote_url:
-                entry["source_url"] = uploaded.remote_url
+            # Deliberately do NOT record the upload destination as the artifact's
+            # global ``source_url`` here. That row is content-addressed and shared
+            # across orgs, so a per-publication URL would leak between tenants.
+            # The download location is published instead as a session-scoped
+            # ``roar.distribution.url`` label (see _set_publish_distribution_labels),
+            # which both the registration-session and this legacy inline path emit.
 
             payloads.append(entry)
 
@@ -1040,6 +1096,48 @@ class PutService:
             return False
         return True
 
+    def _set_publish_distribution_labels(
+        self,
+        uploads: list[_UploadedArtifact],
+        source_type: str | None,
+    ) -> None:
+        """Record each uploaded artifact's distribution URL as a reserved system
+        label (``roar.distribution.url``), scoped to this publication.
+
+        GLaaS reads this as the artifact's AI-BOM ``downloadLocation``. Because
+        the label lives on the ``(artifact, session)`` scope rather than the
+        global content-addressed artifact row, two orgs publishing identical
+        bytes each carry their own download location — no cross-tenant bleed.
+        Gated on a recognized remote ``source_type`` so we never assert a
+        distribution URL for a non-remote destination (e.g. ``memory://``).
+        """
+        if not source_type:
+            return
+        labels_repo = getattr(self._db, "labels", None)
+        if labels_repo is None:
+            return
+        for uploaded in uploads:
+            if not uploaded.remote_url or not uploaded.artifact_id:
+                continue
+            current = labels_repo.get_current("artifact", artifact_id=uploaded.artifact_id)
+            metadata = current.get("metadata") if isinstance(current, dict) else None
+            metadata = dict(metadata) if isinstance(metadata, dict) else {}
+            roar_ns = metadata.get("roar")
+            roar_ns = dict(roar_ns) if isinstance(roar_ns, dict) else {}
+            distribution = roar_ns.get("distribution")
+            distribution = dict(distribution) if isinstance(distribution, dict) else {}
+            if distribution.get("url") == uploaded.remote_url:
+                continue  # already recorded — avoid a redundant label version
+            distribution["url"] = uploaded.remote_url
+            roar_ns["distribution"] = distribution
+            metadata["roar"] = roar_ns
+            labels_repo.create_version(
+                "artifact",
+                metadata,
+                artifact_id=uploaded.artifact_id,
+                write_origin=LABEL_ORIGIN_SYSTEM,
+            )
+
     def _sync_put_job_labels_with_glaas(
         self,
         *,
@@ -1049,16 +1147,25 @@ class PutService:
         job_uid: str,
         remote_job_uid: str,
         registration_errors: list[str],
+        uploads: list[_UploadedArtifact] | None = None,
+        registration_session_id: str | None = None,
     ) -> None:
-        """Sync the local current label document for the publish-time put job."""
+        """Sync the local current label document for the publish-time put job
+        and its published artifacts (carrying ``roar.distribution.url``)."""
+        artifacts = (
+            [{"id": u.artifact_id, "hash": u.hash} for u in uploads if u.artifact_id and u.hash]
+            if uploads
+            else []
+        )
         sync_publish_labels(
             glaas_client=glaas_client,
             db_ctx=self._db,
             session_id=None,
             session_hash=session_hash,
             jobs=[{"id": job_id, "job_uid": job_uid, "remote_job_uid": remote_job_uid}],
-            artifacts=[],
+            artifacts=artifacts,
             errors=registration_errors,
+            registration_session_id=registration_session_id,
         )
 
     def _link_put_job_artifacts_with_glaas(