roar-cli 0.2.8__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {roar_cli-0.2.8 → roar_cli-0.2.9}/PKG-INFO +1 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/pyproject.toml +4 -1
- roar_cli-0.2.9/roar/bin/libroar_tracer_preload.so +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/bin/roar-proxy +0 -0
- roar_cli-0.2.9/roar/bin/roar-tracer +0 -0
- roar_cli-0.2.9/roar/bin/roar-tracer-ebpf +0 -0
- roar_cli-0.2.9/roar/bin/roar-tracer-preload +0 -0
- roar_cli-0.2.9/roar/bin/roard +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/_execution.py +1 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/_ray_job_submit.py +102 -33
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/init.py +0 -2
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/register.py +26 -20
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/context.py +10 -2
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/decorators.py +3 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/config.py +0 -5
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/repositories.py +4 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/config.py +0 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/session.py +43 -2
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/glaas/auth.py +2 -2
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/dag_data_builder.py +48 -4
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/actor.py +5 -11
- roar_cli-0.2.9/roar/ray/collector.py +849 -0
- roar_cli-0.2.9/roar/ray/driver_entrypoint.py +143 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/fragment.py +1 -0
- roar_cli-0.2.9/roar/ray/fragment_reconstituter.py +675 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/glaas_fragment_streamer.py +60 -5
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/node_agent.py +47 -7
- roar_cli-0.2.9/roar/ray/proxy_fragments.py +99 -0
- roar_cli-0.2.9/roar/ray/roar_worker.py +1687 -0
- roar_cli-0.2.9/roar/ray/s3_key_paths.py +57 -0
- roar_cli-0.2.9/roar/services/execution/__init__.py +44 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/coordinator.py +3 -0
- roar_cli-0.2.9/roar/services/execution/inject/sitecustomize.py +1473 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/data_loader.py +4 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/proxy.py +6 -4
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/tracer_backends.py +50 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/composite_builder.py +19 -1
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/register_service.py +335 -26
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/upload/lineage_collector.py +141 -0
- roar_cli-0.2.8/roar/bin/libroar_tracer_preload.so +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer-ebpf +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer-preload +0 -0
- roar_cli-0.2.8/roar/bin/roard +0 -0
- roar_cli-0.2.8/roar/ray/collector.py +0 -1015
- roar_cli-0.2.8/roar/ray/fragment_reconstituter.py +0 -184
- roar_cli-0.2.8/roar/ray/roar_worker.py +0 -607
- roar_cli-0.2.8/roar/ray/worker.py +0 -550
- roar_cli-0.2.8/roar/services/execution/__init__.py +0 -21
- roar_cli-0.2.8/roar/services/execution/inject/sitecustomize.py +0 -851
- {roar_cli-0.2.8 → roar_cli-0.2.9}/LICENSE +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/README.md +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/__main__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/analyzers/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/analyzers/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/analyzers/experiment_trackers.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/auth.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/build.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/config.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/dag.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/env.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/get.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/lineage.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/log.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/pop.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/proxy.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/put.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/reproduce.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/reset.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/run.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/show.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/status.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/cli/commands/tracer.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/bootstrap.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/container.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/di.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/digests.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/dto/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/dto/registration.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/exceptions.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/cloud.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/command.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/config.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/logger.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/presenter.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/provenance.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/registration.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/reproduction.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/run.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/services.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/telemetry.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/upload.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/interfaces/vcs.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/logging.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/artifact.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/command.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/dag.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/dataset_identifier.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/glaas.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/job.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/lineage.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/provenance.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/run.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/session.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/telemetry.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/models/vcs.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/registry.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/settings.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/tracer_modes.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/core/validation.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/context.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/engine.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/hashing/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/hashing/backend.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/models.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/artifact.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/collection.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/composite.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/hash_cache.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/repositories/job.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/schema.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/services/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/services/hashing.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/services/job_recording.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/services/lineage.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/db/services/session.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/filters/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/filters/files.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/filters/omit.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/glaas/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/glaas/transport.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/glaas_client.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/cloud/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/cloud/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/telemetry/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/telemetry/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/telemetry/wandb.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/vcs/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/vcs/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/plugins/vcs/git.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/console.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/dag_renderer.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/formatting.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/null.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/run_report.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/show_renderer.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/presenters/spinner.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/_agent_names.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/ray/fragment_key.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/args.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/backup.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/dag_resolver.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/dataset_identifier.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/execution_service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/inject/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/job_recording.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/assembler.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/build_pip_collector.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/build_tool_collector.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/file_filter.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/package_collector.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/process_summarizer.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/runtime_collector.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/provenance/service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/signal_handler.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/execution/tracer.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/gcs.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/http.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/noop.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/backends/s3.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/get/service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/logging.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/lookup/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/lookup/entity_lookup.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/lookup/step_parser.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/gcs.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/memory.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/noop.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/backends/s3.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/git.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/resolver.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/put/service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/_artifact_ref.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/_dataset_label.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/_dataset_profile.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/artifact.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/coordinator.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/job.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/registration/session.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/environment_setup.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/installers.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/pipeline_executor.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/pipeline_metadata.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/reproduction/service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/secrets/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/secrets/filter_service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/transfer/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/transfer/backend_resolution.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/transfer/common.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/upload/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/upload/service.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/vcs/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/services/vcs/git_access.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/utils/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/utils/cloud.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/roar/utils/git_url.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/Cargo.lock +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/crates/artifact-hash-core/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/crates/artifact-hash-core/src/lib.rs +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/crates/artifact-hash-py/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.9}/rust/crates/artifact-hash-py/src/lib.rs +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "roar-cli"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.9"
|
|
8
8
|
description = "Reproducibility and provenance tracker for ML training pipelines"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="TReqs Team", email="info@treqs.ai" }
|
|
@@ -81,6 +81,7 @@ include = [
|
|
|
81
81
|
{ path = "LICENSE", format = "wheel" },
|
|
82
82
|
{ path = "roar/bin/*", format = "sdist" },
|
|
83
83
|
{ path = "roar/bin/*", format = "wheel" },
|
|
84
|
+
{ path = "roar_inject.pth", format = "wheel" },
|
|
84
85
|
]
|
|
85
86
|
|
|
86
87
|
[tool.pytest.ini_options]
|
|
@@ -97,6 +98,8 @@ markers = [
|
|
|
97
98
|
"cloud: Tests for cloud storage operations",
|
|
98
99
|
"happy_path: Happy path tests for core functionality",
|
|
99
100
|
"ray_e2e: Ray end-to-end tests requiring a running Docker cluster",
|
|
101
|
+
"ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`",
|
|
102
|
+
"ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details",
|
|
100
103
|
]
|
|
101
104
|
addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/integration --ignore=tests/e2e"
|
|
102
105
|
timeout = 60
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -48,7 +48,7 @@ def validate_git_clean() -> str:
|
|
|
48
48
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
49
49
|
# Ray jobs run from extracted working dirs that are not git repos.
|
|
50
50
|
# Allow execution there while preserving git checks everywhere else.
|
|
51
|
-
if "
|
|
51
|
+
if "RAY_JOB_ID" in os.environ:
|
|
52
52
|
return cwd
|
|
53
53
|
raise click.ClickException(
|
|
54
54
|
"roar requires the working directory to be inside a git repository."
|
|
@@ -10,6 +10,13 @@ from pathlib import Path
|
|
|
10
10
|
from ...glaas_client import GlaasClient
|
|
11
11
|
from ...ray.fragment_key import generate_fragment_key, save_key
|
|
12
12
|
|
|
13
|
+
_ROAR_WORKER_PY_EXECUTABLE = "roar-worker"
|
|
14
|
+
_ROAR_WORKER_SETUP_HOOK = "roar.ray.roar_worker._startup"
|
|
15
|
+
_ROAR_DRIVER_ENTRYPOINT_MODULE = "roar.ray.driver_entrypoint"
|
|
16
|
+
_ROAR_JOB_INSTRUMENTED_ENV_VAR = "ROAR_JOB_INSTRUMENTED"
|
|
17
|
+
_ROAR_CLUSTER_GLAAS_URL_ENV = "ROAR_CLUSTER_GLAAS_URL"
|
|
18
|
+
_ROAR_CLUSTER_AWS_ENDPOINT_URL_ENV = "ROAR_CLUSTER_AWS_ENDPOINT_URL"
|
|
19
|
+
|
|
13
20
|
|
|
14
21
|
@dataclass(frozen=True)
|
|
15
22
|
class RayJobSubmitRewrite:
|
|
@@ -30,6 +37,7 @@ def maybe_rewrite_ray_job_submit(command: list[str]) -> RayJobSubmitRewrite:
|
|
|
30
37
|
entrypoint = list(command[separator_index + 1 :])
|
|
31
38
|
if not entrypoint:
|
|
32
39
|
return RayJobSubmitRewrite(command=command)
|
|
40
|
+
entrypoint = _wrap_entrypoint_for_driver_proxy(entrypoint)
|
|
33
41
|
|
|
34
42
|
runtime_env_json_arg = _find_runtime_env_json(before_separator)
|
|
35
43
|
runtime_env = _load_runtime_env(before_separator, runtime_env_json_arg)
|
|
@@ -38,16 +46,54 @@ def maybe_rewrite_ray_job_submit(command: list[str]) -> RayJobSubmitRewrite:
|
|
|
38
46
|
return RayJobSubmitRewrite(command=command)
|
|
39
47
|
|
|
40
48
|
merged_pip = _merge_roar_runtime_env_pip(runtime_env.get("pip"))
|
|
41
|
-
if merged_pip
|
|
49
|
+
if merged_pip:
|
|
42
50
|
runtime_env["pip"] = merged_pip
|
|
51
|
+
else:
|
|
52
|
+
runtime_env.pop("pip", None)
|
|
53
|
+
|
|
54
|
+
# py_executable is intentionally NOT set at job level — it would apply to the
|
|
55
|
+
# JobSupervisor/driver process which doesn't have roar installed yet (pip runs after
|
|
56
|
+
# the supervisor starts). worker_process_setup_hook is sufficient: it runs inside
|
|
57
|
+
# each worker process after the runtime env (and pip) is ready.
|
|
58
|
+
runtime_env["worker_process_setup_hook"] = _ROAR_WORKER_SETUP_HOOK
|
|
43
59
|
|
|
44
60
|
env_vars = dict(runtime_env.get("env_vars", {}) or {})
|
|
61
|
+
env_vars[_ROAR_JOB_INSTRUMENTED_ENV_VAR] = "1"
|
|
62
|
+
env_vars["ROAR_WRAP"] = "1"
|
|
63
|
+
env_vars["ROAR_RAY_NODE_AGENTS"] = "1"
|
|
64
|
+
# Stable job_id shared by driver + workers for node agent name resolution.
|
|
65
|
+
import uuid as _uuid
|
|
66
|
+
|
|
67
|
+
env_vars["ROAR_JOB_ID"] = _uuid.uuid4().hex[:8]
|
|
68
|
+
|
|
69
|
+
# Tell the job driver where roar.db lives (CWD inside the Ray job is the
|
|
70
|
+
# extracted working_dir, not the original project directory).
|
|
71
|
+
roar_dir = os.environ.get("ROAR_PROJECT_DIR", "")
|
|
72
|
+
if not roar_dir:
|
|
73
|
+
# Use CWD — `roar run` is invoked from the project root.
|
|
74
|
+
roar_dir = os.getcwd()
|
|
75
|
+
if roar_dir and os.path.isfile(os.path.join(roar_dir, ".roar", "roar.db")):
|
|
76
|
+
env_vars["ROAR_PROJECT_DIR"] = roar_dir
|
|
77
|
+
|
|
78
|
+
# Route S3 traffic through the per-node proxy (port 19191) for I/O capture.
|
|
79
|
+
# Save the REAL upstream endpoint (not the roar-run local proxy) so the
|
|
80
|
+
# cluster-side proxy can forward to the actual S3 service.
|
|
81
|
+
original_endpoint = (
|
|
82
|
+
os.environ.get("ROAR_UPSTREAM_S3_ENDPOINT") or os.environ.get("AWS_ENDPOINT_URL") or ""
|
|
83
|
+
)
|
|
84
|
+
cluster_upstream_endpoint = _resolve_cluster_upstream_s3_endpoint(original_endpoint)
|
|
85
|
+
if cluster_upstream_endpoint:
|
|
86
|
+
env_vars["ROAR_UPSTREAM_S3_ENDPOINT"] = cluster_upstream_endpoint
|
|
87
|
+
env_vars["ROAR_PROXY_PORT"] = "19191"
|
|
88
|
+
env_vars["AWS_ENDPOINT_URL"] = "http://127.0.0.1:19191"
|
|
89
|
+
|
|
45
90
|
fragment_session_id: str | None = None
|
|
46
91
|
|
|
47
92
|
glaas_url = _resolve_glaas_url()
|
|
48
93
|
if glaas_url:
|
|
49
|
-
|
|
50
|
-
|
|
94
|
+
cluster_glaas_url = _resolve_cluster_glaas_url(glaas_url)
|
|
95
|
+
if cluster_glaas_url:
|
|
96
|
+
env_vars["GLAAS_URL"] = cluster_glaas_url
|
|
51
97
|
|
|
52
98
|
key = generate_fragment_key()
|
|
53
99
|
try:
|
|
@@ -67,7 +113,6 @@ def maybe_rewrite_ray_job_submit(command: list[str]) -> RayJobSubmitRewrite:
|
|
|
67
113
|
runtime_env.pop("env_vars", None)
|
|
68
114
|
|
|
69
115
|
before_separator = _store_runtime_env(before_separator, runtime_env, runtime_env_json_arg)
|
|
70
|
-
entrypoint = _wrap_entrypoint(entrypoint)
|
|
71
116
|
return RayJobSubmitRewrite(
|
|
72
117
|
command=[*before_separator, "--", *entrypoint],
|
|
73
118
|
session_id=fragment_session_id,
|
|
@@ -84,6 +129,17 @@ def _is_ray_job_submit(command: list[str]) -> bool:
|
|
|
84
129
|
return binary == "ray" and noun in {"job", "jobs"} and verb == "submit"
|
|
85
130
|
|
|
86
131
|
|
|
132
|
+
def _wrap_entrypoint_for_driver_proxy(entrypoint: list[str]) -> list[str]:
|
|
133
|
+
if (
|
|
134
|
+
len(entrypoint) >= 3
|
|
135
|
+
and entrypoint[1] == "-m"
|
|
136
|
+
and entrypoint[2] == _ROAR_DRIVER_ENTRYPOINT_MODULE
|
|
137
|
+
):
|
|
138
|
+
return entrypoint
|
|
139
|
+
|
|
140
|
+
return ["python", "-m", _ROAR_DRIVER_ENTRYPOINT_MODULE, "--", *entrypoint]
|
|
141
|
+
|
|
142
|
+
|
|
87
143
|
def _find_runtime_env_json(command: list[str]) -> tuple[int, int | None] | None:
|
|
88
144
|
for index, arg in enumerate(command):
|
|
89
145
|
if arg == "--runtime-env-json":
|
|
@@ -137,27 +193,23 @@ def _store_runtime_env(
|
|
|
137
193
|
return command_out
|
|
138
194
|
|
|
139
195
|
|
|
140
|
-
def _wrap_entrypoint(entrypoint: list[str]) -> list[str]:
|
|
141
|
-
if len(entrypoint) >= 2 and Path(entrypoint[0]).name == "roar" and entrypoint[1] == "run":
|
|
142
|
-
return entrypoint
|
|
143
|
-
return ["roar", "run", *entrypoint]
|
|
144
|
-
|
|
145
|
-
|
|
146
196
|
def _merge_roar_runtime_env_pip(existing_pip: object) -> list[str] | None:
|
|
147
197
|
roar_req = _resolve_roar_requirement()
|
|
148
|
-
if roar_req is None:
|
|
149
|
-
# Local dev mode: vendor wheel present means cluster has roar pre-installed.
|
|
150
|
-
# Skip pip injection — preserve existing pip list unchanged.
|
|
151
|
-
existing = _coerce_runtime_env_pip(existing_pip)
|
|
152
|
-
return existing if existing else None
|
|
153
198
|
dependencies = _coerce_runtime_env_pip(existing_pip)
|
|
154
199
|
dependencies = [
|
|
155
200
|
dependency
|
|
156
201
|
for dependency in dependencies
|
|
157
202
|
if _requirement_name(dependency) not in {"roar-cli", "roar"}
|
|
203
|
+
# Also deduplicate URL-based requirements (e.g. presigned S3 URLs).
|
|
204
|
+
# _requirement_name() returns the full URL for these, so the name-based
|
|
205
|
+
# filter above never matches them — without this check the URL would
|
|
206
|
+
# survive the filter and get appended again, producing duplicates.
|
|
207
|
+
and dependency.strip() != roar_req.strip()
|
|
158
208
|
]
|
|
159
|
-
|
|
160
|
-
|
|
209
|
+
# "skip" means roar is already installed on workers (e.g. Docker image with editable install).
|
|
210
|
+
if roar_req != "skip":
|
|
211
|
+
dependencies.append(roar_req)
|
|
212
|
+
return dependencies if dependencies else None
|
|
161
213
|
|
|
162
214
|
|
|
163
215
|
def _coerce_runtime_env_pip(value: object) -> list[str]:
|
|
@@ -183,24 +235,23 @@ def _requirement_name(requirement: str) -> str:
|
|
|
183
235
|
return text.strip().lower()
|
|
184
236
|
|
|
185
237
|
|
|
186
|
-
def _resolve_roar_requirement() -> str
|
|
187
|
-
import
|
|
238
|
+
def _resolve_roar_requirement() -> str:
|
|
239
|
+
import importlib.metadata as importlib_metadata
|
|
188
240
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
return
|
|
241
|
+
# Allow overriding the pip requirement — useful for testing unreleased wheels via S3 URL
|
|
242
|
+
# without a PyPI publish. Set ROAR_CLUSTER_PIP_REQ=https://... in the environment.
|
|
243
|
+
override = os.environ.get("ROAR_CLUSTER_PIP_REQ", "").strip()
|
|
244
|
+
if override:
|
|
245
|
+
return override
|
|
194
246
|
|
|
195
|
-
|
|
247
|
+
try:
|
|
248
|
+
version = importlib_metadata.version("roar-cli")
|
|
249
|
+
return f"roar-cli=={version}"
|
|
250
|
+
except importlib_metadata.PackageNotFoundError:
|
|
251
|
+
pass
|
|
252
|
+
except Exception:
|
|
253
|
+
pass
|
|
196
254
|
|
|
197
|
-
for package_name in ("roar-cli", "roar"):
|
|
198
|
-
try:
|
|
199
|
-
return f"{package_name}=={importlib_metadata.version(package_name)}"
|
|
200
|
-
except importlib_metadata.PackageNotFoundError:
|
|
201
|
-
continue
|
|
202
|
-
except Exception:
|
|
203
|
-
break
|
|
204
255
|
return "roar-cli"
|
|
205
256
|
|
|
206
257
|
|
|
@@ -209,10 +260,28 @@ def _resolve_glaas_url() -> str | None:
|
|
|
209
260
|
|
|
210
261
|
url = get_glaas_url()
|
|
211
262
|
if not url:
|
|
212
|
-
return
|
|
263
|
+
return "https://api.glaas.ai"
|
|
213
264
|
return str(url)
|
|
214
265
|
|
|
215
266
|
|
|
267
|
+
def _resolve_cluster_glaas_url(host_glaas_url: str | None) -> str | None:
|
|
268
|
+
override = os.environ.get(_ROAR_CLUSTER_GLAAS_URL_ENV, "").strip()
|
|
269
|
+
if override:
|
|
270
|
+
return override
|
|
271
|
+
if not host_glaas_url:
|
|
272
|
+
return None
|
|
273
|
+
return str(host_glaas_url)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _resolve_cluster_upstream_s3_endpoint(host_endpoint: str | None) -> str | None:
|
|
277
|
+
override = os.environ.get(_ROAR_CLUSTER_AWS_ENDPOINT_URL_ENV, "").strip()
|
|
278
|
+
if override:
|
|
279
|
+
return override
|
|
280
|
+
if not host_endpoint:
|
|
281
|
+
return None
|
|
282
|
+
return str(host_endpoint)
|
|
283
|
+
|
|
284
|
+
|
|
216
285
|
def _register_fragment_session(
|
|
217
286
|
glaas_url: str,
|
|
218
287
|
session_id: str,
|
|
@@ -119,8 +119,6 @@ fallback_enabled = true
|
|
|
119
119
|
enabled = true
|
|
120
120
|
# Inject roar-cli into runtime_env.pip for remote workers
|
|
121
121
|
pip_install = true
|
|
122
|
-
# Shared log directory for Ray worker I/O capture
|
|
123
|
-
log_dir = "/shared/.roar-logs"
|
|
124
122
|
# Actor attribution mode for Ray actor methods (per_call | per_actor)
|
|
125
123
|
actor_attribution = "per_call"
|
|
126
124
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Native Click implementation of the register command.
|
|
3
3
|
|
|
4
|
-
Usage: roar register [options] <
|
|
4
|
+
Usage: roar register [options] <target>
|
|
5
5
|
|
|
6
|
-
Registers
|
|
6
|
+
Registers artifact, step, or session lineage with GLaaS.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import click
|
|
@@ -25,7 +25,7 @@ def _confirm_secrets(detected_secrets: list[str]) -> bool:
|
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
@click.command("register")
|
|
28
|
-
@click.argument("
|
|
28
|
+
@click.argument("target", type=click.STRING)
|
|
29
29
|
@click.option(
|
|
30
30
|
"--dry-run",
|
|
31
31
|
is_flag=True,
|
|
@@ -44,15 +44,15 @@ def _confirm_secrets(detected_secrets: list[str]) -> bool:
|
|
|
44
44
|
)
|
|
45
45
|
@click.pass_obj
|
|
46
46
|
@require_init
|
|
47
|
-
def register(
|
|
48
|
-
|
|
49
|
-
) -> None:
|
|
50
|
-
"""Register artifact lineage with GLaaS.
|
|
47
|
+
def register(ctx: RoarContext, target: str, dry_run: bool, yes: bool, as_blake3: bool) -> None:
|
|
48
|
+
"""Register lineage with GLaaS.
|
|
51
49
|
|
|
52
|
-
Submits
|
|
53
|
-
|
|
50
|
+
Submits lineage to the GLaaS server, starting from one of:
|
|
51
|
+
- an artifact path
|
|
52
|
+
- a DAG step reference like ``@4``
|
|
53
|
+
- a local session hash/prefix previously shown by roar
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
Artifact paths must refer to files tracked by roar.
|
|
56
56
|
|
|
57
57
|
If secrets are detected in the data (API keys, tokens, passwords, etc.),
|
|
58
58
|
you will be prompted to confirm. Use --yes to skip the prompt and
|
|
@@ -67,6 +67,10 @@ def register(
|
|
|
67
67
|
|
|
68
68
|
roar register -y model.pt # Skip confirmation prompt
|
|
69
69
|
|
|
70
|
+
roar register @4 # Register the lineage for DAG step 4
|
|
71
|
+
|
|
72
|
+
roar register 8d7a1f2c... # Register a whole local session
|
|
73
|
+
|
|
70
74
|
roar register --as-blake3 model.pt # Upgrade S3 etag hashes
|
|
71
75
|
|
|
72
76
|
roar register outputs/metrics.json # Register from subdirectory
|
|
@@ -74,9 +78,9 @@ def register(
|
|
|
74
78
|
# Create service
|
|
75
79
|
service = RegisterService()
|
|
76
80
|
|
|
77
|
-
# Register the
|
|
78
|
-
result = service.
|
|
79
|
-
|
|
81
|
+
# Register the requested lineage target
|
|
82
|
+
result = service.register_lineage_target(
|
|
83
|
+
target=target,
|
|
80
84
|
roar_dir=ctx.roar_dir,
|
|
81
85
|
cwd=ctx.cwd,
|
|
82
86
|
dry_run=dry_run,
|
|
@@ -105,9 +109,10 @@ def register(
|
|
|
105
109
|
click.echo("")
|
|
106
110
|
click.echo("View on GLaaS:")
|
|
107
111
|
click.echo(f" Session: {web_url}/dag/{result.session_hash}")
|
|
108
|
-
|
|
112
|
+
if result.artifact_hash:
|
|
113
|
+
click.echo(f" Artifact: {web_url}/artifact/{result.artifact_hash}")
|
|
109
114
|
else:
|
|
110
|
-
click.echo(f"Registered lineage for: {
|
|
115
|
+
click.echo(f"Registered lineage for: {target}")
|
|
111
116
|
click.echo(f" Session: {result.session_hash[:12]}...")
|
|
112
117
|
click.echo(f" Jobs: {result.jobs_registered}")
|
|
113
118
|
click.echo(f" Artifacts: {result.artifacts_registered}")
|
|
@@ -122,12 +127,13 @@ def register(
|
|
|
122
127
|
for error in result.error.split("; "):
|
|
123
128
|
click.echo(f" - {error}", err=True)
|
|
124
129
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
130
|
+
if result.artifact_hash:
|
|
131
|
+
click.echo("")
|
|
132
|
+
click.echo("To reproduce this artifact:")
|
|
133
|
+
click.echo(f" roar reproduce {result.artifact_hash}")
|
|
129
134
|
|
|
130
135
|
click.echo("")
|
|
131
136
|
click.echo("View on GLaaS:")
|
|
132
137
|
click.echo(f" Session: {web_url}/dag/{result.session_hash}")
|
|
133
|
-
|
|
138
|
+
if result.artifact_hash:
|
|
139
|
+
click.echo(f" Artifact: {web_url}/artifact/{result.artifact_hash}")
|
|
@@ -7,6 +7,7 @@ passed through the Click command chain via ctx.obj.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import os
|
|
10
11
|
import sys
|
|
11
12
|
from dataclasses import dataclass, field
|
|
12
13
|
from pathlib import Path
|
|
@@ -137,5 +138,12 @@ class RoarContext:
|
|
|
137
138
|
|
|
138
139
|
@property
|
|
139
140
|
def has_repo(self) -> bool:
|
|
140
|
-
"""Check if we're in a git repository.
|
|
141
|
-
|
|
141
|
+
"""Check if we're in a git repository.
|
|
142
|
+
|
|
143
|
+
Returns True when ROAR_JOB_INSTRUMENTED=1 (Ray job driver running
|
|
144
|
+
from an extracted working_dir that is not a git repo).
|
|
145
|
+
"""
|
|
146
|
+
if self.repo_root is not None:
|
|
147
|
+
return True
|
|
148
|
+
# Ray job drivers run from extracted working dirs without .git
|
|
149
|
+
return os.environ.get("ROAR_JOB_INSTRUMENTED") == "1"
|
|
@@ -79,7 +79,9 @@ def require_init(f: F) -> F:
|
|
|
79
79
|
|
|
80
80
|
initialized = bool(ctx.is_initialized) or _has_roar_dir(ctx)
|
|
81
81
|
|
|
82
|
-
if not initialized and
|
|
82
|
+
if not initialized and (
|
|
83
|
+
"RAY_JOB_ID" in os.environ or os.environ.get("ROAR_JOB_INSTRUMENTED") == "1"
|
|
84
|
+
):
|
|
83
85
|
try:
|
|
84
86
|
_auto_init_for_ray_job(ctx)
|
|
85
87
|
except Exception as e:
|
|
@@ -138,11 +138,6 @@ CONFIGURABLE_KEYS = {
|
|
|
138
138
|
"default": True,
|
|
139
139
|
"description": "Inject roar-cli into Ray runtime_env.pip for remote workers",
|
|
140
140
|
},
|
|
141
|
-
"ray.log_dir": {
|
|
142
|
-
"type": str,
|
|
143
|
-
"default": "/shared/.roar-logs",
|
|
144
|
-
"description": "Shared Ray worker log directory for roar collection",
|
|
145
|
-
},
|
|
146
141
|
"ray.actor_attribution": {
|
|
147
142
|
"type": str,
|
|
148
143
|
"default": "per_call",
|
|
@@ -149,6 +149,10 @@ class SessionRepository(Protocol):
|
|
|
149
149
|
"""Get session by ID."""
|
|
150
150
|
...
|
|
151
151
|
|
|
152
|
+
def get_all(self) -> list[dict[str, Any]]:
|
|
153
|
+
"""Get all sessions."""
|
|
154
|
+
...
|
|
155
|
+
|
|
152
156
|
def get_by_hash_prefix(self, hash_prefix: str) -> dict[str, Any] | None:
|
|
153
157
|
"""Get first session matching a hash prefix."""
|
|
154
158
|
...
|
|
@@ -11,12 +11,20 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
import blake3
|
|
14
|
-
from sqlalchemy import delete, func, select, update
|
|
14
|
+
from sqlalchemy import case, delete, func, select, update
|
|
15
15
|
from sqlalchemy.orm import Session as SASession
|
|
16
16
|
|
|
17
17
|
from ...core.interfaces.repositories import SessionRepository
|
|
18
18
|
from ..models import Job, Session
|
|
19
19
|
|
|
20
|
+
_STEP_NOISE_COMMANDS = (
|
|
21
|
+
"ray_task:unknown",
|
|
22
|
+
"ray_task:__init__",
|
|
23
|
+
"ray_task:s3_proxy",
|
|
24
|
+
"ray_task:s3_driver_proxy",
|
|
25
|
+
"ray_task:RoarNodeAgent.__init__",
|
|
26
|
+
)
|
|
27
|
+
|
|
20
28
|
|
|
21
29
|
class SQLAlchemySessionRepository(SessionRepository):
|
|
22
30
|
"""
|
|
@@ -212,6 +220,17 @@ class SQLAlchemySessionRepository(SessionRepository):
|
|
|
212
220
|
session = self._session.get(Session, session_id)
|
|
213
221
|
return self._session_to_dict(session) if session else None
|
|
214
222
|
|
|
223
|
+
def get_all(self) -> list[dict[str, Any]]:
|
|
224
|
+
"""Get all sessions ordered by most recent first."""
|
|
225
|
+
sessions = (
|
|
226
|
+
self._session.execute(
|
|
227
|
+
select(Session).order_by(Session.created_at.desc(), Session.id.desc())
|
|
228
|
+
)
|
|
229
|
+
.scalars()
|
|
230
|
+
.all()
|
|
231
|
+
)
|
|
232
|
+
return [self._session_to_dict(session) for session in sessions]
|
|
233
|
+
|
|
215
234
|
def get_by_hash(self, session_hash: str) -> dict[str, Any] | None:
|
|
216
235
|
"""
|
|
217
236
|
Get a session by its content hash.
|
|
@@ -313,9 +332,30 @@ class SQLAlchemySessionRepository(SessionRepository):
|
|
|
313
332
|
.where(
|
|
314
333
|
Job.session_id == session_id,
|
|
315
334
|
Job.step_number == step_number,
|
|
316
|
-
Job.job_type.is_(None) | (Job.job_type
|
|
335
|
+
Job.job_type.is_(None) | (Job.job_type != "build"),
|
|
336
|
+
)
|
|
337
|
+
.order_by(
|
|
338
|
+
case(
|
|
339
|
+
(Job.job_type.is_(None), 6),
|
|
340
|
+
(Job.job_type == "run", 6),
|
|
341
|
+
(
|
|
342
|
+
Job.command.is_not(None)
|
|
343
|
+
& (~Job.command.in_(_STEP_NOISE_COMMANDS))
|
|
344
|
+
& Job.parent_job_uid.is_not(None)
|
|
345
|
+
& Job.script.is_not(None)
|
|
346
|
+
& (~Job.script.like("%.%")),
|
|
347
|
+
5,
|
|
348
|
+
),
|
|
349
|
+
(
|
|
350
|
+
Job.command.is_not(None) & (~Job.command.in_(_STEP_NOISE_COMMANDS)),
|
|
351
|
+
4,
|
|
352
|
+
),
|
|
353
|
+
(Job.command.in_(_STEP_NOISE_COMMANDS), 1),
|
|
354
|
+
else_=2,
|
|
355
|
+
).desc()
|
|
317
356
|
)
|
|
318
357
|
.order_by(Job.timestamp.desc())
|
|
358
|
+
.order_by(Job.id.desc())
|
|
319
359
|
.limit(1)
|
|
320
360
|
)
|
|
321
361
|
job = self._session.execute(query).scalar_one_or_none()
|
|
@@ -605,6 +645,7 @@ class SQLAlchemySessionRepository(SessionRepository):
|
|
|
605
645
|
return {
|
|
606
646
|
"id": job.id,
|
|
607
647
|
"job_uid": job.job_uid,
|
|
648
|
+
"parent_job_uid": job.parent_job_uid,
|
|
608
649
|
"timestamp": job.timestamp,
|
|
609
650
|
"command": job.command,
|
|
610
651
|
"script": job.script,
|
|
@@ -20,9 +20,9 @@ def get_glaas_url() -> str | None:
|
|
|
20
20
|
"""Get GLaaS server URL from config or environment."""
|
|
21
21
|
from ..config import config_get
|
|
22
22
|
|
|
23
|
-
url =
|
|
23
|
+
url = os.environ.get("GLAAS_URL")
|
|
24
24
|
if not url:
|
|
25
|
-
url =
|
|
25
|
+
url = config_get("glaas.url")
|
|
26
26
|
return url
|
|
27
27
|
|
|
28
28
|
|
|
@@ -4,6 +4,14 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
+
_STEP_NOISE_COMMANDS = {
|
|
8
|
+
"ray_task:unknown",
|
|
9
|
+
"ray_task:__init__",
|
|
10
|
+
"ray_task:s3_proxy",
|
|
11
|
+
"ray_task:s3_driver_proxy",
|
|
12
|
+
"ray_task:RoarNodeAgent.__init__",
|
|
13
|
+
}
|
|
14
|
+
|
|
7
15
|
|
|
8
16
|
class DagDataBuilder:
|
|
9
17
|
"""Build DAG visualization data from session."""
|
|
@@ -115,6 +123,41 @@ class DagDataBuilder:
|
|
|
115
123
|
|
|
116
124
|
return steps_by_number
|
|
117
125
|
|
|
126
|
+
def _step_sort_key(self, step: dict) -> tuple[int, float, int]:
|
|
127
|
+
job_type = step.get("job_type")
|
|
128
|
+
command = str(step.get("command") or "")
|
|
129
|
+
script = str(step.get("script") or "")
|
|
130
|
+
parent_job_uid = str(step.get("parent_job_uid") or "")
|
|
131
|
+
is_phase_wrapper = (
|
|
132
|
+
command.startswith("ray_task:")
|
|
133
|
+
and command not in _STEP_NOISE_COMMANDS
|
|
134
|
+
and bool(parent_job_uid)
|
|
135
|
+
and bool(script)
|
|
136
|
+
and "." not in script
|
|
137
|
+
)
|
|
138
|
+
if job_type in (None, "run"):
|
|
139
|
+
priority = 6
|
|
140
|
+
elif is_phase_wrapper:
|
|
141
|
+
priority = 5
|
|
142
|
+
elif command and command not in _STEP_NOISE_COMMANDS:
|
|
143
|
+
priority = 4
|
|
144
|
+
elif command in _STEP_NOISE_COMMANDS:
|
|
145
|
+
priority = 1
|
|
146
|
+
else:
|
|
147
|
+
priority = 2
|
|
148
|
+
return (
|
|
149
|
+
priority,
|
|
150
|
+
float(step.get("timestamp") or 0.0),
|
|
151
|
+
int(step.get("id") or 0),
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
def _representative_steps(self, steps_by_number: dict[int, list[dict]]) -> dict[int, dict]:
|
|
155
|
+
return {
|
|
156
|
+
num: max(group, key=self._step_sort_key)
|
|
157
|
+
for num, group in steps_by_number.items()
|
|
158
|
+
if group
|
|
159
|
+
}
|
|
160
|
+
|
|
118
161
|
def _select_steps(
|
|
119
162
|
self,
|
|
120
163
|
steps_by_number: dict[int, list[dict]],
|
|
@@ -124,13 +167,14 @@ class DagDataBuilder:
|
|
|
124
167
|
"""Pick which steps to show based on expanded mode."""
|
|
125
168
|
if expanded:
|
|
126
169
|
return steps
|
|
127
|
-
|
|
170
|
+
representative_by_step = self._representative_steps(steps_by_number)
|
|
171
|
+
return [representative_by_step[num] for num in sorted(representative_by_step.keys())]
|
|
128
172
|
|
|
129
173
|
def _collect_artifacts(
|
|
130
174
|
self, steps_by_number: dict[int, list[dict]]
|
|
131
175
|
) -> tuple[dict[str, dict], dict[str, list[str]]]:
|
|
132
176
|
"""Build all_artifacts and artifacts_by_path from latest step outputs."""
|
|
133
|
-
latest_by_step
|
|
177
|
+
latest_by_step = self._representative_steps(steps_by_number)
|
|
134
178
|
|
|
135
179
|
all_artifacts: dict[str, dict] = {}
|
|
136
180
|
artifacts_by_path: dict[str, list[str]] = {}
|
|
@@ -179,7 +223,7 @@ class DagDataBuilder:
|
|
|
179
223
|
all_artifacts: dict[str, dict],
|
|
180
224
|
) -> dict[str, list[int]]:
|
|
181
225
|
"""Build consumer relationships from step inputs."""
|
|
182
|
-
latest_by_step
|
|
226
|
+
latest_by_step = self._representative_steps(steps_by_number)
|
|
183
227
|
|
|
184
228
|
artifact_consumers: dict[str, list[int]] = {}
|
|
185
229
|
|
|
@@ -206,7 +250,7 @@ class DagDataBuilder:
|
|
|
206
250
|
expanded: bool,
|
|
207
251
|
) -> list[dict]:
|
|
208
252
|
"""Build node data for each step to show."""
|
|
209
|
-
latest_by_step
|
|
253
|
+
latest_by_step = self._representative_steps(steps_by_number)
|
|
210
254
|
nodes = []
|
|
211
255
|
|
|
212
256
|
for step in steps_to_show:
|
|
@@ -13,10 +13,7 @@ class RoarLogCollectorActor:
|
|
|
13
13
|
token: str | None = None,
|
|
14
14
|
glaas_url: str | None = None,
|
|
15
15
|
) -> None:
|
|
16
|
-
self._events: list[dict] = []
|
|
17
|
-
self._fragments: list[dict] = []
|
|
18
16
|
self._streamer: GlaasFragmentStreamer | None = None
|
|
19
|
-
|
|
20
17
|
if session_id and token and glaas_url:
|
|
21
18
|
self._streamer = GlaasFragmentStreamer(
|
|
22
19
|
session_id=session_id,
|
|
@@ -24,20 +21,17 @@ class RoarLogCollectorActor:
|
|
|
24
21
|
glaas_url=glaas_url,
|
|
25
22
|
)
|
|
26
23
|
|
|
27
|
-
def
|
|
28
|
-
|
|
24
|
+
def ping(self) -> bool:
|
|
25
|
+
return True
|
|
29
26
|
|
|
30
|
-
def
|
|
31
|
-
|
|
27
|
+
def append_batch(self, _events: list[dict]) -> None:
|
|
28
|
+
# Deprecated compatibility shim for legacy worker hooks.
|
|
29
|
+
return None
|
|
32
30
|
|
|
33
31
|
def append_fragment(self, fragment: dict) -> None:
|
|
34
|
-
self._fragments.append(fragment)
|
|
35
32
|
if self._streamer is not None:
|
|
36
33
|
self._streamer.append_fragment(fragment)
|
|
37
34
|
|
|
38
|
-
def get_all_fragments(self) -> list[dict]:
|
|
39
|
-
return list(self._fragments)
|
|
40
|
-
|
|
41
35
|
def flush_to_glaas(self) -> bool:
|
|
42
36
|
if self._streamer is None:
|
|
43
37
|
return True
|