roar-cli 0.2.8__tar.gz → 0.2.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {roar_cli-0.2.8 → roar_cli-0.2.10}/PKG-INFO +47 -23
- {roar_cli-0.2.8 → roar_cli-0.2.10}/README.md +45 -22
- {roar_cli-0.2.8 → roar_cli-0.2.10}/pyproject.toml +13 -3
- roar_cli-0.2.10/roar/application/__init__.py +1 -0
- roar_cli-0.2.10/roar/application/get/__init__.py +7 -0
- roar_cli-0.2.10/roar/application/get/requests.py +22 -0
- roar_cli-0.2.10/roar/application/get/results.py +35 -0
- roar_cli-0.2.10/roar/application/get/service.py +169 -0
- roar_cli-0.2.10/roar/application/get/transfer.py +327 -0
- roar_cli-0.2.10/roar/application/git.py +194 -0
- roar_cli-0.2.10/roar/application/labels.py +398 -0
- roar_cli-0.2.10/roar/application/publish/__init__.py +1 -0
- roar_cli-0.2.10/roar/application/publish/blake3_upgrade.py +258 -0
- roar_cli-0.2.10/roar/application/publish/collection.py +314 -0
- {roar_cli-0.2.8/roar/services/put → roar_cli-0.2.10/roar/application/publish}/composite_builder.py +21 -3
- roar_cli-0.2.10/roar/application/publish/composites.py +43 -0
- roar_cli-0.2.10/roar/application/publish/datasets.py +92 -0
- roar_cli-0.2.10/roar/application/publish/job_links.py +136 -0
- roar_cli-0.2.10/roar/application/publish/job_preparation.py +201 -0
- roar_cli-0.2.8/roar/services/upload/lineage_collector.py → roar_cli-0.2.10/roar/application/publish/lineage.py +215 -16
- roar_cli-0.2.10/roar/application/publish/lineage_composites.py +165 -0
- roar_cli-0.2.10/roar/application/publish/metadata.py +151 -0
- roar_cli-0.2.10/roar/application/publish/put_composites.py +258 -0
- roar_cli-0.2.10/roar/application/publish/put_execution.py +627 -0
- roar_cli-0.2.10/roar/application/publish/put_preparation.py +110 -0
- roar_cli-0.2.10/roar/application/publish/register_execution.py +281 -0
- roar_cli-0.2.10/roar/application/publish/register_preparation.py +75 -0
- roar_cli-0.2.10/roar/application/publish/registration.py +513 -0
- roar_cli-0.2.10/roar/application/publish/requests.py +34 -0
- roar_cli-0.2.10/roar/application/publish/results.py +66 -0
- roar_cli-0.2.10/roar/application/publish/runtime.py +33 -0
- roar_cli-0.2.10/roar/application/publish/secrets.py +71 -0
- roar_cli-0.2.10/roar/application/publish/service.py +176 -0
- roar_cli-0.2.10/roar/application/publish/session.py +84 -0
- roar_cli-0.2.8/roar/services/put/resolver.py → roar_cli-0.2.10/roar/application/publish/source_resolution.py +1 -6
- roar_cli-0.2.10/roar/application/publish/targets.py +143 -0
- roar_cli-0.2.10/roar/application/query/__init__.py +67 -0
- roar_cli-0.2.10/roar/application/query/dag.py +82 -0
- roar_cli-0.2.10/roar/application/query/label.py +114 -0
- roar_cli-0.2.10/roar/application/query/lineage.py +121 -0
- roar_cli-0.2.10/roar/application/query/log.py +94 -0
- roar_cli-0.2.10/roar/application/query/requests.py +78 -0
- roar_cli-0.2.10/roar/application/query/results.py +294 -0
- roar_cli-0.2.10/roar/application/query/show.py +325 -0
- roar_cli-0.2.10/roar/application/query/status.py +90 -0
- roar_cli-0.2.10/roar/application/reproduce/__init__.py +14 -0
- roar_cli-0.2.10/roar/application/reproduce/environment.py +91 -0
- roar_cli-0.2.10/roar/application/reproduce/lookup.py +144 -0
- roar_cli-0.2.10/roar/application/reproduce/requests.py +20 -0
- roar_cli-0.2.10/roar/application/reproduce/results.py +39 -0
- roar_cli-0.2.10/roar/application/reproduce/service.py +357 -0
- roar_cli-0.2.10/roar/application/run/__init__.py +13 -0
- roar_cli-0.2.8/roar/services/execution/dag_resolver.py → roar_cli-0.2.10/roar/application/run/dag_references.py +1 -1
- roar_cli-0.2.10/roar/application/run/execution.py +148 -0
- roar_cli-0.2.10/roar/application/run/requests.py +30 -0
- roar_cli-0.2.10/roar/application/run/service.py +187 -0
- roar_cli-0.2.10/roar/backends/__init__.py +1 -0
- roar_cli-0.2.10/roar/backends/local/__init__.py +1 -0
- roar_cli-0.2.10/roar/backends/local/plugin.py +47 -0
- roar_cli-0.2.10/roar/backends/ray/__init__.py +3 -0
- roar_cli-0.2.10/roar/backends/ray/collector.py +398 -0
- roar_cli-0.2.10/roar/backends/ray/config.py +89 -0
- roar_cli-0.2.10/roar/backends/ray/constants.py +17 -0
- roar_cli-0.2.10/roar/backends/ray/env_contract.py +80 -0
- roar_cli-0.2.10/roar/backends/ray/fragment.py +135 -0
- roar_cli-0.2.10/roar/backends/ray/fragment_reconstituter.py +691 -0
- roar_cli-0.2.10/roar/backends/ray/node_agent.py +92 -0
- roar_cli-0.2.10/roar/backends/ray/plugin.py +150 -0
- roar_cli-0.2.10/roar/backends/ray/proxy_fragments.py +85 -0
- roar_cli-0.2.10/roar/backends/ray/roar_worker.py +1662 -0
- roar_cli-0.2.10/roar/backends/ray/runtime_hooks.py +1127 -0
- roar_cli-0.2.10/roar/backends/ray/s3_key_paths.py +57 -0
- roar_cli-0.2.8/roar/cli/commands/_ray_job_submit.py → roar_cli-0.2.10/roar/backends/ray/submit.py +127 -66
- roar_cli-0.2.10/roar/backends/ray/submit_context.py +141 -0
- roar_cli-0.2.10/roar/bin/libroar_tracer_preload.so +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/bin/roar-proxy +0 -0
- roar_cli-0.2.10/roar/bin/roar-tracer +0 -0
- roar_cli-0.2.10/roar/bin/roar-tracer-ebpf +0 -0
- roar_cli-0.2.10/roar/bin/roar-tracer-preload +0 -0
- roar_cli-0.2.10/roar/bin/roard +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/__init__.py +2 -1
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/auth.py +9 -8
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/build.py +18 -27
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/config.py +1 -1
- roar_cli-0.2.10/roar/cli/commands/dag.py +103 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/env.py +1 -1
- roar_cli-0.2.10/roar/cli/commands/get.py +163 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/init.py +15 -12
- roar_cli-0.2.10/roar/cli/commands/label.py +134 -0
- roar_cli-0.2.10/roar/cli/commands/lineage.py +62 -0
- roar_cli-0.2.10/roar/cli/commands/log.py +28 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/proxy.py +7 -7
- roar_cli-0.2.10/roar/cli/commands/put.py +168 -0
- roar_cli-0.2.10/roar/cli/commands/register.py +143 -0
- roar_cli-0.2.10/roar/cli/commands/reproduce.py +94 -0
- roar_cli-0.2.10/roar/cli/commands/run.py +113 -0
- roar_cli-0.2.10/roar/cli/commands/show.py +38 -0
- roar_cli-0.2.10/roar/cli/commands/status.py +17 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/tracer.py +2 -2
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/context.py +11 -3
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/decorators.py +14 -8
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/__init__.py +1 -25
- roar_cli-0.2.10/roar/core/bootstrap.py +85 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/__init__.py +5 -14
- roar_cli-0.2.10/roar/core/interfaces/lineage.py +43 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/provenance.py +12 -10
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/repositories.py +49 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/reproduction.py +0 -30
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/run.py +10 -49
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/services.py +2 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/telemetry.py +5 -2
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/vcs.py +5 -2
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/core}/logging.py +45 -45
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/__init__.py +0 -31
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/dag.py +4 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/run.py +2 -0
- roar_cli-0.2.10/roar/core/operation_metadata.py +11 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/__init__.py +2 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/context.py +13 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/hashing/__init__.py +2 -0
- roar_cli-0.2.10/roar/db/hashing/blake3.py +22 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/models.py +60 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/__init__.py +3 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/artifact.py +99 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/job.py +61 -3
- roar_cli-0.2.10/roar/db/repositories/label.py +124 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/session.py +78 -23
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/schema.py +10 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/services/job_recording.py +38 -13
- roar_cli-0.2.10/roar/execution/__init__.py +1 -0
- roar_cli-0.2.10/roar/execution/cluster/__init__.py +1 -0
- roar_cli-0.2.10/roar/execution/cluster/bridge.py +233 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/cluster}/proxy.py +6 -4
- roar_cli-0.2.10/roar/execution/cluster/proxy_config.py +24 -0
- roar_cli-0.2.10/roar/execution/fragments/__init__.py +1 -0
- roar_cli-0.2.10/roar/execution/fragments/lineage.py +761 -0
- roar_cli-0.2.10/roar/execution/fragments/models.py +294 -0
- roar_cli-0.2.10/roar/execution/fragments/reconstitution.py +71 -0
- roar_cli-0.2.8/roar/ray/fragment_key.py → roar_cli-0.2.10/roar/execution/fragments/sessions.py +8 -8
- roar_cli-0.2.10/roar/execution/fragments/transport.py +54 -0
- roar_cli-0.2.10/roar/execution/framework/__init__.py +41 -0
- roar_cli-0.2.10/roar/execution/framework/contract.py +160 -0
- roar_cli-0.2.10/roar/execution/framework/planning.py +45 -0
- roar_cli-0.2.10/roar/execution/framework/registry.py +353 -0
- roar_cli-0.2.10/roar/execution/framework/runtime_imports.py +90 -0
- roar_cli-0.2.10/roar/execution/provenance/__init__.py +5 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/assembler.py +4 -4
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/build_pip_collector.py +2 -2
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/build_tool_collector.py +2 -2
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/data_loader.py +17 -4
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/file_filter.py +3 -3
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/package_collector.py +3 -3
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/runtime_collector.py +3 -3
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/service.py +11 -10
- roar_cli-0.2.10/roar/execution/recording/__init__.py +25 -0
- roar_cli-0.2.10/roar/execution/recording/dataset_metadata.py +108 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/recording}/job_recording.py +203 -13
- roar_cli-0.2.10/roar/execution/reproduction/__init__.py +9 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/execution}/reproduction/environment_setup.py +1 -1
- roar_cli-0.2.10/roar/execution/runtime/__init__.py +47 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/runtime}/backup.py +2 -2
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/runtime}/coordinator.py +41 -9
- roar_cli-0.2.10/roar/execution/runtime/driver_entrypoint.py +147 -0
- roar_cli-0.2.10/roar/execution/runtime/host_execution.py +39 -0
- roar_cli-0.2.10/roar/execution/runtime/inject/__init__.py +1 -0
- roar_cli-0.2.10/roar/execution/runtime/inject/sitecustomize.py +30 -0
- roar_cli-0.2.10/roar/execution/runtime/inject/support.py +51 -0
- roar_cli-0.2.10/roar/execution/runtime/inject/tracker.py +194 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/runtime}/tracer.py +7 -6
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/runtime}/tracer_backends.py +50 -1
- roar_cli-0.2.10/roar/execution/runtime/worker_bootstrap.py +135 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/filters/files.py +29 -8
- roar_cli-0.2.10/roar/integrations/__init__.py +27 -0
- roar_cli-0.2.10/roar/integrations/config/__init__.py +70 -0
- roar_cli-0.2.8/roar/config.py → roar_cli-0.2.10/roar/integrations/config/access.py +41 -28
- roar_cli-0.2.8/roar/core/settings.py → roar_cli-0.2.10/roar/integrations/config/loader.py +16 -15
- roar_cli-0.2.8/roar/core/models/config.py → roar_cli-0.2.10/roar/integrations/config/schema.py +53 -20
- roar_cli-0.2.10/roar/integrations/discovery.py +54 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/__init__.py +3 -7
- roar_cli-0.2.10/roar/integrations/download/get.py +40 -0
- roar_cli-0.2.10/roar/integrations/git/__init__.py +10 -0
- {roar_cli-0.2.8/roar/plugins/vcs → roar_cli-0.2.10/roar/integrations/git}/base.py +9 -12
- roar_cli-0.2.8/roar/services/transfer/common.py → roar_cli-0.2.10/roar/integrations/git/context.py +5 -40
- roar_cli-0.2.8/roar/plugins/vcs/git.py → roar_cli-0.2.10/roar/integrations/git/provider.py +1 -1
- {roar_cli-0.2.8/roar → roar_cli-0.2.10/roar/integrations}/glaas/__init__.py +14 -0
- {roar_cli-0.2.8/roar → roar_cli-0.2.10/roar/integrations}/glaas/auth.py +3 -3
- roar_cli-0.2.8/roar/glaas_client.py → roar_cli-0.2.10/roar/integrations/glaas/client.py +15 -6
- roar_cli-0.2.10/roar/integrations/glaas/fragment_streamer.py +135 -0
- roar_cli-0.2.10/roar/integrations/glaas/registration/__init__.py +13 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/integrations/glaas}/registration/artifact.py +6 -6
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/integrations/glaas}/registration/coordinator.py +4 -4
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/integrations/glaas}/registration/job.py +6 -6
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/integrations/glaas}/registration/session.py +6 -6
- roar_cli-0.2.10/roar/integrations/glaas/transport.py +153 -0
- roar_cli-0.2.10/roar/integrations/registry.py +103 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/__init__.py +6 -7
- roar_cli-0.2.10/roar/integrations/storage/publish.py +40 -0
- {roar_cli-0.2.8/roar/plugins → roar_cli-0.2.10/roar/integrations}/telemetry/__init__.py +1 -5
- {roar_cli-0.2.8/roar/plugins → roar_cli-0.2.10/roar/integrations}/telemetry/base.py +2 -1
- {roar_cli-0.2.8/roar/plugins → roar_cli-0.2.10/roar/integrations}/telemetry/wandb.py +1 -1
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/dag_data_builder.py +72 -8
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/dag_renderer.py +9 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/run_report.py +1 -1
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/show_renderer.py +37 -17
- roar_cli-0.2.8/roar/bin/libroar_tracer_preload.so +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer-ebpf +0 -0
- roar_cli-0.2.8/roar/bin/roar-tracer-preload +0 -0
- roar_cli-0.2.8/roar/bin/roard +0 -0
- roar_cli-0.2.8/roar/cli/commands/_execution.py +0 -241
- roar_cli-0.2.8/roar/cli/commands/dag.py +0 -175
- roar_cli-0.2.8/roar/cli/commands/get.py +0 -293
- roar_cli-0.2.8/roar/cli/commands/lineage.py +0 -179
- roar_cli-0.2.8/roar/cli/commands/log.py +0 -117
- roar_cli-0.2.8/roar/cli/commands/put.py +0 -325
- roar_cli-0.2.8/roar/cli/commands/register.py +0 -133
- roar_cli-0.2.8/roar/cli/commands/reproduce.py +0 -281
- roar_cli-0.2.8/roar/cli/commands/run.py +0 -272
- roar_cli-0.2.8/roar/cli/commands/show.py +0 -334
- roar_cli-0.2.8/roar/cli/commands/status.py +0 -88
- roar_cli-0.2.8/roar/core/bootstrap.py +0 -100
- roar_cli-0.2.8/roar/core/container.py +0 -423
- roar_cli-0.2.8/roar/core/di.py +0 -118
- roar_cli-0.2.8/roar/core/interfaces/cloud.py +0 -135
- roar_cli-0.2.8/roar/core/interfaces/command.py +0 -76
- roar_cli-0.2.8/roar/core/interfaces/upload.py +0 -83
- roar_cli-0.2.8/roar/core/logging.py +0 -34
- roar_cli-0.2.8/roar/core/models/command.py +0 -89
- roar_cli-0.2.8/roar/core/registry.py +0 -258
- roar_cli-0.2.8/roar/glaas/transport.py +0 -127
- roar_cli-0.2.8/roar/plugins/__init__.py +0 -16
- roar_cli-0.2.8/roar/plugins/cloud/__init__.py +0 -13
- roar_cli-0.2.8/roar/plugins/cloud/base.py +0 -302
- roar_cli-0.2.8/roar/plugins/vcs/__init__.py +0 -14
- roar_cli-0.2.8/roar/ray/__init__.py +0 -6
- roar_cli-0.2.8/roar/ray/actor.py +0 -44
- roar_cli-0.2.8/roar/ray/collector.py +0 -1015
- roar_cli-0.2.8/roar/ray/fragment.py +0 -50
- roar_cli-0.2.8/roar/ray/fragment_reconstituter.py +0 -184
- roar_cli-0.2.8/roar/ray/glaas_fragment_streamer.py +0 -80
- roar_cli-0.2.8/roar/ray/node_agent.py +0 -139
- roar_cli-0.2.8/roar/ray/roar_worker.py +0 -607
- roar_cli-0.2.8/roar/ray/worker.py +0 -550
- roar_cli-0.2.8/roar/services/__init__.py +0 -8
- roar_cli-0.2.8/roar/services/execution/__init__.py +0 -21
- roar_cli-0.2.8/roar/services/execution/args.py +0 -163
- roar_cli-0.2.8/roar/services/execution/execution_service.py +0 -203
- roar_cli-0.2.8/roar/services/execution/inject/__init__.py +0 -1
- roar_cli-0.2.8/roar/services/execution/inject/sitecustomize.py +0 -851
- roar_cli-0.2.8/roar/services/execution/provenance/__init__.py +0 -10
- roar_cli-0.2.8/roar/services/get/__init__.py +0 -10
- roar_cli-0.2.8/roar/services/get/service.py +0 -444
- roar_cli-0.2.8/roar/services/lookup/__init__.py +0 -19
- roar_cli-0.2.8/roar/services/lookup/entity_lookup.py +0 -263
- roar_cli-0.2.8/roar/services/lookup/step_parser.py +0 -156
- roar_cli-0.2.8/roar/services/put/__init__.py +0 -18
- roar_cli-0.2.8/roar/services/put/git.py +0 -132
- roar_cli-0.2.8/roar/services/put/service.py +0 -1563
- roar_cli-0.2.8/roar/services/registration/__init__.py +0 -22
- roar_cli-0.2.8/roar/services/registration/_dataset_label.py +0 -52
- roar_cli-0.2.8/roar/services/registration/register_service.py +0 -749
- roar_cli-0.2.8/roar/services/reproduction/__init__.py +0 -21
- roar_cli-0.2.8/roar/services/reproduction/service.py +0 -457
- roar_cli-0.2.8/roar/services/secrets/__init__.py +0 -5
- roar_cli-0.2.8/roar/services/secrets/filter_service.py +0 -133
- roar_cli-0.2.8/roar/services/transfer/__init__.py +0 -18
- roar_cli-0.2.8/roar/services/upload/__init__.py +0 -18
- roar_cli-0.2.8/roar/services/upload/service.py +0 -256
- roar_cli-0.2.8/roar/services/vcs/__init__.py +0 -15
- roar_cli-0.2.8/roar/services/vcs/git_access.py +0 -206
- {roar_cli-0.2.8 → roar_cli-0.2.10}/LICENSE +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/__main__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/analyzers/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/analyzers/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/analyzers/experiment_trackers.py +0 -0
- {roar_cli-0.2.8/roar → roar_cli-0.2.10/roar/backends}/ray/_agent_names.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/pop.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/cli/commands/reset.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/digests.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/dto/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/dto/registration.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/exceptions.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/config.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/logger.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/presenter.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/interfaces/registration.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/artifact.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/base.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/dataset_identifier.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/glaas.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/job.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/lineage.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/provenance.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/session.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/telemetry.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/models/vcs.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/tracer_modes.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/core/validation.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/engine.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/hashing/backend.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/collection.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/composite.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/repositories/hash_cache.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/services/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/services/hashing.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/services/lineage.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/db/services/session.py +0 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar}/execution/provenance/process_summarizer.py +0 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/recording}/dataset_identifier.py +0 -0
- /roar_cli-0.2.8/roar/services/registration/_dataset_profile.py → /roar_cli-0.2.10/roar/execution/recording/dataset_profile.py +0 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/execution}/reproduction/installers.py +0 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/execution}/reproduction/pipeline_executor.py +0 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/execution}/reproduction/pipeline_metadata.py +0 -0
- {roar_cli-0.2.8/roar/services/execution → roar_cli-0.2.10/roar/execution/runtime}/signal_handler.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/filters/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/filters/omit.py +0 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/base.py +0 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/gcs.py +0 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/http.py +0 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/noop.py +0 -0
- {roar_cli-0.2.8/roar/services/get/backends → roar_cli-0.2.10/roar/integrations/download}/s3.py +0 -0
- {roar_cli-0.2.8/roar/services → roar_cli-0.2.10/roar/integrations/glaas}/registration/_artifact_ref.py +0 -0
- /roar_cli-0.2.8/roar/services/transfer/backend_resolution.py → /roar_cli-0.2.10/roar/integrations/resolution.py +0 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/base.py +0 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/gcs.py +0 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/memory.py +0 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/noop.py +0 -0
- {roar_cli-0.2.8/roar/services/put/backends → roar_cli-0.2.10/roar/integrations/storage}/s3.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/console.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/formatting.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/null.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/presenters/spinner.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/utils/__init__.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/utils/cloud.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/roar/utils/git_url.py +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/Cargo.lock +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/crates/artifact-hash-core/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/crates/artifact-hash-core/src/lib.rs +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/crates/artifact-hash-py/Cargo.toml +0 -0
- {roar_cli-0.2.8 → roar_cli-0.2.10}/rust/crates/artifact-hash-py/src/lib.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: roar-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.10
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -15,6 +15,7 @@ Classifier: Topic :: Software Development :: Version Control
|
|
|
15
15
|
Classifier: Typing :: Typed
|
|
16
16
|
Requires-Dist: blake3>=0.3.0
|
|
17
17
|
Requires-Dist: click>=8.1.0
|
|
18
|
+
Requires-Dist: cryptography>=42.0.0
|
|
18
19
|
Requires-Dist: dependency-injector>=4.40.0
|
|
19
20
|
Requires-Dist: msgpack>=1.0.0
|
|
20
21
|
Requires-Dist: sqlalchemy>=2.0.0
|
|
@@ -66,12 +67,12 @@ Requires Python 3.10+.
|
|
|
66
67
|
|
|
67
68
|
### Platform Support
|
|
68
69
|
|
|
69
|
-
| Platform | Status
|
|
70
|
-
| ------------- |
|
|
71
|
-
| Linux x86_64 | ✅ Full support
|
|
72
|
-
| Linux aarch64 | ✅ Full support
|
|
70
|
+
| Platform | Status |
|
|
71
|
+
| ------------- | ----------------------------------------------------------- |
|
|
72
|
+
| Linux x86_64 | ✅ Full support |
|
|
73
|
+
| Linux aarch64 | ✅ Full support |
|
|
73
74
|
| macOS | 🚧 Experimental ([limitations](#macos-tracing-limitations)) |
|
|
74
|
-
| Windows | Coming soon
|
|
75
|
+
| Windows | Coming soon |
|
|
75
76
|
|
|
76
77
|
PyPI wheels are published for Linux and macOS (`x86_64` and `arm64`).
|
|
77
78
|
|
|
@@ -107,11 +108,11 @@ roar run python evaluate.py --model model.pt --output metrics.json
|
|
|
107
108
|
|
|
108
109
|
### Backends
|
|
109
110
|
|
|
110
|
-
| Backend | Binary
|
|
111
|
-
|
|
|
112
|
-
| eBPF
|
|
111
|
+
| Backend | Binary | Platforms | Notes |
|
|
112
|
+
| ------- | ------------------------------------------------ | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
113
|
+
| eBPF | `roar-tracer-ebpf` | Linux | Fastest, but requires permissions and kernel support. |
|
|
113
114
|
| preload | `roar-tracer-preload` + `libroar_tracer_preload` | macOS, Linux | Uses `DYLD_INSERT_LIBRARIES` (macOS) or `LD_PRELOAD` (Linux). Not compatible with processes that ignore preload env vars (e.g., SIP/hardened runtime on macOS), or fully-static binaries (common with Go). |
|
|
114
|
-
| ptrace
|
|
115
|
+
| ptrace | `roar-tracer` | Linux | Slowest, broadest compatibility on Linux. |
|
|
115
116
|
|
|
116
117
|
### Building
|
|
117
118
|
|
|
@@ -259,19 +260,19 @@ roar config set <key> <value>
|
|
|
259
260
|
|
|
260
261
|
Run `roar config list` to see all available options with descriptions. Common options:
|
|
261
262
|
|
|
262
|
-
| Key | Default
|
|
263
|
-
| ------------------------------ |
|
|
264
|
-
| `output.track_repo_files` | false
|
|
265
|
-
| `output.quiet` | false
|
|
266
|
-
| `filters.ignore_system_reads` | true
|
|
267
|
-
| `filters.ignore_package_reads` | true
|
|
268
|
-
| `filters.ignore_torch_cache` | true
|
|
269
|
-
| `filters.ignore_tmp_files` | true
|
|
263
|
+
| Key | Default | Description |
|
|
264
|
+
| ------------------------------ | ---------------------- | --------------------------------------- |
|
|
265
|
+
| `output.track_repo_files` | false | Include repo files in provenance |
|
|
266
|
+
| `output.quiet` | false | Suppress written files report |
|
|
267
|
+
| `filters.ignore_system_reads` | true | Ignore /sys, /etc, /sbin reads |
|
|
268
|
+
| `filters.ignore_package_reads` | true | Ignore installed package reads |
|
|
269
|
+
| `filters.ignore_torch_cache` | true | Ignore torch/triton cache |
|
|
270
|
+
| `filters.ignore_tmp_files` | true | Ignore /tmp files |
|
|
270
271
|
| `glaas.url` | <https://api.glaas.ai> | GLaaS server URL |
|
|
271
272
|
| `glaas.web_url` | <https://glaas.ai> | GLaaS web UI URL |
|
|
272
|
-
| `registration.omit.enabled` | true
|
|
273
|
-
| `hash.primary` | blake3
|
|
274
|
-
| `logging.level` | warning
|
|
273
|
+
| `registration.omit.enabled` | true | Enable secret filtering |
|
|
274
|
+
| `hash.primary` | blake3 | Primary hash algorithm |
|
|
275
|
+
| `logging.level` | warning | Log level (debug, info, warning, error) |
|
|
275
276
|
|
|
276
277
|
### `roar dag`
|
|
277
278
|
|
|
@@ -305,14 +306,30 @@ roar log # Show recent job history
|
|
|
305
306
|
|
|
306
307
|
### `roar register`
|
|
307
308
|
|
|
308
|
-
Register artifact lineage with GLaaS.
|
|
309
|
+
Register session, job, step, or artifact lineage with GLaaS.
|
|
309
310
|
|
|
310
311
|
```bash
|
|
311
312
|
roar register model.pt # Register model lineage
|
|
312
313
|
roar register --dry-run model.pt # Preview without registering
|
|
313
314
|
roar register -y model.pt # Skip confirmation prompt
|
|
315
|
+
roar register @4 # Register lineage for DAG step 4
|
|
316
|
+
roar register deadbeef # Register lineage for a local job UID
|
|
317
|
+
roar register 7f1e...c9a4 # Register lineage for a tracked artifact hash
|
|
318
|
+
roar register 8d7a1f2c... # Register a whole local session
|
|
319
|
+
roar register s3://bucket/run/out # Register a tracked remote S3 artifact
|
|
314
320
|
```
|
|
315
321
|
|
|
322
|
+
**Supported targets:**
|
|
323
|
+
|
|
324
|
+
- Local artifact path: `model.pt`, `./outputs/metrics.json`
|
|
325
|
+
- Tracked artifact hash: primitive or composite
|
|
326
|
+
- Local job UID: full UID or unique prefix
|
|
327
|
+
- Step reference: `@N` or `@BN`
|
|
328
|
+
- Local session hash: full hash or unique prefix
|
|
329
|
+
- Tracked remote path: `s3://...`
|
|
330
|
+
|
|
331
|
+
For bare 8-character hex targets, `roar register` prefers a matching local job UID before falling back to session-hash-prefix resolution.
|
|
332
|
+
|
|
316
333
|
### `roar put`
|
|
317
334
|
|
|
318
335
|
Upload artifacts to cloud storage and register lineage with GLaaS.
|
|
@@ -324,11 +341,13 @@ roar put @2 s3://bucket/outputs/ -m "Step 2 outputs"
|
|
|
324
341
|
```
|
|
325
342
|
|
|
326
343
|
**Options:**
|
|
344
|
+
|
|
327
345
|
- `-m, --message` — Description of the upload (required)
|
|
328
346
|
- `--dry-run` — Preview without uploading
|
|
329
347
|
- `--no-tag` — Skip git tagging
|
|
330
348
|
|
|
331
349
|
**Source formats:**
|
|
350
|
+
|
|
332
351
|
- File path: `model.pt`, `./data/output.csv`
|
|
333
352
|
- Directory: `./checkpoints/` (uploads all files recursively)
|
|
334
353
|
- Job reference: `@2` (uploads outputs from step 2)
|
|
@@ -346,6 +365,7 @@ roar get s3://bucket/checkpoints/ ./local/ # Download all files under prefix
|
|
|
346
365
|
```
|
|
347
366
|
|
|
348
367
|
**Options:**
|
|
368
|
+
|
|
349
369
|
- `-m, --message` — Annotation for this download
|
|
350
370
|
- `--hash` — Expected BLAKE3 hash (for verification)
|
|
351
371
|
- `--tag` — Create a git tag for this download
|
|
@@ -393,6 +413,7 @@ roar pop -y # Pop without confirmation (skip prompt)
|
|
|
393
413
|
```
|
|
394
414
|
|
|
395
415
|
**What it does:**
|
|
416
|
+
|
|
396
417
|
- Removes the last job from the session history
|
|
397
418
|
- Deletes output artifacts created by that job (unless they're packages/system files)
|
|
398
419
|
- Does not affect the original input files
|
|
@@ -443,7 +464,7 @@ Add `.roar/` to your `.gitignore` (roar offers to do this during `roar init`).
|
|
|
443
464
|
|
|
444
465
|
## GLaaS Server
|
|
445
466
|
|
|
446
|
-
Roar can register
|
|
467
|
+
Roar can register sessions, jobs, steps, and artifacts with a GLaaS (Global Lineage-as-a-Service) server using the `roar register` command.
|
|
447
468
|
|
|
448
469
|
### Server Setup
|
|
449
470
|
|
|
@@ -479,6 +500,9 @@ roar auth register
|
|
|
479
500
|
roar auth test
|
|
480
501
|
```
|
|
481
502
|
|
|
503
|
+
> [!TIP]
|
|
504
|
+
> Roar activity can be registered without authentication. Unauthenticated registrations are attributed to a public "anonymous" user, but are not guaranteed persistence. For persistent attribution, we recommend setting up `roar auth`.
|
|
505
|
+
|
|
482
506
|
## Development
|
|
483
507
|
|
|
484
508
|
### Prerequisites
|
|
@@ -20,12 +20,12 @@ Requires Python 3.10+.
|
|
|
20
20
|
|
|
21
21
|
### Platform Support
|
|
22
22
|
|
|
23
|
-
| Platform | Status
|
|
24
|
-
| ------------- |
|
|
25
|
-
| Linux x86_64 | ✅ Full support
|
|
26
|
-
| Linux aarch64 | ✅ Full support
|
|
23
|
+
| Platform | Status |
|
|
24
|
+
| ------------- | ----------------------------------------------------------- |
|
|
25
|
+
| Linux x86_64 | ✅ Full support |
|
|
26
|
+
| Linux aarch64 | ✅ Full support |
|
|
27
27
|
| macOS | 🚧 Experimental ([limitations](#macos-tracing-limitations)) |
|
|
28
|
-
| Windows | Coming soon
|
|
28
|
+
| Windows | Coming soon |
|
|
29
29
|
|
|
30
30
|
PyPI wheels are published for Linux and macOS (`x86_64` and `arm64`).
|
|
31
31
|
|
|
@@ -61,11 +61,11 @@ roar run python evaluate.py --model model.pt --output metrics.json
|
|
|
61
61
|
|
|
62
62
|
### Backends
|
|
63
63
|
|
|
64
|
-
| Backend | Binary
|
|
65
|
-
|
|
|
66
|
-
| eBPF
|
|
64
|
+
| Backend | Binary | Platforms | Notes |
|
|
65
|
+
| ------- | ------------------------------------------------ | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
66
|
+
| eBPF | `roar-tracer-ebpf` | Linux | Fastest, but requires permissions and kernel support. |
|
|
67
67
|
| preload | `roar-tracer-preload` + `libroar_tracer_preload` | macOS, Linux | Uses `DYLD_INSERT_LIBRARIES` (macOS) or `LD_PRELOAD` (Linux). Not compatible with processes that ignore preload env vars (e.g., SIP/hardened runtime on macOS), or fully-static binaries (common with Go). |
|
|
68
|
-
| ptrace
|
|
68
|
+
| ptrace | `roar-tracer` | Linux | Slowest, broadest compatibility on Linux. |
|
|
69
69
|
|
|
70
70
|
### Building
|
|
71
71
|
|
|
@@ -213,19 +213,19 @@ roar config set <key> <value>
|
|
|
213
213
|
|
|
214
214
|
Run `roar config list` to see all available options with descriptions. Common options:
|
|
215
215
|
|
|
216
|
-
| Key | Default
|
|
217
|
-
| ------------------------------ |
|
|
218
|
-
| `output.track_repo_files` | false
|
|
219
|
-
| `output.quiet` | false
|
|
220
|
-
| `filters.ignore_system_reads` | true
|
|
221
|
-
| `filters.ignore_package_reads` | true
|
|
222
|
-
| `filters.ignore_torch_cache` | true
|
|
223
|
-
| `filters.ignore_tmp_files` | true
|
|
216
|
+
| Key | Default | Description |
|
|
217
|
+
| ------------------------------ | ---------------------- | --------------------------------------- |
|
|
218
|
+
| `output.track_repo_files` | false | Include repo files in provenance |
|
|
219
|
+
| `output.quiet` | false | Suppress written files report |
|
|
220
|
+
| `filters.ignore_system_reads` | true | Ignore /sys, /etc, /sbin reads |
|
|
221
|
+
| `filters.ignore_package_reads` | true | Ignore installed package reads |
|
|
222
|
+
| `filters.ignore_torch_cache` | true | Ignore torch/triton cache |
|
|
223
|
+
| `filters.ignore_tmp_files` | true | Ignore /tmp files |
|
|
224
224
|
| `glaas.url` | <https://api.glaas.ai> | GLaaS server URL |
|
|
225
225
|
| `glaas.web_url` | <https://glaas.ai> | GLaaS web UI URL |
|
|
226
|
-
| `registration.omit.enabled` | true
|
|
227
|
-
| `hash.primary` | blake3
|
|
228
|
-
| `logging.level` | warning
|
|
226
|
+
| `registration.omit.enabled` | true | Enable secret filtering |
|
|
227
|
+
| `hash.primary` | blake3 | Primary hash algorithm |
|
|
228
|
+
| `logging.level` | warning | Log level (debug, info, warning, error) |
|
|
229
229
|
|
|
230
230
|
### `roar dag`
|
|
231
231
|
|
|
@@ -259,14 +259,30 @@ roar log # Show recent job history
|
|
|
259
259
|
|
|
260
260
|
### `roar register`
|
|
261
261
|
|
|
262
|
-
Register artifact lineage with GLaaS.
|
|
262
|
+
Register session, job, step, or artifact lineage with GLaaS.
|
|
263
263
|
|
|
264
264
|
```bash
|
|
265
265
|
roar register model.pt # Register model lineage
|
|
266
266
|
roar register --dry-run model.pt # Preview without registering
|
|
267
267
|
roar register -y model.pt # Skip confirmation prompt
|
|
268
|
+
roar register @4 # Register lineage for DAG step 4
|
|
269
|
+
roar register deadbeef # Register lineage for a local job UID
|
|
270
|
+
roar register 7f1e...c9a4 # Register lineage for a tracked artifact hash
|
|
271
|
+
roar register 8d7a1f2c... # Register a whole local session
|
|
272
|
+
roar register s3://bucket/run/out # Register a tracked remote S3 artifact
|
|
268
273
|
```
|
|
269
274
|
|
|
275
|
+
**Supported targets:**
|
|
276
|
+
|
|
277
|
+
- Local artifact path: `model.pt`, `./outputs/metrics.json`
|
|
278
|
+
- Tracked artifact hash: primitive or composite
|
|
279
|
+
- Local job UID: full UID or unique prefix
|
|
280
|
+
- Step reference: `@N` or `@BN`
|
|
281
|
+
- Local session hash: full hash or unique prefix
|
|
282
|
+
- Tracked remote path: `s3://...`
|
|
283
|
+
|
|
284
|
+
For bare 8-character hex targets, `roar register` prefers a matching local job UID before falling back to session-hash-prefix resolution.
|
|
285
|
+
|
|
270
286
|
### `roar put`
|
|
271
287
|
|
|
272
288
|
Upload artifacts to cloud storage and register lineage with GLaaS.
|
|
@@ -278,11 +294,13 @@ roar put @2 s3://bucket/outputs/ -m "Step 2 outputs"
|
|
|
278
294
|
```
|
|
279
295
|
|
|
280
296
|
**Options:**
|
|
297
|
+
|
|
281
298
|
- `-m, --message` — Description of the upload (required)
|
|
282
299
|
- `--dry-run` — Preview without uploading
|
|
283
300
|
- `--no-tag` — Skip git tagging
|
|
284
301
|
|
|
285
302
|
**Source formats:**
|
|
303
|
+
|
|
286
304
|
- File path: `model.pt`, `./data/output.csv`
|
|
287
305
|
- Directory: `./checkpoints/` (uploads all files recursively)
|
|
288
306
|
- Job reference: `@2` (uploads outputs from step 2)
|
|
@@ -300,6 +318,7 @@ roar get s3://bucket/checkpoints/ ./local/ # Download all files under prefix
|
|
|
300
318
|
```
|
|
301
319
|
|
|
302
320
|
**Options:**
|
|
321
|
+
|
|
303
322
|
- `-m, --message` — Annotation for this download
|
|
304
323
|
- `--hash` — Expected BLAKE3 hash (for verification)
|
|
305
324
|
- `--tag` — Create a git tag for this download
|
|
@@ -347,6 +366,7 @@ roar pop -y # Pop without confirmation (skip prompt)
|
|
|
347
366
|
```
|
|
348
367
|
|
|
349
368
|
**What it does:**
|
|
369
|
+
|
|
350
370
|
- Removes the last job from the session history
|
|
351
371
|
- Deletes output artifacts created by that job (unless they're packages/system files)
|
|
352
372
|
- Does not affect the original input files
|
|
@@ -397,7 +417,7 @@ Add `.roar/` to your `.gitignore` (roar offers to do this during `roar init`).
|
|
|
397
417
|
|
|
398
418
|
## GLaaS Server
|
|
399
419
|
|
|
400
|
-
Roar can register
|
|
420
|
+
Roar can register sessions, jobs, steps, and artifacts with a GLaaS (Global Lineage-as-a-Service) server using the `roar register` command.
|
|
401
421
|
|
|
402
422
|
### Server Setup
|
|
403
423
|
|
|
@@ -433,6 +453,9 @@ roar auth register
|
|
|
433
453
|
roar auth test
|
|
434
454
|
```
|
|
435
455
|
|
|
456
|
+
> [!TIP]
|
|
457
|
+
> Roar activity can be registered without authentication. Unauthenticated registrations are attributed to a public "anonymous" user, but are not guaranteed persistence. For persistent attribution, we recommend setting up `roar auth`.
|
|
458
|
+
|
|
436
459
|
## Development
|
|
437
460
|
|
|
438
461
|
### Prerequisites
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "roar-cli"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.10"
|
|
8
8
|
description = "Reproducibility and provenance tracker for ML training pipelines"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="TReqs Team", email="info@treqs.ai" }
|
|
@@ -38,6 +38,7 @@ classifiers = [
|
|
|
38
38
|
dependencies = [
|
|
39
39
|
"blake3>=0.3.0",
|
|
40
40
|
"click>=8.1.0",
|
|
41
|
+
"cryptography>=42.0.0",
|
|
41
42
|
"dependency-injector>=4.40.0",
|
|
42
43
|
"msgpack>=1.0.0",
|
|
43
44
|
"sqlalchemy>=2.0.0",
|
|
@@ -69,7 +70,10 @@ dev = [
|
|
|
69
70
|
[project.scripts]
|
|
70
71
|
# This registers `roar` on PATH after pip install
|
|
71
72
|
roar = "roar.__main__:main"
|
|
72
|
-
roar-worker = "roar.
|
|
73
|
+
roar-worker = "roar.execution.runtime.worker_bootstrap:main"
|
|
74
|
+
|
|
75
|
+
[project.entry-points."roar.execution_backends"]
|
|
76
|
+
ray = "roar.backends.ray.plugin:register"
|
|
73
77
|
|
|
74
78
|
[tool.maturin]
|
|
75
79
|
manifest-path = "rust/crates/artifact-hash-py/Cargo.toml"
|
|
@@ -81,6 +85,7 @@ include = [
|
|
|
81
85
|
{ path = "LICENSE", format = "wheel" },
|
|
82
86
|
{ path = "roar/bin/*", format = "sdist" },
|
|
83
87
|
{ path = "roar/bin/*", format = "wheel" },
|
|
88
|
+
{ path = "roar_inject.pth", format = "wheel" },
|
|
84
89
|
]
|
|
85
90
|
|
|
86
91
|
[tool.pytest.ini_options]
|
|
@@ -96,9 +101,13 @@ markers = [
|
|
|
96
101
|
"ebpf: Tests requiring eBPF daemon/root privileges",
|
|
97
102
|
"cloud: Tests for cloud storage operations",
|
|
98
103
|
"happy_path: Happy path tests for core functionality",
|
|
104
|
+
"diagnostic: Opt-in diagnostics or aspirational performance budgets outside the default gate",
|
|
105
|
+
"large_pipeline: Stress-style pipeline coverage with larger DAG fixtures",
|
|
99
106
|
"ray_e2e: Ray end-to-end tests requiring a running Docker cluster",
|
|
107
|
+
"ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`",
|
|
108
|
+
"ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details",
|
|
100
109
|
]
|
|
101
|
-
addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/integration --ignore=tests/e2e"
|
|
110
|
+
addopts = "-v --strict-markers -n auto --dist loadfile --ignore=tests/ebpf --ignore=tests/live_glaas --ignore=tests/benchmarks --ignore=tests/e2e --ignore=tests/integration/test_cli_startup.py --ignore=tests/execution/runtime/test_sitecustomize_perf.py --ignore-glob=tests/backends/*/e2e --ignore-glob=tests/backends/*/live"
|
|
102
111
|
timeout = 60
|
|
103
112
|
filterwarnings = [
|
|
104
113
|
"ignore::DeprecationWarning",
|
|
@@ -130,6 +139,7 @@ ignore = ["E501", "B008", "SIM108"]
|
|
|
130
139
|
|
|
131
140
|
[tool.ruff.lint.per-file-ignores]
|
|
132
141
|
"tests/**/*.py" = ["B011", "B017"]
|
|
142
|
+
"tests/backends/*/e2e/jobs/**/*.py" = ["I001"]
|
|
133
143
|
|
|
134
144
|
[tool.ruff.lint.isort]
|
|
135
145
|
known-first-party = ["roar"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Application workflow entrypoints."""
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Application entrypoints for `roar get` workflows."""
|
|
2
|
+
|
|
3
|
+
from .requests import GetRequest
|
|
4
|
+
from .results import GetDownloadedFile, GetDryRunItem, GetResponse
|
|
5
|
+
from .service import get_artifacts
|
|
6
|
+
|
|
7
|
+
__all__ = ["GetDownloadedFile", "GetDryRunItem", "GetRequest", "GetResponse", "get_artifacts"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Request DTOs for `roar get` application flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class GetRequest:
|
|
11
|
+
"""Application request for a get workflow."""
|
|
12
|
+
|
|
13
|
+
source: str
|
|
14
|
+
destination: Path
|
|
15
|
+
roar_dir: Path
|
|
16
|
+
cwd: Path
|
|
17
|
+
repo_root: Path | None = None
|
|
18
|
+
message: str | None = None
|
|
19
|
+
expected_hash: str | None = None
|
|
20
|
+
dry_run: bool = False
|
|
21
|
+
force: bool = False
|
|
22
|
+
tag: bool = False
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Typed result DTOs for `roar get` application flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class GetDownloadedFile:
|
|
10
|
+
remote_url: str
|
|
11
|
+
local_path: str
|
|
12
|
+
hash: str | None = None
|
|
13
|
+
size: int | None = None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class GetDryRunItem:
|
|
18
|
+
remote_url: str
|
|
19
|
+
local_path: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class GetResponse:
|
|
24
|
+
"""Application response for a get workflow."""
|
|
25
|
+
|
|
26
|
+
success: bool
|
|
27
|
+
source: str
|
|
28
|
+
job_id: int | None = None
|
|
29
|
+
job_uid: str | None = None
|
|
30
|
+
downloaded_files: list[GetDownloadedFile] = field(default_factory=list)
|
|
31
|
+
dry_run: bool = False
|
|
32
|
+
would_download: list[GetDryRunItem] = field(default_factory=list)
|
|
33
|
+
git_tag: str | None = None
|
|
34
|
+
warnings: list[str] = field(default_factory=list)
|
|
35
|
+
error: str | None = None
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Application orchestration for `roar get` workflows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from ...application.git import build_roar_git_tag_name, create_roar_git_tag, resolve_git_state
|
|
8
|
+
from ...core.bootstrap import bootstrap
|
|
9
|
+
from ...core.logging import get_logger
|
|
10
|
+
from ...core.operation_metadata import build_operation_metadata_json
|
|
11
|
+
from ...db.context import create_database_context
|
|
12
|
+
from ...execution.recording import LocalJobRecorder, LocalRecordedArtifact
|
|
13
|
+
from ...integrations.download import parse_source, resolve_download_backend
|
|
14
|
+
from .requests import GetRequest
|
|
15
|
+
from .results import GetDownloadedFile, GetResponse
|
|
16
|
+
from .transfer import GetService, GetTransferResult
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_artifacts(request: GetRequest) -> GetResponse:
|
|
20
|
+
"""Execute the `roar get` application workflow."""
|
|
21
|
+
bootstrap(request.roar_dir)
|
|
22
|
+
logger = get_logger()
|
|
23
|
+
|
|
24
|
+
parsed_source = parse_source(request.source)
|
|
25
|
+
backend = resolve_download_backend(request.source)
|
|
26
|
+
repo_root = request.repo_root or request.cwd
|
|
27
|
+
is_prefix = request.source.rstrip("/") != request.source or parsed_source.is_prefix
|
|
28
|
+
|
|
29
|
+
git_commit = None
|
|
30
|
+
if not request.dry_run:
|
|
31
|
+
try:
|
|
32
|
+
git_commit = resolve_git_state(repo_root).commit
|
|
33
|
+
logger.debug("Git commit: %s", git_commit)
|
|
34
|
+
except Exception as exc:
|
|
35
|
+
logger.debug("Git operation failed (non-fatal for get): %s", exc)
|
|
36
|
+
|
|
37
|
+
with create_database_context(request.roar_dir) as db_ctx:
|
|
38
|
+
service = GetService(
|
|
39
|
+
backend=backend,
|
|
40
|
+
source=parsed_source,
|
|
41
|
+
repo_root=repo_root,
|
|
42
|
+
)
|
|
43
|
+
transfer_result = service.get(
|
|
44
|
+
destination=request.destination,
|
|
45
|
+
expected_hash=request.expected_hash,
|
|
46
|
+
dry_run=request.dry_run,
|
|
47
|
+
force=request.force,
|
|
48
|
+
is_prefix=is_prefix,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
result = _materialize_get_result(
|
|
52
|
+
db_ctx=db_ctx,
|
|
53
|
+
request=request,
|
|
54
|
+
parsed_source=parsed_source,
|
|
55
|
+
transfer_result=transfer_result,
|
|
56
|
+
git_commit=git_commit,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
git_tag_name = None
|
|
60
|
+
warnings: list[str] = []
|
|
61
|
+
if request.tag and git_commit and result.success and not result.dry_run:
|
|
62
|
+
git_tag_name = build_roar_git_tag_name(git_commit)
|
|
63
|
+
try:
|
|
64
|
+
success, tag_error = create_roar_git_tag(repo_root, git_tag_name)
|
|
65
|
+
if not success:
|
|
66
|
+
git_tag_name = None
|
|
67
|
+
if tag_error:
|
|
68
|
+
warnings.append(f"Could not create git tag: {tag_error}")
|
|
69
|
+
except Exception as exc:
|
|
70
|
+
git_tag_name = None
|
|
71
|
+
warnings.append(f"Could not create git tag: {exc}")
|
|
72
|
+
|
|
73
|
+
return GetResponse(
|
|
74
|
+
success=result.success,
|
|
75
|
+
source=request.source,
|
|
76
|
+
job_id=result.job_id,
|
|
77
|
+
job_uid=result.job_uid,
|
|
78
|
+
downloaded_files=result.downloaded_files,
|
|
79
|
+
dry_run=result.dry_run,
|
|
80
|
+
would_download=result.would_download,
|
|
81
|
+
git_tag=git_tag_name,
|
|
82
|
+
warnings=warnings,
|
|
83
|
+
error=result.error,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _materialize_get_result(
|
|
88
|
+
*,
|
|
89
|
+
db_ctx,
|
|
90
|
+
request: GetRequest,
|
|
91
|
+
parsed_source,
|
|
92
|
+
transfer_result: GetTransferResult,
|
|
93
|
+
git_commit: str | None,
|
|
94
|
+
) -> GetResponse:
|
|
95
|
+
if transfer_result.dry_run or not transfer_result.success:
|
|
96
|
+
return GetResponse(
|
|
97
|
+
success=transfer_result.success,
|
|
98
|
+
source=request.source,
|
|
99
|
+
downloaded_files=transfer_result.downloaded_files,
|
|
100
|
+
dry_run=transfer_result.dry_run,
|
|
101
|
+
would_download=transfer_result.would_download,
|
|
102
|
+
error=transfer_result.error,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
metadata_json = _build_get_operation_metadata_json(
|
|
106
|
+
request=request,
|
|
107
|
+
parsed_source=parsed_source,
|
|
108
|
+
downloaded_files=transfer_result.downloaded_files,
|
|
109
|
+
git_commit=git_commit,
|
|
110
|
+
)
|
|
111
|
+
recorder = LocalJobRecorder()
|
|
112
|
+
output_artifacts = [
|
|
113
|
+
LocalRecordedArtifact(
|
|
114
|
+
path=file_info.local_path,
|
|
115
|
+
hashes={"blake3": str(file_info.hash)},
|
|
116
|
+
size=int(file_info.size or 0),
|
|
117
|
+
)
|
|
118
|
+
for file_info in transfer_result.downloaded_files
|
|
119
|
+
]
|
|
120
|
+
job_id, job_uid = recorder.record(
|
|
121
|
+
db_ctx,
|
|
122
|
+
command=_build_get_command(request),
|
|
123
|
+
timestamp=time.time(),
|
|
124
|
+
metadata=metadata_json,
|
|
125
|
+
execution_backend="local",
|
|
126
|
+
execution_role="host",
|
|
127
|
+
job_type="get",
|
|
128
|
+
output_artifacts=output_artifacts,
|
|
129
|
+
exit_code=0,
|
|
130
|
+
)
|
|
131
|
+
return GetResponse(
|
|
132
|
+
success=True,
|
|
133
|
+
source=request.source,
|
|
134
|
+
job_id=job_id,
|
|
135
|
+
job_uid=job_uid,
|
|
136
|
+
downloaded_files=transfer_result.downloaded_files,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _build_get_command(request: GetRequest) -> str:
|
|
141
|
+
command = f"roar get {request.source}"
|
|
142
|
+
if request.message:
|
|
143
|
+
command += f' -m "{request.message}"'
|
|
144
|
+
return command
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _build_get_operation_metadata_json(
|
|
148
|
+
*,
|
|
149
|
+
request: GetRequest,
|
|
150
|
+
parsed_source,
|
|
151
|
+
downloaded_files: list[GetDownloadedFile],
|
|
152
|
+
git_commit: str | None,
|
|
153
|
+
) -> str:
|
|
154
|
+
artifact_urls: dict[str, str] = {}
|
|
155
|
+
for file_info in downloaded_files:
|
|
156
|
+
artifact_urls[file_info.local_path] = file_info.remote_url
|
|
157
|
+
|
|
158
|
+
return build_operation_metadata_json(
|
|
159
|
+
"get",
|
|
160
|
+
{
|
|
161
|
+
"source": request.source,
|
|
162
|
+
"source_type": parsed_source.scheme,
|
|
163
|
+
"message": request.message,
|
|
164
|
+
"artifacts": artifact_urls,
|
|
165
|
+
"git_commit": git_commit,
|
|
166
|
+
"git_tag": None,
|
|
167
|
+
"timestamp": time.time(),
|
|
168
|
+
},
|
|
169
|
+
)
|