wandb 0.18.2__py3-none-musllinux_1_2_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package_readme.md +89 -0
- wandb/__init__.py +245 -0
- wandb/__init__.pyi +1139 -0
- wandb/__main__.py +3 -0
- wandb/_globals.py +19 -0
- wandb/agents/__init__.py +0 -0
- wandb/agents/pyagent.py +363 -0
- wandb/analytics/__init__.py +3 -0
- wandb/analytics/sentry.py +266 -0
- wandb/apis/__init__.py +48 -0
- wandb/apis/attrs.py +40 -0
- wandb/apis/importers/__init__.py +1 -0
- wandb/apis/importers/internals/internal.py +385 -0
- wandb/apis/importers/internals/protocols.py +99 -0
- wandb/apis/importers/internals/util.py +78 -0
- wandb/apis/importers/mlflow.py +254 -0
- wandb/apis/importers/validation.py +108 -0
- wandb/apis/importers/wandb.py +1603 -0
- wandb/apis/internal.py +232 -0
- wandb/apis/normalize.py +89 -0
- wandb/apis/paginator.py +81 -0
- wandb/apis/public/__init__.py +34 -0
- wandb/apis/public/api.py +1305 -0
- wandb/apis/public/artifacts.py +1090 -0
- wandb/apis/public/const.py +4 -0
- wandb/apis/public/files.py +195 -0
- wandb/apis/public/history.py +149 -0
- wandb/apis/public/jobs.py +659 -0
- wandb/apis/public/projects.py +154 -0
- wandb/apis/public/query_generator.py +166 -0
- wandb/apis/public/reports.py +469 -0
- wandb/apis/public/runs.py +914 -0
- wandb/apis/public/sweeps.py +240 -0
- wandb/apis/public/teams.py +198 -0
- wandb/apis/public/users.py +136 -0
- wandb/apis/reports/__init__.py +1 -0
- wandb/apis/reports/v1/__init__.py +8 -0
- wandb/apis/reports/v2/__init__.py +8 -0
- wandb/apis/workspaces/__init__.py +8 -0
- wandb/beta/workflows.py +288 -0
- wandb/bin/nvidia_gpu_stats +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/__init__.py +0 -0
- wandb/cli/cli.py +3004 -0
- wandb/data_types.py +63 -0
- wandb/docker/__init__.py +342 -0
- wandb/docker/auth.py +436 -0
- wandb/docker/wandb-entrypoint.sh +33 -0
- wandb/docker/www_authenticate.py +94 -0
- wandb/env.py +514 -0
- wandb/errors/__init__.py +17 -0
- wandb/errors/errors.py +37 -0
- wandb/errors/term.py +103 -0
- wandb/errors/util.py +57 -0
- wandb/errors/warnings.py +2 -0
- wandb/filesync/__init__.py +0 -0
- wandb/filesync/dir_watcher.py +403 -0
- wandb/filesync/stats.py +100 -0
- wandb/filesync/step_checksum.py +142 -0
- wandb/filesync/step_prepare.py +179 -0
- wandb/filesync/step_upload.py +290 -0
- wandb/filesync/upload_job.py +142 -0
- wandb/integration/__init__.py +0 -0
- wandb/integration/catboost/__init__.py +5 -0
- wandb/integration/catboost/catboost.py +178 -0
- wandb/integration/cohere/__init__.py +3 -0
- wandb/integration/cohere/cohere.py +21 -0
- wandb/integration/cohere/resolver.py +347 -0
- wandb/integration/diffusers/__init__.py +3 -0
- wandb/integration/diffusers/autologger.py +76 -0
- wandb/integration/diffusers/pipeline_resolver.py +50 -0
- wandb/integration/diffusers/resolvers/__init__.py +9 -0
- wandb/integration/diffusers/resolvers/multimodal.py +882 -0
- wandb/integration/diffusers/resolvers/utils.py +102 -0
- wandb/integration/fastai/__init__.py +249 -0
- wandb/integration/gym/__init__.py +105 -0
- wandb/integration/huggingface/__init__.py +3 -0
- wandb/integration/huggingface/huggingface.py +18 -0
- wandb/integration/huggingface/resolver.py +213 -0
- wandb/integration/keras/__init__.py +11 -0
- wandb/integration/keras/callbacks/__init__.py +5 -0
- wandb/integration/keras/callbacks/metrics_logger.py +136 -0
- wandb/integration/keras/callbacks/model_checkpoint.py +195 -0
- wandb/integration/keras/callbacks/tables_builder.py +226 -0
- wandb/integration/keras/keras.py +1091 -0
- wandb/integration/kfp/__init__.py +6 -0
- wandb/integration/kfp/helpers.py +28 -0
- wandb/integration/kfp/kfp_patch.py +324 -0
- wandb/integration/kfp/wandb_logging.py +182 -0
- wandb/integration/langchain/__init__.py +3 -0
- wandb/integration/langchain/wandb_tracer.py +48 -0
- wandb/integration/lightgbm/__init__.py +239 -0
- wandb/integration/lightning/__init__.py +0 -0
- wandb/integration/lightning/fabric/__init__.py +3 -0
- wandb/integration/lightning/fabric/logger.py +762 -0
- wandb/integration/magic.py +556 -0
- wandb/integration/metaflow/__init__.py +3 -0
- wandb/integration/metaflow/metaflow.py +383 -0
- wandb/integration/openai/__init__.py +3 -0
- wandb/integration/openai/fine_tuning.py +480 -0
- wandb/integration/openai/openai.py +22 -0
- wandb/integration/openai/resolver.py +240 -0
- wandb/integration/prodigy/__init__.py +3 -0
- wandb/integration/prodigy/prodigy.py +299 -0
- wandb/integration/sacred/__init__.py +117 -0
- wandb/integration/sagemaker/__init__.py +12 -0
- wandb/integration/sagemaker/auth.py +28 -0
- wandb/integration/sagemaker/config.py +49 -0
- wandb/integration/sagemaker/files.py +3 -0
- wandb/integration/sagemaker/resources.py +34 -0
- wandb/integration/sb3/__init__.py +3 -0
- wandb/integration/sb3/sb3.py +153 -0
- wandb/integration/sklearn/__init__.py +37 -0
- wandb/integration/sklearn/calculate/__init__.py +32 -0
- wandb/integration/sklearn/calculate/calibration_curves.py +125 -0
- wandb/integration/sklearn/calculate/class_proportions.py +68 -0
- wandb/integration/sklearn/calculate/confusion_matrix.py +93 -0
- wandb/integration/sklearn/calculate/decision_boundaries.py +40 -0
- wandb/integration/sklearn/calculate/elbow_curve.py +55 -0
- wandb/integration/sklearn/calculate/feature_importances.py +67 -0
- wandb/integration/sklearn/calculate/learning_curve.py +64 -0
- wandb/integration/sklearn/calculate/outlier_candidates.py +69 -0
- wandb/integration/sklearn/calculate/residuals.py +86 -0
- wandb/integration/sklearn/calculate/silhouette.py +118 -0
- wandb/integration/sklearn/calculate/summary_metrics.py +62 -0
- wandb/integration/sklearn/plot/__init__.py +35 -0
- wandb/integration/sklearn/plot/classifier.py +329 -0
- wandb/integration/sklearn/plot/clusterer.py +146 -0
- wandb/integration/sklearn/plot/regressor.py +121 -0
- wandb/integration/sklearn/plot/shared.py +91 -0
- wandb/integration/sklearn/utils.py +183 -0
- wandb/integration/tensorboard/__init__.py +10 -0
- wandb/integration/tensorboard/log.py +355 -0
- wandb/integration/tensorboard/monkeypatch.py +185 -0
- wandb/integration/tensorflow/__init__.py +5 -0
- wandb/integration/tensorflow/estimator_hook.py +54 -0
- wandb/integration/torch/__init__.py +0 -0
- wandb/integration/torch/wandb_torch.py +554 -0
- wandb/integration/ultralytics/__init__.py +11 -0
- wandb/integration/ultralytics/bbox_utils.py +208 -0
- wandb/integration/ultralytics/callback.py +524 -0
- wandb/integration/ultralytics/classification_utils.py +83 -0
- wandb/integration/ultralytics/mask_utils.py +202 -0
- wandb/integration/ultralytics/pose_utils.py +103 -0
- wandb/integration/xgboost/__init__.py +11 -0
- wandb/integration/xgboost/xgboost.py +189 -0
- wandb/integration/yolov8/__init__.py +0 -0
- wandb/integration/yolov8/yolov8.py +284 -0
- wandb/jupyter.py +515 -0
- wandb/magic.py +3 -0
- wandb/mpmain/__init__.py +0 -0
- wandb/mpmain/__main__.py +1 -0
- wandb/old/__init__.py +0 -0
- wandb/old/core.py +53 -0
- wandb/old/settings.py +173 -0
- wandb/old/summary.py +440 -0
- wandb/plot/__init__.py +19 -0
- wandb/plot/bar.py +45 -0
- wandb/plot/confusion_matrix.py +100 -0
- wandb/plot/histogram.py +39 -0
- wandb/plot/line.py +43 -0
- wandb/plot/line_series.py +88 -0
- wandb/plot/pr_curve.py +136 -0
- wandb/plot/roc_curve.py +118 -0
- wandb/plot/scatter.py +32 -0
- wandb/plot/utils.py +183 -0
- wandb/plot/viz.py +123 -0
- wandb/proto/__init__.py +0 -0
- wandb/proto/v3/__init__.py +0 -0
- wandb/proto/v3/wandb_base_pb2.py +55 -0
- wandb/proto/v3/wandb_internal_pb2.py +1608 -0
- wandb/proto/v3/wandb_server_pb2.py +208 -0
- wandb/proto/v3/wandb_settings_pb2.py +112 -0
- wandb/proto/v3/wandb_telemetry_pb2.py +106 -0
- wandb/proto/v4/__init__.py +0 -0
- wandb/proto/v4/wandb_base_pb2.py +30 -0
- wandb/proto/v4/wandb_internal_pb2.py +360 -0
- wandb/proto/v4/wandb_server_pb2.py +63 -0
- wandb/proto/v4/wandb_settings_pb2.py +45 -0
- wandb/proto/v4/wandb_telemetry_pb2.py +41 -0
- wandb/proto/v5/wandb_base_pb2.py +31 -0
- wandb/proto/v5/wandb_internal_pb2.py +361 -0
- wandb/proto/v5/wandb_server_pb2.py +64 -0
- wandb/proto/v5/wandb_settings_pb2.py +46 -0
- wandb/proto/v5/wandb_telemetry_pb2.py +42 -0
- wandb/proto/wandb_base_pb2.py +10 -0
- wandb/proto/wandb_deprecated.py +53 -0
- wandb/proto/wandb_generate_deprecated.py +34 -0
- wandb/proto/wandb_generate_proto.py +49 -0
- wandb/proto/wandb_internal_pb2.py +16 -0
- wandb/proto/wandb_server_pb2.py +10 -0
- wandb/proto/wandb_settings_pb2.py +10 -0
- wandb/proto/wandb_telemetry_pb2.py +10 -0
- wandb/py.typed +0 -0
- wandb/sdk/__init__.py +37 -0
- wandb/sdk/artifacts/__init__.py +0 -0
- wandb/sdk/artifacts/_validators.py +90 -0
- wandb/sdk/artifacts/artifact.py +2389 -0
- wandb/sdk/artifacts/artifact_download_logger.py +43 -0
- wandb/sdk/artifacts/artifact_file_cache.py +253 -0
- wandb/sdk/artifacts/artifact_instance_cache.py +17 -0
- wandb/sdk/artifacts/artifact_manifest.py +74 -0
- wandb/sdk/artifacts/artifact_manifest_entry.py +249 -0
- wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +92 -0
- wandb/sdk/artifacts/artifact_saver.py +269 -0
- wandb/sdk/artifacts/artifact_state.py +11 -0
- wandb/sdk/artifacts/artifact_ttl.py +7 -0
- wandb/sdk/artifacts/exceptions.py +57 -0
- wandb/sdk/artifacts/staging.py +25 -0
- wandb/sdk/artifacts/storage_handler.py +62 -0
- wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +208 -0
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +228 -0
- wandb/sdk/artifacts/storage_handlers/http_handler.py +114 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +141 -0
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +56 -0
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +300 -0
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +72 -0
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +135 -0
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +74 -0
- wandb/sdk/artifacts/storage_layout.py +6 -0
- wandb/sdk/artifacts/storage_policies/__init__.py +4 -0
- wandb/sdk/artifacts/storage_policies/register.py +1 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +378 -0
- wandb/sdk/artifacts/storage_policy.py +72 -0
- wandb/sdk/backend/__init__.py +0 -0
- wandb/sdk/backend/backend.py +222 -0
- wandb/sdk/data_types/__init__.py +0 -0
- wandb/sdk/data_types/_dtypes.py +914 -0
- wandb/sdk/data_types/_private.py +10 -0
- wandb/sdk/data_types/audio.py +165 -0
- wandb/sdk/data_types/base_types/__init__.py +0 -0
- wandb/sdk/data_types/base_types/json_metadata.py +55 -0
- wandb/sdk/data_types/base_types/media.py +315 -0
- wandb/sdk/data_types/base_types/wb_value.py +272 -0
- wandb/sdk/data_types/bokeh.py +70 -0
- wandb/sdk/data_types/graph.py +405 -0
- wandb/sdk/data_types/helper_types/__init__.py +0 -0
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +295 -0
- wandb/sdk/data_types/helper_types/classes.py +159 -0
- wandb/sdk/data_types/helper_types/image_mask.py +235 -0
- wandb/sdk/data_types/histogram.py +96 -0
- wandb/sdk/data_types/html.py +115 -0
- wandb/sdk/data_types/image.py +845 -0
- wandb/sdk/data_types/molecule.py +241 -0
- wandb/sdk/data_types/object_3d.py +474 -0
- wandb/sdk/data_types/plotly.py +82 -0
- wandb/sdk/data_types/saved_model.py +446 -0
- wandb/sdk/data_types/table.py +1204 -0
- wandb/sdk/data_types/trace_tree.py +438 -0
- wandb/sdk/data_types/utils.py +229 -0
- wandb/sdk/data_types/video.py +247 -0
- wandb/sdk/integration_utils/__init__.py +0 -0
- wandb/sdk/integration_utils/auto_logging.py +239 -0
- wandb/sdk/integration_utils/data_logging.py +475 -0
- wandb/sdk/interface/__init__.py +0 -0
- wandb/sdk/interface/constants.py +4 -0
- wandb/sdk/interface/interface.py +972 -0
- wandb/sdk/interface/interface_queue.py +59 -0
- wandb/sdk/interface/interface_relay.py +53 -0
- wandb/sdk/interface/interface_shared.py +537 -0
- wandb/sdk/interface/interface_sock.py +61 -0
- wandb/sdk/interface/message_future.py +27 -0
- wandb/sdk/interface/message_future_poll.py +50 -0
- wandb/sdk/interface/router.py +118 -0
- wandb/sdk/interface/router_queue.py +44 -0
- wandb/sdk/interface/router_relay.py +39 -0
- wandb/sdk/interface/router_sock.py +36 -0
- wandb/sdk/interface/summary_record.py +67 -0
- wandb/sdk/internal/__init__.py +0 -0
- wandb/sdk/internal/context.py +89 -0
- wandb/sdk/internal/datastore.py +297 -0
- wandb/sdk/internal/file_pusher.py +181 -0
- wandb/sdk/internal/file_stream.py +695 -0
- wandb/sdk/internal/flow_control.py +263 -0
- wandb/sdk/internal/handler.py +901 -0
- wandb/sdk/internal/internal.py +417 -0
- wandb/sdk/internal/internal_api.py +4358 -0
- wandb/sdk/internal/internal_util.py +100 -0
- wandb/sdk/internal/job_builder.py +629 -0
- wandb/sdk/internal/profiler.py +78 -0
- wandb/sdk/internal/progress.py +83 -0
- wandb/sdk/internal/run.py +25 -0
- wandb/sdk/internal/sample.py +70 -0
- wandb/sdk/internal/sender.py +1686 -0
- wandb/sdk/internal/sender_config.py +197 -0
- wandb/sdk/internal/settings_static.py +90 -0
- wandb/sdk/internal/system/__init__.py +0 -0
- wandb/sdk/internal/system/assets/__init__.py +27 -0
- wandb/sdk/internal/system/assets/aggregators.py +37 -0
- wandb/sdk/internal/system/assets/asset_registry.py +20 -0
- wandb/sdk/internal/system/assets/cpu.py +163 -0
- wandb/sdk/internal/system/assets/disk.py +210 -0
- wandb/sdk/internal/system/assets/gpu.py +416 -0
- wandb/sdk/internal/system/assets/gpu_amd.py +239 -0
- wandb/sdk/internal/system/assets/gpu_apple.py +177 -0
- wandb/sdk/internal/system/assets/interfaces.py +207 -0
- wandb/sdk/internal/system/assets/ipu.py +177 -0
- wandb/sdk/internal/system/assets/memory.py +166 -0
- wandb/sdk/internal/system/assets/network.py +125 -0
- wandb/sdk/internal/system/assets/open_metrics.py +299 -0
- wandb/sdk/internal/system/assets/tpu.py +154 -0
- wandb/sdk/internal/system/assets/trainium.py +399 -0
- wandb/sdk/internal/system/env_probe_helpers.py +13 -0
- wandb/sdk/internal/system/system_info.py +249 -0
- wandb/sdk/internal/system/system_monitor.py +229 -0
- wandb/sdk/internal/tb_watcher.py +518 -0
- wandb/sdk/internal/thread_local_settings.py +18 -0
- wandb/sdk/internal/writer.py +206 -0
- wandb/sdk/launch/__init__.py +14 -0
- wandb/sdk/launch/_launch.py +330 -0
- wandb/sdk/launch/_launch_add.py +255 -0
- wandb/sdk/launch/_project_spec.py +566 -0
- wandb/sdk/launch/agent/__init__.py +5 -0
- wandb/sdk/launch/agent/agent.py +924 -0
- wandb/sdk/launch/agent/config.py +296 -0
- wandb/sdk/launch/agent/job_status_tracker.py +53 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/__init__.py +0 -0
- wandb/sdk/launch/builder/abstract.py +156 -0
- wandb/sdk/launch/builder/build.py +297 -0
- wandb/sdk/launch/builder/context_manager.py +235 -0
- wandb/sdk/launch/builder/docker_builder.py +177 -0
- wandb/sdk/launch/builder/kaniko_builder.py +595 -0
- wandb/sdk/launch/builder/noop.py +58 -0
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +188 -0
- wandb/sdk/launch/builder/templates/dockerfile.py +92 -0
- wandb/sdk/launch/create_job.py +528 -0
- wandb/sdk/launch/environment/abstract.py +29 -0
- wandb/sdk/launch/environment/aws_environment.py +322 -0
- wandb/sdk/launch/environment/azure_environment.py +105 -0
- wandb/sdk/launch/environment/gcp_environment.py +335 -0
- wandb/sdk/launch/environment/local_environment.py +66 -0
- wandb/sdk/launch/errors.py +19 -0
- wandb/sdk/launch/git_reference.py +109 -0
- wandb/sdk/launch/inputs/files.py +148 -0
- wandb/sdk/launch/inputs/internal.py +315 -0
- wandb/sdk/launch/inputs/manage.py +113 -0
- wandb/sdk/launch/inputs/schema.py +39 -0
- wandb/sdk/launch/loader.py +249 -0
- wandb/sdk/launch/registry/abstract.py +48 -0
- wandb/sdk/launch/registry/anon.py +29 -0
- wandb/sdk/launch/registry/azure_container_registry.py +124 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +192 -0
- wandb/sdk/launch/registry/google_artifact_registry.py +219 -0
- wandb/sdk/launch/registry/local_registry.py +67 -0
- wandb/sdk/launch/runner/__init__.py +0 -0
- wandb/sdk/launch/runner/abstract.py +195 -0
- wandb/sdk/launch/runner/kubernetes_monitor.py +474 -0
- wandb/sdk/launch/runner/kubernetes_runner.py +963 -0
- wandb/sdk/launch/runner/local_container.py +301 -0
- wandb/sdk/launch/runner/local_process.py +78 -0
- wandb/sdk/launch/runner/sagemaker_runner.py +426 -0
- wandb/sdk/launch/runner/vertex_runner.py +230 -0
- wandb/sdk/launch/sweeps/__init__.py +39 -0
- wandb/sdk/launch/sweeps/scheduler.py +742 -0
- wandb/sdk/launch/sweeps/scheduler_sweep.py +91 -0
- wandb/sdk/launch/sweeps/utils.py +316 -0
- wandb/sdk/launch/utils.py +746 -0
- wandb/sdk/launch/wandb_reference.py +138 -0
- wandb/sdk/lib/__init__.py +5 -0
- wandb/sdk/lib/_settings_toposort_generate.py +159 -0
- wandb/sdk/lib/_settings_toposort_generated.py +250 -0
- wandb/sdk/lib/_wburls_generate.py +25 -0
- wandb/sdk/lib/_wburls_generated.py +22 -0
- wandb/sdk/lib/apikey.py +273 -0
- wandb/sdk/lib/capped_dict.py +26 -0
- wandb/sdk/lib/config_util.py +101 -0
- wandb/sdk/lib/credentials.py +141 -0
- wandb/sdk/lib/deprecate.py +42 -0
- wandb/sdk/lib/disabled.py +29 -0
- wandb/sdk/lib/exit_hooks.py +54 -0
- wandb/sdk/lib/file_stream_utils.py +118 -0
- wandb/sdk/lib/filenames.py +64 -0
- wandb/sdk/lib/filesystem.py +372 -0
- wandb/sdk/lib/fsm.py +174 -0
- wandb/sdk/lib/gitlib.py +239 -0
- wandb/sdk/lib/gql_request.py +65 -0
- wandb/sdk/lib/handler_util.py +21 -0
- wandb/sdk/lib/hashutil.py +84 -0
- wandb/sdk/lib/import_hooks.py +275 -0
- wandb/sdk/lib/ipython.py +146 -0
- wandb/sdk/lib/json_util.py +80 -0
- wandb/sdk/lib/lazyloader.py +63 -0
- wandb/sdk/lib/mailbox.py +460 -0
- wandb/sdk/lib/module.py +69 -0
- wandb/sdk/lib/paths.py +106 -0
- wandb/sdk/lib/preinit.py +42 -0
- wandb/sdk/lib/printer.py +313 -0
- wandb/sdk/lib/proto_util.py +90 -0
- wandb/sdk/lib/redirect.py +845 -0
- wandb/sdk/lib/reporting.py +99 -0
- wandb/sdk/lib/retry.py +289 -0
- wandb/sdk/lib/run_moment.py +78 -0
- wandb/sdk/lib/runid.py +12 -0
- wandb/sdk/lib/server.py +52 -0
- wandb/sdk/lib/service_connection.py +216 -0
- wandb/sdk/lib/service_token.py +94 -0
- wandb/sdk/lib/sock_client.py +295 -0
- wandb/sdk/lib/sparkline.py +45 -0
- wandb/sdk/lib/telemetry.py +100 -0
- wandb/sdk/lib/timed_input.py +133 -0
- wandb/sdk/lib/timer.py +19 -0
- wandb/sdk/lib/tracelog.py +255 -0
- wandb/sdk/lib/wburls.py +46 -0
- wandb/sdk/service/__init__.py +0 -0
- wandb/sdk/service/_startup_debug.py +22 -0
- wandb/sdk/service/port_file.py +53 -0
- wandb/sdk/service/server.py +116 -0
- wandb/sdk/service/server_sock.py +276 -0
- wandb/sdk/service/service.py +242 -0
- wandb/sdk/service/streams.py +417 -0
- wandb/sdk/verify/__init__.py +0 -0
- wandb/sdk/verify/verify.py +501 -0
- wandb/sdk/wandb_alerts.py +12 -0
- wandb/sdk/wandb_config.py +322 -0
- wandb/sdk/wandb_helper.py +54 -0
- wandb/sdk/wandb_init.py +1266 -0
- wandb/sdk/wandb_login.py +349 -0
- wandb/sdk/wandb_metric.py +110 -0
- wandb/sdk/wandb_require.py +97 -0
- wandb/sdk/wandb_require_helpers.py +44 -0
- wandb/sdk/wandb_run.py +4236 -0
- wandb/sdk/wandb_settings.py +2001 -0
- wandb/sdk/wandb_setup.py +409 -0
- wandb/sdk/wandb_summary.py +150 -0
- wandb/sdk/wandb_sweep.py +119 -0
- wandb/sdk/wandb_sync.py +81 -0
- wandb/sdk/wandb_watch.py +144 -0
- wandb/sklearn.py +35 -0
- wandb/sync/__init__.py +3 -0
- wandb/sync/sync.py +443 -0
- wandb/trigger.py +29 -0
- wandb/util.py +1956 -0
- wandb/vendor/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/setup.py +40 -0
- wandb/vendor/gql-0.2.0/tests/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/tests/starwars/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py +96 -0
- wandb/vendor/gql-0.2.0/tests/starwars/schema.py +146 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py +293 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_query.py +355 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py +171 -0
- wandb/vendor/gql-0.2.0/tests/test_client.py +31 -0
- wandb/vendor/gql-0.2.0/tests/test_transport.py +89 -0
- wandb/vendor/gql-0.2.0/wandb_gql/__init__.py +4 -0
- wandb/vendor/gql-0.2.0/wandb_gql/client.py +75 -0
- wandb/vendor/gql-0.2.0/wandb_gql/dsl.py +152 -0
- wandb/vendor/gql-0.2.0/wandb_gql/gql.py +10 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py +6 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py +15 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py +46 -0
- wandb/vendor/gql-0.2.0/wandb_gql/utils.py +21 -0
- wandb/vendor/graphql-core-1.1/setup.py +86 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py +287 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py +6 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py +42 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py +11 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py +29 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py +36 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py +26 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py +311 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py +398 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py +53 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py +22 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py +32 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py +7 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py +35 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py +6 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py +66 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py +252 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py +151 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py +7 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py +57 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py +145 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py +60 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py +1349 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py +19 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py +435 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py +30 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py +779 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py +193 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py +18 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py +222 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py +82 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py +17 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py +28 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py +40 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py +8 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py +43 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py +78 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py +67 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py +619 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py +132 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py +440 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py +131 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py +100 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py +145 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py +9 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py +65 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py +49 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py +24 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py +75 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py +291 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py +250 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py +9 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py +357 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py +27 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py +90 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py +67 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py +66 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py +168 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py +56 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py +69 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py +149 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py +69 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py +4 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py +79 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py +24 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py +8 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py +44 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py +113 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py +70 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py +97 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py +19 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py +43 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py +23 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py +59 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py +36 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py +38 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py +37 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py +529 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py +44 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py +46 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py +32 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py +28 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py +31 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py +27 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py +53 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py +158 -0
- wandb/vendor/promise-2.3.0/conftest.py +30 -0
- wandb/vendor/promise-2.3.0/setup.py +64 -0
- wandb/vendor/promise-2.3.0/tests/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/tests/conftest.py +8 -0
- wandb/vendor/promise-2.3.0/tests/test_awaitable.py +32 -0
- wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py +47 -0
- wandb/vendor/promise-2.3.0/tests/test_benchmark.py +116 -0
- wandb/vendor/promise-2.3.0/tests/test_complex_threads.py +23 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader.py +452 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py +99 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py +65 -0
- wandb/vendor/promise-2.3.0/tests/test_extra.py +670 -0
- wandb/vendor/promise-2.3.0/tests/test_issues.py +132 -0
- wandb/vendor/promise-2.3.0/tests/test_promise_list.py +70 -0
- wandb/vendor/promise-2.3.0/tests/test_spec.py +584 -0
- wandb/vendor/promise-2.3.0/tests/test_thread_safety.py +115 -0
- wandb/vendor/promise-2.3.0/tests/utils.py +3 -0
- wandb/vendor/promise-2.3.0/wandb_promise/__init__.py +38 -0
- wandb/vendor/promise-2.3.0/wandb_promise/async_.py +135 -0
- wandb/vendor/promise-2.3.0/wandb_promise/compat.py +32 -0
- wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py +326 -0
- wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py +12 -0
- wandb/vendor/promise-2.3.0/wandb_promise/promise.py +848 -0
- wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py +151 -0
- wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py +83 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py +22 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py +21 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py +27 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py +18 -0
- wandb/vendor/promise-2.3.0/wandb_promise/utils.py +56 -0
- wandb/vendor/pygments/__init__.py +90 -0
- wandb/vendor/pygments/cmdline.py +568 -0
- wandb/vendor/pygments/console.py +74 -0
- wandb/vendor/pygments/filter.py +74 -0
- wandb/vendor/pygments/filters/__init__.py +350 -0
- wandb/vendor/pygments/formatter.py +95 -0
- wandb/vendor/pygments/formatters/__init__.py +153 -0
- wandb/vendor/pygments/formatters/_mapping.py +85 -0
- wandb/vendor/pygments/formatters/bbcode.py +109 -0
- wandb/vendor/pygments/formatters/html.py +851 -0
- wandb/vendor/pygments/formatters/img.py +600 -0
- wandb/vendor/pygments/formatters/irc.py +182 -0
- wandb/vendor/pygments/formatters/latex.py +482 -0
- wandb/vendor/pygments/formatters/other.py +160 -0
- wandb/vendor/pygments/formatters/rtf.py +147 -0
- wandb/vendor/pygments/formatters/svg.py +153 -0
- wandb/vendor/pygments/formatters/terminal.py +136 -0
- wandb/vendor/pygments/formatters/terminal256.py +309 -0
- wandb/vendor/pygments/lexer.py +871 -0
- wandb/vendor/pygments/lexers/__init__.py +329 -0
- wandb/vendor/pygments/lexers/_asy_builtins.py +1645 -0
- wandb/vendor/pygments/lexers/_cl_builtins.py +232 -0
- wandb/vendor/pygments/lexers/_cocoa_builtins.py +72 -0
- wandb/vendor/pygments/lexers/_csound_builtins.py +1346 -0
- wandb/vendor/pygments/lexers/_lasso_builtins.py +5327 -0
- wandb/vendor/pygments/lexers/_lua_builtins.py +295 -0
- wandb/vendor/pygments/lexers/_mapping.py +500 -0
- wandb/vendor/pygments/lexers/_mql_builtins.py +1172 -0
- wandb/vendor/pygments/lexers/_openedge_builtins.py +2547 -0
- wandb/vendor/pygments/lexers/_php_builtins.py +4756 -0
- wandb/vendor/pygments/lexers/_postgres_builtins.py +621 -0
- wandb/vendor/pygments/lexers/_scilab_builtins.py +3094 -0
- wandb/vendor/pygments/lexers/_sourcemod_builtins.py +1163 -0
- wandb/vendor/pygments/lexers/_stan_builtins.py +532 -0
- wandb/vendor/pygments/lexers/_stata_builtins.py +419 -0
- wandb/vendor/pygments/lexers/_tsql_builtins.py +1004 -0
- wandb/vendor/pygments/lexers/_vim_builtins.py +1939 -0
- wandb/vendor/pygments/lexers/actionscript.py +240 -0
- wandb/vendor/pygments/lexers/agile.py +24 -0
- wandb/vendor/pygments/lexers/algebra.py +221 -0
- wandb/vendor/pygments/lexers/ambient.py +76 -0
- wandb/vendor/pygments/lexers/ampl.py +87 -0
- wandb/vendor/pygments/lexers/apl.py +101 -0
- wandb/vendor/pygments/lexers/archetype.py +318 -0
- wandb/vendor/pygments/lexers/asm.py +641 -0
- wandb/vendor/pygments/lexers/automation.py +374 -0
- wandb/vendor/pygments/lexers/basic.py +500 -0
- wandb/vendor/pygments/lexers/bibtex.py +160 -0
- wandb/vendor/pygments/lexers/business.py +612 -0
- wandb/vendor/pygments/lexers/c_cpp.py +252 -0
- wandb/vendor/pygments/lexers/c_like.py +541 -0
- wandb/vendor/pygments/lexers/capnproto.py +78 -0
- wandb/vendor/pygments/lexers/chapel.py +102 -0
- wandb/vendor/pygments/lexers/clean.py +288 -0
- wandb/vendor/pygments/lexers/compiled.py +34 -0
- wandb/vendor/pygments/lexers/configs.py +833 -0
- wandb/vendor/pygments/lexers/console.py +114 -0
- wandb/vendor/pygments/lexers/crystal.py +393 -0
- wandb/vendor/pygments/lexers/csound.py +366 -0
- wandb/vendor/pygments/lexers/css.py +689 -0
- wandb/vendor/pygments/lexers/d.py +251 -0
- wandb/vendor/pygments/lexers/dalvik.py +125 -0
- wandb/vendor/pygments/lexers/data.py +555 -0
- wandb/vendor/pygments/lexers/diff.py +165 -0
- wandb/vendor/pygments/lexers/dotnet.py +691 -0
- wandb/vendor/pygments/lexers/dsls.py +878 -0
- wandb/vendor/pygments/lexers/dylan.py +289 -0
- wandb/vendor/pygments/lexers/ecl.py +125 -0
- wandb/vendor/pygments/lexers/eiffel.py +65 -0
- wandb/vendor/pygments/lexers/elm.py +121 -0
- wandb/vendor/pygments/lexers/erlang.py +533 -0
- wandb/vendor/pygments/lexers/esoteric.py +277 -0
- wandb/vendor/pygments/lexers/ezhil.py +69 -0
- wandb/vendor/pygments/lexers/factor.py +344 -0
- wandb/vendor/pygments/lexers/fantom.py +250 -0
- wandb/vendor/pygments/lexers/felix.py +273 -0
- wandb/vendor/pygments/lexers/forth.py +177 -0
- wandb/vendor/pygments/lexers/fortran.py +205 -0
- wandb/vendor/pygments/lexers/foxpro.py +428 -0
- wandb/vendor/pygments/lexers/functional.py +21 -0
- wandb/vendor/pygments/lexers/go.py +101 -0
- wandb/vendor/pygments/lexers/grammar_notation.py +213 -0
- wandb/vendor/pygments/lexers/graph.py +80 -0
- wandb/vendor/pygments/lexers/graphics.py +553 -0
- wandb/vendor/pygments/lexers/haskell.py +843 -0
- wandb/vendor/pygments/lexers/haxe.py +936 -0
- wandb/vendor/pygments/lexers/hdl.py +382 -0
- wandb/vendor/pygments/lexers/hexdump.py +103 -0
- wandb/vendor/pygments/lexers/html.py +602 -0
- wandb/vendor/pygments/lexers/idl.py +270 -0
- wandb/vendor/pygments/lexers/igor.py +288 -0
- wandb/vendor/pygments/lexers/inferno.py +96 -0
- wandb/vendor/pygments/lexers/installers.py +322 -0
- wandb/vendor/pygments/lexers/int_fiction.py +1343 -0
- wandb/vendor/pygments/lexers/iolang.py +63 -0
- wandb/vendor/pygments/lexers/j.py +146 -0
- wandb/vendor/pygments/lexers/javascript.py +1525 -0
- wandb/vendor/pygments/lexers/julia.py +333 -0
- wandb/vendor/pygments/lexers/jvm.py +1573 -0
- wandb/vendor/pygments/lexers/lisp.py +2621 -0
- wandb/vendor/pygments/lexers/make.py +202 -0
- wandb/vendor/pygments/lexers/markup.py +595 -0
- wandb/vendor/pygments/lexers/math.py +21 -0
- wandb/vendor/pygments/lexers/matlab.py +663 -0
- wandb/vendor/pygments/lexers/ml.py +769 -0
- wandb/vendor/pygments/lexers/modeling.py +358 -0
- wandb/vendor/pygments/lexers/modula2.py +1561 -0
- wandb/vendor/pygments/lexers/monte.py +204 -0
- wandb/vendor/pygments/lexers/ncl.py +894 -0
- wandb/vendor/pygments/lexers/nimrod.py +159 -0
- wandb/vendor/pygments/lexers/nit.py +64 -0
- wandb/vendor/pygments/lexers/nix.py +136 -0
- wandb/vendor/pygments/lexers/oberon.py +105 -0
- wandb/vendor/pygments/lexers/objective.py +504 -0
- wandb/vendor/pygments/lexers/ooc.py +85 -0
- wandb/vendor/pygments/lexers/other.py +41 -0
- wandb/vendor/pygments/lexers/parasail.py +79 -0
- wandb/vendor/pygments/lexers/parsers.py +835 -0
- wandb/vendor/pygments/lexers/pascal.py +644 -0
- wandb/vendor/pygments/lexers/pawn.py +199 -0
- wandb/vendor/pygments/lexers/perl.py +620 -0
- wandb/vendor/pygments/lexers/php.py +267 -0
- wandb/vendor/pygments/lexers/praat.py +294 -0
- wandb/vendor/pygments/lexers/prolog.py +306 -0
- wandb/vendor/pygments/lexers/python.py +939 -0
- wandb/vendor/pygments/lexers/qvt.py +152 -0
- wandb/vendor/pygments/lexers/r.py +453 -0
- wandb/vendor/pygments/lexers/rdf.py +270 -0
- wandb/vendor/pygments/lexers/rebol.py +431 -0
- wandb/vendor/pygments/lexers/resource.py +85 -0
- wandb/vendor/pygments/lexers/rnc.py +67 -0
- wandb/vendor/pygments/lexers/roboconf.py +82 -0
- wandb/vendor/pygments/lexers/robotframework.py +560 -0
- wandb/vendor/pygments/lexers/ruby.py +519 -0
- wandb/vendor/pygments/lexers/rust.py +220 -0
- wandb/vendor/pygments/lexers/sas.py +228 -0
- wandb/vendor/pygments/lexers/scripting.py +1222 -0
- wandb/vendor/pygments/lexers/shell.py +794 -0
- wandb/vendor/pygments/lexers/smalltalk.py +195 -0
- wandb/vendor/pygments/lexers/smv.py +79 -0
- wandb/vendor/pygments/lexers/snobol.py +83 -0
- wandb/vendor/pygments/lexers/special.py +103 -0
- wandb/vendor/pygments/lexers/sql.py +681 -0
- wandb/vendor/pygments/lexers/stata.py +108 -0
- wandb/vendor/pygments/lexers/supercollider.py +90 -0
- wandb/vendor/pygments/lexers/tcl.py +145 -0
- wandb/vendor/pygments/lexers/templates.py +2283 -0
- wandb/vendor/pygments/lexers/testing.py +207 -0
- wandb/vendor/pygments/lexers/text.py +25 -0
- wandb/vendor/pygments/lexers/textedit.py +169 -0
- wandb/vendor/pygments/lexers/textfmts.py +297 -0
- wandb/vendor/pygments/lexers/theorem.py +458 -0
- wandb/vendor/pygments/lexers/trafficscript.py +54 -0
- wandb/vendor/pygments/lexers/typoscript.py +226 -0
- wandb/vendor/pygments/lexers/urbi.py +133 -0
- wandb/vendor/pygments/lexers/varnish.py +190 -0
- wandb/vendor/pygments/lexers/verification.py +111 -0
- wandb/vendor/pygments/lexers/web.py +24 -0
- wandb/vendor/pygments/lexers/webmisc.py +988 -0
- wandb/vendor/pygments/lexers/whiley.py +116 -0
- wandb/vendor/pygments/lexers/x10.py +69 -0
- wandb/vendor/pygments/modeline.py +44 -0
- wandb/vendor/pygments/plugin.py +68 -0
- wandb/vendor/pygments/regexopt.py +92 -0
- wandb/vendor/pygments/scanner.py +105 -0
- wandb/vendor/pygments/sphinxext.py +158 -0
- wandb/vendor/pygments/style.py +155 -0
- wandb/vendor/pygments/styles/__init__.py +80 -0
- wandb/vendor/pygments/styles/abap.py +29 -0
- wandb/vendor/pygments/styles/algol.py +63 -0
- wandb/vendor/pygments/styles/algol_nu.py +63 -0
- wandb/vendor/pygments/styles/arduino.py +98 -0
- wandb/vendor/pygments/styles/autumn.py +65 -0
- wandb/vendor/pygments/styles/borland.py +51 -0
- wandb/vendor/pygments/styles/bw.py +49 -0
- wandb/vendor/pygments/styles/colorful.py +81 -0
- wandb/vendor/pygments/styles/default.py +73 -0
- wandb/vendor/pygments/styles/emacs.py +72 -0
- wandb/vendor/pygments/styles/friendly.py +72 -0
- wandb/vendor/pygments/styles/fruity.py +42 -0
- wandb/vendor/pygments/styles/igor.py +29 -0
- wandb/vendor/pygments/styles/lovelace.py +97 -0
- wandb/vendor/pygments/styles/manni.py +75 -0
- wandb/vendor/pygments/styles/monokai.py +106 -0
- wandb/vendor/pygments/styles/murphy.py +80 -0
- wandb/vendor/pygments/styles/native.py +65 -0
- wandb/vendor/pygments/styles/paraiso_dark.py +125 -0
- wandb/vendor/pygments/styles/paraiso_light.py +125 -0
- wandb/vendor/pygments/styles/pastie.py +75 -0
- wandb/vendor/pygments/styles/perldoc.py +69 -0
- wandb/vendor/pygments/styles/rainbow_dash.py +89 -0
- wandb/vendor/pygments/styles/rrt.py +33 -0
- wandb/vendor/pygments/styles/sas.py +44 -0
- wandb/vendor/pygments/styles/stata.py +40 -0
- wandb/vendor/pygments/styles/tango.py +141 -0
- wandb/vendor/pygments/styles/trac.py +63 -0
- wandb/vendor/pygments/styles/vim.py +63 -0
- wandb/vendor/pygments/styles/vs.py +38 -0
- wandb/vendor/pygments/styles/xcode.py +51 -0
- wandb/vendor/pygments/token.py +213 -0
- wandb/vendor/pygments/unistring.py +217 -0
- wandb/vendor/pygments/util.py +388 -0
- wandb/vendor/pynvml/__init__.py +0 -0
- wandb/vendor/pynvml/pynvml.py +4779 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py +17 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py +615 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py +98 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py +369 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py +172 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py +239 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py +218 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py +81 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py +575 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py +730 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py +145 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py +133 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py +348 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py +265 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py +174 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py +151 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py +249 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py +29 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py +198 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py +88 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py +293 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py +157 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py +41 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py +40 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py +57 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py +64 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py +123 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py +28 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py +577 -0
- wandb/wandb_agent.py +588 -0
- wandb/wandb_controller.py +721 -0
- wandb/wandb_run.py +9 -0
- wandb-0.18.2.dist-info/METADATA +213 -0
- wandb-0.18.2.dist-info/RECORD +827 -0
- wandb-0.18.2.dist-info/WHEEL +5 -0
- wandb-0.18.2.dist-info/entry_points.txt +3 -0
- wandb-0.18.2.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,56 @@
|
|
1
|
+
"""Multi storage handler."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
6
|
+
from urllib.parse import urlparse
|
7
|
+
|
8
|
+
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
9
|
+
from wandb.sdk.lib.paths import FilePathStr, URIStr
|
10
|
+
|
11
|
+
if TYPE_CHECKING:
|
12
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
13
|
+
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
14
|
+
|
15
|
+
|
16
|
+
class MultiHandler(StorageHandler):
|
17
|
+
_handlers: list[StorageHandler]
|
18
|
+
|
19
|
+
def __init__(
|
20
|
+
self,
|
21
|
+
handlers: list[StorageHandler] | None = None,
|
22
|
+
default_handler: StorageHandler | None = None,
|
23
|
+
) -> None:
|
24
|
+
self._handlers = handlers or []
|
25
|
+
self._default_handler = default_handler
|
26
|
+
|
27
|
+
def _get_handler(self, url: FilePathStr | URIStr) -> StorageHandler:
|
28
|
+
parsed_url = urlparse(url)
|
29
|
+
for handler in self._handlers:
|
30
|
+
if handler.can_handle(parsed_url):
|
31
|
+
return handler
|
32
|
+
if self._default_handler is not None:
|
33
|
+
return self._default_handler
|
34
|
+
raise ValueError('No storage handler registered for url "{}"'.format(str(url)))
|
35
|
+
|
36
|
+
def load_path(
|
37
|
+
self,
|
38
|
+
manifest_entry: ArtifactManifestEntry,
|
39
|
+
local: bool = False,
|
40
|
+
) -> URIStr | FilePathStr:
|
41
|
+
assert manifest_entry.ref is not None
|
42
|
+
handler = self._get_handler(manifest_entry.ref)
|
43
|
+
return handler.load_path(manifest_entry, local=local)
|
44
|
+
|
45
|
+
def store_path(
|
46
|
+
self,
|
47
|
+
artifact: Artifact,
|
48
|
+
path: URIStr | FilePathStr,
|
49
|
+
name: str | None = None,
|
50
|
+
checksum: bool = True,
|
51
|
+
max_objects: int | None = None,
|
52
|
+
) -> Sequence[ArtifactManifestEntry]:
|
53
|
+
handler = self._get_handler(path)
|
54
|
+
return handler.store_path(
|
55
|
+
artifact, path, name=name, checksum=checksum, max_objects=max_objects
|
56
|
+
)
|
@@ -0,0 +1,300 @@
|
|
1
|
+
"""S3 storage handler."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
import os
|
6
|
+
import time
|
7
|
+
from pathlib import PurePosixPath
|
8
|
+
from typing import TYPE_CHECKING, Sequence
|
9
|
+
from urllib.parse import parse_qsl, urlparse
|
10
|
+
|
11
|
+
from wandb import util
|
12
|
+
from wandb.errors import CommError
|
13
|
+
from wandb.errors.term import termlog
|
14
|
+
from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
|
15
|
+
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
16
|
+
from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
|
17
|
+
from wandb.sdk.lib.hashutil import ETag
|
18
|
+
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
19
|
+
|
20
|
+
if TYPE_CHECKING:
|
21
|
+
from urllib.parse import ParseResult
|
22
|
+
|
23
|
+
# We could probably use https://pypi.org/project/boto3-stubs/ or something
|
24
|
+
# instead of `type:ignore`ing these boto imports, but it's nontrivial:
|
25
|
+
# for some reason, despite being actively maintained as of 2022-09-30,
|
26
|
+
# the latest release of boto3-stubs doesn't include all the features we use.
|
27
|
+
import boto3 # type: ignore
|
28
|
+
import boto3.resources.base # type: ignore
|
29
|
+
import boto3.s3 # type: ignore
|
30
|
+
import boto3.session # type: ignore
|
31
|
+
|
32
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
33
|
+
|
34
|
+
|
35
|
+
class S3Handler(StorageHandler):
|
36
|
+
_s3: boto3.resources.base.ServiceResource | None
|
37
|
+
_scheme: str
|
38
|
+
|
39
|
+
def __init__(self, scheme: str | None = None) -> None:
|
40
|
+
self._scheme = scheme or "s3"
|
41
|
+
self._s3 = None
|
42
|
+
self._cache = get_artifact_file_cache()
|
43
|
+
|
44
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
45
|
+
return parsed_url.scheme == self._scheme
|
46
|
+
|
47
|
+
def init_boto(self) -> boto3.resources.base.ServiceResource:
|
48
|
+
if self._s3 is not None:
|
49
|
+
return self._s3
|
50
|
+
boto: boto3 = util.get_module(
|
51
|
+
"boto3",
|
52
|
+
required="s3:// references requires the boto3 library, run pip install wandb[aws]",
|
53
|
+
lazy=False,
|
54
|
+
)
|
55
|
+
self._s3 = boto.session.Session().resource(
|
56
|
+
"s3",
|
57
|
+
endpoint_url=os.getenv("AWS_S3_ENDPOINT_URL"),
|
58
|
+
region_name=os.getenv("AWS_REGION"),
|
59
|
+
)
|
60
|
+
self._botocore = util.get_module("botocore")
|
61
|
+
return self._s3
|
62
|
+
|
63
|
+
def _parse_uri(self, uri: str) -> tuple[str, str, str | None]:
|
64
|
+
url = urlparse(uri)
|
65
|
+
query = dict(parse_qsl(url.query))
|
66
|
+
|
67
|
+
bucket = url.netloc
|
68
|
+
key = url.path[1:] # strip leading slash
|
69
|
+
version = query.get("versionId")
|
70
|
+
|
71
|
+
return bucket, key, version
|
72
|
+
|
73
|
+
def load_path(
|
74
|
+
self,
|
75
|
+
manifest_entry: ArtifactManifestEntry,
|
76
|
+
local: bool = False,
|
77
|
+
) -> URIStr | FilePathStr:
|
78
|
+
if not local:
|
79
|
+
assert manifest_entry.ref is not None
|
80
|
+
return manifest_entry.ref
|
81
|
+
|
82
|
+
assert manifest_entry.ref is not None
|
83
|
+
|
84
|
+
path, hit, cache_open = self._cache.check_etag_obj_path(
|
85
|
+
URIStr(manifest_entry.ref),
|
86
|
+
ETag(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
|
87
|
+
manifest_entry.size if manifest_entry.size is not None else 0,
|
88
|
+
)
|
89
|
+
if hit:
|
90
|
+
return path
|
91
|
+
|
92
|
+
self.init_boto()
|
93
|
+
assert self._s3 is not None # mypy: unwraps optionality
|
94
|
+
bucket, key, _ = self._parse_uri(manifest_entry.ref)
|
95
|
+
version = manifest_entry.extra.get("versionID")
|
96
|
+
|
97
|
+
extra_args = {}
|
98
|
+
if version:
|
99
|
+
obj_version = self._s3.ObjectVersion(bucket, key, version)
|
100
|
+
extra_args["VersionId"] = version
|
101
|
+
obj = obj_version.Object()
|
102
|
+
else:
|
103
|
+
obj = self._s3.Object(bucket, key)
|
104
|
+
|
105
|
+
try:
|
106
|
+
etag = (
|
107
|
+
obj_version.head()["ETag"][1:-1] # escape leading and trailing
|
108
|
+
if version
|
109
|
+
else self._etag_from_obj(obj)
|
110
|
+
)
|
111
|
+
except self._botocore.exceptions.ClientError as e:
|
112
|
+
if e.response["Error"]["Code"] == "404":
|
113
|
+
raise FileNotFoundError(
|
114
|
+
f"Unable to find {manifest_entry.path} at s3://{bucket}/{key}"
|
115
|
+
) from e
|
116
|
+
raise
|
117
|
+
|
118
|
+
if etag != manifest_entry.digest:
|
119
|
+
# Try to match the etag with some other version.
|
120
|
+
if version:
|
121
|
+
raise ValueError(
|
122
|
+
f"Digest mismatch for object {manifest_entry.ref} with version {version}: expected {manifest_entry.digest} but found {etag}"
|
123
|
+
)
|
124
|
+
obj = None
|
125
|
+
object_versions = self._s3.Bucket(bucket).object_versions.filter(Prefix=key)
|
126
|
+
for object_version in object_versions:
|
127
|
+
if manifest_entry.extra.get("etag") == self._etag_from_obj(
|
128
|
+
object_version
|
129
|
+
):
|
130
|
+
obj = object_version.Object()
|
131
|
+
extra_args["VersionId"] = object_version.version_id
|
132
|
+
break
|
133
|
+
if obj is None:
|
134
|
+
raise FileNotFoundError(
|
135
|
+
"Couldn't find object version for {}/{} matching etag {}".format(
|
136
|
+
bucket, key, manifest_entry.extra.get("etag")
|
137
|
+
)
|
138
|
+
)
|
139
|
+
|
140
|
+
with cache_open(mode="wb") as f:
|
141
|
+
obj.download_fileobj(f, ExtraArgs=extra_args)
|
142
|
+
return path
|
143
|
+
|
144
|
+
def store_path(
|
145
|
+
self,
|
146
|
+
artifact: Artifact,
|
147
|
+
path: URIStr | FilePathStr,
|
148
|
+
name: StrPath | None = None,
|
149
|
+
checksum: bool = True,
|
150
|
+
max_objects: int | None = None,
|
151
|
+
) -> Sequence[ArtifactManifestEntry]:
|
152
|
+
self.init_boto()
|
153
|
+
assert self._s3 is not None # mypy: unwraps optionality
|
154
|
+
|
155
|
+
# The passed in path might have query string parameters.
|
156
|
+
# We only need to care about a subset, like version, when
|
157
|
+
# parsing. Once we have that, we can store the rest of the
|
158
|
+
# metadata in the artifact entry itself.
|
159
|
+
bucket, key, version = self._parse_uri(path)
|
160
|
+
path = URIStr(f"{self._scheme}://{bucket}/{key}")
|
161
|
+
|
162
|
+
max_objects = max_objects or DEFAULT_MAX_OBJECTS
|
163
|
+
if not checksum:
|
164
|
+
entry_path = name or (key if key != "" else bucket)
|
165
|
+
return [ArtifactManifestEntry(path=entry_path, ref=path, digest=path)]
|
166
|
+
|
167
|
+
# If an explicit version is specified, use that. Otherwise, use the head version.
|
168
|
+
objs = (
|
169
|
+
[self._s3.ObjectVersion(bucket, key, version).Object()]
|
170
|
+
if version
|
171
|
+
else [self._s3.Object(bucket, key)]
|
172
|
+
)
|
173
|
+
start_time = None
|
174
|
+
multi = False
|
175
|
+
if key != "":
|
176
|
+
try:
|
177
|
+
objs[0].load()
|
178
|
+
# S3 doesn't have real folders, however there are cases where the folder key has a valid file which will not
|
179
|
+
# trigger a recursive upload.
|
180
|
+
# we should check the object's metadata says it is a directory and do a multi file upload if it is
|
181
|
+
if "x-directory" in objs[0].content_type:
|
182
|
+
multi = True
|
183
|
+
except self._botocore.exceptions.ClientError as e:
|
184
|
+
if e.response["Error"]["Code"] == "404":
|
185
|
+
multi = True
|
186
|
+
else:
|
187
|
+
raise CommError(
|
188
|
+
f"Unable to connect to S3 ({e.response['Error']['Code']}): "
|
189
|
+
f"{e.response['Error']['Message']}. Check that your "
|
190
|
+
"authentication credentials are valid and that your region is "
|
191
|
+
"set correctly."
|
192
|
+
)
|
193
|
+
else:
|
194
|
+
multi = True
|
195
|
+
|
196
|
+
if multi:
|
197
|
+
start_time = time.time()
|
198
|
+
termlog(
|
199
|
+
'Generating checksum for up to %i objects in "%s/%s"... '
|
200
|
+
% (max_objects, bucket, key),
|
201
|
+
newline=False,
|
202
|
+
)
|
203
|
+
if key != "":
|
204
|
+
objs = (
|
205
|
+
self._s3.Bucket(bucket)
|
206
|
+
.objects.filter(Prefix=key)
|
207
|
+
.limit(max_objects)
|
208
|
+
)
|
209
|
+
else:
|
210
|
+
objs = self._s3.Bucket(bucket).objects.limit(max_objects)
|
211
|
+
# Weird iterator scoping makes us assign this to a local function
|
212
|
+
size = self._size_from_obj
|
213
|
+
entries = [
|
214
|
+
self._entry_from_obj(obj, path, name, prefix=key, multi=multi)
|
215
|
+
for obj in objs
|
216
|
+
if size(obj) > 0
|
217
|
+
]
|
218
|
+
if start_time is not None:
|
219
|
+
termlog("Done. %.1fs" % (time.time() - start_time), prefix=False)
|
220
|
+
if len(entries) > max_objects:
|
221
|
+
raise ValueError(
|
222
|
+
"Exceeded %i objects tracked, pass max_objects to add_reference"
|
223
|
+
% max_objects
|
224
|
+
)
|
225
|
+
return entries
|
226
|
+
|
227
|
+
def _size_from_obj(self, obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> int:
|
228
|
+
# ObjectSummary has size, Object has content_length
|
229
|
+
size: int
|
230
|
+
if hasattr(obj, "size"):
|
231
|
+
size = obj.size
|
232
|
+
else:
|
233
|
+
size = obj.content_length
|
234
|
+
return size
|
235
|
+
|
236
|
+
def _entry_from_obj(
|
237
|
+
self,
|
238
|
+
obj: boto3.s3.Object | boto3.s3.ObjectSummary,
|
239
|
+
path: str,
|
240
|
+
name: StrPath | None = None,
|
241
|
+
prefix: str = "",
|
242
|
+
multi: bool = False,
|
243
|
+
) -> ArtifactManifestEntry:
|
244
|
+
"""Create an ArtifactManifestEntry from an S3 object.
|
245
|
+
|
246
|
+
Arguments:
|
247
|
+
obj: The S3 object
|
248
|
+
path: The S3-style path (e.g.: "s3://bucket/file.txt")
|
249
|
+
name: The user assigned name, or None if not specified
|
250
|
+
prefix: The prefix to add (will be the same as `path` for directories)
|
251
|
+
multi: Whether or not this is a multi-object add.
|
252
|
+
"""
|
253
|
+
bucket, key, _ = self._parse_uri(path)
|
254
|
+
|
255
|
+
# Always use posix paths, since that's what S3 uses.
|
256
|
+
posix_key = PurePosixPath(obj.key) # the bucket key
|
257
|
+
posix_path = PurePosixPath(bucket) / key # the path, with the scheme stripped
|
258
|
+
posix_prefix = PurePosixPath(prefix) # the prefix, if adding a prefix
|
259
|
+
posix_name = PurePosixPath(name or "")
|
260
|
+
posix_ref = posix_path
|
261
|
+
|
262
|
+
if name is None:
|
263
|
+
# We're adding a directory (prefix), so calculate a relative path.
|
264
|
+
if str(posix_prefix) in str(posix_key) and posix_prefix != posix_key:
|
265
|
+
posix_name = posix_key.relative_to(posix_prefix)
|
266
|
+
posix_ref = posix_path / posix_name
|
267
|
+
else:
|
268
|
+
posix_name = PurePosixPath(posix_key.name)
|
269
|
+
posix_ref = posix_path
|
270
|
+
elif multi:
|
271
|
+
# We're adding a directory with a name override.
|
272
|
+
relpath = posix_key.relative_to(posix_prefix)
|
273
|
+
posix_name = posix_name / relpath
|
274
|
+
posix_ref = posix_path / relpath
|
275
|
+
return ArtifactManifestEntry(
|
276
|
+
path=posix_name,
|
277
|
+
ref=URIStr(f"{self._scheme}://{str(posix_ref)}"),
|
278
|
+
digest=ETag(self._etag_from_obj(obj)),
|
279
|
+
size=self._size_from_obj(obj),
|
280
|
+
extra=self._extra_from_obj(obj),
|
281
|
+
)
|
282
|
+
|
283
|
+
@staticmethod
|
284
|
+
def _etag_from_obj(obj: boto3.s3.Object | boto3.s3.ObjectSummary) -> ETag:
|
285
|
+
etag: ETag
|
286
|
+
etag = obj.e_tag[1:-1] # escape leading and trailing quote
|
287
|
+
return etag
|
288
|
+
|
289
|
+
def _extra_from_obj(
|
290
|
+
self, obj: boto3.s3.Object | boto3.s3.ObjectSummary
|
291
|
+
) -> dict[str, str]:
|
292
|
+
extra = {
|
293
|
+
"etag": obj.e_tag[1:-1], # escape leading and trailing quote
|
294
|
+
}
|
295
|
+
if not hasattr(obj, "version_id"):
|
296
|
+
# Convert ObjectSummary to Object to get the version_id.
|
297
|
+
obj = self._s3.Object(obj.bucket_name, obj.key) # type: ignore[union-attr]
|
298
|
+
if hasattr(obj, "version_id") and obj.version_id and obj.version_id != "null":
|
299
|
+
extra["versionID"] = obj.version_id
|
300
|
+
return extra
|
@@ -0,0 +1,72 @@
|
|
1
|
+
"""Tracking storage handler."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from typing import TYPE_CHECKING, Sequence
|
6
|
+
from urllib.parse import urlparse
|
7
|
+
|
8
|
+
from wandb.errors.term import termwarn
|
9
|
+
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
10
|
+
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
11
|
+
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
12
|
+
|
13
|
+
if TYPE_CHECKING:
|
14
|
+
from urllib.parse import ParseResult
|
15
|
+
|
16
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
17
|
+
|
18
|
+
|
19
|
+
class TrackingHandler(StorageHandler):
|
20
|
+
def __init__(self, scheme: str | None = None) -> None:
|
21
|
+
"""Track paths with no modification or special processing.
|
22
|
+
|
23
|
+
Useful when paths being tracked are on file systems mounted at a standardized
|
24
|
+
location.
|
25
|
+
|
26
|
+
For example, if the data to track is located on an NFS share mounted on
|
27
|
+
`/data`, then it is sufficient to just track the paths.
|
28
|
+
"""
|
29
|
+
self._scheme = scheme or ""
|
30
|
+
|
31
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
32
|
+
return parsed_url.scheme == self._scheme
|
33
|
+
|
34
|
+
def load_path(
|
35
|
+
self,
|
36
|
+
manifest_entry: ArtifactManifestEntry,
|
37
|
+
local: bool = False,
|
38
|
+
) -> URIStr | FilePathStr:
|
39
|
+
if local:
|
40
|
+
# Likely a user error. The tracking handler is
|
41
|
+
# oblivious to the underlying paths, so it has
|
42
|
+
# no way of actually loading it.
|
43
|
+
url = urlparse(manifest_entry.ref)
|
44
|
+
raise ValueError(
|
45
|
+
f"Cannot download file at path {str(manifest_entry.ref)}, scheme {str(url.scheme)} not recognized"
|
46
|
+
)
|
47
|
+
# TODO(spencerpearson): should this go through util.to_native_slash_path
|
48
|
+
# instead of just getting typecast?
|
49
|
+
return FilePathStr(manifest_entry.path)
|
50
|
+
|
51
|
+
def store_path(
|
52
|
+
self,
|
53
|
+
artifact: Artifact,
|
54
|
+
path: URIStr | FilePathStr,
|
55
|
+
name: StrPath | None = None,
|
56
|
+
checksum: bool = True,
|
57
|
+
max_objects: int | None = None,
|
58
|
+
) -> Sequence[ArtifactManifestEntry]:
|
59
|
+
url = urlparse(path)
|
60
|
+
if name is None:
|
61
|
+
raise ValueError(
|
62
|
+
'You must pass name="<entry_name>" when tracking references with unknown schemes. ref: {}'.format(
|
63
|
+
path
|
64
|
+
)
|
65
|
+
)
|
66
|
+
termwarn(
|
67
|
+
"Artifact references with unsupported schemes cannot be checksummed: {}".format(
|
68
|
+
path
|
69
|
+
)
|
70
|
+
)
|
71
|
+
name = name or url.path[1:] # strip leading slash
|
72
|
+
return [ArtifactManifestEntry(path=name, ref=path, digest=path)]
|
@@ -0,0 +1,135 @@
|
|
1
|
+
"""WB artifact storage handler."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
import os
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
7
|
+
from urllib.parse import urlparse
|
8
|
+
|
9
|
+
import wandb
|
10
|
+
from wandb import util
|
11
|
+
from wandb.apis import PublicApi
|
12
|
+
from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
|
13
|
+
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
14
|
+
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
15
|
+
from wandb.sdk.lib.hashutil import B64MD5, b64_to_hex_id, hex_to_b64_id
|
16
|
+
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
17
|
+
|
18
|
+
if TYPE_CHECKING:
|
19
|
+
from urllib.parse import ParseResult
|
20
|
+
|
21
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
22
|
+
|
23
|
+
|
24
|
+
class WBArtifactHandler(StorageHandler):
|
25
|
+
"""Handles loading and storing Artifact reference-type files."""
|
26
|
+
|
27
|
+
_client: PublicApi | None
|
28
|
+
|
29
|
+
def __init__(self) -> None:
|
30
|
+
self._scheme = "wandb-artifact"
|
31
|
+
self._cache = get_artifact_file_cache()
|
32
|
+
self._client = None
|
33
|
+
|
34
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
35
|
+
return parsed_url.scheme == self._scheme
|
36
|
+
|
37
|
+
@property
|
38
|
+
def client(self) -> PublicApi:
|
39
|
+
if self._client is None:
|
40
|
+
self._client = PublicApi()
|
41
|
+
return self._client
|
42
|
+
|
43
|
+
def load_path(
|
44
|
+
self,
|
45
|
+
manifest_entry: ArtifactManifestEntry,
|
46
|
+
local: bool = False,
|
47
|
+
) -> URIStr | FilePathStr:
|
48
|
+
"""Load the file in the specified artifact given its corresponding entry.
|
49
|
+
|
50
|
+
Download the referenced artifact; create and return a new symlink to the caller.
|
51
|
+
|
52
|
+
Arguments:
|
53
|
+
manifest_entry (ArtifactManifestEntry): The index entry to load
|
54
|
+
|
55
|
+
Returns:
|
56
|
+
(os.PathLike): A path to the file represented by `index_entry`
|
57
|
+
"""
|
58
|
+
# We don't check for cache hits here. Since we have 0 for size (since this
|
59
|
+
# is a cross-artifact reference which and we've made the choice to store 0
|
60
|
+
# in the size field), we can't confirm if the file is complete. So we just
|
61
|
+
# rely on the dep_artifact entry's download() method to do its own cache
|
62
|
+
# check.
|
63
|
+
|
64
|
+
# Parse the reference path and download the artifact if needed
|
65
|
+
artifact_id = util.host_from_path(manifest_entry.ref)
|
66
|
+
artifact_file_path = util.uri_from_path(manifest_entry.ref)
|
67
|
+
|
68
|
+
dep_artifact = wandb.Artifact._from_id(
|
69
|
+
hex_to_b64_id(artifact_id), self.client.client
|
70
|
+
)
|
71
|
+
assert dep_artifact is not None
|
72
|
+
link_target_path: URIStr | FilePathStr
|
73
|
+
if local:
|
74
|
+
link_target_path = dep_artifact.get_entry(artifact_file_path).download()
|
75
|
+
else:
|
76
|
+
link_target_path = dep_artifact.get_entry(artifact_file_path).ref_target()
|
77
|
+
|
78
|
+
return link_target_path
|
79
|
+
|
80
|
+
def store_path(
|
81
|
+
self,
|
82
|
+
artifact: Artifact,
|
83
|
+
path: URIStr | FilePathStr,
|
84
|
+
name: StrPath | None = None,
|
85
|
+
checksum: bool = True,
|
86
|
+
max_objects: int | None = None,
|
87
|
+
) -> Sequence[ArtifactManifestEntry]:
|
88
|
+
"""Store the file or directory at the given path into the specified artifact.
|
89
|
+
|
90
|
+
Recursively resolves the reference until the result is a concrete asset.
|
91
|
+
|
92
|
+
Arguments:
|
93
|
+
artifact: The artifact doing the storing path (str): The path to store name
|
94
|
+
(str): If specified, the logical name that should map to `path`
|
95
|
+
|
96
|
+
Returns:
|
97
|
+
(list[ArtifactManifestEntry]): A list of manifest entries to store within
|
98
|
+
the artifact
|
99
|
+
"""
|
100
|
+
# Recursively resolve the reference until a concrete asset is found
|
101
|
+
# TODO: Consider resolving server-side for performance improvements.
|
102
|
+
iter_path: URIStr | FilePathStr | None = path
|
103
|
+
while iter_path is not None and urlparse(iter_path).scheme == self._scheme:
|
104
|
+
artifact_id = util.host_from_path(iter_path)
|
105
|
+
artifact_file_path = util.uri_from_path(iter_path)
|
106
|
+
target_artifact = wandb.Artifact._from_id(
|
107
|
+
hex_to_b64_id(artifact_id), self.client.client
|
108
|
+
)
|
109
|
+
assert target_artifact is not None
|
110
|
+
|
111
|
+
entry = target_artifact.manifest.get_entry_by_path(artifact_file_path)
|
112
|
+
assert entry is not None
|
113
|
+
iter_path = entry.ref
|
114
|
+
|
115
|
+
# Create the path reference
|
116
|
+
assert target_artifact is not None
|
117
|
+
assert target_artifact.id is not None
|
118
|
+
path = URIStr(
|
119
|
+
"{}://{}/{}".format(
|
120
|
+
self._scheme,
|
121
|
+
b64_to_hex_id(B64MD5(target_artifact.id)),
|
122
|
+
artifact_file_path,
|
123
|
+
)
|
124
|
+
)
|
125
|
+
|
126
|
+
# Return the new entry
|
127
|
+
assert entry is not None
|
128
|
+
return [
|
129
|
+
ArtifactManifestEntry(
|
130
|
+
path=name or os.path.basename(path),
|
131
|
+
ref=path,
|
132
|
+
size=0,
|
133
|
+
digest=entry.digest,
|
134
|
+
)
|
135
|
+
]
|
@@ -0,0 +1,74 @@
|
|
1
|
+
"""WB local artifact storage handler."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
import os
|
6
|
+
from typing import TYPE_CHECKING, Sequence
|
7
|
+
|
8
|
+
import wandb
|
9
|
+
from wandb import util
|
10
|
+
from wandb.sdk.artifacts.artifact_instance_cache import artifact_instance_cache
|
11
|
+
from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
|
12
|
+
from wandb.sdk.artifacts.storage_handler import StorageHandler
|
13
|
+
from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from urllib.parse import ParseResult
|
17
|
+
|
18
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
19
|
+
|
20
|
+
|
21
|
+
class WBLocalArtifactHandler(StorageHandler):
|
22
|
+
"""Handles loading and storing Artifact reference-type files."""
|
23
|
+
|
24
|
+
def __init__(self) -> None:
|
25
|
+
self._scheme = "wandb-client-artifact"
|
26
|
+
|
27
|
+
def can_handle(self, parsed_url: ParseResult) -> bool:
|
28
|
+
return parsed_url.scheme == self._scheme
|
29
|
+
|
30
|
+
def load_path(
|
31
|
+
self,
|
32
|
+
manifest_entry: ArtifactManifestEntry,
|
33
|
+
local: bool = False,
|
34
|
+
) -> URIStr | FilePathStr:
|
35
|
+
raise NotImplementedError(
|
36
|
+
"Should not be loading a path for an artifact entry with unresolved client id."
|
37
|
+
)
|
38
|
+
|
39
|
+
def store_path(
|
40
|
+
self,
|
41
|
+
artifact: Artifact,
|
42
|
+
path: URIStr | FilePathStr,
|
43
|
+
name: StrPath | None = None,
|
44
|
+
checksum: bool = True,
|
45
|
+
max_objects: int | None = None,
|
46
|
+
) -> Sequence[ArtifactManifestEntry]:
|
47
|
+
"""Store the file or directory at the given path within the specified artifact.
|
48
|
+
|
49
|
+
Arguments:
|
50
|
+
artifact: The artifact doing the storing
|
51
|
+
path (str): The path to store
|
52
|
+
name (str): If specified, the logical name that should map to `path`
|
53
|
+
|
54
|
+
Returns:
|
55
|
+
(list[ArtifactManifestEntry]): A list of manifest entries to store within the artifact
|
56
|
+
"""
|
57
|
+
client_id = util.host_from_path(path)
|
58
|
+
target_path = util.uri_from_path(path)
|
59
|
+
target_artifact = artifact_instance_cache.get(client_id)
|
60
|
+
if not isinstance(target_artifact, wandb.Artifact):
|
61
|
+
raise RuntimeError("Local Artifact not found - invalid reference")
|
62
|
+
target_entry = target_artifact._manifest.entries[target_path] # type: ignore
|
63
|
+
if target_entry is None:
|
64
|
+
raise RuntimeError("Local entry not found - invalid reference")
|
65
|
+
|
66
|
+
# Return the new entry
|
67
|
+
return [
|
68
|
+
ArtifactManifestEntry(
|
69
|
+
path=name or os.path.basename(path),
|
70
|
+
ref=path,
|
71
|
+
size=0,
|
72
|
+
digest=target_entry.digest,
|
73
|
+
)
|
74
|
+
]
|
@@ -0,0 +1 @@
|
|
1
|
+
WANDB_STORAGE_POLICY = "wandb-storage-policy-v1"
|