wandb 0.18.2__py3-none-musllinux_1_2_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package_readme.md +89 -0
- wandb/__init__.py +245 -0
- wandb/__init__.pyi +1139 -0
- wandb/__main__.py +3 -0
- wandb/_globals.py +19 -0
- wandb/agents/__init__.py +0 -0
- wandb/agents/pyagent.py +363 -0
- wandb/analytics/__init__.py +3 -0
- wandb/analytics/sentry.py +266 -0
- wandb/apis/__init__.py +48 -0
- wandb/apis/attrs.py +40 -0
- wandb/apis/importers/__init__.py +1 -0
- wandb/apis/importers/internals/internal.py +385 -0
- wandb/apis/importers/internals/protocols.py +99 -0
- wandb/apis/importers/internals/util.py +78 -0
- wandb/apis/importers/mlflow.py +254 -0
- wandb/apis/importers/validation.py +108 -0
- wandb/apis/importers/wandb.py +1603 -0
- wandb/apis/internal.py +232 -0
- wandb/apis/normalize.py +89 -0
- wandb/apis/paginator.py +81 -0
- wandb/apis/public/__init__.py +34 -0
- wandb/apis/public/api.py +1305 -0
- wandb/apis/public/artifacts.py +1090 -0
- wandb/apis/public/const.py +4 -0
- wandb/apis/public/files.py +195 -0
- wandb/apis/public/history.py +149 -0
- wandb/apis/public/jobs.py +659 -0
- wandb/apis/public/projects.py +154 -0
- wandb/apis/public/query_generator.py +166 -0
- wandb/apis/public/reports.py +469 -0
- wandb/apis/public/runs.py +914 -0
- wandb/apis/public/sweeps.py +240 -0
- wandb/apis/public/teams.py +198 -0
- wandb/apis/public/users.py +136 -0
- wandb/apis/reports/__init__.py +1 -0
- wandb/apis/reports/v1/__init__.py +8 -0
- wandb/apis/reports/v2/__init__.py +8 -0
- wandb/apis/workspaces/__init__.py +8 -0
- wandb/beta/workflows.py +288 -0
- wandb/bin/nvidia_gpu_stats +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/__init__.py +0 -0
- wandb/cli/cli.py +3004 -0
- wandb/data_types.py +63 -0
- wandb/docker/__init__.py +342 -0
- wandb/docker/auth.py +436 -0
- wandb/docker/wandb-entrypoint.sh +33 -0
- wandb/docker/www_authenticate.py +94 -0
- wandb/env.py +514 -0
- wandb/errors/__init__.py +17 -0
- wandb/errors/errors.py +37 -0
- wandb/errors/term.py +103 -0
- wandb/errors/util.py +57 -0
- wandb/errors/warnings.py +2 -0
- wandb/filesync/__init__.py +0 -0
- wandb/filesync/dir_watcher.py +403 -0
- wandb/filesync/stats.py +100 -0
- wandb/filesync/step_checksum.py +142 -0
- wandb/filesync/step_prepare.py +179 -0
- wandb/filesync/step_upload.py +290 -0
- wandb/filesync/upload_job.py +142 -0
- wandb/integration/__init__.py +0 -0
- wandb/integration/catboost/__init__.py +5 -0
- wandb/integration/catboost/catboost.py +178 -0
- wandb/integration/cohere/__init__.py +3 -0
- wandb/integration/cohere/cohere.py +21 -0
- wandb/integration/cohere/resolver.py +347 -0
- wandb/integration/diffusers/__init__.py +3 -0
- wandb/integration/diffusers/autologger.py +76 -0
- wandb/integration/diffusers/pipeline_resolver.py +50 -0
- wandb/integration/diffusers/resolvers/__init__.py +9 -0
- wandb/integration/diffusers/resolvers/multimodal.py +882 -0
- wandb/integration/diffusers/resolvers/utils.py +102 -0
- wandb/integration/fastai/__init__.py +249 -0
- wandb/integration/gym/__init__.py +105 -0
- wandb/integration/huggingface/__init__.py +3 -0
- wandb/integration/huggingface/huggingface.py +18 -0
- wandb/integration/huggingface/resolver.py +213 -0
- wandb/integration/keras/__init__.py +11 -0
- wandb/integration/keras/callbacks/__init__.py +5 -0
- wandb/integration/keras/callbacks/metrics_logger.py +136 -0
- wandb/integration/keras/callbacks/model_checkpoint.py +195 -0
- wandb/integration/keras/callbacks/tables_builder.py +226 -0
- wandb/integration/keras/keras.py +1091 -0
- wandb/integration/kfp/__init__.py +6 -0
- wandb/integration/kfp/helpers.py +28 -0
- wandb/integration/kfp/kfp_patch.py +324 -0
- wandb/integration/kfp/wandb_logging.py +182 -0
- wandb/integration/langchain/__init__.py +3 -0
- wandb/integration/langchain/wandb_tracer.py +48 -0
- wandb/integration/lightgbm/__init__.py +239 -0
- wandb/integration/lightning/__init__.py +0 -0
- wandb/integration/lightning/fabric/__init__.py +3 -0
- wandb/integration/lightning/fabric/logger.py +762 -0
- wandb/integration/magic.py +556 -0
- wandb/integration/metaflow/__init__.py +3 -0
- wandb/integration/metaflow/metaflow.py +383 -0
- wandb/integration/openai/__init__.py +3 -0
- wandb/integration/openai/fine_tuning.py +480 -0
- wandb/integration/openai/openai.py +22 -0
- wandb/integration/openai/resolver.py +240 -0
- wandb/integration/prodigy/__init__.py +3 -0
- wandb/integration/prodigy/prodigy.py +299 -0
- wandb/integration/sacred/__init__.py +117 -0
- wandb/integration/sagemaker/__init__.py +12 -0
- wandb/integration/sagemaker/auth.py +28 -0
- wandb/integration/sagemaker/config.py +49 -0
- wandb/integration/sagemaker/files.py +3 -0
- wandb/integration/sagemaker/resources.py +34 -0
- wandb/integration/sb3/__init__.py +3 -0
- wandb/integration/sb3/sb3.py +153 -0
- wandb/integration/sklearn/__init__.py +37 -0
- wandb/integration/sklearn/calculate/__init__.py +32 -0
- wandb/integration/sklearn/calculate/calibration_curves.py +125 -0
- wandb/integration/sklearn/calculate/class_proportions.py +68 -0
- wandb/integration/sklearn/calculate/confusion_matrix.py +93 -0
- wandb/integration/sklearn/calculate/decision_boundaries.py +40 -0
- wandb/integration/sklearn/calculate/elbow_curve.py +55 -0
- wandb/integration/sklearn/calculate/feature_importances.py +67 -0
- wandb/integration/sklearn/calculate/learning_curve.py +64 -0
- wandb/integration/sklearn/calculate/outlier_candidates.py +69 -0
- wandb/integration/sklearn/calculate/residuals.py +86 -0
- wandb/integration/sklearn/calculate/silhouette.py +118 -0
- wandb/integration/sklearn/calculate/summary_metrics.py +62 -0
- wandb/integration/sklearn/plot/__init__.py +35 -0
- wandb/integration/sklearn/plot/classifier.py +329 -0
- wandb/integration/sklearn/plot/clusterer.py +146 -0
- wandb/integration/sklearn/plot/regressor.py +121 -0
- wandb/integration/sklearn/plot/shared.py +91 -0
- wandb/integration/sklearn/utils.py +183 -0
- wandb/integration/tensorboard/__init__.py +10 -0
- wandb/integration/tensorboard/log.py +355 -0
- wandb/integration/tensorboard/monkeypatch.py +185 -0
- wandb/integration/tensorflow/__init__.py +5 -0
- wandb/integration/tensorflow/estimator_hook.py +54 -0
- wandb/integration/torch/__init__.py +0 -0
- wandb/integration/torch/wandb_torch.py +554 -0
- wandb/integration/ultralytics/__init__.py +11 -0
- wandb/integration/ultralytics/bbox_utils.py +208 -0
- wandb/integration/ultralytics/callback.py +524 -0
- wandb/integration/ultralytics/classification_utils.py +83 -0
- wandb/integration/ultralytics/mask_utils.py +202 -0
- wandb/integration/ultralytics/pose_utils.py +103 -0
- wandb/integration/xgboost/__init__.py +11 -0
- wandb/integration/xgboost/xgboost.py +189 -0
- wandb/integration/yolov8/__init__.py +0 -0
- wandb/integration/yolov8/yolov8.py +284 -0
- wandb/jupyter.py +515 -0
- wandb/magic.py +3 -0
- wandb/mpmain/__init__.py +0 -0
- wandb/mpmain/__main__.py +1 -0
- wandb/old/__init__.py +0 -0
- wandb/old/core.py +53 -0
- wandb/old/settings.py +173 -0
- wandb/old/summary.py +440 -0
- wandb/plot/__init__.py +19 -0
- wandb/plot/bar.py +45 -0
- wandb/plot/confusion_matrix.py +100 -0
- wandb/plot/histogram.py +39 -0
- wandb/plot/line.py +43 -0
- wandb/plot/line_series.py +88 -0
- wandb/plot/pr_curve.py +136 -0
- wandb/plot/roc_curve.py +118 -0
- wandb/plot/scatter.py +32 -0
- wandb/plot/utils.py +183 -0
- wandb/plot/viz.py +123 -0
- wandb/proto/__init__.py +0 -0
- wandb/proto/v3/__init__.py +0 -0
- wandb/proto/v3/wandb_base_pb2.py +55 -0
- wandb/proto/v3/wandb_internal_pb2.py +1608 -0
- wandb/proto/v3/wandb_server_pb2.py +208 -0
- wandb/proto/v3/wandb_settings_pb2.py +112 -0
- wandb/proto/v3/wandb_telemetry_pb2.py +106 -0
- wandb/proto/v4/__init__.py +0 -0
- wandb/proto/v4/wandb_base_pb2.py +30 -0
- wandb/proto/v4/wandb_internal_pb2.py +360 -0
- wandb/proto/v4/wandb_server_pb2.py +63 -0
- wandb/proto/v4/wandb_settings_pb2.py +45 -0
- wandb/proto/v4/wandb_telemetry_pb2.py +41 -0
- wandb/proto/v5/wandb_base_pb2.py +31 -0
- wandb/proto/v5/wandb_internal_pb2.py +361 -0
- wandb/proto/v5/wandb_server_pb2.py +64 -0
- wandb/proto/v5/wandb_settings_pb2.py +46 -0
- wandb/proto/v5/wandb_telemetry_pb2.py +42 -0
- wandb/proto/wandb_base_pb2.py +10 -0
- wandb/proto/wandb_deprecated.py +53 -0
- wandb/proto/wandb_generate_deprecated.py +34 -0
- wandb/proto/wandb_generate_proto.py +49 -0
- wandb/proto/wandb_internal_pb2.py +16 -0
- wandb/proto/wandb_server_pb2.py +10 -0
- wandb/proto/wandb_settings_pb2.py +10 -0
- wandb/proto/wandb_telemetry_pb2.py +10 -0
- wandb/py.typed +0 -0
- wandb/sdk/__init__.py +37 -0
- wandb/sdk/artifacts/__init__.py +0 -0
- wandb/sdk/artifacts/_validators.py +90 -0
- wandb/sdk/artifacts/artifact.py +2389 -0
- wandb/sdk/artifacts/artifact_download_logger.py +43 -0
- wandb/sdk/artifacts/artifact_file_cache.py +253 -0
- wandb/sdk/artifacts/artifact_instance_cache.py +17 -0
- wandb/sdk/artifacts/artifact_manifest.py +74 -0
- wandb/sdk/artifacts/artifact_manifest_entry.py +249 -0
- wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +92 -0
- wandb/sdk/artifacts/artifact_saver.py +269 -0
- wandb/sdk/artifacts/artifact_state.py +11 -0
- wandb/sdk/artifacts/artifact_ttl.py +7 -0
- wandb/sdk/artifacts/exceptions.py +57 -0
- wandb/sdk/artifacts/staging.py +25 -0
- wandb/sdk/artifacts/storage_handler.py +62 -0
- wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +208 -0
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +228 -0
- wandb/sdk/artifacts/storage_handlers/http_handler.py +114 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +141 -0
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +56 -0
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +300 -0
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +72 -0
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +135 -0
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +74 -0
- wandb/sdk/artifacts/storage_layout.py +6 -0
- wandb/sdk/artifacts/storage_policies/__init__.py +4 -0
- wandb/sdk/artifacts/storage_policies/register.py +1 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +378 -0
- wandb/sdk/artifacts/storage_policy.py +72 -0
- wandb/sdk/backend/__init__.py +0 -0
- wandb/sdk/backend/backend.py +222 -0
- wandb/sdk/data_types/__init__.py +0 -0
- wandb/sdk/data_types/_dtypes.py +914 -0
- wandb/sdk/data_types/_private.py +10 -0
- wandb/sdk/data_types/audio.py +165 -0
- wandb/sdk/data_types/base_types/__init__.py +0 -0
- wandb/sdk/data_types/base_types/json_metadata.py +55 -0
- wandb/sdk/data_types/base_types/media.py +315 -0
- wandb/sdk/data_types/base_types/wb_value.py +272 -0
- wandb/sdk/data_types/bokeh.py +70 -0
- wandb/sdk/data_types/graph.py +405 -0
- wandb/sdk/data_types/helper_types/__init__.py +0 -0
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +295 -0
- wandb/sdk/data_types/helper_types/classes.py +159 -0
- wandb/sdk/data_types/helper_types/image_mask.py +235 -0
- wandb/sdk/data_types/histogram.py +96 -0
- wandb/sdk/data_types/html.py +115 -0
- wandb/sdk/data_types/image.py +845 -0
- wandb/sdk/data_types/molecule.py +241 -0
- wandb/sdk/data_types/object_3d.py +474 -0
- wandb/sdk/data_types/plotly.py +82 -0
- wandb/sdk/data_types/saved_model.py +446 -0
- wandb/sdk/data_types/table.py +1204 -0
- wandb/sdk/data_types/trace_tree.py +438 -0
- wandb/sdk/data_types/utils.py +229 -0
- wandb/sdk/data_types/video.py +247 -0
- wandb/sdk/integration_utils/__init__.py +0 -0
- wandb/sdk/integration_utils/auto_logging.py +239 -0
- wandb/sdk/integration_utils/data_logging.py +475 -0
- wandb/sdk/interface/__init__.py +0 -0
- wandb/sdk/interface/constants.py +4 -0
- wandb/sdk/interface/interface.py +972 -0
- wandb/sdk/interface/interface_queue.py +59 -0
- wandb/sdk/interface/interface_relay.py +53 -0
- wandb/sdk/interface/interface_shared.py +537 -0
- wandb/sdk/interface/interface_sock.py +61 -0
- wandb/sdk/interface/message_future.py +27 -0
- wandb/sdk/interface/message_future_poll.py +50 -0
- wandb/sdk/interface/router.py +118 -0
- wandb/sdk/interface/router_queue.py +44 -0
- wandb/sdk/interface/router_relay.py +39 -0
- wandb/sdk/interface/router_sock.py +36 -0
- wandb/sdk/interface/summary_record.py +67 -0
- wandb/sdk/internal/__init__.py +0 -0
- wandb/sdk/internal/context.py +89 -0
- wandb/sdk/internal/datastore.py +297 -0
- wandb/sdk/internal/file_pusher.py +181 -0
- wandb/sdk/internal/file_stream.py +695 -0
- wandb/sdk/internal/flow_control.py +263 -0
- wandb/sdk/internal/handler.py +901 -0
- wandb/sdk/internal/internal.py +417 -0
- wandb/sdk/internal/internal_api.py +4358 -0
- wandb/sdk/internal/internal_util.py +100 -0
- wandb/sdk/internal/job_builder.py +629 -0
- wandb/sdk/internal/profiler.py +78 -0
- wandb/sdk/internal/progress.py +83 -0
- wandb/sdk/internal/run.py +25 -0
- wandb/sdk/internal/sample.py +70 -0
- wandb/sdk/internal/sender.py +1686 -0
- wandb/sdk/internal/sender_config.py +197 -0
- wandb/sdk/internal/settings_static.py +90 -0
- wandb/sdk/internal/system/__init__.py +0 -0
- wandb/sdk/internal/system/assets/__init__.py +27 -0
- wandb/sdk/internal/system/assets/aggregators.py +37 -0
- wandb/sdk/internal/system/assets/asset_registry.py +20 -0
- wandb/sdk/internal/system/assets/cpu.py +163 -0
- wandb/sdk/internal/system/assets/disk.py +210 -0
- wandb/sdk/internal/system/assets/gpu.py +416 -0
- wandb/sdk/internal/system/assets/gpu_amd.py +239 -0
- wandb/sdk/internal/system/assets/gpu_apple.py +177 -0
- wandb/sdk/internal/system/assets/interfaces.py +207 -0
- wandb/sdk/internal/system/assets/ipu.py +177 -0
- wandb/sdk/internal/system/assets/memory.py +166 -0
- wandb/sdk/internal/system/assets/network.py +125 -0
- wandb/sdk/internal/system/assets/open_metrics.py +299 -0
- wandb/sdk/internal/system/assets/tpu.py +154 -0
- wandb/sdk/internal/system/assets/trainium.py +399 -0
- wandb/sdk/internal/system/env_probe_helpers.py +13 -0
- wandb/sdk/internal/system/system_info.py +249 -0
- wandb/sdk/internal/system/system_monitor.py +229 -0
- wandb/sdk/internal/tb_watcher.py +518 -0
- wandb/sdk/internal/thread_local_settings.py +18 -0
- wandb/sdk/internal/writer.py +206 -0
- wandb/sdk/launch/__init__.py +14 -0
- wandb/sdk/launch/_launch.py +330 -0
- wandb/sdk/launch/_launch_add.py +255 -0
- wandb/sdk/launch/_project_spec.py +566 -0
- wandb/sdk/launch/agent/__init__.py +5 -0
- wandb/sdk/launch/agent/agent.py +924 -0
- wandb/sdk/launch/agent/config.py +296 -0
- wandb/sdk/launch/agent/job_status_tracker.py +53 -0
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
- wandb/sdk/launch/builder/__init__.py +0 -0
- wandb/sdk/launch/builder/abstract.py +156 -0
- wandb/sdk/launch/builder/build.py +297 -0
- wandb/sdk/launch/builder/context_manager.py +235 -0
- wandb/sdk/launch/builder/docker_builder.py +177 -0
- wandb/sdk/launch/builder/kaniko_builder.py +595 -0
- wandb/sdk/launch/builder/noop.py +58 -0
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +188 -0
- wandb/sdk/launch/builder/templates/dockerfile.py +92 -0
- wandb/sdk/launch/create_job.py +528 -0
- wandb/sdk/launch/environment/abstract.py +29 -0
- wandb/sdk/launch/environment/aws_environment.py +322 -0
- wandb/sdk/launch/environment/azure_environment.py +105 -0
- wandb/sdk/launch/environment/gcp_environment.py +335 -0
- wandb/sdk/launch/environment/local_environment.py +66 -0
- wandb/sdk/launch/errors.py +19 -0
- wandb/sdk/launch/git_reference.py +109 -0
- wandb/sdk/launch/inputs/files.py +148 -0
- wandb/sdk/launch/inputs/internal.py +315 -0
- wandb/sdk/launch/inputs/manage.py +113 -0
- wandb/sdk/launch/inputs/schema.py +39 -0
- wandb/sdk/launch/loader.py +249 -0
- wandb/sdk/launch/registry/abstract.py +48 -0
- wandb/sdk/launch/registry/anon.py +29 -0
- wandb/sdk/launch/registry/azure_container_registry.py +124 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +192 -0
- wandb/sdk/launch/registry/google_artifact_registry.py +219 -0
- wandb/sdk/launch/registry/local_registry.py +67 -0
- wandb/sdk/launch/runner/__init__.py +0 -0
- wandb/sdk/launch/runner/abstract.py +195 -0
- wandb/sdk/launch/runner/kubernetes_monitor.py +474 -0
- wandb/sdk/launch/runner/kubernetes_runner.py +963 -0
- wandb/sdk/launch/runner/local_container.py +301 -0
- wandb/sdk/launch/runner/local_process.py +78 -0
- wandb/sdk/launch/runner/sagemaker_runner.py +426 -0
- wandb/sdk/launch/runner/vertex_runner.py +230 -0
- wandb/sdk/launch/sweeps/__init__.py +39 -0
- wandb/sdk/launch/sweeps/scheduler.py +742 -0
- wandb/sdk/launch/sweeps/scheduler_sweep.py +91 -0
- wandb/sdk/launch/sweeps/utils.py +316 -0
- wandb/sdk/launch/utils.py +746 -0
- wandb/sdk/launch/wandb_reference.py +138 -0
- wandb/sdk/lib/__init__.py +5 -0
- wandb/sdk/lib/_settings_toposort_generate.py +159 -0
- wandb/sdk/lib/_settings_toposort_generated.py +250 -0
- wandb/sdk/lib/_wburls_generate.py +25 -0
- wandb/sdk/lib/_wburls_generated.py +22 -0
- wandb/sdk/lib/apikey.py +273 -0
- wandb/sdk/lib/capped_dict.py +26 -0
- wandb/sdk/lib/config_util.py +101 -0
- wandb/sdk/lib/credentials.py +141 -0
- wandb/sdk/lib/deprecate.py +42 -0
- wandb/sdk/lib/disabled.py +29 -0
- wandb/sdk/lib/exit_hooks.py +54 -0
- wandb/sdk/lib/file_stream_utils.py +118 -0
- wandb/sdk/lib/filenames.py +64 -0
- wandb/sdk/lib/filesystem.py +372 -0
- wandb/sdk/lib/fsm.py +174 -0
- wandb/sdk/lib/gitlib.py +239 -0
- wandb/sdk/lib/gql_request.py +65 -0
- wandb/sdk/lib/handler_util.py +21 -0
- wandb/sdk/lib/hashutil.py +84 -0
- wandb/sdk/lib/import_hooks.py +275 -0
- wandb/sdk/lib/ipython.py +146 -0
- wandb/sdk/lib/json_util.py +80 -0
- wandb/sdk/lib/lazyloader.py +63 -0
- wandb/sdk/lib/mailbox.py +460 -0
- wandb/sdk/lib/module.py +69 -0
- wandb/sdk/lib/paths.py +106 -0
- wandb/sdk/lib/preinit.py +42 -0
- wandb/sdk/lib/printer.py +313 -0
- wandb/sdk/lib/proto_util.py +90 -0
- wandb/sdk/lib/redirect.py +845 -0
- wandb/sdk/lib/reporting.py +99 -0
- wandb/sdk/lib/retry.py +289 -0
- wandb/sdk/lib/run_moment.py +78 -0
- wandb/sdk/lib/runid.py +12 -0
- wandb/sdk/lib/server.py +52 -0
- wandb/sdk/lib/service_connection.py +216 -0
- wandb/sdk/lib/service_token.py +94 -0
- wandb/sdk/lib/sock_client.py +295 -0
- wandb/sdk/lib/sparkline.py +45 -0
- wandb/sdk/lib/telemetry.py +100 -0
- wandb/sdk/lib/timed_input.py +133 -0
- wandb/sdk/lib/timer.py +19 -0
- wandb/sdk/lib/tracelog.py +255 -0
- wandb/sdk/lib/wburls.py +46 -0
- wandb/sdk/service/__init__.py +0 -0
- wandb/sdk/service/_startup_debug.py +22 -0
- wandb/sdk/service/port_file.py +53 -0
- wandb/sdk/service/server.py +116 -0
- wandb/sdk/service/server_sock.py +276 -0
- wandb/sdk/service/service.py +242 -0
- wandb/sdk/service/streams.py +417 -0
- wandb/sdk/verify/__init__.py +0 -0
- wandb/sdk/verify/verify.py +501 -0
- wandb/sdk/wandb_alerts.py +12 -0
- wandb/sdk/wandb_config.py +322 -0
- wandb/sdk/wandb_helper.py +54 -0
- wandb/sdk/wandb_init.py +1266 -0
- wandb/sdk/wandb_login.py +349 -0
- wandb/sdk/wandb_metric.py +110 -0
- wandb/sdk/wandb_require.py +97 -0
- wandb/sdk/wandb_require_helpers.py +44 -0
- wandb/sdk/wandb_run.py +4236 -0
- wandb/sdk/wandb_settings.py +2001 -0
- wandb/sdk/wandb_setup.py +409 -0
- wandb/sdk/wandb_summary.py +150 -0
- wandb/sdk/wandb_sweep.py +119 -0
- wandb/sdk/wandb_sync.py +81 -0
- wandb/sdk/wandb_watch.py +144 -0
- wandb/sklearn.py +35 -0
- wandb/sync/__init__.py +3 -0
- wandb/sync/sync.py +443 -0
- wandb/trigger.py +29 -0
- wandb/util.py +1956 -0
- wandb/vendor/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/setup.py +40 -0
- wandb/vendor/gql-0.2.0/tests/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/tests/starwars/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py +96 -0
- wandb/vendor/gql-0.2.0/tests/starwars/schema.py +146 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py +293 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_query.py +355 -0
- wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py +171 -0
- wandb/vendor/gql-0.2.0/tests/test_client.py +31 -0
- wandb/vendor/gql-0.2.0/tests/test_transport.py +89 -0
- wandb/vendor/gql-0.2.0/wandb_gql/__init__.py +4 -0
- wandb/vendor/gql-0.2.0/wandb_gql/client.py +75 -0
- wandb/vendor/gql-0.2.0/wandb_gql/dsl.py +152 -0
- wandb/vendor/gql-0.2.0/wandb_gql/gql.py +10 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py +0 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py +6 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py +15 -0
- wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py +46 -0
- wandb/vendor/gql-0.2.0/wandb_gql/utils.py +21 -0
- wandb/vendor/graphql-core-1.1/setup.py +86 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py +287 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py +6 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py +42 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py +11 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py +29 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py +36 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py +26 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py +311 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py +398 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py +53 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py +22 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py +32 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py +7 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py +35 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py +6 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py +66 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py +252 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py +151 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py +7 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py +57 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py +145 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py +60 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py +1349 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py +19 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py +435 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py +30 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py +779 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py +193 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py +18 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py +222 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py +82 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py +17 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py +28 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py +40 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py +8 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py +43 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py +78 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py +67 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py +619 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py +132 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py +440 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py +131 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py +100 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py +145 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py +0 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py +9 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py +65 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py +49 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py +24 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py +75 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py +291 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py +250 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py +9 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py +357 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py +27 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py +90 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py +67 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py +66 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py +168 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py +56 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py +69 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py +149 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py +69 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py +4 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py +79 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py +24 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py +8 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py +44 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py +113 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py +70 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py +97 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py +19 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py +43 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py +23 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py +59 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py +36 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py +38 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py +37 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py +529 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py +44 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py +46 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py +32 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py +28 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py +33 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py +31 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py +27 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py +21 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py +53 -0
- wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py +158 -0
- wandb/vendor/promise-2.3.0/conftest.py +30 -0
- wandb/vendor/promise-2.3.0/setup.py +64 -0
- wandb/vendor/promise-2.3.0/tests/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/tests/conftest.py +8 -0
- wandb/vendor/promise-2.3.0/tests/test_awaitable.py +32 -0
- wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py +47 -0
- wandb/vendor/promise-2.3.0/tests/test_benchmark.py +116 -0
- wandb/vendor/promise-2.3.0/tests/test_complex_threads.py +23 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader.py +452 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py +99 -0
- wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py +65 -0
- wandb/vendor/promise-2.3.0/tests/test_extra.py +670 -0
- wandb/vendor/promise-2.3.0/tests/test_issues.py +132 -0
- wandb/vendor/promise-2.3.0/tests/test_promise_list.py +70 -0
- wandb/vendor/promise-2.3.0/tests/test_spec.py +584 -0
- wandb/vendor/promise-2.3.0/tests/test_thread_safety.py +115 -0
- wandb/vendor/promise-2.3.0/tests/utils.py +3 -0
- wandb/vendor/promise-2.3.0/wandb_promise/__init__.py +38 -0
- wandb/vendor/promise-2.3.0/wandb_promise/async_.py +135 -0
- wandb/vendor/promise-2.3.0/wandb_promise/compat.py +32 -0
- wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py +326 -0
- wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py +12 -0
- wandb/vendor/promise-2.3.0/wandb_promise/promise.py +848 -0
- wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py +151 -0
- wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py +83 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py +0 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py +22 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py +21 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py +27 -0
- wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py +18 -0
- wandb/vendor/promise-2.3.0/wandb_promise/utils.py +56 -0
- wandb/vendor/pygments/__init__.py +90 -0
- wandb/vendor/pygments/cmdline.py +568 -0
- wandb/vendor/pygments/console.py +74 -0
- wandb/vendor/pygments/filter.py +74 -0
- wandb/vendor/pygments/filters/__init__.py +350 -0
- wandb/vendor/pygments/formatter.py +95 -0
- wandb/vendor/pygments/formatters/__init__.py +153 -0
- wandb/vendor/pygments/formatters/_mapping.py +85 -0
- wandb/vendor/pygments/formatters/bbcode.py +109 -0
- wandb/vendor/pygments/formatters/html.py +851 -0
- wandb/vendor/pygments/formatters/img.py +600 -0
- wandb/vendor/pygments/formatters/irc.py +182 -0
- wandb/vendor/pygments/formatters/latex.py +482 -0
- wandb/vendor/pygments/formatters/other.py +160 -0
- wandb/vendor/pygments/formatters/rtf.py +147 -0
- wandb/vendor/pygments/formatters/svg.py +153 -0
- wandb/vendor/pygments/formatters/terminal.py +136 -0
- wandb/vendor/pygments/formatters/terminal256.py +309 -0
- wandb/vendor/pygments/lexer.py +871 -0
- wandb/vendor/pygments/lexers/__init__.py +329 -0
- wandb/vendor/pygments/lexers/_asy_builtins.py +1645 -0
- wandb/vendor/pygments/lexers/_cl_builtins.py +232 -0
- wandb/vendor/pygments/lexers/_cocoa_builtins.py +72 -0
- wandb/vendor/pygments/lexers/_csound_builtins.py +1346 -0
- wandb/vendor/pygments/lexers/_lasso_builtins.py +5327 -0
- wandb/vendor/pygments/lexers/_lua_builtins.py +295 -0
- wandb/vendor/pygments/lexers/_mapping.py +500 -0
- wandb/vendor/pygments/lexers/_mql_builtins.py +1172 -0
- wandb/vendor/pygments/lexers/_openedge_builtins.py +2547 -0
- wandb/vendor/pygments/lexers/_php_builtins.py +4756 -0
- wandb/vendor/pygments/lexers/_postgres_builtins.py +621 -0
- wandb/vendor/pygments/lexers/_scilab_builtins.py +3094 -0
- wandb/vendor/pygments/lexers/_sourcemod_builtins.py +1163 -0
- wandb/vendor/pygments/lexers/_stan_builtins.py +532 -0
- wandb/vendor/pygments/lexers/_stata_builtins.py +419 -0
- wandb/vendor/pygments/lexers/_tsql_builtins.py +1004 -0
- wandb/vendor/pygments/lexers/_vim_builtins.py +1939 -0
- wandb/vendor/pygments/lexers/actionscript.py +240 -0
- wandb/vendor/pygments/lexers/agile.py +24 -0
- wandb/vendor/pygments/lexers/algebra.py +221 -0
- wandb/vendor/pygments/lexers/ambient.py +76 -0
- wandb/vendor/pygments/lexers/ampl.py +87 -0
- wandb/vendor/pygments/lexers/apl.py +101 -0
- wandb/vendor/pygments/lexers/archetype.py +318 -0
- wandb/vendor/pygments/lexers/asm.py +641 -0
- wandb/vendor/pygments/lexers/automation.py +374 -0
- wandb/vendor/pygments/lexers/basic.py +500 -0
- wandb/vendor/pygments/lexers/bibtex.py +160 -0
- wandb/vendor/pygments/lexers/business.py +612 -0
- wandb/vendor/pygments/lexers/c_cpp.py +252 -0
- wandb/vendor/pygments/lexers/c_like.py +541 -0
- wandb/vendor/pygments/lexers/capnproto.py +78 -0
- wandb/vendor/pygments/lexers/chapel.py +102 -0
- wandb/vendor/pygments/lexers/clean.py +288 -0
- wandb/vendor/pygments/lexers/compiled.py +34 -0
- wandb/vendor/pygments/lexers/configs.py +833 -0
- wandb/vendor/pygments/lexers/console.py +114 -0
- wandb/vendor/pygments/lexers/crystal.py +393 -0
- wandb/vendor/pygments/lexers/csound.py +366 -0
- wandb/vendor/pygments/lexers/css.py +689 -0
- wandb/vendor/pygments/lexers/d.py +251 -0
- wandb/vendor/pygments/lexers/dalvik.py +125 -0
- wandb/vendor/pygments/lexers/data.py +555 -0
- wandb/vendor/pygments/lexers/diff.py +165 -0
- wandb/vendor/pygments/lexers/dotnet.py +691 -0
- wandb/vendor/pygments/lexers/dsls.py +878 -0
- wandb/vendor/pygments/lexers/dylan.py +289 -0
- wandb/vendor/pygments/lexers/ecl.py +125 -0
- wandb/vendor/pygments/lexers/eiffel.py +65 -0
- wandb/vendor/pygments/lexers/elm.py +121 -0
- wandb/vendor/pygments/lexers/erlang.py +533 -0
- wandb/vendor/pygments/lexers/esoteric.py +277 -0
- wandb/vendor/pygments/lexers/ezhil.py +69 -0
- wandb/vendor/pygments/lexers/factor.py +344 -0
- wandb/vendor/pygments/lexers/fantom.py +250 -0
- wandb/vendor/pygments/lexers/felix.py +273 -0
- wandb/vendor/pygments/lexers/forth.py +177 -0
- wandb/vendor/pygments/lexers/fortran.py +205 -0
- wandb/vendor/pygments/lexers/foxpro.py +428 -0
- wandb/vendor/pygments/lexers/functional.py +21 -0
- wandb/vendor/pygments/lexers/go.py +101 -0
- wandb/vendor/pygments/lexers/grammar_notation.py +213 -0
- wandb/vendor/pygments/lexers/graph.py +80 -0
- wandb/vendor/pygments/lexers/graphics.py +553 -0
- wandb/vendor/pygments/lexers/haskell.py +843 -0
- wandb/vendor/pygments/lexers/haxe.py +936 -0
- wandb/vendor/pygments/lexers/hdl.py +382 -0
- wandb/vendor/pygments/lexers/hexdump.py +103 -0
- wandb/vendor/pygments/lexers/html.py +602 -0
- wandb/vendor/pygments/lexers/idl.py +270 -0
- wandb/vendor/pygments/lexers/igor.py +288 -0
- wandb/vendor/pygments/lexers/inferno.py +96 -0
- wandb/vendor/pygments/lexers/installers.py +322 -0
- wandb/vendor/pygments/lexers/int_fiction.py +1343 -0
- wandb/vendor/pygments/lexers/iolang.py +63 -0
- wandb/vendor/pygments/lexers/j.py +146 -0
- wandb/vendor/pygments/lexers/javascript.py +1525 -0
- wandb/vendor/pygments/lexers/julia.py +333 -0
- wandb/vendor/pygments/lexers/jvm.py +1573 -0
- wandb/vendor/pygments/lexers/lisp.py +2621 -0
- wandb/vendor/pygments/lexers/make.py +202 -0
- wandb/vendor/pygments/lexers/markup.py +595 -0
- wandb/vendor/pygments/lexers/math.py +21 -0
- wandb/vendor/pygments/lexers/matlab.py +663 -0
- wandb/vendor/pygments/lexers/ml.py +769 -0
- wandb/vendor/pygments/lexers/modeling.py +358 -0
- wandb/vendor/pygments/lexers/modula2.py +1561 -0
- wandb/vendor/pygments/lexers/monte.py +204 -0
- wandb/vendor/pygments/lexers/ncl.py +894 -0
- wandb/vendor/pygments/lexers/nimrod.py +159 -0
- wandb/vendor/pygments/lexers/nit.py +64 -0
- wandb/vendor/pygments/lexers/nix.py +136 -0
- wandb/vendor/pygments/lexers/oberon.py +105 -0
- wandb/vendor/pygments/lexers/objective.py +504 -0
- wandb/vendor/pygments/lexers/ooc.py +85 -0
- wandb/vendor/pygments/lexers/other.py +41 -0
- wandb/vendor/pygments/lexers/parasail.py +79 -0
- wandb/vendor/pygments/lexers/parsers.py +835 -0
- wandb/vendor/pygments/lexers/pascal.py +644 -0
- wandb/vendor/pygments/lexers/pawn.py +199 -0
- wandb/vendor/pygments/lexers/perl.py +620 -0
- wandb/vendor/pygments/lexers/php.py +267 -0
- wandb/vendor/pygments/lexers/praat.py +294 -0
- wandb/vendor/pygments/lexers/prolog.py +306 -0
- wandb/vendor/pygments/lexers/python.py +939 -0
- wandb/vendor/pygments/lexers/qvt.py +152 -0
- wandb/vendor/pygments/lexers/r.py +453 -0
- wandb/vendor/pygments/lexers/rdf.py +270 -0
- wandb/vendor/pygments/lexers/rebol.py +431 -0
- wandb/vendor/pygments/lexers/resource.py +85 -0
- wandb/vendor/pygments/lexers/rnc.py +67 -0
- wandb/vendor/pygments/lexers/roboconf.py +82 -0
- wandb/vendor/pygments/lexers/robotframework.py +560 -0
- wandb/vendor/pygments/lexers/ruby.py +519 -0
- wandb/vendor/pygments/lexers/rust.py +220 -0
- wandb/vendor/pygments/lexers/sas.py +228 -0
- wandb/vendor/pygments/lexers/scripting.py +1222 -0
- wandb/vendor/pygments/lexers/shell.py +794 -0
- wandb/vendor/pygments/lexers/smalltalk.py +195 -0
- wandb/vendor/pygments/lexers/smv.py +79 -0
- wandb/vendor/pygments/lexers/snobol.py +83 -0
- wandb/vendor/pygments/lexers/special.py +103 -0
- wandb/vendor/pygments/lexers/sql.py +681 -0
- wandb/vendor/pygments/lexers/stata.py +108 -0
- wandb/vendor/pygments/lexers/supercollider.py +90 -0
- wandb/vendor/pygments/lexers/tcl.py +145 -0
- wandb/vendor/pygments/lexers/templates.py +2283 -0
- wandb/vendor/pygments/lexers/testing.py +207 -0
- wandb/vendor/pygments/lexers/text.py +25 -0
- wandb/vendor/pygments/lexers/textedit.py +169 -0
- wandb/vendor/pygments/lexers/textfmts.py +297 -0
- wandb/vendor/pygments/lexers/theorem.py +458 -0
- wandb/vendor/pygments/lexers/trafficscript.py +54 -0
- wandb/vendor/pygments/lexers/typoscript.py +226 -0
- wandb/vendor/pygments/lexers/urbi.py +133 -0
- wandb/vendor/pygments/lexers/varnish.py +190 -0
- wandb/vendor/pygments/lexers/verification.py +111 -0
- wandb/vendor/pygments/lexers/web.py +24 -0
- wandb/vendor/pygments/lexers/webmisc.py +988 -0
- wandb/vendor/pygments/lexers/whiley.py +116 -0
- wandb/vendor/pygments/lexers/x10.py +69 -0
- wandb/vendor/pygments/modeline.py +44 -0
- wandb/vendor/pygments/plugin.py +68 -0
- wandb/vendor/pygments/regexopt.py +92 -0
- wandb/vendor/pygments/scanner.py +105 -0
- wandb/vendor/pygments/sphinxext.py +158 -0
- wandb/vendor/pygments/style.py +155 -0
- wandb/vendor/pygments/styles/__init__.py +80 -0
- wandb/vendor/pygments/styles/abap.py +29 -0
- wandb/vendor/pygments/styles/algol.py +63 -0
- wandb/vendor/pygments/styles/algol_nu.py +63 -0
- wandb/vendor/pygments/styles/arduino.py +98 -0
- wandb/vendor/pygments/styles/autumn.py +65 -0
- wandb/vendor/pygments/styles/borland.py +51 -0
- wandb/vendor/pygments/styles/bw.py +49 -0
- wandb/vendor/pygments/styles/colorful.py +81 -0
- wandb/vendor/pygments/styles/default.py +73 -0
- wandb/vendor/pygments/styles/emacs.py +72 -0
- wandb/vendor/pygments/styles/friendly.py +72 -0
- wandb/vendor/pygments/styles/fruity.py +42 -0
- wandb/vendor/pygments/styles/igor.py +29 -0
- wandb/vendor/pygments/styles/lovelace.py +97 -0
- wandb/vendor/pygments/styles/manni.py +75 -0
- wandb/vendor/pygments/styles/monokai.py +106 -0
- wandb/vendor/pygments/styles/murphy.py +80 -0
- wandb/vendor/pygments/styles/native.py +65 -0
- wandb/vendor/pygments/styles/paraiso_dark.py +125 -0
- wandb/vendor/pygments/styles/paraiso_light.py +125 -0
- wandb/vendor/pygments/styles/pastie.py +75 -0
- wandb/vendor/pygments/styles/perldoc.py +69 -0
- wandb/vendor/pygments/styles/rainbow_dash.py +89 -0
- wandb/vendor/pygments/styles/rrt.py +33 -0
- wandb/vendor/pygments/styles/sas.py +44 -0
- wandb/vendor/pygments/styles/stata.py +40 -0
- wandb/vendor/pygments/styles/tango.py +141 -0
- wandb/vendor/pygments/styles/trac.py +63 -0
- wandb/vendor/pygments/styles/vim.py +63 -0
- wandb/vendor/pygments/styles/vs.py +38 -0
- wandb/vendor/pygments/styles/xcode.py +51 -0
- wandb/vendor/pygments/token.py +213 -0
- wandb/vendor/pygments/unistring.py +217 -0
- wandb/vendor/pygments/util.py +388 -0
- wandb/vendor/pynvml/__init__.py +0 -0
- wandb/vendor/pynvml/pynvml.py +4779 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py +17 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py +615 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py +98 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py +369 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py +172 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py +239 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py +218 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py +81 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py +575 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py +730 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py +145 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py +133 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py +348 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py +265 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py +174 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py +151 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py +249 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py +29 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py +198 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py +88 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py +293 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py +157 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py +41 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py +40 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py +57 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py +64 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py +123 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py +28 -0
- wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py +577 -0
- wandb/wandb_agent.py +588 -0
- wandb/wandb_controller.py +721 -0
- wandb/wandb_run.py +9 -0
- wandb-0.18.2.dist-info/METADATA +213 -0
- wandb-0.18.2.dist-info/RECORD +827 -0
- wandb-0.18.2.dist-info/WHEEL +5 -0
- wandb-0.18.2.dist-info/entry_points.txt +3 -0
- wandb-0.18.2.dist-info/licenses/LICENSE +21 -0
wandb/util.py
ADDED
@@ -0,0 +1,1956 @@
|
|
1
|
+
import colorsys
|
2
|
+
import contextlib
|
3
|
+
import dataclasses
|
4
|
+
import enum
|
5
|
+
import functools
|
6
|
+
import gzip
|
7
|
+
import importlib
|
8
|
+
import importlib.util
|
9
|
+
import itertools
|
10
|
+
import json
|
11
|
+
import logging
|
12
|
+
import math
|
13
|
+
import numbers
|
14
|
+
import os
|
15
|
+
import pathlib
|
16
|
+
import platform
|
17
|
+
import queue
|
18
|
+
import random
|
19
|
+
import re
|
20
|
+
import secrets
|
21
|
+
import shlex
|
22
|
+
import socket
|
23
|
+
import string
|
24
|
+
import sys
|
25
|
+
import tarfile
|
26
|
+
import tempfile
|
27
|
+
import threading
|
28
|
+
import time
|
29
|
+
import types
|
30
|
+
import urllib
|
31
|
+
from dataclasses import asdict, is_dataclass
|
32
|
+
from datetime import date, datetime, timedelta
|
33
|
+
from importlib import import_module
|
34
|
+
from sys import getsizeof
|
35
|
+
from types import ModuleType
|
36
|
+
from typing import (
|
37
|
+
IO,
|
38
|
+
TYPE_CHECKING,
|
39
|
+
Any,
|
40
|
+
Callable,
|
41
|
+
Dict,
|
42
|
+
Generator,
|
43
|
+
Iterable,
|
44
|
+
List,
|
45
|
+
Mapping,
|
46
|
+
Optional,
|
47
|
+
Sequence,
|
48
|
+
TextIO,
|
49
|
+
Tuple,
|
50
|
+
TypeVar,
|
51
|
+
Union,
|
52
|
+
)
|
53
|
+
|
54
|
+
import requests
|
55
|
+
import yaml
|
56
|
+
|
57
|
+
import wandb
|
58
|
+
import wandb.env
|
59
|
+
from wandb.errors import (
|
60
|
+
AuthenticationError,
|
61
|
+
CommError,
|
62
|
+
UsageError,
|
63
|
+
WandbCoreNotAvailableError,
|
64
|
+
term,
|
65
|
+
)
|
66
|
+
from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
|
67
|
+
from wandb.sdk.lib import filesystem, runid
|
68
|
+
from wandb.sdk.lib.json_util import dump, dumps
|
69
|
+
from wandb.sdk.lib.paths import FilePathStr, StrPath
|
70
|
+
|
71
|
+
if TYPE_CHECKING:
|
72
|
+
import packaging.version # type: ignore[import-not-found]
|
73
|
+
|
74
|
+
import wandb.sdk.internal.settings_static
|
75
|
+
import wandb.sdk.wandb_settings
|
76
|
+
from wandb.sdk.artifacts.artifact import Artifact
|
77
|
+
|
78
|
+
CheckRetryFnType = Callable[[Exception], Union[bool, timedelta]]
|
79
|
+
T = TypeVar("T")
|
80
|
+
|
81
|
+
|
82
|
+
logger = logging.getLogger(__name__)
|
83
|
+
_not_importable = set()
|
84
|
+
|
85
|
+
MAX_LINE_BYTES = (10 << 20) - (100 << 10) # imposed by back end
|
86
|
+
IS_GIT = os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git"))
|
87
|
+
RE_WINFNAMES = re.compile(r'[<>:"\\?*]')
|
88
|
+
|
89
|
+
# From https://docs.docker.com/engine/reference/commandline/tag/
|
90
|
+
# "Name components may contain lowercase letters, digits and separators.
|
91
|
+
# A separator is defined as a period, one or two underscores, or one or more dashes.
|
92
|
+
# A name component may not start or end with a separator."
|
93
|
+
DOCKER_IMAGE_NAME_SEPARATOR = "(?:__|[._]|[-]+)"
|
94
|
+
RE_DOCKER_IMAGE_NAME_SEPARATOR_START = re.compile("^" + DOCKER_IMAGE_NAME_SEPARATOR)
|
95
|
+
RE_DOCKER_IMAGE_NAME_SEPARATOR_END = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "$")
|
96
|
+
RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "{2,}")
|
97
|
+
RE_DOCKER_IMAGE_NAME_CHARS = re.compile(r"[^a-z0-9._\-]")
|
98
|
+
|
99
|
+
# these match the environments for gorilla
|
100
|
+
if IS_GIT:
|
101
|
+
SENTRY_ENV = "development"
|
102
|
+
else:
|
103
|
+
SENTRY_ENV = "production"
|
104
|
+
|
105
|
+
|
106
|
+
PLATFORM_WINDOWS = "windows"
|
107
|
+
PLATFORM_LINUX = "linux"
|
108
|
+
PLATFORM_BSD = "bsd"
|
109
|
+
PLATFORM_DARWIN = "darwin"
|
110
|
+
PLATFORM_UNKNOWN = "unknown"
|
111
|
+
|
112
|
+
LAUNCH_JOB_ARTIFACT_SLOT_NAME = "_wandb_job"
|
113
|
+
|
114
|
+
|
115
|
+
def get_platform_name() -> str:
|
116
|
+
if sys.platform.startswith("win"):
|
117
|
+
return PLATFORM_WINDOWS
|
118
|
+
elif sys.platform.startswith("darwin"):
|
119
|
+
return PLATFORM_DARWIN
|
120
|
+
elif sys.platform.startswith("linux"):
|
121
|
+
return PLATFORM_LINUX
|
122
|
+
elif sys.platform.startswith(
|
123
|
+
(
|
124
|
+
"dragonfly",
|
125
|
+
"freebsd",
|
126
|
+
"netbsd",
|
127
|
+
"openbsd",
|
128
|
+
)
|
129
|
+
):
|
130
|
+
return PLATFORM_BSD
|
131
|
+
else:
|
132
|
+
return PLATFORM_UNKNOWN
|
133
|
+
|
134
|
+
|
135
|
+
POW_10_BYTES = [
|
136
|
+
("B", 10**0),
|
137
|
+
("KB", 10**3),
|
138
|
+
("MB", 10**6),
|
139
|
+
("GB", 10**9),
|
140
|
+
("TB", 10**12),
|
141
|
+
("PB", 10**15),
|
142
|
+
("EB", 10**18),
|
143
|
+
]
|
144
|
+
|
145
|
+
POW_2_BYTES = [
|
146
|
+
("B", 2**0),
|
147
|
+
("KiB", 2**10),
|
148
|
+
("MiB", 2**20),
|
149
|
+
("GiB", 2**30),
|
150
|
+
("TiB", 2**40),
|
151
|
+
("PiB", 2**50),
|
152
|
+
("EiB", 2**60),
|
153
|
+
]
|
154
|
+
|
155
|
+
|
156
|
+
def vendor_setup() -> Callable:
|
157
|
+
"""Create a function that restores user paths after vendor imports.
|
158
|
+
|
159
|
+
This enables us to use the vendor directory for packages we don't depend on. Call
|
160
|
+
the returned function after imports are complete. If you don't you may modify the
|
161
|
+
user's path which is never good.
|
162
|
+
|
163
|
+
Usage:
|
164
|
+
|
165
|
+
```python
|
166
|
+
reset_path = vendor_setup()
|
167
|
+
# do any vendor imports...
|
168
|
+
reset_path()
|
169
|
+
```
|
170
|
+
"""
|
171
|
+
original_path = [directory for directory in sys.path]
|
172
|
+
|
173
|
+
def reset_import_path() -> None:
|
174
|
+
sys.path = original_path
|
175
|
+
|
176
|
+
parent_dir = os.path.abspath(os.path.dirname(__file__))
|
177
|
+
vendor_dir = os.path.join(parent_dir, "vendor")
|
178
|
+
vendor_packages = (
|
179
|
+
"gql-0.2.0",
|
180
|
+
"graphql-core-1.1",
|
181
|
+
"watchdog_0_9_0",
|
182
|
+
"promise-2.3.0",
|
183
|
+
)
|
184
|
+
package_dirs = [os.path.join(vendor_dir, p) for p in vendor_packages]
|
185
|
+
for p in [vendor_dir] + package_dirs:
|
186
|
+
if p not in sys.path:
|
187
|
+
sys.path.insert(1, p)
|
188
|
+
|
189
|
+
return reset_import_path
|
190
|
+
|
191
|
+
|
192
|
+
def vendor_import(name: str) -> Any:
|
193
|
+
reset_path = vendor_setup()
|
194
|
+
module = import_module(name)
|
195
|
+
reset_path()
|
196
|
+
return module
|
197
|
+
|
198
|
+
|
199
|
+
class LazyModuleState:
|
200
|
+
def __init__(self, module: types.ModuleType) -> None:
|
201
|
+
self.module = module
|
202
|
+
self.load_started = False
|
203
|
+
self.lock = threading.RLock()
|
204
|
+
|
205
|
+
def load(self) -> None:
|
206
|
+
with self.lock:
|
207
|
+
if self.load_started:
|
208
|
+
return
|
209
|
+
self.load_started = True
|
210
|
+
assert self.module.__spec__ is not None
|
211
|
+
assert self.module.__spec__.loader is not None
|
212
|
+
self.module.__spec__.loader.exec_module(self.module)
|
213
|
+
self.module.__class__ = types.ModuleType
|
214
|
+
|
215
|
+
|
216
|
+
class LazyModule(types.ModuleType):
|
217
|
+
def __getattribute__(self, name: str) -> Any:
|
218
|
+
state = object.__getattribute__(self, "__lazy_module_state__")
|
219
|
+
state.load()
|
220
|
+
return object.__getattribute__(self, name)
|
221
|
+
|
222
|
+
def __setattr__(self, name: str, value: Any) -> None:
|
223
|
+
state = object.__getattribute__(self, "__lazy_module_state__")
|
224
|
+
state.load()
|
225
|
+
object.__setattr__(self, name, value)
|
226
|
+
|
227
|
+
def __delattr__(self, name: str) -> None:
|
228
|
+
state = object.__getattribute__(self, "__lazy_module_state__")
|
229
|
+
state.load()
|
230
|
+
object.__delattr__(self, name)
|
231
|
+
|
232
|
+
|
233
|
+
def import_module_lazy(name: str) -> types.ModuleType:
|
234
|
+
"""Import a module lazily, only when it is used.
|
235
|
+
|
236
|
+
Inspired by importlib.util.LazyLoader, but improved so that the module loading is
|
237
|
+
thread-safe. Circular dependency between modules can lead to a deadlock if the two
|
238
|
+
modules are loaded from different threads.
|
239
|
+
|
240
|
+
:param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
|
241
|
+
"""
|
242
|
+
try:
|
243
|
+
return sys.modules[name]
|
244
|
+
except KeyError:
|
245
|
+
spec = importlib.util.find_spec(name)
|
246
|
+
if spec is None:
|
247
|
+
raise ModuleNotFoundError
|
248
|
+
module = importlib.util.module_from_spec(spec)
|
249
|
+
module.__lazy_module_state__ = LazyModuleState(module) # type: ignore
|
250
|
+
module.__class__ = LazyModule
|
251
|
+
sys.modules[name] = module
|
252
|
+
return module
|
253
|
+
|
254
|
+
|
255
|
+
def get_module(
|
256
|
+
name: str,
|
257
|
+
required: Optional[Union[str, bool]] = None,
|
258
|
+
lazy: bool = True,
|
259
|
+
) -> Any:
|
260
|
+
"""Return module or None. Absolute import is required.
|
261
|
+
|
262
|
+
:param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
|
263
|
+
:param (str) required: A string to raise a ValueError if missing
|
264
|
+
:param (bool) lazy: If True, return a lazy loader for the module.
|
265
|
+
:return: (module|None) If import succeeds, the module will be returned.
|
266
|
+
"""
|
267
|
+
if name not in _not_importable:
|
268
|
+
try:
|
269
|
+
if not lazy:
|
270
|
+
return import_module(name)
|
271
|
+
else:
|
272
|
+
return import_module_lazy(name)
|
273
|
+
except Exception:
|
274
|
+
_not_importable.add(name)
|
275
|
+
msg = f"Error importing optional module {name}"
|
276
|
+
if required:
|
277
|
+
logger.exception(msg)
|
278
|
+
if required and name in _not_importable:
|
279
|
+
raise wandb.Error(required)
|
280
|
+
|
281
|
+
|
282
|
+
def get_optional_module(name) -> Optional["importlib.ModuleInterface"]: # type: ignore
|
283
|
+
return get_module(name)
|
284
|
+
|
285
|
+
|
286
|
+
np = get_module("numpy")
|
287
|
+
|
288
|
+
pd_available = False
|
289
|
+
pandas_spec = importlib.util.find_spec("pandas")
|
290
|
+
if pandas_spec is not None:
|
291
|
+
pd_available = True
|
292
|
+
|
293
|
+
# TODO: Revisit these limits
|
294
|
+
VALUE_BYTES_LIMIT = 100000
|
295
|
+
|
296
|
+
|
297
|
+
def app_url(api_url: str) -> str:
|
298
|
+
"""Return the frontend app url without a trailing slash."""
|
299
|
+
# TODO: move me to settings
|
300
|
+
app_url = wandb.env.get_app_url()
|
301
|
+
if app_url is not None:
|
302
|
+
return str(app_url.strip("/"))
|
303
|
+
if "://api.wandb.test" in api_url:
|
304
|
+
# dev mode
|
305
|
+
return api_url.replace("://api.", "://app.").strip("/")
|
306
|
+
elif "://api.wandb." in api_url:
|
307
|
+
# cloud
|
308
|
+
return api_url.replace("://api.", "://").strip("/")
|
309
|
+
elif "://api." in api_url:
|
310
|
+
# onprem cloud
|
311
|
+
return api_url.replace("://api.", "://app.").strip("/")
|
312
|
+
# wandb/local
|
313
|
+
return api_url
|
314
|
+
|
315
|
+
|
316
|
+
def get_full_typename(o: Any) -> Any:
|
317
|
+
"""Determine types based on type names.
|
318
|
+
|
319
|
+
Avoids needing to to import (and therefore depend on) PyTorch, TensorFlow, etc.
|
320
|
+
"""
|
321
|
+
instance_name = o.__class__.__module__ + "." + o.__class__.__name__
|
322
|
+
if instance_name in ["builtins.module", "__builtin__.module"]:
|
323
|
+
return o.__name__
|
324
|
+
else:
|
325
|
+
return instance_name
|
326
|
+
|
327
|
+
|
328
|
+
def get_h5_typename(o: Any) -> Any:
|
329
|
+
typename = get_full_typename(o)
|
330
|
+
if is_tf_tensor_typename(typename):
|
331
|
+
return "tensorflow.Tensor"
|
332
|
+
elif is_pytorch_tensor_typename(typename):
|
333
|
+
return "torch.Tensor"
|
334
|
+
else:
|
335
|
+
return o.__class__.__module__.split(".")[0] + "." + o.__class__.__name__
|
336
|
+
|
337
|
+
|
338
|
+
def is_uri(string: str) -> bool:
|
339
|
+
parsed_uri = urllib.parse.urlparse(string)
|
340
|
+
return len(parsed_uri.scheme) > 0
|
341
|
+
|
342
|
+
|
343
|
+
def local_file_uri_to_path(uri: str) -> str:
|
344
|
+
"""Convert URI to local filesystem path.
|
345
|
+
|
346
|
+
No-op if the uri does not have the expected scheme.
|
347
|
+
"""
|
348
|
+
path = urllib.parse.urlparse(uri).path if uri.startswith("file:") else uri
|
349
|
+
return urllib.request.url2pathname(path)
|
350
|
+
|
351
|
+
|
352
|
+
def get_local_path_or_none(path_or_uri: str) -> Optional[str]:
|
353
|
+
"""Return path if local, None otherwise.
|
354
|
+
|
355
|
+
Return None if the argument is a local path (not a scheme or file:///). Otherwise
|
356
|
+
return `path_or_uri`.
|
357
|
+
"""
|
358
|
+
parsed_uri = urllib.parse.urlparse(path_or_uri)
|
359
|
+
if (
|
360
|
+
len(parsed_uri.scheme) == 0
|
361
|
+
or parsed_uri.scheme == "file"
|
362
|
+
and len(parsed_uri.netloc) == 0
|
363
|
+
):
|
364
|
+
return local_file_uri_to_path(path_or_uri)
|
365
|
+
else:
|
366
|
+
return None
|
367
|
+
|
368
|
+
|
369
|
+
def make_tarfile(
|
370
|
+
output_filename: str,
|
371
|
+
source_dir: str,
|
372
|
+
archive_name: str,
|
373
|
+
custom_filter: Optional[Callable] = None,
|
374
|
+
) -> None:
|
375
|
+
# Helper for filtering out modification timestamps
|
376
|
+
def _filter_timestamps(tar_info: "tarfile.TarInfo") -> Optional["tarfile.TarInfo"]:
|
377
|
+
tar_info.mtime = 0
|
378
|
+
return tar_info if custom_filter is None else custom_filter(tar_info)
|
379
|
+
|
380
|
+
descriptor, unzipped_filename = tempfile.mkstemp()
|
381
|
+
try:
|
382
|
+
with tarfile.open(unzipped_filename, "w") as tar:
|
383
|
+
tar.add(source_dir, arcname=archive_name, filter=_filter_timestamps)
|
384
|
+
# When gzipping the tar, don't include the tar's filename or modification time in the
|
385
|
+
# zipped archive (see https://docs.python.org/3/library/gzip.html#gzip.GzipFile)
|
386
|
+
with gzip.GzipFile(
|
387
|
+
filename="", fileobj=open(output_filename, "wb"), mode="wb", mtime=0
|
388
|
+
) as gzipped_tar, open(unzipped_filename, "rb") as tar_file:
|
389
|
+
gzipped_tar.write(tar_file.read())
|
390
|
+
finally:
|
391
|
+
os.close(descriptor)
|
392
|
+
os.remove(unzipped_filename)
|
393
|
+
|
394
|
+
|
395
|
+
def is_tf_tensor(obj: Any) -> bool:
|
396
|
+
import tensorflow # type: ignore
|
397
|
+
|
398
|
+
return isinstance(obj, tensorflow.Tensor)
|
399
|
+
|
400
|
+
|
401
|
+
def is_tf_tensor_typename(typename: str) -> bool:
|
402
|
+
return typename.startswith("tensorflow.") and (
|
403
|
+
"Tensor" in typename or "Variable" in typename
|
404
|
+
)
|
405
|
+
|
406
|
+
|
407
|
+
def is_tf_eager_tensor_typename(typename: str) -> bool:
|
408
|
+
return typename.startswith("tensorflow.") and ("EagerTensor" in typename)
|
409
|
+
|
410
|
+
|
411
|
+
def is_pytorch_tensor(obj: Any) -> bool:
|
412
|
+
import torch # type: ignore
|
413
|
+
|
414
|
+
return isinstance(obj, torch.Tensor)
|
415
|
+
|
416
|
+
|
417
|
+
def is_pytorch_tensor_typename(typename: str) -> bool:
|
418
|
+
return typename.startswith("torch.") and (
|
419
|
+
"Tensor" in typename or "Variable" in typename
|
420
|
+
)
|
421
|
+
|
422
|
+
|
423
|
+
def is_jax_tensor_typename(typename: str) -> bool:
|
424
|
+
return typename.startswith("jaxlib.") and "Array" in typename
|
425
|
+
|
426
|
+
|
427
|
+
def get_jax_tensor(obj: Any) -> Optional[Any]:
|
428
|
+
import jax # type: ignore
|
429
|
+
|
430
|
+
return jax.device_get(obj)
|
431
|
+
|
432
|
+
|
433
|
+
def is_fastai_tensor_typename(typename: str) -> bool:
|
434
|
+
return typename.startswith("fastai.") and ("Tensor" in typename)
|
435
|
+
|
436
|
+
|
437
|
+
def is_pandas_data_frame_typename(typename: str) -> bool:
|
438
|
+
return typename.startswith("pandas.") and "DataFrame" in typename
|
439
|
+
|
440
|
+
|
441
|
+
def is_matplotlib_typename(typename: str) -> bool:
|
442
|
+
return typename.startswith("matplotlib.")
|
443
|
+
|
444
|
+
|
445
|
+
def is_plotly_typename(typename: str) -> bool:
|
446
|
+
return typename.startswith("plotly.")
|
447
|
+
|
448
|
+
|
449
|
+
def is_plotly_figure_typename(typename: str) -> bool:
|
450
|
+
return typename.startswith("plotly.") and typename.endswith(".Figure")
|
451
|
+
|
452
|
+
|
453
|
+
def is_numpy_array(obj: Any) -> bool:
|
454
|
+
return np and isinstance(obj, np.ndarray)
|
455
|
+
|
456
|
+
|
457
|
+
def is_pandas_data_frame(obj: Any) -> bool:
|
458
|
+
if pd_available:
|
459
|
+
import pandas as pd
|
460
|
+
|
461
|
+
return isinstance(obj, pd.DataFrame)
|
462
|
+
else:
|
463
|
+
return is_pandas_data_frame_typename(get_full_typename(obj))
|
464
|
+
|
465
|
+
|
466
|
+
def ensure_matplotlib_figure(obj: Any) -> Any:
|
467
|
+
"""Extract the current figure from a matplotlib object.
|
468
|
+
|
469
|
+
Return the object itself if it's a figure.
|
470
|
+
Raises ValueError if the object can't be converted.
|
471
|
+
"""
|
472
|
+
import matplotlib # type: ignore
|
473
|
+
from matplotlib.figure import Figure # type: ignore
|
474
|
+
|
475
|
+
# there are combinations of plotly and matplotlib versions that don't work well together,
|
476
|
+
# this patches matplotlib to add a removed method that plotly assumes exists
|
477
|
+
from matplotlib.spines import Spine # type: ignore
|
478
|
+
|
479
|
+
def is_frame_like(self: Any) -> bool:
|
480
|
+
"""Return True if directly on axes frame.
|
481
|
+
|
482
|
+
This is useful for determining if a spine is the edge of an
|
483
|
+
old style MPL plot. If so, this function will return True.
|
484
|
+
"""
|
485
|
+
position = self._position or ("outward", 0.0)
|
486
|
+
if isinstance(position, str):
|
487
|
+
if position == "center":
|
488
|
+
position = ("axes", 0.5)
|
489
|
+
elif position == "zero":
|
490
|
+
position = ("data", 0)
|
491
|
+
if len(position) != 2:
|
492
|
+
raise ValueError("position should be 2-tuple")
|
493
|
+
position_type, amount = position # type: ignore
|
494
|
+
if position_type == "outward" and amount == 0:
|
495
|
+
return True
|
496
|
+
else:
|
497
|
+
return False
|
498
|
+
|
499
|
+
Spine.is_frame_like = is_frame_like
|
500
|
+
|
501
|
+
if obj == matplotlib.pyplot:
|
502
|
+
obj = obj.gcf()
|
503
|
+
elif not isinstance(obj, Figure):
|
504
|
+
if hasattr(obj, "figure"):
|
505
|
+
obj = obj.figure
|
506
|
+
# Some matplotlib objects have a figure function
|
507
|
+
if not isinstance(obj, Figure):
|
508
|
+
raise ValueError(
|
509
|
+
"Only matplotlib.pyplot or matplotlib.pyplot.Figure objects are accepted."
|
510
|
+
)
|
511
|
+
return obj
|
512
|
+
|
513
|
+
|
514
|
+
def matplotlib_to_plotly(obj: Any) -> Any:
|
515
|
+
obj = ensure_matplotlib_figure(obj)
|
516
|
+
tools = get_module(
|
517
|
+
"plotly.tools",
|
518
|
+
required=(
|
519
|
+
"plotly is required to log interactive plots, install with: "
|
520
|
+
"`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
|
521
|
+
),
|
522
|
+
)
|
523
|
+
return tools.mpl_to_plotly(obj)
|
524
|
+
|
525
|
+
|
526
|
+
def matplotlib_contains_images(obj: Any) -> bool:
|
527
|
+
obj = ensure_matplotlib_figure(obj)
|
528
|
+
return any(len(ax.images) > 0 for ax in obj.axes)
|
529
|
+
|
530
|
+
|
531
|
+
def _numpy_generic_convert(obj: Any) -> Any:
|
532
|
+
obj = obj.item()
|
533
|
+
if isinstance(obj, float) and math.isnan(obj):
|
534
|
+
obj = None
|
535
|
+
elif isinstance(obj, np.generic) and (
|
536
|
+
obj.dtype.kind == "f" or obj.dtype == "bfloat16"
|
537
|
+
):
|
538
|
+
# obj is a numpy float with precision greater than that of native python float
|
539
|
+
# (i.e., float96 or float128) or it is of custom type such as bfloat16.
|
540
|
+
# in these cases, obj.item() does not return a native
|
541
|
+
# python float (in the first case - to avoid loss of precision,
|
542
|
+
# so we need to explicitly cast this down to a 64bit float)
|
543
|
+
obj = float(obj)
|
544
|
+
return obj
|
545
|
+
|
546
|
+
|
547
|
+
def _sanitize_numpy_keys(
|
548
|
+
d: Dict,
|
549
|
+
visited: Optional[Dict[int, Dict]] = None,
|
550
|
+
) -> Tuple[Dict, bool]:
|
551
|
+
"""Returns a dictionary where all NumPy keys are converted.
|
552
|
+
|
553
|
+
Args:
|
554
|
+
d: The dictionary to sanitize.
|
555
|
+
|
556
|
+
Returns:
|
557
|
+
A sanitized dictionary, and a boolean indicating whether anything was
|
558
|
+
changed.
|
559
|
+
"""
|
560
|
+
out: Dict[Any, Any] = dict()
|
561
|
+
converted = False
|
562
|
+
|
563
|
+
# Work with recursive dictionaries: if a dictionary has already been
|
564
|
+
# converted, reuse its converted value to retain the recursive structure
|
565
|
+
# of the input.
|
566
|
+
if visited is None:
|
567
|
+
visited = {id(d): out}
|
568
|
+
elif id(d) in visited:
|
569
|
+
return visited[id(d)], False
|
570
|
+
visited[id(d)] = out
|
571
|
+
|
572
|
+
for key, value in d.items():
|
573
|
+
if isinstance(value, dict):
|
574
|
+
value, converted_value = _sanitize_numpy_keys(value, visited)
|
575
|
+
converted |= converted_value
|
576
|
+
if isinstance(key, np.generic):
|
577
|
+
key = _numpy_generic_convert(key)
|
578
|
+
converted = True
|
579
|
+
out[key] = value
|
580
|
+
|
581
|
+
return out, converted
|
582
|
+
|
583
|
+
|
584
|
+
def json_friendly( # noqa: C901
|
585
|
+
obj: Any,
|
586
|
+
) -> Union[Tuple[Any, bool], Tuple[Union[None, str, float], bool]]:
|
587
|
+
"""Convert an object into something that's more becoming of JSON."""
|
588
|
+
converted = True
|
589
|
+
typename = get_full_typename(obj)
|
590
|
+
|
591
|
+
if is_tf_eager_tensor_typename(typename):
|
592
|
+
obj = obj.numpy()
|
593
|
+
elif is_tf_tensor_typename(typename):
|
594
|
+
try:
|
595
|
+
obj = obj.eval()
|
596
|
+
except RuntimeError:
|
597
|
+
obj = obj.numpy()
|
598
|
+
elif is_pytorch_tensor_typename(typename) or is_fastai_tensor_typename(typename):
|
599
|
+
try:
|
600
|
+
if obj.requires_grad:
|
601
|
+
obj = obj.detach()
|
602
|
+
except AttributeError:
|
603
|
+
pass # before 0.4 is only present on variables
|
604
|
+
|
605
|
+
try:
|
606
|
+
obj = obj.data
|
607
|
+
except RuntimeError:
|
608
|
+
pass # happens for Tensors before 0.4
|
609
|
+
|
610
|
+
if obj.size():
|
611
|
+
obj = obj.cpu().detach().numpy()
|
612
|
+
else:
|
613
|
+
return obj.item(), True
|
614
|
+
elif is_jax_tensor_typename(typename):
|
615
|
+
obj = get_jax_tensor(obj)
|
616
|
+
|
617
|
+
if is_numpy_array(obj):
|
618
|
+
if obj.size == 1:
|
619
|
+
obj = obj.flatten()[0]
|
620
|
+
elif obj.size <= 32:
|
621
|
+
obj = obj.tolist()
|
622
|
+
elif np and isinstance(obj, np.generic):
|
623
|
+
obj = _numpy_generic_convert(obj)
|
624
|
+
elif isinstance(obj, bytes):
|
625
|
+
obj = obj.decode("utf-8")
|
626
|
+
elif isinstance(obj, (datetime, date)):
|
627
|
+
obj = obj.isoformat()
|
628
|
+
elif callable(obj):
|
629
|
+
obj = (
|
630
|
+
f"{obj.__module__}.{obj.__qualname__}"
|
631
|
+
if hasattr(obj, "__qualname__") and hasattr(obj, "__module__")
|
632
|
+
else str(obj)
|
633
|
+
)
|
634
|
+
elif isinstance(obj, float) and math.isnan(obj):
|
635
|
+
obj = None
|
636
|
+
elif isinstance(obj, dict) and np:
|
637
|
+
obj, converted = _sanitize_numpy_keys(obj)
|
638
|
+
elif isinstance(obj, set):
|
639
|
+
# set is not json serializable, so we convert it to tuple
|
640
|
+
obj = tuple(obj)
|
641
|
+
elif isinstance(obj, enum.Enum):
|
642
|
+
obj = obj.name
|
643
|
+
else:
|
644
|
+
converted = False
|
645
|
+
if getsizeof(obj) > VALUE_BYTES_LIMIT:
|
646
|
+
wandb.termwarn(
|
647
|
+
"Serializing object of type {} that is {} bytes".format(
|
648
|
+
type(obj).__name__, getsizeof(obj)
|
649
|
+
)
|
650
|
+
)
|
651
|
+
return obj, converted
|
652
|
+
|
653
|
+
|
654
|
+
def json_friendly_val(val: Any) -> Any:
|
655
|
+
"""Make any value (including dict, slice, sequence, dataclass) JSON friendly."""
|
656
|
+
converted: Union[dict, list]
|
657
|
+
if isinstance(val, dict):
|
658
|
+
converted = {}
|
659
|
+
for key, value in val.items():
|
660
|
+
converted[key] = json_friendly_val(value)
|
661
|
+
return converted
|
662
|
+
if isinstance(val, slice):
|
663
|
+
converted = dict(
|
664
|
+
slice_start=val.start, slice_step=val.step, slice_stop=val.stop
|
665
|
+
)
|
666
|
+
return converted
|
667
|
+
val, _ = json_friendly(val)
|
668
|
+
if isinstance(val, Sequence) and not isinstance(val, str):
|
669
|
+
converted = []
|
670
|
+
for value in val:
|
671
|
+
converted.append(json_friendly_val(value))
|
672
|
+
return converted
|
673
|
+
if is_dataclass(val) and not isinstance(val, type):
|
674
|
+
converted = asdict(val)
|
675
|
+
return converted
|
676
|
+
else:
|
677
|
+
if val.__class__.__module__ not in ("builtins", "__builtin__"):
|
678
|
+
val = str(val)
|
679
|
+
return val
|
680
|
+
|
681
|
+
|
682
|
+
def alias_is_version_index(alias: str) -> bool:
|
683
|
+
return len(alias) >= 2 and alias[0] == "v" and alias[1:].isnumeric()
|
684
|
+
|
685
|
+
|
686
|
+
def convert_plots(obj: Any) -> Any:
|
687
|
+
if is_matplotlib_typename(get_full_typename(obj)):
|
688
|
+
tools = get_module(
|
689
|
+
"plotly.tools",
|
690
|
+
required=(
|
691
|
+
"plotly is required to log interactive plots, install with: "
|
692
|
+
"`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
|
693
|
+
),
|
694
|
+
)
|
695
|
+
obj = tools.mpl_to_plotly(obj)
|
696
|
+
|
697
|
+
if is_plotly_typename(get_full_typename(obj)):
|
698
|
+
return {"_type": "plotly", "plot": obj.to_plotly_json()}
|
699
|
+
else:
|
700
|
+
return obj
|
701
|
+
|
702
|
+
|
703
|
+
def maybe_compress_history(obj: Any) -> Tuple[Any, bool]:
|
704
|
+
if np and isinstance(obj, np.ndarray) and obj.size > 32:
|
705
|
+
return wandb.Histogram(obj, num_bins=32).to_json(), True
|
706
|
+
else:
|
707
|
+
return obj, False
|
708
|
+
|
709
|
+
|
710
|
+
def maybe_compress_summary(obj: Any, h5_typename: str) -> Tuple[Any, bool]:
|
711
|
+
if np and isinstance(obj, np.ndarray) and obj.size > 32:
|
712
|
+
return (
|
713
|
+
{
|
714
|
+
"_type": h5_typename, # may not be ndarray
|
715
|
+
"var": np.var(obj).item(),
|
716
|
+
"mean": np.mean(obj).item(),
|
717
|
+
"min": np.amin(obj).item(),
|
718
|
+
"max": np.amax(obj).item(),
|
719
|
+
"10%": np.percentile(obj, 10),
|
720
|
+
"25%": np.percentile(obj, 25),
|
721
|
+
"75%": np.percentile(obj, 75),
|
722
|
+
"90%": np.percentile(obj, 90),
|
723
|
+
"size": obj.size,
|
724
|
+
},
|
725
|
+
True,
|
726
|
+
)
|
727
|
+
else:
|
728
|
+
return obj, False
|
729
|
+
|
730
|
+
|
731
|
+
def launch_browser(attempt_launch_browser: bool = True) -> bool:
|
732
|
+
"""Decide if we should launch a browser."""
|
733
|
+
_display_variables = ["DISPLAY", "WAYLAND_DISPLAY", "MIR_SOCKET"]
|
734
|
+
_webbrowser_names_blocklist = ["www-browser", "lynx", "links", "elinks", "w3m"]
|
735
|
+
|
736
|
+
import webbrowser
|
737
|
+
|
738
|
+
launch_browser = attempt_launch_browser
|
739
|
+
if launch_browser:
|
740
|
+
if "linux" in sys.platform and not any(
|
741
|
+
os.getenv(var) for var in _display_variables
|
742
|
+
):
|
743
|
+
launch_browser = False
|
744
|
+
try:
|
745
|
+
browser = webbrowser.get()
|
746
|
+
if hasattr(browser, "name") and browser.name in _webbrowser_names_blocklist:
|
747
|
+
launch_browser = False
|
748
|
+
except webbrowser.Error:
|
749
|
+
launch_browser = False
|
750
|
+
|
751
|
+
return launch_browser
|
752
|
+
|
753
|
+
|
754
|
+
def generate_id(length: int = 8) -> str:
|
755
|
+
# Do not use this; use wandb.sdk.lib.runid.generate_id instead.
|
756
|
+
# This is kept only for legacy code.
|
757
|
+
return runid.generate_id(length)
|
758
|
+
|
759
|
+
|
760
|
+
def parse_tfjob_config() -> Any:
|
761
|
+
"""Attempt to parse TFJob config, returning False if it can't find it."""
|
762
|
+
if os.getenv("TF_CONFIG"):
|
763
|
+
try:
|
764
|
+
return json.loads(os.environ["TF_CONFIG"])
|
765
|
+
except ValueError:
|
766
|
+
return False
|
767
|
+
else:
|
768
|
+
return False
|
769
|
+
|
770
|
+
|
771
|
+
class WandBJSONEncoder(json.JSONEncoder):
|
772
|
+
"""A JSON Encoder that handles some extra types."""
|
773
|
+
|
774
|
+
def default(self, obj: Any) -> Any:
|
775
|
+
if hasattr(obj, "json_encode"):
|
776
|
+
return obj.json_encode()
|
777
|
+
# if hasattr(obj, 'to_json'):
|
778
|
+
# return obj.to_json()
|
779
|
+
tmp_obj, converted = json_friendly(obj)
|
780
|
+
if converted:
|
781
|
+
return tmp_obj
|
782
|
+
return json.JSONEncoder.default(self, obj)
|
783
|
+
|
784
|
+
|
785
|
+
class WandBJSONEncoderOld(json.JSONEncoder):
|
786
|
+
"""A JSON Encoder that handles some extra types."""
|
787
|
+
|
788
|
+
def default(self, obj: Any) -> Any:
|
789
|
+
tmp_obj, converted = json_friendly(obj)
|
790
|
+
tmp_obj, compressed = maybe_compress_summary(tmp_obj, get_h5_typename(obj))
|
791
|
+
if converted:
|
792
|
+
return tmp_obj
|
793
|
+
return json.JSONEncoder.default(self, tmp_obj)
|
794
|
+
|
795
|
+
|
796
|
+
class WandBHistoryJSONEncoder(json.JSONEncoder):
|
797
|
+
"""A JSON Encoder that handles some extra types.
|
798
|
+
|
799
|
+
This encoder turns numpy like objects with a size > 32 into histograms.
|
800
|
+
"""
|
801
|
+
|
802
|
+
def default(self, obj: Any) -> Any:
|
803
|
+
obj, converted = json_friendly(obj)
|
804
|
+
obj, compressed = maybe_compress_history(obj)
|
805
|
+
if converted:
|
806
|
+
return obj
|
807
|
+
return json.JSONEncoder.default(self, obj)
|
808
|
+
|
809
|
+
|
810
|
+
class JSONEncoderUncompressed(json.JSONEncoder):
|
811
|
+
"""A JSON Encoder that handles some extra types.
|
812
|
+
|
813
|
+
This encoder turns numpy like objects with a size > 32 into histograms.
|
814
|
+
"""
|
815
|
+
|
816
|
+
def default(self, obj: Any) -> Any:
|
817
|
+
if is_numpy_array(obj):
|
818
|
+
return obj.tolist()
|
819
|
+
elif np and isinstance(obj, np.generic):
|
820
|
+
obj = obj.item()
|
821
|
+
return json.JSONEncoder.default(self, obj)
|
822
|
+
|
823
|
+
|
824
|
+
def json_dump_safer(obj: Any, fp: IO[str], **kwargs: Any) -> None:
|
825
|
+
"""Convert obj to json, with some extra encodable types."""
|
826
|
+
return dump(obj, fp, cls=WandBJSONEncoder, **kwargs)
|
827
|
+
|
828
|
+
|
829
|
+
def json_dumps_safer(obj: Any, **kwargs: Any) -> str:
|
830
|
+
"""Convert obj to json, with some extra encodable types."""
|
831
|
+
return dumps(obj, cls=WandBJSONEncoder, **kwargs)
|
832
|
+
|
833
|
+
|
834
|
+
# This is used for dumping raw json into files
|
835
|
+
def json_dump_uncompressed(obj: Any, fp: IO[str], **kwargs: Any) -> None:
|
836
|
+
"""Convert obj to json, with some extra encodable types."""
|
837
|
+
return dump(obj, fp, cls=JSONEncoderUncompressed, **kwargs)
|
838
|
+
|
839
|
+
|
840
|
+
def json_dumps_safer_history(obj: Any, **kwargs: Any) -> str:
|
841
|
+
"""Convert obj to json, with some extra encodable types, including histograms."""
|
842
|
+
return dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs)
|
843
|
+
|
844
|
+
|
845
|
+
def make_json_if_not_number(
|
846
|
+
v: Union[int, float, str, Mapping, Sequence],
|
847
|
+
) -> Union[int, float, str]:
|
848
|
+
"""If v is not a basic type convert it to json."""
|
849
|
+
if isinstance(v, (float, int)):
|
850
|
+
return v
|
851
|
+
return json_dumps_safer(v)
|
852
|
+
|
853
|
+
|
854
|
+
def make_safe_for_json(obj: Any) -> Any:
|
855
|
+
"""Replace invalid json floats with strings. Also converts to lists and dicts."""
|
856
|
+
if isinstance(obj, Mapping):
|
857
|
+
return {k: make_safe_for_json(v) for k, v in obj.items()}
|
858
|
+
elif isinstance(obj, str):
|
859
|
+
# str's are Sequence, so we need to short-circuit
|
860
|
+
return obj
|
861
|
+
elif isinstance(obj, Sequence):
|
862
|
+
return [make_safe_for_json(v) for v in obj]
|
863
|
+
elif isinstance(obj, float):
|
864
|
+
# W&B backend and UI handle these strings
|
865
|
+
if obj != obj: # standard way to check for NaN
|
866
|
+
return "NaN"
|
867
|
+
elif obj == float("+inf"):
|
868
|
+
return "Infinity"
|
869
|
+
elif obj == float("-inf"):
|
870
|
+
return "-Infinity"
|
871
|
+
return obj
|
872
|
+
|
873
|
+
|
874
|
+
def no_retry_4xx(e: Exception) -> bool:
|
875
|
+
if not isinstance(e, requests.HTTPError):
|
876
|
+
return True
|
877
|
+
assert e.response is not None
|
878
|
+
if not (400 <= e.response.status_code < 500) or e.response.status_code == 429:
|
879
|
+
return True
|
880
|
+
body = json.loads(e.response.content)
|
881
|
+
raise UsageError(body["errors"][0]["message"])
|
882
|
+
|
883
|
+
|
884
|
+
def no_retry_auth(e: Any) -> bool:
|
885
|
+
if hasattr(e, "exception"):
|
886
|
+
e = e.exception
|
887
|
+
if not isinstance(e, requests.HTTPError):
|
888
|
+
return True
|
889
|
+
if e.response is None:
|
890
|
+
return True
|
891
|
+
# Don't retry bad request errors; raise immediately
|
892
|
+
if e.response.status_code in (400, 409):
|
893
|
+
return False
|
894
|
+
# Retry all non-forbidden/unauthorized/not-found errors.
|
895
|
+
if e.response.status_code not in (401, 403, 404):
|
896
|
+
return True
|
897
|
+
# Crash w/message on forbidden/unauthorized errors.
|
898
|
+
if e.response.status_code == 401:
|
899
|
+
raise AuthenticationError(
|
900
|
+
"The API key you provided is either invalid or missing. "
|
901
|
+
f"If the `{wandb.env.API_KEY}` environment variable is set, make sure it is correct. "
|
902
|
+
"Otherwise, to resolve this issue, you may try running the 'wandb login --relogin' command. "
|
903
|
+
"If you are using a local server, make sure that you're using the correct hostname. "
|
904
|
+
"If you're not sure, you can try logging in again using the 'wandb login --relogin --host [hostname]' command."
|
905
|
+
f"(Error {e.response.status_code}: {e.response.reason})"
|
906
|
+
)
|
907
|
+
elif wandb.run:
|
908
|
+
raise CommError(f"Permission denied to access {wandb.run.path}")
|
909
|
+
else:
|
910
|
+
raise CommError(
|
911
|
+
"It appears that you do not have permission to access the requested resource. "
|
912
|
+
"Please reach out to the project owner to grant you access. "
|
913
|
+
"If you have the correct permissions, verify that there are no issues with your networking setup."
|
914
|
+
f"(Error {e.response.status_code}: {e.response.reason})"
|
915
|
+
)
|
916
|
+
|
917
|
+
|
918
|
+
def check_retry_conflict(e: Any) -> Optional[bool]:
|
919
|
+
"""Check if the exception is a conflict type so it can be retried.
|
920
|
+
|
921
|
+
Returns:
|
922
|
+
True - Should retry this operation
|
923
|
+
False - Should not retry this operation
|
924
|
+
None - No decision, let someone else decide
|
925
|
+
"""
|
926
|
+
if hasattr(e, "exception"):
|
927
|
+
e = e.exception
|
928
|
+
if isinstance(e, requests.HTTPError) and e.response is not None:
|
929
|
+
if e.response.status_code == 409:
|
930
|
+
return True
|
931
|
+
return None
|
932
|
+
|
933
|
+
|
934
|
+
def check_retry_conflict_or_gone(e: Any) -> Optional[bool]:
|
935
|
+
"""Check if the exception is a conflict or gone type, so it can be retried or not.
|
936
|
+
|
937
|
+
Returns:
|
938
|
+
True - Should retry this operation
|
939
|
+
False - Should not retry this operation
|
940
|
+
None - No decision, let someone else decide
|
941
|
+
"""
|
942
|
+
if hasattr(e, "exception"):
|
943
|
+
e = e.exception
|
944
|
+
if isinstance(e, requests.HTTPError) and e.response is not None:
|
945
|
+
if e.response.status_code == 409:
|
946
|
+
return True
|
947
|
+
if e.response.status_code == 410:
|
948
|
+
return False
|
949
|
+
return None
|
950
|
+
|
951
|
+
|
952
|
+
def make_check_retry_fn(
|
953
|
+
fallback_retry_fn: CheckRetryFnType,
|
954
|
+
check_fn: Callable[[Exception], Optional[bool]],
|
955
|
+
check_timedelta: Optional[timedelta] = None,
|
956
|
+
) -> CheckRetryFnType:
|
957
|
+
"""Return a check_retry_fn which can be used by lib.Retry().
|
958
|
+
|
959
|
+
Arguments:
|
960
|
+
fallback_fn: Use this function if check_fn didn't decide if a retry should happen.
|
961
|
+
check_fn: Function which returns bool if retry should happen or None if unsure.
|
962
|
+
check_timedelta: Optional retry timeout if we check_fn matches the exception
|
963
|
+
"""
|
964
|
+
|
965
|
+
def check_retry_fn(e: Exception) -> Union[bool, timedelta]:
|
966
|
+
check = check_fn(e)
|
967
|
+
if check is None:
|
968
|
+
return fallback_retry_fn(e)
|
969
|
+
if check is False:
|
970
|
+
return False
|
971
|
+
if check_timedelta:
|
972
|
+
return check_timedelta
|
973
|
+
return True
|
974
|
+
|
975
|
+
return check_retry_fn
|
976
|
+
|
977
|
+
|
978
|
+
def find_runner(program: str) -> Union[None, list, List[str]]:
|
979
|
+
"""Return a command that will run program.
|
980
|
+
|
981
|
+
Arguments:
|
982
|
+
program: The string name of the program to try to run.
|
983
|
+
|
984
|
+
Returns:
|
985
|
+
commandline list of strings to run the program (eg. with subprocess.call()) or None
|
986
|
+
"""
|
987
|
+
if os.path.isfile(program) and not os.access(program, os.X_OK):
|
988
|
+
# program is a path to a non-executable file
|
989
|
+
try:
|
990
|
+
opened = open(program)
|
991
|
+
except OSError: # PermissionError doesn't exist in 2.7
|
992
|
+
return None
|
993
|
+
first_line = opened.readline().strip()
|
994
|
+
if first_line.startswith("#!"):
|
995
|
+
return shlex.split(first_line[2:])
|
996
|
+
if program.endswith(".py"):
|
997
|
+
return [sys.executable]
|
998
|
+
return None
|
999
|
+
|
1000
|
+
|
1001
|
+
def downsample(values: Sequence, target_length: int) -> list:
|
1002
|
+
"""Downsample 1d values to target_length, including start and end.
|
1003
|
+
|
1004
|
+
Algorithm just rounds index down.
|
1005
|
+
|
1006
|
+
Values can be any sequence, including a generator.
|
1007
|
+
"""
|
1008
|
+
if not target_length > 1:
|
1009
|
+
raise UsageError("target_length must be > 1")
|
1010
|
+
values = list(values)
|
1011
|
+
if len(values) < target_length:
|
1012
|
+
return values
|
1013
|
+
ratio = float(len(values) - 1) / (target_length - 1)
|
1014
|
+
result = []
|
1015
|
+
for i in range(target_length):
|
1016
|
+
result.append(values[int(i * ratio)])
|
1017
|
+
return result
|
1018
|
+
|
1019
|
+
|
1020
|
+
def has_num(dictionary: Mapping, key: Any) -> bool:
|
1021
|
+
return key in dictionary and isinstance(dictionary[key], numbers.Number)
|
1022
|
+
|
1023
|
+
|
1024
|
+
def get_log_file_path() -> str:
|
1025
|
+
"""Log file path used in error messages.
|
1026
|
+
|
1027
|
+
It would probably be better if this pointed to a log file in a
|
1028
|
+
run directory.
|
1029
|
+
"""
|
1030
|
+
# TODO(jhr, cvp): refactor
|
1031
|
+
if wandb.run is not None:
|
1032
|
+
return wandb.run._settings.log_internal
|
1033
|
+
return os.path.join("wandb", "debug-internal.log")
|
1034
|
+
|
1035
|
+
|
1036
|
+
def docker_image_regex(image: str) -> Any:
|
1037
|
+
"""Regex match for valid docker image names."""
|
1038
|
+
if image:
|
1039
|
+
return re.match(
|
1040
|
+
r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-zA-Z0-9-]{1,63}(?<!-))*(?::[0-9]{1,5})?/)?((?![._-])(?:[a-z0-9._-]*)(?<![._-])(?:/(?![._-])[a-z0-9._-]*(?<![._-]))*)(?::(?![.-])[a-zA-Z0-9_.-]{1,128})?$",
|
1041
|
+
image,
|
1042
|
+
)
|
1043
|
+
return None
|
1044
|
+
|
1045
|
+
|
1046
|
+
def image_from_docker_args(args: List[str]) -> Optional[str]:
|
1047
|
+
"""Scan docker run args and attempt to find the most likely docker image argument.
|
1048
|
+
|
1049
|
+
It excludes any arguments that start with a dash, and the argument after it if it
|
1050
|
+
isn't a boolean switch. This can be improved, we currently fallback gracefully when
|
1051
|
+
this fails.
|
1052
|
+
"""
|
1053
|
+
bool_args = [
|
1054
|
+
"-t",
|
1055
|
+
"--tty",
|
1056
|
+
"--rm",
|
1057
|
+
"--privileged",
|
1058
|
+
"--oom-kill-disable",
|
1059
|
+
"--no-healthcheck",
|
1060
|
+
"-i",
|
1061
|
+
"--interactive",
|
1062
|
+
"--init",
|
1063
|
+
"--help",
|
1064
|
+
"--detach",
|
1065
|
+
"-d",
|
1066
|
+
"--sig-proxy",
|
1067
|
+
"-it",
|
1068
|
+
"-itd",
|
1069
|
+
]
|
1070
|
+
last_flag = -2
|
1071
|
+
last_arg = ""
|
1072
|
+
possible_images = []
|
1073
|
+
if len(args) > 0 and args[0] == "run":
|
1074
|
+
args.pop(0)
|
1075
|
+
for i, arg in enumerate(args):
|
1076
|
+
if arg.startswith("-"):
|
1077
|
+
last_flag = i
|
1078
|
+
last_arg = arg
|
1079
|
+
elif "@sha256:" in arg:
|
1080
|
+
# Because our regex doesn't match digests
|
1081
|
+
possible_images.append(arg)
|
1082
|
+
elif docker_image_regex(arg):
|
1083
|
+
if last_flag == i - 2:
|
1084
|
+
possible_images.append(arg)
|
1085
|
+
elif "=" in last_arg:
|
1086
|
+
possible_images.append(arg)
|
1087
|
+
elif last_arg in bool_args and last_flag == i - 1:
|
1088
|
+
possible_images.append(arg)
|
1089
|
+
most_likely = None
|
1090
|
+
for img in possible_images:
|
1091
|
+
if ":" in img or "@" in img or "/" in img:
|
1092
|
+
most_likely = img
|
1093
|
+
break
|
1094
|
+
if most_likely is None and len(possible_images) > 0:
|
1095
|
+
most_likely = possible_images[0]
|
1096
|
+
return most_likely
|
1097
|
+
|
1098
|
+
|
1099
|
+
def load_yaml(file: Any) -> Any:
|
1100
|
+
return yaml.safe_load(file)
|
1101
|
+
|
1102
|
+
|
1103
|
+
def image_id_from_k8s() -> Optional[str]:
|
1104
|
+
"""Ping the k8s metadata service for the image id.
|
1105
|
+
|
1106
|
+
Specify the KUBERNETES_NAMESPACE environment variable if your pods are not in the
|
1107
|
+
default namespace:
|
1108
|
+
|
1109
|
+
- name: KUBERNETES_NAMESPACE valueFrom:
|
1110
|
+
fieldRef:
|
1111
|
+
fieldPath: metadata.namespace
|
1112
|
+
"""
|
1113
|
+
token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
1114
|
+
|
1115
|
+
if not os.path.exists(token_path):
|
1116
|
+
return None
|
1117
|
+
|
1118
|
+
try:
|
1119
|
+
with open(token_path) as token_file:
|
1120
|
+
token = token_file.read()
|
1121
|
+
except FileNotFoundError:
|
1122
|
+
logger.warning(f"Token file not found at {token_path}.")
|
1123
|
+
return None
|
1124
|
+
except PermissionError as e:
|
1125
|
+
current_uid = os.getuid()
|
1126
|
+
warning = (
|
1127
|
+
f"Unable to read the token file at {token_path} due to permission error ({e})."
|
1128
|
+
f"The current user id is {current_uid}. "
|
1129
|
+
"Consider changing the securityContext to run the container as the current user."
|
1130
|
+
)
|
1131
|
+
logger.warning(warning)
|
1132
|
+
wandb.termwarn(warning)
|
1133
|
+
return None
|
1134
|
+
|
1135
|
+
if not token:
|
1136
|
+
return None
|
1137
|
+
|
1138
|
+
k8s_server = "https://{}:{}/api/v1/namespaces/{}/pods/{}".format(
|
1139
|
+
os.getenv("KUBERNETES_SERVICE_HOST"),
|
1140
|
+
os.getenv("KUBERNETES_PORT_443_TCP_PORT"),
|
1141
|
+
os.getenv("KUBERNETES_NAMESPACE", "default"),
|
1142
|
+
os.getenv("HOSTNAME"),
|
1143
|
+
)
|
1144
|
+
try:
|
1145
|
+
res = requests.get(
|
1146
|
+
k8s_server,
|
1147
|
+
verify="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
|
1148
|
+
timeout=3,
|
1149
|
+
headers={"Authorization": f"Bearer {token}"},
|
1150
|
+
)
|
1151
|
+
res.raise_for_status()
|
1152
|
+
except requests.RequestException:
|
1153
|
+
return None
|
1154
|
+
try:
|
1155
|
+
return str( # noqa: B005
|
1156
|
+
res.json()["status"]["containerStatuses"][0]["imageID"]
|
1157
|
+
).strip("docker-pullable://")
|
1158
|
+
except (ValueError, KeyError, IndexError):
|
1159
|
+
logger.exception("Error checking kubernetes for image id")
|
1160
|
+
return None
|
1161
|
+
|
1162
|
+
|
1163
|
+
def async_call(
|
1164
|
+
target: Callable, timeout: Optional[Union[int, float]] = None
|
1165
|
+
) -> Callable:
|
1166
|
+
"""Wrap a method to run in the background with an optional timeout.
|
1167
|
+
|
1168
|
+
Returns a new method that will call the original with any args, waiting for upto
|
1169
|
+
timeout seconds. This new method blocks on the original and returns the result or
|
1170
|
+
None if timeout was reached, along with the thread. You can check thread.is_alive()
|
1171
|
+
to determine if a timeout was reached. If an exception is thrown in the thread, we
|
1172
|
+
reraise it.
|
1173
|
+
"""
|
1174
|
+
q: queue.Queue = queue.Queue()
|
1175
|
+
|
1176
|
+
def wrapped_target(q: "queue.Queue", *args: Any, **kwargs: Any) -> Any:
|
1177
|
+
try:
|
1178
|
+
q.put(target(*args, **kwargs))
|
1179
|
+
except Exception as e:
|
1180
|
+
q.put(e)
|
1181
|
+
|
1182
|
+
def wrapper(
|
1183
|
+
*args: Any, **kwargs: Any
|
1184
|
+
) -> Union[Tuple[Exception, "threading.Thread"], Tuple[None, "threading.Thread"]]:
|
1185
|
+
thread = threading.Thread(
|
1186
|
+
target=wrapped_target, args=(q,) + args, kwargs=kwargs
|
1187
|
+
)
|
1188
|
+
thread.daemon = True
|
1189
|
+
thread.start()
|
1190
|
+
try:
|
1191
|
+
result = q.get(True, timeout)
|
1192
|
+
if isinstance(result, Exception):
|
1193
|
+
raise result.with_traceback(sys.exc_info()[2])
|
1194
|
+
return result, thread
|
1195
|
+
except queue.Empty:
|
1196
|
+
return None, thread
|
1197
|
+
|
1198
|
+
return wrapper
|
1199
|
+
|
1200
|
+
|
1201
|
+
def read_many_from_queue(
|
1202
|
+
q: "queue.Queue", max_items: int, queue_timeout: Union[int, float]
|
1203
|
+
) -> list:
|
1204
|
+
try:
|
1205
|
+
item = q.get(True, queue_timeout)
|
1206
|
+
except queue.Empty:
|
1207
|
+
return []
|
1208
|
+
items = [item]
|
1209
|
+
for _ in range(max_items):
|
1210
|
+
try:
|
1211
|
+
item = q.get_nowait()
|
1212
|
+
except queue.Empty:
|
1213
|
+
return items
|
1214
|
+
items.append(item)
|
1215
|
+
return items
|
1216
|
+
|
1217
|
+
|
1218
|
+
def stopwatch_now() -> float:
|
1219
|
+
"""Get a time value for interval comparisons.
|
1220
|
+
|
1221
|
+
When possible it is a monotonic clock to prevent backwards time issues.
|
1222
|
+
"""
|
1223
|
+
return time.monotonic()
|
1224
|
+
|
1225
|
+
|
1226
|
+
def class_colors(class_count: int) -> List[List[int]]:
|
1227
|
+
# make class 0 black, and the rest equally spaced fully saturated hues
|
1228
|
+
return [[0, 0, 0]] + [
|
1229
|
+
colorsys.hsv_to_rgb(i / (class_count - 1.0), 1.0, 1.0) # type: ignore
|
1230
|
+
for i in range(class_count - 1)
|
1231
|
+
]
|
1232
|
+
|
1233
|
+
|
1234
|
+
def _prompt_choice(
|
1235
|
+
input_timeout: Union[int, float, None] = None,
|
1236
|
+
jupyter: bool = False,
|
1237
|
+
) -> str:
|
1238
|
+
input_fn: Callable = input
|
1239
|
+
prompt = term.LOG_STRING
|
1240
|
+
if input_timeout is not None:
|
1241
|
+
# delayed import to mitigate risk of timed_input complexity
|
1242
|
+
from wandb.sdk.lib import timed_input
|
1243
|
+
|
1244
|
+
input_fn = functools.partial(timed_input.timed_input, timeout=input_timeout)
|
1245
|
+
# timed_input doesn't handle enhanced prompts
|
1246
|
+
if platform.system() == "Windows":
|
1247
|
+
prompt = "wandb"
|
1248
|
+
|
1249
|
+
text = f"{prompt}: Enter your choice: "
|
1250
|
+
if input_fn == input:
|
1251
|
+
choice = input_fn(text)
|
1252
|
+
else:
|
1253
|
+
choice = input_fn(text, jupyter=jupyter)
|
1254
|
+
return choice # type: ignore
|
1255
|
+
|
1256
|
+
|
1257
|
+
def prompt_choices(
|
1258
|
+
choices: Sequence[str],
|
1259
|
+
input_timeout: Union[int, float, None] = None,
|
1260
|
+
jupyter: bool = False,
|
1261
|
+
) -> str:
|
1262
|
+
"""Allow a user to choose from a list of options."""
|
1263
|
+
for i, choice in enumerate(choices):
|
1264
|
+
wandb.termlog(f"({i+1}) {choice}")
|
1265
|
+
|
1266
|
+
idx = -1
|
1267
|
+
while idx < 0 or idx > len(choices) - 1:
|
1268
|
+
choice = _prompt_choice(input_timeout=input_timeout, jupyter=jupyter)
|
1269
|
+
if not choice:
|
1270
|
+
continue
|
1271
|
+
idx = -1
|
1272
|
+
try:
|
1273
|
+
idx = int(choice) - 1
|
1274
|
+
except ValueError:
|
1275
|
+
pass
|
1276
|
+
if idx < 0 or idx > len(choices) - 1:
|
1277
|
+
wandb.termwarn("Invalid choice")
|
1278
|
+
result = choices[idx]
|
1279
|
+
wandb.termlog(f"You chose {result!r}")
|
1280
|
+
return result
|
1281
|
+
|
1282
|
+
|
1283
|
+
def guess_data_type(shape: Sequence[int], risky: bool = False) -> Optional[str]:
|
1284
|
+
"""Infer the type of data based on the shape of the tensors.
|
1285
|
+
|
1286
|
+
Arguments:
|
1287
|
+
shape (Sequence[int]): The shape of the data
|
1288
|
+
risky(bool): some guesses are more likely to be wrong.
|
1289
|
+
"""
|
1290
|
+
# (samples,) or (samples,logits)
|
1291
|
+
if len(shape) in (1, 2):
|
1292
|
+
return "label"
|
1293
|
+
# Assume image mask like fashion mnist: (no color channel)
|
1294
|
+
# This is risky because RNNs often have 3 dim tensors: batch, time, channels
|
1295
|
+
if risky and len(shape) == 3:
|
1296
|
+
return "image"
|
1297
|
+
if len(shape) == 4:
|
1298
|
+
if shape[-1] in (1, 3, 4):
|
1299
|
+
# (samples, height, width, Y \ RGB \ RGBA)
|
1300
|
+
return "image"
|
1301
|
+
else:
|
1302
|
+
# (samples, height, width, logits)
|
1303
|
+
return "segmentation_mask"
|
1304
|
+
return None
|
1305
|
+
|
1306
|
+
|
1307
|
+
def download_file_from_url(
|
1308
|
+
dest_path: str, source_url: str, api_key: Optional[str] = None
|
1309
|
+
) -> None:
|
1310
|
+
auth = None
|
1311
|
+
if not _thread_local_api_settings.cookies:
|
1312
|
+
auth = ("api", api_key or "")
|
1313
|
+
response = requests.get(
|
1314
|
+
source_url,
|
1315
|
+
auth=auth,
|
1316
|
+
headers=_thread_local_api_settings.headers,
|
1317
|
+
cookies=_thread_local_api_settings.cookies,
|
1318
|
+
stream=True,
|
1319
|
+
timeout=5,
|
1320
|
+
)
|
1321
|
+
response.raise_for_status()
|
1322
|
+
|
1323
|
+
if os.sep in dest_path:
|
1324
|
+
filesystem.mkdir_exists_ok(os.path.dirname(dest_path))
|
1325
|
+
with fsync_open(dest_path, "wb") as file:
|
1326
|
+
for data in response.iter_content(chunk_size=1024):
|
1327
|
+
file.write(data)
|
1328
|
+
|
1329
|
+
|
1330
|
+
def download_file_into_memory(source_url: str, api_key: Optional[str] = None) -> bytes:
|
1331
|
+
auth = None
|
1332
|
+
if not _thread_local_api_settings.cookies:
|
1333
|
+
auth = ("api", api_key or "")
|
1334
|
+
response = requests.get(
|
1335
|
+
source_url,
|
1336
|
+
auth=auth,
|
1337
|
+
headers=_thread_local_api_settings.headers,
|
1338
|
+
cookies=_thread_local_api_settings.cookies,
|
1339
|
+
stream=True,
|
1340
|
+
timeout=5,
|
1341
|
+
)
|
1342
|
+
response.raise_for_status()
|
1343
|
+
return response.content
|
1344
|
+
|
1345
|
+
|
1346
|
+
def isatty(ob: IO) -> bool:
|
1347
|
+
return hasattr(ob, "isatty") and ob.isatty()
|
1348
|
+
|
1349
|
+
|
1350
|
+
def to_human_size(size: int, units: Optional[List[Tuple[str, Any]]] = None) -> str:
|
1351
|
+
units = units or POW_10_BYTES
|
1352
|
+
unit, value = units[0]
|
1353
|
+
factor = round(float(size) / value, 1)
|
1354
|
+
return (
|
1355
|
+
f"{factor}{unit}"
|
1356
|
+
if factor < 1024 or len(units) == 1
|
1357
|
+
else to_human_size(size, units[1:])
|
1358
|
+
)
|
1359
|
+
|
1360
|
+
|
1361
|
+
def from_human_size(size: str, units: Optional[List[Tuple[str, Any]]] = None) -> int:
|
1362
|
+
units = units or POW_10_BYTES
|
1363
|
+
units_dict = {unit.upper(): value for (unit, value) in units}
|
1364
|
+
regex = re.compile(
|
1365
|
+
r"(\d+\.?\d*)\s*({})?".format("|".join(units_dict.keys())), re.IGNORECASE
|
1366
|
+
)
|
1367
|
+
match = re.match(regex, size)
|
1368
|
+
if not match:
|
1369
|
+
raise ValueError("size must be of the form `10`, `10B` or `10 B`.")
|
1370
|
+
factor, unit = (
|
1371
|
+
float(match.group(1)),
|
1372
|
+
units_dict[match.group(2).upper()] if match.group(2) else 1,
|
1373
|
+
)
|
1374
|
+
return int(factor * unit)
|
1375
|
+
|
1376
|
+
|
1377
|
+
def auto_project_name(program: Optional[str]) -> str:
|
1378
|
+
# if we're in git, set project name to git repo name + relative path within repo
|
1379
|
+
from wandb.sdk.lib.gitlib import GitRepo
|
1380
|
+
|
1381
|
+
root_dir = GitRepo().root_dir
|
1382
|
+
if root_dir is None:
|
1383
|
+
return "uncategorized"
|
1384
|
+
# On windows, GitRepo returns paths in unix style, but os.path is windows
|
1385
|
+
# style. Coerce here.
|
1386
|
+
root_dir = to_native_slash_path(root_dir)
|
1387
|
+
repo_name = os.path.basename(root_dir)
|
1388
|
+
if program is None:
|
1389
|
+
return str(repo_name)
|
1390
|
+
if not os.path.isabs(program):
|
1391
|
+
program = os.path.join(os.curdir, program)
|
1392
|
+
prog_dir = os.path.dirname(os.path.abspath(program))
|
1393
|
+
if not prog_dir.startswith(root_dir):
|
1394
|
+
return str(repo_name)
|
1395
|
+
project = repo_name
|
1396
|
+
sub_path = os.path.relpath(prog_dir, root_dir)
|
1397
|
+
if sub_path != ".":
|
1398
|
+
project += "-" + sub_path
|
1399
|
+
return str(project.replace(os.sep, "_"))
|
1400
|
+
|
1401
|
+
|
1402
|
+
# TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
|
1403
|
+
def to_forward_slash_path(path: str) -> str:
|
1404
|
+
if platform.system() == "Windows":
|
1405
|
+
path = path.replace("\\", "/")
|
1406
|
+
return path
|
1407
|
+
|
1408
|
+
|
1409
|
+
# TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
|
1410
|
+
def to_native_slash_path(path: str) -> FilePathStr:
|
1411
|
+
return FilePathStr(path.replace("/", os.sep))
|
1412
|
+
|
1413
|
+
|
1414
|
+
def check_and_warn_old(files: List[str]) -> bool:
|
1415
|
+
if "wandb-metadata.json" in files:
|
1416
|
+
wandb.termwarn("These runs were logged with a previous version of wandb.")
|
1417
|
+
wandb.termwarn(
|
1418
|
+
"Run pip install wandb<0.10.0 to get the old library and sync your runs."
|
1419
|
+
)
|
1420
|
+
return True
|
1421
|
+
return False
|
1422
|
+
|
1423
|
+
|
1424
|
+
class ImportMetaHook:
|
1425
|
+
def __init__(self) -> None:
|
1426
|
+
self.modules: Dict[str, ModuleType] = dict()
|
1427
|
+
self.on_import: Dict[str, list] = dict()
|
1428
|
+
|
1429
|
+
def add(self, fullname: str, on_import: Callable) -> None:
|
1430
|
+
self.on_import.setdefault(fullname, []).append(on_import)
|
1431
|
+
|
1432
|
+
def install(self) -> None:
|
1433
|
+
sys.meta_path.insert(0, self) # type: ignore
|
1434
|
+
|
1435
|
+
def uninstall(self) -> None:
|
1436
|
+
sys.meta_path.remove(self) # type: ignore
|
1437
|
+
|
1438
|
+
def find_module(
|
1439
|
+
self, fullname: str, path: Optional[str] = None
|
1440
|
+
) -> Optional["ImportMetaHook"]:
|
1441
|
+
if fullname in self.on_import:
|
1442
|
+
return self
|
1443
|
+
return None
|
1444
|
+
|
1445
|
+
def load_module(self, fullname: str) -> ModuleType:
|
1446
|
+
self.uninstall()
|
1447
|
+
mod = importlib.import_module(fullname)
|
1448
|
+
self.install()
|
1449
|
+
self.modules[fullname] = mod
|
1450
|
+
on_imports = self.on_import.get(fullname)
|
1451
|
+
if on_imports:
|
1452
|
+
for f in on_imports:
|
1453
|
+
f()
|
1454
|
+
return mod
|
1455
|
+
|
1456
|
+
def get_modules(self) -> Tuple[str, ...]:
|
1457
|
+
return tuple(self.modules)
|
1458
|
+
|
1459
|
+
def get_module(self, module: str) -> ModuleType:
|
1460
|
+
return self.modules[module]
|
1461
|
+
|
1462
|
+
|
1463
|
+
_import_hook: Optional[ImportMetaHook] = None
|
1464
|
+
|
1465
|
+
|
1466
|
+
def add_import_hook(fullname: str, on_import: Callable) -> None:
|
1467
|
+
global _import_hook
|
1468
|
+
if _import_hook is None:
|
1469
|
+
_import_hook = ImportMetaHook()
|
1470
|
+
_import_hook.install()
|
1471
|
+
_import_hook.add(fullname, on_import)
|
1472
|
+
|
1473
|
+
|
1474
|
+
def host_from_path(path: Optional[str]) -> str:
|
1475
|
+
"""Return the host of the path."""
|
1476
|
+
url = urllib.parse.urlparse(path)
|
1477
|
+
return str(url.netloc)
|
1478
|
+
|
1479
|
+
|
1480
|
+
def uri_from_path(path: Optional[str]) -> str:
|
1481
|
+
"""Return the URI of the path."""
|
1482
|
+
url = urllib.parse.urlparse(path)
|
1483
|
+
uri = url.path if url.path[0] != "/" else url.path[1:]
|
1484
|
+
return str(uri)
|
1485
|
+
|
1486
|
+
|
1487
|
+
def is_unicode_safe(stream: TextIO) -> bool:
|
1488
|
+
"""Return True if the stream supports UTF-8."""
|
1489
|
+
encoding = getattr(stream, "encoding", None)
|
1490
|
+
return encoding.lower() in {"utf-8", "utf_8"} if encoding else False
|
1491
|
+
|
1492
|
+
|
1493
|
+
def _has_internet() -> bool:
|
1494
|
+
"""Attempt to open a DNS connection to Googles root servers."""
|
1495
|
+
try:
|
1496
|
+
s = socket.create_connection(("8.8.8.8", 53), 0.5)
|
1497
|
+
s.close()
|
1498
|
+
return True
|
1499
|
+
except OSError:
|
1500
|
+
return False
|
1501
|
+
|
1502
|
+
|
1503
|
+
def rand_alphanumeric(
|
1504
|
+
length: int = 8, rand: Optional[Union[ModuleType, random.Random]] = None
|
1505
|
+
) -> str:
|
1506
|
+
wandb.termerror("rand_alphanumeric is deprecated, use 'secrets.token_hex'")
|
1507
|
+
rand = rand or random
|
1508
|
+
return "".join(rand.choice("0123456789ABCDEF") for _ in range(length))
|
1509
|
+
|
1510
|
+
|
1511
|
+
@contextlib.contextmanager
|
1512
|
+
def fsync_open(
|
1513
|
+
path: StrPath, mode: str = "w", encoding: Optional[str] = None
|
1514
|
+
) -> Generator[IO[Any], None, None]:
|
1515
|
+
"""Open a path for I/O and guarantee that the file is flushed and synced."""
|
1516
|
+
with open(path, mode, encoding=encoding) as f:
|
1517
|
+
yield f
|
1518
|
+
|
1519
|
+
f.flush()
|
1520
|
+
os.fsync(f.fileno())
|
1521
|
+
|
1522
|
+
|
1523
|
+
def _is_kaggle() -> bool:
|
1524
|
+
return (
|
1525
|
+
os.getenv("KAGGLE_KERNEL_RUN_TYPE") is not None
|
1526
|
+
or "kaggle_environments" in sys.modules
|
1527
|
+
)
|
1528
|
+
|
1529
|
+
|
1530
|
+
def _is_likely_kaggle() -> bool:
|
1531
|
+
# Telemetry to mark first runs from Kagglers.
|
1532
|
+
return (
|
1533
|
+
_is_kaggle()
|
1534
|
+
or os.path.exists(
|
1535
|
+
os.path.expanduser(os.path.join("~", ".kaggle", "kaggle.json"))
|
1536
|
+
)
|
1537
|
+
or "kaggle" in sys.modules
|
1538
|
+
)
|
1539
|
+
|
1540
|
+
|
1541
|
+
def _is_databricks() -> bool:
|
1542
|
+
# check if we are running inside a databricks notebook by
|
1543
|
+
# inspecting sys.modules, searching for dbutils and verifying that
|
1544
|
+
# it has the appropriate structure
|
1545
|
+
|
1546
|
+
if "dbutils" in sys.modules:
|
1547
|
+
dbutils = sys.modules["dbutils"]
|
1548
|
+
if hasattr(dbutils, "shell"):
|
1549
|
+
shell = dbutils.shell
|
1550
|
+
if hasattr(shell, "sc"):
|
1551
|
+
sc = shell.sc
|
1552
|
+
if hasattr(sc, "appName"):
|
1553
|
+
return bool(sc.appName == "Databricks Shell")
|
1554
|
+
return False
|
1555
|
+
|
1556
|
+
|
1557
|
+
def _is_py_requirements_or_dockerfile(path: str) -> bool:
|
1558
|
+
file = os.path.basename(path)
|
1559
|
+
return (
|
1560
|
+
file.endswith(".py")
|
1561
|
+
or file.startswith("Dockerfile")
|
1562
|
+
or file == "requirements.txt"
|
1563
|
+
)
|
1564
|
+
|
1565
|
+
|
1566
|
+
def check_windows_valid_filename(path: Union[int, str]) -> bool:
|
1567
|
+
return not bool(re.search(RE_WINFNAMES, path)) # type: ignore
|
1568
|
+
|
1569
|
+
|
1570
|
+
def artifact_to_json(artifact: "Artifact") -> Dict[str, Any]:
|
1571
|
+
return {
|
1572
|
+
"_type": "artifactVersion",
|
1573
|
+
"_version": "v0",
|
1574
|
+
"id": artifact.id,
|
1575
|
+
"version": artifact.source_version,
|
1576
|
+
"sequenceName": artifact.source_name.split(":")[0],
|
1577
|
+
"usedAs": artifact.use_as,
|
1578
|
+
}
|
1579
|
+
|
1580
|
+
|
1581
|
+
def check_dict_contains_nested_artifact(d: dict, nested: bool = False) -> bool:
|
1582
|
+
for item in d.values():
|
1583
|
+
if isinstance(item, dict):
|
1584
|
+
contains_artifacts = check_dict_contains_nested_artifact(item, True)
|
1585
|
+
if contains_artifacts:
|
1586
|
+
return True
|
1587
|
+
elif (isinstance(item, wandb.Artifact) or _is_artifact_string(item)) and nested:
|
1588
|
+
return True
|
1589
|
+
return False
|
1590
|
+
|
1591
|
+
|
1592
|
+
def load_json_yaml_dict(config: str) -> Any:
|
1593
|
+
ext = os.path.splitext(config)[-1]
|
1594
|
+
if ext == ".json":
|
1595
|
+
with open(config) as f:
|
1596
|
+
return json.load(f)
|
1597
|
+
elif ext == ".yaml":
|
1598
|
+
with open(config) as f:
|
1599
|
+
return yaml.safe_load(f)
|
1600
|
+
else:
|
1601
|
+
try:
|
1602
|
+
return json.loads(config)
|
1603
|
+
except ValueError:
|
1604
|
+
return None
|
1605
|
+
|
1606
|
+
|
1607
|
+
def _parse_entity_project_item(path: str) -> tuple:
|
1608
|
+
"""Parse paths with the following formats: {item}, {project}/{item}, & {entity}/{project}/{item}.
|
1609
|
+
|
1610
|
+
Args:
|
1611
|
+
path: `str`, input path; must be between 0 and 3 in length.
|
1612
|
+
|
1613
|
+
Returns:
|
1614
|
+
tuple of length 3 - (item, project, entity)
|
1615
|
+
|
1616
|
+
Example:
|
1617
|
+
alias, project, entity = _parse_entity_project_item("myproj/mymodel:best")
|
1618
|
+
|
1619
|
+
assert entity == ""
|
1620
|
+
assert project == "myproj"
|
1621
|
+
assert alias == "mymodel:best"
|
1622
|
+
|
1623
|
+
"""
|
1624
|
+
words = path.split("/")
|
1625
|
+
if len(words) > 3:
|
1626
|
+
raise ValueError(
|
1627
|
+
"Invalid path: must be str the form {item}, {project}/{item}, or {entity}/{project}/{item}"
|
1628
|
+
)
|
1629
|
+
padded_words = [""] * (3 - len(words)) + words
|
1630
|
+
return tuple(reversed(padded_words))
|
1631
|
+
|
1632
|
+
|
1633
|
+
def _resolve_aliases(aliases: Optional[Union[str, Iterable[str]]]) -> List[str]:
|
1634
|
+
"""Add the 'latest' alias and ensure that all aliases are unique.
|
1635
|
+
|
1636
|
+
Takes in `aliases` which can be None, str, or List[str] and returns List[str].
|
1637
|
+
Ensures that "latest" is always present in the returned list.
|
1638
|
+
|
1639
|
+
Args:
|
1640
|
+
aliases: `Optional[Union[str, List[str]]]`
|
1641
|
+
|
1642
|
+
Returns:
|
1643
|
+
List[str], with "latest" always present.
|
1644
|
+
|
1645
|
+
Usage:
|
1646
|
+
|
1647
|
+
```python
|
1648
|
+
aliases = _resolve_aliases(["best", "dev"])
|
1649
|
+
assert aliases == ["best", "dev", "latest"]
|
1650
|
+
|
1651
|
+
aliases = _resolve_aliases("boom")
|
1652
|
+
assert aliases == ["boom", "latest"]
|
1653
|
+
```
|
1654
|
+
"""
|
1655
|
+
aliases = aliases or ["latest"]
|
1656
|
+
|
1657
|
+
if isinstance(aliases, str):
|
1658
|
+
aliases = [aliases]
|
1659
|
+
|
1660
|
+
try:
|
1661
|
+
return list(set(aliases) | {"latest"})
|
1662
|
+
except TypeError as exc:
|
1663
|
+
raise ValueError("`aliases` must be Iterable or None") from exc
|
1664
|
+
|
1665
|
+
|
1666
|
+
def _is_artifact_object(v: Any) -> bool:
|
1667
|
+
return isinstance(v, wandb.Artifact)
|
1668
|
+
|
1669
|
+
|
1670
|
+
def _is_artifact_string(v: Any) -> bool:
|
1671
|
+
return isinstance(v, str) and v.startswith("wandb-artifact://")
|
1672
|
+
|
1673
|
+
|
1674
|
+
def _is_artifact_version_weave_dict(v: Any) -> bool:
|
1675
|
+
return isinstance(v, dict) and v.get("_type") == "artifactVersion"
|
1676
|
+
|
1677
|
+
|
1678
|
+
def _is_artifact_representation(v: Any) -> bool:
|
1679
|
+
return (
|
1680
|
+
_is_artifact_object(v)
|
1681
|
+
or _is_artifact_string(v)
|
1682
|
+
or _is_artifact_version_weave_dict(v)
|
1683
|
+
)
|
1684
|
+
|
1685
|
+
|
1686
|
+
def parse_artifact_string(v: str) -> Tuple[str, Optional[str], bool]:
|
1687
|
+
if not v.startswith("wandb-artifact://"):
|
1688
|
+
raise ValueError(f"Invalid artifact string: {v}")
|
1689
|
+
parsed_v = v[len("wandb-artifact://") :]
|
1690
|
+
base_uri = None
|
1691
|
+
url_info = urllib.parse.urlparse(parsed_v)
|
1692
|
+
if url_info.scheme != "":
|
1693
|
+
base_uri = f"{url_info.scheme}://{url_info.netloc}"
|
1694
|
+
parts = url_info.path.split("/")[1:]
|
1695
|
+
else:
|
1696
|
+
parts = parsed_v.split("/")
|
1697
|
+
if parts[0] == "_id":
|
1698
|
+
# for now can't fetch paths but this will be supported in the future
|
1699
|
+
# when we allow passing typed media objects, this can be extended
|
1700
|
+
# to include paths
|
1701
|
+
return parts[1], base_uri, True
|
1702
|
+
|
1703
|
+
if len(parts) < 3:
|
1704
|
+
raise ValueError(f"Invalid artifact string: {v}")
|
1705
|
+
|
1706
|
+
# for now can't fetch paths but this will be supported in the future
|
1707
|
+
# when we allow passing typed media objects, this can be extended
|
1708
|
+
# to include paths
|
1709
|
+
entity, project, name_and_alias_or_version = parts[:3]
|
1710
|
+
return f"{entity}/{project}/{name_and_alias_or_version}", base_uri, False
|
1711
|
+
|
1712
|
+
|
1713
|
+
def _get_max_cli_version() -> Union[str, None]:
|
1714
|
+
max_cli_version = wandb.api.max_cli_version()
|
1715
|
+
return str(max_cli_version) if max_cli_version is not None else None
|
1716
|
+
|
1717
|
+
|
1718
|
+
def _is_offline() -> bool:
|
1719
|
+
return ( # type: ignore[no-any-return]
|
1720
|
+
wandb.run is not None and wandb.run.settings._offline
|
1721
|
+
) or wandb.setup().settings._offline # type: ignore
|
1722
|
+
|
1723
|
+
|
1724
|
+
def ensure_text(
|
1725
|
+
string: Union[str, bytes], encoding: str = "utf-8", errors: str = "strict"
|
1726
|
+
) -> str:
|
1727
|
+
"""Coerce s to str."""
|
1728
|
+
if isinstance(string, bytes):
|
1729
|
+
return string.decode(encoding, errors)
|
1730
|
+
elif isinstance(string, str):
|
1731
|
+
return string
|
1732
|
+
else:
|
1733
|
+
raise TypeError(f"not expecting type {type(string)!r}")
|
1734
|
+
|
1735
|
+
|
1736
|
+
def make_artifact_name_safe(name: str) -> str:
|
1737
|
+
"""Make an artifact name safe for use in artifacts."""
|
1738
|
+
# artifact names may only contain alphanumeric characters, dashes, underscores, and dots.
|
1739
|
+
cleaned = re.sub(r"[^a-zA-Z0-9_\-.]", "_", name)
|
1740
|
+
if len(cleaned) <= 128:
|
1741
|
+
return cleaned
|
1742
|
+
# truncate with dots in the middle using regex
|
1743
|
+
return re.sub(r"(^.{63}).*(.{63}$)", r"\g<1>..\g<2>", cleaned)
|
1744
|
+
|
1745
|
+
|
1746
|
+
def make_docker_image_name_safe(name: str) -> str:
|
1747
|
+
"""Make a docker image name safe for use in artifacts."""
|
1748
|
+
safe_chars = RE_DOCKER_IMAGE_NAME_CHARS.sub("__", name.lower())
|
1749
|
+
deduped = RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT.sub("__", safe_chars)
|
1750
|
+
trimmed_start = RE_DOCKER_IMAGE_NAME_SEPARATOR_START.sub("", deduped)
|
1751
|
+
trimmed = RE_DOCKER_IMAGE_NAME_SEPARATOR_END.sub("", trimmed_start)
|
1752
|
+
return trimmed if trimmed else "image"
|
1753
|
+
|
1754
|
+
|
1755
|
+
def merge_dicts(
|
1756
|
+
source: Dict[str, Any],
|
1757
|
+
destination: Dict[str, Any],
|
1758
|
+
) -> Dict[str, Any]:
|
1759
|
+
"""Recursively merge two dictionaries.
|
1760
|
+
|
1761
|
+
This mutates the destination and its nested dictionaries and lists.
|
1762
|
+
|
1763
|
+
Instances of `dict` are recursively merged and instances of `list`
|
1764
|
+
are appended to the destination. If the destination type is not
|
1765
|
+
`dict` or `list`, respectively, the key is overwritten with the
|
1766
|
+
source value.
|
1767
|
+
|
1768
|
+
For all other types, the source value overwrites the destination value.
|
1769
|
+
"""
|
1770
|
+
for key, value in source.items():
|
1771
|
+
if isinstance(value, dict):
|
1772
|
+
node = destination.get(key)
|
1773
|
+
if isinstance(node, dict):
|
1774
|
+
merge_dicts(value, node)
|
1775
|
+
else:
|
1776
|
+
destination[key] = value
|
1777
|
+
|
1778
|
+
elif isinstance(value, list):
|
1779
|
+
dest_value = destination.get(key)
|
1780
|
+
if isinstance(dest_value, list):
|
1781
|
+
dest_value.extend(value)
|
1782
|
+
else:
|
1783
|
+
destination[key] = value
|
1784
|
+
|
1785
|
+
else:
|
1786
|
+
destination[key] = value
|
1787
|
+
|
1788
|
+
return destination
|
1789
|
+
|
1790
|
+
|
1791
|
+
def coalesce(*arg: Any) -> Any:
|
1792
|
+
"""Return the first non-none value in the list of arguments.
|
1793
|
+
|
1794
|
+
Similar to ?? in C#.
|
1795
|
+
"""
|
1796
|
+
return next((a for a in arg if a is not None), None)
|
1797
|
+
|
1798
|
+
|
1799
|
+
def recursive_cast_dictlike_to_dict(d: Dict[str, Any]) -> Dict[str, Any]:
|
1800
|
+
for k, v in d.items():
|
1801
|
+
if isinstance(v, dict):
|
1802
|
+
recursive_cast_dictlike_to_dict(v)
|
1803
|
+
elif hasattr(v, "keys"):
|
1804
|
+
d[k] = dict(v)
|
1805
|
+
recursive_cast_dictlike_to_dict(d[k])
|
1806
|
+
return d
|
1807
|
+
|
1808
|
+
|
1809
|
+
def remove_keys_with_none_values(
|
1810
|
+
d: Union[Dict[str, Any], Any],
|
1811
|
+
) -> Union[Dict[str, Any], Any]:
|
1812
|
+
# otherwise iterrows will create a bunch of ugly charts
|
1813
|
+
if not isinstance(d, dict):
|
1814
|
+
return d
|
1815
|
+
|
1816
|
+
if isinstance(d, dict):
|
1817
|
+
new_dict = {}
|
1818
|
+
for k, v in d.items():
|
1819
|
+
new_v = remove_keys_with_none_values(v)
|
1820
|
+
if new_v is not None and not (isinstance(new_v, dict) and len(new_v) == 0):
|
1821
|
+
new_dict[k] = new_v
|
1822
|
+
return new_dict if new_dict else None
|
1823
|
+
|
1824
|
+
|
1825
|
+
def batched(n: int, iterable: Iterable[T]) -> Generator[List[T], None, None]:
|
1826
|
+
i = iter(iterable)
|
1827
|
+
batch = list(itertools.islice(i, n))
|
1828
|
+
while batch:
|
1829
|
+
yield batch
|
1830
|
+
batch = list(itertools.islice(i, n))
|
1831
|
+
|
1832
|
+
|
1833
|
+
def random_string(length: int = 12) -> str:
|
1834
|
+
"""Generate a random string of a given length.
|
1835
|
+
|
1836
|
+
:param length: Length of the string to generate.
|
1837
|
+
:return: Random string.
|
1838
|
+
"""
|
1839
|
+
return "".join(
|
1840
|
+
secrets.choice(string.ascii_lowercase + string.digits) for _ in range(length)
|
1841
|
+
)
|
1842
|
+
|
1843
|
+
|
1844
|
+
def sample_with_exponential_decay_weights(
|
1845
|
+
xs: Union[Iterable, Iterable[Iterable]],
|
1846
|
+
ys: Iterable[Iterable],
|
1847
|
+
keys: Optional[Iterable] = None,
|
1848
|
+
sample_size: int = 1500,
|
1849
|
+
) -> Tuple[List, List, Optional[List]]:
|
1850
|
+
"""Sample from a list of lists with weights that decay exponentially.
|
1851
|
+
|
1852
|
+
May be used with the wandb.plot.line_series function.
|
1853
|
+
"""
|
1854
|
+
xs_array = np.array(xs)
|
1855
|
+
ys_array = np.array(ys)
|
1856
|
+
keys_array = np.array(keys) if keys else None
|
1857
|
+
weights = np.exp(-np.arange(len(xs_array)) / len(xs_array))
|
1858
|
+
weights /= np.sum(weights)
|
1859
|
+
sampled_indices = np.random.choice(len(xs_array), size=sample_size, p=weights)
|
1860
|
+
sampled_xs = xs_array[sampled_indices].tolist()
|
1861
|
+
sampled_ys = ys_array[sampled_indices].tolist()
|
1862
|
+
sampled_keys = keys_array[sampled_indices].tolist() if keys else None
|
1863
|
+
|
1864
|
+
return sampled_xs, sampled_ys, sampled_keys
|
1865
|
+
|
1866
|
+
|
1867
|
+
@dataclasses.dataclass(frozen=True)
|
1868
|
+
class InstalledDistribution:
|
1869
|
+
"""An installed distribution.
|
1870
|
+
|
1871
|
+
Attributes:
|
1872
|
+
key: The distribution name as it would be imported.
|
1873
|
+
version: The distribution's version string.
|
1874
|
+
"""
|
1875
|
+
|
1876
|
+
key: str
|
1877
|
+
version: str
|
1878
|
+
|
1879
|
+
|
1880
|
+
def working_set() -> Iterable[InstalledDistribution]:
|
1881
|
+
"""Return the working set of installed distributions.
|
1882
|
+
|
1883
|
+
Uses importlib.metadata in Python versions above 3.7, and importlib_metadata otherwise.
|
1884
|
+
"""
|
1885
|
+
try:
|
1886
|
+
from importlib.metadata import distributions
|
1887
|
+
except ImportError:
|
1888
|
+
from importlib_metadata import distributions # type: ignore
|
1889
|
+
|
1890
|
+
for d in distributions():
|
1891
|
+
try:
|
1892
|
+
# In some distributions, the "Name" attribute may not be present,
|
1893
|
+
# which can raise a KeyError. To handle this, we catch the exception
|
1894
|
+
# and skip those distributions.
|
1895
|
+
# For additional context, see: https://github.com/python/importlib_metadata/issues/371.
|
1896
|
+
yield InstalledDistribution(key=d.metadata["Name"], version=d.version)
|
1897
|
+
except KeyError:
|
1898
|
+
pass
|
1899
|
+
|
1900
|
+
|
1901
|
+
def parse_version(version: str) -> "packaging.version.Version":
|
1902
|
+
"""Parse a version string into a version object.
|
1903
|
+
|
1904
|
+
This function is a wrapper around the `packaging.version.parse` function, which
|
1905
|
+
is used to parse version strings into version objects. If the `packaging` library
|
1906
|
+
is not installed, it falls back to the `pkg_resources` library.
|
1907
|
+
"""
|
1908
|
+
try:
|
1909
|
+
from packaging.version import parse as parse_version # type: ignore
|
1910
|
+
except ImportError:
|
1911
|
+
from pkg_resources import parse_version # type: ignore[assignment]
|
1912
|
+
|
1913
|
+
return parse_version(version)
|
1914
|
+
|
1915
|
+
|
1916
|
+
def get_core_path() -> str:
|
1917
|
+
"""Returns the path to the wandb-core binary.
|
1918
|
+
|
1919
|
+
The path can be set explicitly via the _WANDB_CORE_PATH environment
|
1920
|
+
variable. Otherwise, the path to the binary in the current package
|
1921
|
+
is returned.
|
1922
|
+
|
1923
|
+
Returns:
|
1924
|
+
str: The path to the wandb-core package.
|
1925
|
+
|
1926
|
+
Raises:
|
1927
|
+
WandbCoreNotAvailableError: If wandb-core was not built for the current system.
|
1928
|
+
"""
|
1929
|
+
# NOTE: Environment variable _WANDB_CORE_PATH is a temporary development feature
|
1930
|
+
# to assist in running the core service from a live development directory.
|
1931
|
+
path_from_env: str = os.environ.get("_WANDB_CORE_PATH", "")
|
1932
|
+
if path_from_env:
|
1933
|
+
wandb.termwarn(
|
1934
|
+
f"Using wandb-core from path `_WANDB_CORE_PATH={path_from_env}`. "
|
1935
|
+
"This is a development feature and may not work as expected."
|
1936
|
+
)
|
1937
|
+
return path_from_env
|
1938
|
+
|
1939
|
+
bin_path = pathlib.Path(__file__).parent / "bin" / "wandb-core"
|
1940
|
+
if not bin_path.exists():
|
1941
|
+
raise WandbCoreNotAvailableError(
|
1942
|
+
f"Looks like wandb-core is not compiled for your system ({platform.platform()}):"
|
1943
|
+
" Please contact support at support@wandb.com to request `wandb-core`"
|
1944
|
+
" support for your system."
|
1945
|
+
)
|
1946
|
+
|
1947
|
+
return str(bin_path)
|
1948
|
+
|
1949
|
+
|
1950
|
+
class NonOctalStringDumper(yaml.Dumper):
|
1951
|
+
"""Prevents strings containing non-octal values like "008" and "009" from being converted to numbers in in the yaml string saved as the sweep config."""
|
1952
|
+
|
1953
|
+
def represent_scalar(self, tag, value, style=None):
|
1954
|
+
if tag == "tag:yaml.org,2002:str" and value.startswith("0") and len(value) > 1:
|
1955
|
+
return super().represent_scalar(tag, value, style="'")
|
1956
|
+
return super().represent_scalar(tag, value, style)
|