wandb 0.18.2__py3-none-musllinux_1_2_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. package_readme.md +89 -0
  2. wandb/__init__.py +245 -0
  3. wandb/__init__.pyi +1139 -0
  4. wandb/__main__.py +3 -0
  5. wandb/_globals.py +19 -0
  6. wandb/agents/__init__.py +0 -0
  7. wandb/agents/pyagent.py +363 -0
  8. wandb/analytics/__init__.py +3 -0
  9. wandb/analytics/sentry.py +266 -0
  10. wandb/apis/__init__.py +48 -0
  11. wandb/apis/attrs.py +40 -0
  12. wandb/apis/importers/__init__.py +1 -0
  13. wandb/apis/importers/internals/internal.py +385 -0
  14. wandb/apis/importers/internals/protocols.py +99 -0
  15. wandb/apis/importers/internals/util.py +78 -0
  16. wandb/apis/importers/mlflow.py +254 -0
  17. wandb/apis/importers/validation.py +108 -0
  18. wandb/apis/importers/wandb.py +1603 -0
  19. wandb/apis/internal.py +232 -0
  20. wandb/apis/normalize.py +89 -0
  21. wandb/apis/paginator.py +81 -0
  22. wandb/apis/public/__init__.py +34 -0
  23. wandb/apis/public/api.py +1305 -0
  24. wandb/apis/public/artifacts.py +1090 -0
  25. wandb/apis/public/const.py +4 -0
  26. wandb/apis/public/files.py +195 -0
  27. wandb/apis/public/history.py +149 -0
  28. wandb/apis/public/jobs.py +659 -0
  29. wandb/apis/public/projects.py +154 -0
  30. wandb/apis/public/query_generator.py +166 -0
  31. wandb/apis/public/reports.py +469 -0
  32. wandb/apis/public/runs.py +914 -0
  33. wandb/apis/public/sweeps.py +240 -0
  34. wandb/apis/public/teams.py +198 -0
  35. wandb/apis/public/users.py +136 -0
  36. wandb/apis/reports/__init__.py +1 -0
  37. wandb/apis/reports/v1/__init__.py +8 -0
  38. wandb/apis/reports/v2/__init__.py +8 -0
  39. wandb/apis/workspaces/__init__.py +8 -0
  40. wandb/beta/workflows.py +288 -0
  41. wandb/bin/nvidia_gpu_stats +0 -0
  42. wandb/bin/wandb-core +0 -0
  43. wandb/cli/__init__.py +0 -0
  44. wandb/cli/cli.py +3004 -0
  45. wandb/data_types.py +63 -0
  46. wandb/docker/__init__.py +342 -0
  47. wandb/docker/auth.py +436 -0
  48. wandb/docker/wandb-entrypoint.sh +33 -0
  49. wandb/docker/www_authenticate.py +94 -0
  50. wandb/env.py +514 -0
  51. wandb/errors/__init__.py +17 -0
  52. wandb/errors/errors.py +37 -0
  53. wandb/errors/term.py +103 -0
  54. wandb/errors/util.py +57 -0
  55. wandb/errors/warnings.py +2 -0
  56. wandb/filesync/__init__.py +0 -0
  57. wandb/filesync/dir_watcher.py +403 -0
  58. wandb/filesync/stats.py +100 -0
  59. wandb/filesync/step_checksum.py +142 -0
  60. wandb/filesync/step_prepare.py +179 -0
  61. wandb/filesync/step_upload.py +290 -0
  62. wandb/filesync/upload_job.py +142 -0
  63. wandb/integration/__init__.py +0 -0
  64. wandb/integration/catboost/__init__.py +5 -0
  65. wandb/integration/catboost/catboost.py +178 -0
  66. wandb/integration/cohere/__init__.py +3 -0
  67. wandb/integration/cohere/cohere.py +21 -0
  68. wandb/integration/cohere/resolver.py +347 -0
  69. wandb/integration/diffusers/__init__.py +3 -0
  70. wandb/integration/diffusers/autologger.py +76 -0
  71. wandb/integration/diffusers/pipeline_resolver.py +50 -0
  72. wandb/integration/diffusers/resolvers/__init__.py +9 -0
  73. wandb/integration/diffusers/resolvers/multimodal.py +882 -0
  74. wandb/integration/diffusers/resolvers/utils.py +102 -0
  75. wandb/integration/fastai/__init__.py +249 -0
  76. wandb/integration/gym/__init__.py +105 -0
  77. wandb/integration/huggingface/__init__.py +3 -0
  78. wandb/integration/huggingface/huggingface.py +18 -0
  79. wandb/integration/huggingface/resolver.py +213 -0
  80. wandb/integration/keras/__init__.py +11 -0
  81. wandb/integration/keras/callbacks/__init__.py +5 -0
  82. wandb/integration/keras/callbacks/metrics_logger.py +136 -0
  83. wandb/integration/keras/callbacks/model_checkpoint.py +195 -0
  84. wandb/integration/keras/callbacks/tables_builder.py +226 -0
  85. wandb/integration/keras/keras.py +1091 -0
  86. wandb/integration/kfp/__init__.py +6 -0
  87. wandb/integration/kfp/helpers.py +28 -0
  88. wandb/integration/kfp/kfp_patch.py +324 -0
  89. wandb/integration/kfp/wandb_logging.py +182 -0
  90. wandb/integration/langchain/__init__.py +3 -0
  91. wandb/integration/langchain/wandb_tracer.py +48 -0
  92. wandb/integration/lightgbm/__init__.py +239 -0
  93. wandb/integration/lightning/__init__.py +0 -0
  94. wandb/integration/lightning/fabric/__init__.py +3 -0
  95. wandb/integration/lightning/fabric/logger.py +762 -0
  96. wandb/integration/magic.py +556 -0
  97. wandb/integration/metaflow/__init__.py +3 -0
  98. wandb/integration/metaflow/metaflow.py +383 -0
  99. wandb/integration/openai/__init__.py +3 -0
  100. wandb/integration/openai/fine_tuning.py +480 -0
  101. wandb/integration/openai/openai.py +22 -0
  102. wandb/integration/openai/resolver.py +240 -0
  103. wandb/integration/prodigy/__init__.py +3 -0
  104. wandb/integration/prodigy/prodigy.py +299 -0
  105. wandb/integration/sacred/__init__.py +117 -0
  106. wandb/integration/sagemaker/__init__.py +12 -0
  107. wandb/integration/sagemaker/auth.py +28 -0
  108. wandb/integration/sagemaker/config.py +49 -0
  109. wandb/integration/sagemaker/files.py +3 -0
  110. wandb/integration/sagemaker/resources.py +34 -0
  111. wandb/integration/sb3/__init__.py +3 -0
  112. wandb/integration/sb3/sb3.py +153 -0
  113. wandb/integration/sklearn/__init__.py +37 -0
  114. wandb/integration/sklearn/calculate/__init__.py +32 -0
  115. wandb/integration/sklearn/calculate/calibration_curves.py +125 -0
  116. wandb/integration/sklearn/calculate/class_proportions.py +68 -0
  117. wandb/integration/sklearn/calculate/confusion_matrix.py +93 -0
  118. wandb/integration/sklearn/calculate/decision_boundaries.py +40 -0
  119. wandb/integration/sklearn/calculate/elbow_curve.py +55 -0
  120. wandb/integration/sklearn/calculate/feature_importances.py +67 -0
  121. wandb/integration/sklearn/calculate/learning_curve.py +64 -0
  122. wandb/integration/sklearn/calculate/outlier_candidates.py +69 -0
  123. wandb/integration/sklearn/calculate/residuals.py +86 -0
  124. wandb/integration/sklearn/calculate/silhouette.py +118 -0
  125. wandb/integration/sklearn/calculate/summary_metrics.py +62 -0
  126. wandb/integration/sklearn/plot/__init__.py +35 -0
  127. wandb/integration/sklearn/plot/classifier.py +329 -0
  128. wandb/integration/sklearn/plot/clusterer.py +146 -0
  129. wandb/integration/sklearn/plot/regressor.py +121 -0
  130. wandb/integration/sklearn/plot/shared.py +91 -0
  131. wandb/integration/sklearn/utils.py +183 -0
  132. wandb/integration/tensorboard/__init__.py +10 -0
  133. wandb/integration/tensorboard/log.py +355 -0
  134. wandb/integration/tensorboard/monkeypatch.py +185 -0
  135. wandb/integration/tensorflow/__init__.py +5 -0
  136. wandb/integration/tensorflow/estimator_hook.py +54 -0
  137. wandb/integration/torch/__init__.py +0 -0
  138. wandb/integration/torch/wandb_torch.py +554 -0
  139. wandb/integration/ultralytics/__init__.py +11 -0
  140. wandb/integration/ultralytics/bbox_utils.py +208 -0
  141. wandb/integration/ultralytics/callback.py +524 -0
  142. wandb/integration/ultralytics/classification_utils.py +83 -0
  143. wandb/integration/ultralytics/mask_utils.py +202 -0
  144. wandb/integration/ultralytics/pose_utils.py +103 -0
  145. wandb/integration/xgboost/__init__.py +11 -0
  146. wandb/integration/xgboost/xgboost.py +189 -0
  147. wandb/integration/yolov8/__init__.py +0 -0
  148. wandb/integration/yolov8/yolov8.py +284 -0
  149. wandb/jupyter.py +515 -0
  150. wandb/magic.py +3 -0
  151. wandb/mpmain/__init__.py +0 -0
  152. wandb/mpmain/__main__.py +1 -0
  153. wandb/old/__init__.py +0 -0
  154. wandb/old/core.py +53 -0
  155. wandb/old/settings.py +173 -0
  156. wandb/old/summary.py +440 -0
  157. wandb/plot/__init__.py +19 -0
  158. wandb/plot/bar.py +45 -0
  159. wandb/plot/confusion_matrix.py +100 -0
  160. wandb/plot/histogram.py +39 -0
  161. wandb/plot/line.py +43 -0
  162. wandb/plot/line_series.py +88 -0
  163. wandb/plot/pr_curve.py +136 -0
  164. wandb/plot/roc_curve.py +118 -0
  165. wandb/plot/scatter.py +32 -0
  166. wandb/plot/utils.py +183 -0
  167. wandb/plot/viz.py +123 -0
  168. wandb/proto/__init__.py +0 -0
  169. wandb/proto/v3/__init__.py +0 -0
  170. wandb/proto/v3/wandb_base_pb2.py +55 -0
  171. wandb/proto/v3/wandb_internal_pb2.py +1608 -0
  172. wandb/proto/v3/wandb_server_pb2.py +208 -0
  173. wandb/proto/v3/wandb_settings_pb2.py +112 -0
  174. wandb/proto/v3/wandb_telemetry_pb2.py +106 -0
  175. wandb/proto/v4/__init__.py +0 -0
  176. wandb/proto/v4/wandb_base_pb2.py +30 -0
  177. wandb/proto/v4/wandb_internal_pb2.py +360 -0
  178. wandb/proto/v4/wandb_server_pb2.py +63 -0
  179. wandb/proto/v4/wandb_settings_pb2.py +45 -0
  180. wandb/proto/v4/wandb_telemetry_pb2.py +41 -0
  181. wandb/proto/v5/wandb_base_pb2.py +31 -0
  182. wandb/proto/v5/wandb_internal_pb2.py +361 -0
  183. wandb/proto/v5/wandb_server_pb2.py +64 -0
  184. wandb/proto/v5/wandb_settings_pb2.py +46 -0
  185. wandb/proto/v5/wandb_telemetry_pb2.py +42 -0
  186. wandb/proto/wandb_base_pb2.py +10 -0
  187. wandb/proto/wandb_deprecated.py +53 -0
  188. wandb/proto/wandb_generate_deprecated.py +34 -0
  189. wandb/proto/wandb_generate_proto.py +49 -0
  190. wandb/proto/wandb_internal_pb2.py +16 -0
  191. wandb/proto/wandb_server_pb2.py +10 -0
  192. wandb/proto/wandb_settings_pb2.py +10 -0
  193. wandb/proto/wandb_telemetry_pb2.py +10 -0
  194. wandb/py.typed +0 -0
  195. wandb/sdk/__init__.py +37 -0
  196. wandb/sdk/artifacts/__init__.py +0 -0
  197. wandb/sdk/artifacts/_validators.py +90 -0
  198. wandb/sdk/artifacts/artifact.py +2389 -0
  199. wandb/sdk/artifacts/artifact_download_logger.py +43 -0
  200. wandb/sdk/artifacts/artifact_file_cache.py +253 -0
  201. wandb/sdk/artifacts/artifact_instance_cache.py +17 -0
  202. wandb/sdk/artifacts/artifact_manifest.py +74 -0
  203. wandb/sdk/artifacts/artifact_manifest_entry.py +249 -0
  204. wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
  205. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +92 -0
  206. wandb/sdk/artifacts/artifact_saver.py +269 -0
  207. wandb/sdk/artifacts/artifact_state.py +11 -0
  208. wandb/sdk/artifacts/artifact_ttl.py +7 -0
  209. wandb/sdk/artifacts/exceptions.py +57 -0
  210. wandb/sdk/artifacts/staging.py +25 -0
  211. wandb/sdk/artifacts/storage_handler.py +62 -0
  212. wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
  213. wandb/sdk/artifacts/storage_handlers/azure_handler.py +208 -0
  214. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +228 -0
  215. wandb/sdk/artifacts/storage_handlers/http_handler.py +114 -0
  216. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +141 -0
  217. wandb/sdk/artifacts/storage_handlers/multi_handler.py +56 -0
  218. wandb/sdk/artifacts/storage_handlers/s3_handler.py +300 -0
  219. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +72 -0
  220. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +135 -0
  221. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +74 -0
  222. wandb/sdk/artifacts/storage_layout.py +6 -0
  223. wandb/sdk/artifacts/storage_policies/__init__.py +4 -0
  224. wandb/sdk/artifacts/storage_policies/register.py +1 -0
  225. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +378 -0
  226. wandb/sdk/artifacts/storage_policy.py +72 -0
  227. wandb/sdk/backend/__init__.py +0 -0
  228. wandb/sdk/backend/backend.py +222 -0
  229. wandb/sdk/data_types/__init__.py +0 -0
  230. wandb/sdk/data_types/_dtypes.py +914 -0
  231. wandb/sdk/data_types/_private.py +10 -0
  232. wandb/sdk/data_types/audio.py +165 -0
  233. wandb/sdk/data_types/base_types/__init__.py +0 -0
  234. wandb/sdk/data_types/base_types/json_metadata.py +55 -0
  235. wandb/sdk/data_types/base_types/media.py +315 -0
  236. wandb/sdk/data_types/base_types/wb_value.py +272 -0
  237. wandb/sdk/data_types/bokeh.py +70 -0
  238. wandb/sdk/data_types/graph.py +405 -0
  239. wandb/sdk/data_types/helper_types/__init__.py +0 -0
  240. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +295 -0
  241. wandb/sdk/data_types/helper_types/classes.py +159 -0
  242. wandb/sdk/data_types/helper_types/image_mask.py +235 -0
  243. wandb/sdk/data_types/histogram.py +96 -0
  244. wandb/sdk/data_types/html.py +115 -0
  245. wandb/sdk/data_types/image.py +845 -0
  246. wandb/sdk/data_types/molecule.py +241 -0
  247. wandb/sdk/data_types/object_3d.py +474 -0
  248. wandb/sdk/data_types/plotly.py +82 -0
  249. wandb/sdk/data_types/saved_model.py +446 -0
  250. wandb/sdk/data_types/table.py +1204 -0
  251. wandb/sdk/data_types/trace_tree.py +438 -0
  252. wandb/sdk/data_types/utils.py +229 -0
  253. wandb/sdk/data_types/video.py +247 -0
  254. wandb/sdk/integration_utils/__init__.py +0 -0
  255. wandb/sdk/integration_utils/auto_logging.py +239 -0
  256. wandb/sdk/integration_utils/data_logging.py +475 -0
  257. wandb/sdk/interface/__init__.py +0 -0
  258. wandb/sdk/interface/constants.py +4 -0
  259. wandb/sdk/interface/interface.py +972 -0
  260. wandb/sdk/interface/interface_queue.py +59 -0
  261. wandb/sdk/interface/interface_relay.py +53 -0
  262. wandb/sdk/interface/interface_shared.py +537 -0
  263. wandb/sdk/interface/interface_sock.py +61 -0
  264. wandb/sdk/interface/message_future.py +27 -0
  265. wandb/sdk/interface/message_future_poll.py +50 -0
  266. wandb/sdk/interface/router.py +118 -0
  267. wandb/sdk/interface/router_queue.py +44 -0
  268. wandb/sdk/interface/router_relay.py +39 -0
  269. wandb/sdk/interface/router_sock.py +36 -0
  270. wandb/sdk/interface/summary_record.py +67 -0
  271. wandb/sdk/internal/__init__.py +0 -0
  272. wandb/sdk/internal/context.py +89 -0
  273. wandb/sdk/internal/datastore.py +297 -0
  274. wandb/sdk/internal/file_pusher.py +181 -0
  275. wandb/sdk/internal/file_stream.py +695 -0
  276. wandb/sdk/internal/flow_control.py +263 -0
  277. wandb/sdk/internal/handler.py +901 -0
  278. wandb/sdk/internal/internal.py +417 -0
  279. wandb/sdk/internal/internal_api.py +4358 -0
  280. wandb/sdk/internal/internal_util.py +100 -0
  281. wandb/sdk/internal/job_builder.py +629 -0
  282. wandb/sdk/internal/profiler.py +78 -0
  283. wandb/sdk/internal/progress.py +83 -0
  284. wandb/sdk/internal/run.py +25 -0
  285. wandb/sdk/internal/sample.py +70 -0
  286. wandb/sdk/internal/sender.py +1686 -0
  287. wandb/sdk/internal/sender_config.py +197 -0
  288. wandb/sdk/internal/settings_static.py +90 -0
  289. wandb/sdk/internal/system/__init__.py +0 -0
  290. wandb/sdk/internal/system/assets/__init__.py +27 -0
  291. wandb/sdk/internal/system/assets/aggregators.py +37 -0
  292. wandb/sdk/internal/system/assets/asset_registry.py +20 -0
  293. wandb/sdk/internal/system/assets/cpu.py +163 -0
  294. wandb/sdk/internal/system/assets/disk.py +210 -0
  295. wandb/sdk/internal/system/assets/gpu.py +416 -0
  296. wandb/sdk/internal/system/assets/gpu_amd.py +239 -0
  297. wandb/sdk/internal/system/assets/gpu_apple.py +177 -0
  298. wandb/sdk/internal/system/assets/interfaces.py +207 -0
  299. wandb/sdk/internal/system/assets/ipu.py +177 -0
  300. wandb/sdk/internal/system/assets/memory.py +166 -0
  301. wandb/sdk/internal/system/assets/network.py +125 -0
  302. wandb/sdk/internal/system/assets/open_metrics.py +299 -0
  303. wandb/sdk/internal/system/assets/tpu.py +154 -0
  304. wandb/sdk/internal/system/assets/trainium.py +399 -0
  305. wandb/sdk/internal/system/env_probe_helpers.py +13 -0
  306. wandb/sdk/internal/system/system_info.py +249 -0
  307. wandb/sdk/internal/system/system_monitor.py +229 -0
  308. wandb/sdk/internal/tb_watcher.py +518 -0
  309. wandb/sdk/internal/thread_local_settings.py +18 -0
  310. wandb/sdk/internal/writer.py +206 -0
  311. wandb/sdk/launch/__init__.py +14 -0
  312. wandb/sdk/launch/_launch.py +330 -0
  313. wandb/sdk/launch/_launch_add.py +255 -0
  314. wandb/sdk/launch/_project_spec.py +566 -0
  315. wandb/sdk/launch/agent/__init__.py +5 -0
  316. wandb/sdk/launch/agent/agent.py +924 -0
  317. wandb/sdk/launch/agent/config.py +296 -0
  318. wandb/sdk/launch/agent/job_status_tracker.py +53 -0
  319. wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
  320. wandb/sdk/launch/builder/__init__.py +0 -0
  321. wandb/sdk/launch/builder/abstract.py +156 -0
  322. wandb/sdk/launch/builder/build.py +297 -0
  323. wandb/sdk/launch/builder/context_manager.py +235 -0
  324. wandb/sdk/launch/builder/docker_builder.py +177 -0
  325. wandb/sdk/launch/builder/kaniko_builder.py +595 -0
  326. wandb/sdk/launch/builder/noop.py +58 -0
  327. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +188 -0
  328. wandb/sdk/launch/builder/templates/dockerfile.py +92 -0
  329. wandb/sdk/launch/create_job.py +528 -0
  330. wandb/sdk/launch/environment/abstract.py +29 -0
  331. wandb/sdk/launch/environment/aws_environment.py +322 -0
  332. wandb/sdk/launch/environment/azure_environment.py +105 -0
  333. wandb/sdk/launch/environment/gcp_environment.py +335 -0
  334. wandb/sdk/launch/environment/local_environment.py +66 -0
  335. wandb/sdk/launch/errors.py +19 -0
  336. wandb/sdk/launch/git_reference.py +109 -0
  337. wandb/sdk/launch/inputs/files.py +148 -0
  338. wandb/sdk/launch/inputs/internal.py +315 -0
  339. wandb/sdk/launch/inputs/manage.py +113 -0
  340. wandb/sdk/launch/inputs/schema.py +39 -0
  341. wandb/sdk/launch/loader.py +249 -0
  342. wandb/sdk/launch/registry/abstract.py +48 -0
  343. wandb/sdk/launch/registry/anon.py +29 -0
  344. wandb/sdk/launch/registry/azure_container_registry.py +124 -0
  345. wandb/sdk/launch/registry/elastic_container_registry.py +192 -0
  346. wandb/sdk/launch/registry/google_artifact_registry.py +219 -0
  347. wandb/sdk/launch/registry/local_registry.py +67 -0
  348. wandb/sdk/launch/runner/__init__.py +0 -0
  349. wandb/sdk/launch/runner/abstract.py +195 -0
  350. wandb/sdk/launch/runner/kubernetes_monitor.py +474 -0
  351. wandb/sdk/launch/runner/kubernetes_runner.py +963 -0
  352. wandb/sdk/launch/runner/local_container.py +301 -0
  353. wandb/sdk/launch/runner/local_process.py +78 -0
  354. wandb/sdk/launch/runner/sagemaker_runner.py +426 -0
  355. wandb/sdk/launch/runner/vertex_runner.py +230 -0
  356. wandb/sdk/launch/sweeps/__init__.py +39 -0
  357. wandb/sdk/launch/sweeps/scheduler.py +742 -0
  358. wandb/sdk/launch/sweeps/scheduler_sweep.py +91 -0
  359. wandb/sdk/launch/sweeps/utils.py +316 -0
  360. wandb/sdk/launch/utils.py +746 -0
  361. wandb/sdk/launch/wandb_reference.py +138 -0
  362. wandb/sdk/lib/__init__.py +5 -0
  363. wandb/sdk/lib/_settings_toposort_generate.py +159 -0
  364. wandb/sdk/lib/_settings_toposort_generated.py +250 -0
  365. wandb/sdk/lib/_wburls_generate.py +25 -0
  366. wandb/sdk/lib/_wburls_generated.py +22 -0
  367. wandb/sdk/lib/apikey.py +273 -0
  368. wandb/sdk/lib/capped_dict.py +26 -0
  369. wandb/sdk/lib/config_util.py +101 -0
  370. wandb/sdk/lib/credentials.py +141 -0
  371. wandb/sdk/lib/deprecate.py +42 -0
  372. wandb/sdk/lib/disabled.py +29 -0
  373. wandb/sdk/lib/exit_hooks.py +54 -0
  374. wandb/sdk/lib/file_stream_utils.py +118 -0
  375. wandb/sdk/lib/filenames.py +64 -0
  376. wandb/sdk/lib/filesystem.py +372 -0
  377. wandb/sdk/lib/fsm.py +174 -0
  378. wandb/sdk/lib/gitlib.py +239 -0
  379. wandb/sdk/lib/gql_request.py +65 -0
  380. wandb/sdk/lib/handler_util.py +21 -0
  381. wandb/sdk/lib/hashutil.py +84 -0
  382. wandb/sdk/lib/import_hooks.py +275 -0
  383. wandb/sdk/lib/ipython.py +146 -0
  384. wandb/sdk/lib/json_util.py +80 -0
  385. wandb/sdk/lib/lazyloader.py +63 -0
  386. wandb/sdk/lib/mailbox.py +460 -0
  387. wandb/sdk/lib/module.py +69 -0
  388. wandb/sdk/lib/paths.py +106 -0
  389. wandb/sdk/lib/preinit.py +42 -0
  390. wandb/sdk/lib/printer.py +313 -0
  391. wandb/sdk/lib/proto_util.py +90 -0
  392. wandb/sdk/lib/redirect.py +845 -0
  393. wandb/sdk/lib/reporting.py +99 -0
  394. wandb/sdk/lib/retry.py +289 -0
  395. wandb/sdk/lib/run_moment.py +78 -0
  396. wandb/sdk/lib/runid.py +12 -0
  397. wandb/sdk/lib/server.py +52 -0
  398. wandb/sdk/lib/service_connection.py +216 -0
  399. wandb/sdk/lib/service_token.py +94 -0
  400. wandb/sdk/lib/sock_client.py +295 -0
  401. wandb/sdk/lib/sparkline.py +45 -0
  402. wandb/sdk/lib/telemetry.py +100 -0
  403. wandb/sdk/lib/timed_input.py +133 -0
  404. wandb/sdk/lib/timer.py +19 -0
  405. wandb/sdk/lib/tracelog.py +255 -0
  406. wandb/sdk/lib/wburls.py +46 -0
  407. wandb/sdk/service/__init__.py +0 -0
  408. wandb/sdk/service/_startup_debug.py +22 -0
  409. wandb/sdk/service/port_file.py +53 -0
  410. wandb/sdk/service/server.py +116 -0
  411. wandb/sdk/service/server_sock.py +276 -0
  412. wandb/sdk/service/service.py +242 -0
  413. wandb/sdk/service/streams.py +417 -0
  414. wandb/sdk/verify/__init__.py +0 -0
  415. wandb/sdk/verify/verify.py +501 -0
  416. wandb/sdk/wandb_alerts.py +12 -0
  417. wandb/sdk/wandb_config.py +322 -0
  418. wandb/sdk/wandb_helper.py +54 -0
  419. wandb/sdk/wandb_init.py +1266 -0
  420. wandb/sdk/wandb_login.py +349 -0
  421. wandb/sdk/wandb_metric.py +110 -0
  422. wandb/sdk/wandb_require.py +97 -0
  423. wandb/sdk/wandb_require_helpers.py +44 -0
  424. wandb/sdk/wandb_run.py +4236 -0
  425. wandb/sdk/wandb_settings.py +2001 -0
  426. wandb/sdk/wandb_setup.py +409 -0
  427. wandb/sdk/wandb_summary.py +150 -0
  428. wandb/sdk/wandb_sweep.py +119 -0
  429. wandb/sdk/wandb_sync.py +81 -0
  430. wandb/sdk/wandb_watch.py +144 -0
  431. wandb/sklearn.py +35 -0
  432. wandb/sync/__init__.py +3 -0
  433. wandb/sync/sync.py +443 -0
  434. wandb/trigger.py +29 -0
  435. wandb/util.py +1956 -0
  436. wandb/vendor/__init__.py +0 -0
  437. wandb/vendor/gql-0.2.0/setup.py +40 -0
  438. wandb/vendor/gql-0.2.0/tests/__init__.py +0 -0
  439. wandb/vendor/gql-0.2.0/tests/starwars/__init__.py +0 -0
  440. wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py +96 -0
  441. wandb/vendor/gql-0.2.0/tests/starwars/schema.py +146 -0
  442. wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py +293 -0
  443. wandb/vendor/gql-0.2.0/tests/starwars/test_query.py +355 -0
  444. wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py +171 -0
  445. wandb/vendor/gql-0.2.0/tests/test_client.py +31 -0
  446. wandb/vendor/gql-0.2.0/tests/test_transport.py +89 -0
  447. wandb/vendor/gql-0.2.0/wandb_gql/__init__.py +4 -0
  448. wandb/vendor/gql-0.2.0/wandb_gql/client.py +75 -0
  449. wandb/vendor/gql-0.2.0/wandb_gql/dsl.py +152 -0
  450. wandb/vendor/gql-0.2.0/wandb_gql/gql.py +10 -0
  451. wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py +0 -0
  452. wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py +6 -0
  453. wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py +15 -0
  454. wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py +46 -0
  455. wandb/vendor/gql-0.2.0/wandb_gql/utils.py +21 -0
  456. wandb/vendor/graphql-core-1.1/setup.py +86 -0
  457. wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py +287 -0
  458. wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py +6 -0
  459. wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py +42 -0
  460. wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py +11 -0
  461. wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py +29 -0
  462. wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py +36 -0
  463. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py +26 -0
  464. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py +311 -0
  465. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py +398 -0
  466. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py +0 -0
  467. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py +53 -0
  468. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py +22 -0
  469. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py +32 -0
  470. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py +7 -0
  471. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py +35 -0
  472. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py +6 -0
  473. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py +0 -0
  474. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py +66 -0
  475. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py +252 -0
  476. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py +151 -0
  477. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py +7 -0
  478. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py +57 -0
  479. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py +145 -0
  480. wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py +60 -0
  481. wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py +0 -0
  482. wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py +1349 -0
  483. wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py +19 -0
  484. wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py +435 -0
  485. wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py +30 -0
  486. wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py +779 -0
  487. wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py +193 -0
  488. wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py +18 -0
  489. wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py +222 -0
  490. wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py +82 -0
  491. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py +0 -0
  492. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py +17 -0
  493. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py +28 -0
  494. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py +40 -0
  495. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py +8 -0
  496. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py +43 -0
  497. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py +78 -0
  498. wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py +67 -0
  499. wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py +619 -0
  500. wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py +132 -0
  501. wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py +440 -0
  502. wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py +131 -0
  503. wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py +100 -0
  504. wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py +145 -0
  505. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py +0 -0
  506. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py +9 -0
  507. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py +65 -0
  508. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py +49 -0
  509. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py +24 -0
  510. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py +75 -0
  511. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py +291 -0
  512. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py +250 -0
  513. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py +9 -0
  514. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py +357 -0
  515. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py +27 -0
  516. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py +21 -0
  517. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py +90 -0
  518. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py +67 -0
  519. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py +66 -0
  520. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py +21 -0
  521. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py +168 -0
  522. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py +56 -0
  523. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py +69 -0
  524. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py +21 -0
  525. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py +149 -0
  526. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py +69 -0
  527. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py +4 -0
  528. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py +79 -0
  529. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py +24 -0
  530. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py +8 -0
  531. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py +44 -0
  532. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py +113 -0
  533. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py +33 -0
  534. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py +70 -0
  535. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py +97 -0
  536. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py +19 -0
  537. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py +43 -0
  538. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py +23 -0
  539. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py +59 -0
  540. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py +36 -0
  541. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py +38 -0
  542. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py +37 -0
  543. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py +529 -0
  544. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py +44 -0
  545. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py +46 -0
  546. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py +33 -0
  547. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py +32 -0
  548. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py +28 -0
  549. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py +33 -0
  550. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py +31 -0
  551. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py +27 -0
  552. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py +21 -0
  553. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py +53 -0
  554. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py +158 -0
  555. wandb/vendor/promise-2.3.0/conftest.py +30 -0
  556. wandb/vendor/promise-2.3.0/setup.py +64 -0
  557. wandb/vendor/promise-2.3.0/tests/__init__.py +0 -0
  558. wandb/vendor/promise-2.3.0/tests/conftest.py +8 -0
  559. wandb/vendor/promise-2.3.0/tests/test_awaitable.py +32 -0
  560. wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py +47 -0
  561. wandb/vendor/promise-2.3.0/tests/test_benchmark.py +116 -0
  562. wandb/vendor/promise-2.3.0/tests/test_complex_threads.py +23 -0
  563. wandb/vendor/promise-2.3.0/tests/test_dataloader.py +452 -0
  564. wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py +99 -0
  565. wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py +65 -0
  566. wandb/vendor/promise-2.3.0/tests/test_extra.py +670 -0
  567. wandb/vendor/promise-2.3.0/tests/test_issues.py +132 -0
  568. wandb/vendor/promise-2.3.0/tests/test_promise_list.py +70 -0
  569. wandb/vendor/promise-2.3.0/tests/test_spec.py +584 -0
  570. wandb/vendor/promise-2.3.0/tests/test_thread_safety.py +115 -0
  571. wandb/vendor/promise-2.3.0/tests/utils.py +3 -0
  572. wandb/vendor/promise-2.3.0/wandb_promise/__init__.py +38 -0
  573. wandb/vendor/promise-2.3.0/wandb_promise/async_.py +135 -0
  574. wandb/vendor/promise-2.3.0/wandb_promise/compat.py +32 -0
  575. wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py +326 -0
  576. wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py +12 -0
  577. wandb/vendor/promise-2.3.0/wandb_promise/promise.py +848 -0
  578. wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py +151 -0
  579. wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py +0 -0
  580. wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py +83 -0
  581. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py +0 -0
  582. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py +22 -0
  583. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py +21 -0
  584. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py +27 -0
  585. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py +18 -0
  586. wandb/vendor/promise-2.3.0/wandb_promise/utils.py +56 -0
  587. wandb/vendor/pygments/__init__.py +90 -0
  588. wandb/vendor/pygments/cmdline.py +568 -0
  589. wandb/vendor/pygments/console.py +74 -0
  590. wandb/vendor/pygments/filter.py +74 -0
  591. wandb/vendor/pygments/filters/__init__.py +350 -0
  592. wandb/vendor/pygments/formatter.py +95 -0
  593. wandb/vendor/pygments/formatters/__init__.py +153 -0
  594. wandb/vendor/pygments/formatters/_mapping.py +85 -0
  595. wandb/vendor/pygments/formatters/bbcode.py +109 -0
  596. wandb/vendor/pygments/formatters/html.py +851 -0
  597. wandb/vendor/pygments/formatters/img.py +600 -0
  598. wandb/vendor/pygments/formatters/irc.py +182 -0
  599. wandb/vendor/pygments/formatters/latex.py +482 -0
  600. wandb/vendor/pygments/formatters/other.py +160 -0
  601. wandb/vendor/pygments/formatters/rtf.py +147 -0
  602. wandb/vendor/pygments/formatters/svg.py +153 -0
  603. wandb/vendor/pygments/formatters/terminal.py +136 -0
  604. wandb/vendor/pygments/formatters/terminal256.py +309 -0
  605. wandb/vendor/pygments/lexer.py +871 -0
  606. wandb/vendor/pygments/lexers/__init__.py +329 -0
  607. wandb/vendor/pygments/lexers/_asy_builtins.py +1645 -0
  608. wandb/vendor/pygments/lexers/_cl_builtins.py +232 -0
  609. wandb/vendor/pygments/lexers/_cocoa_builtins.py +72 -0
  610. wandb/vendor/pygments/lexers/_csound_builtins.py +1346 -0
  611. wandb/vendor/pygments/lexers/_lasso_builtins.py +5327 -0
  612. wandb/vendor/pygments/lexers/_lua_builtins.py +295 -0
  613. wandb/vendor/pygments/lexers/_mapping.py +500 -0
  614. wandb/vendor/pygments/lexers/_mql_builtins.py +1172 -0
  615. wandb/vendor/pygments/lexers/_openedge_builtins.py +2547 -0
  616. wandb/vendor/pygments/lexers/_php_builtins.py +4756 -0
  617. wandb/vendor/pygments/lexers/_postgres_builtins.py +621 -0
  618. wandb/vendor/pygments/lexers/_scilab_builtins.py +3094 -0
  619. wandb/vendor/pygments/lexers/_sourcemod_builtins.py +1163 -0
  620. wandb/vendor/pygments/lexers/_stan_builtins.py +532 -0
  621. wandb/vendor/pygments/lexers/_stata_builtins.py +419 -0
  622. wandb/vendor/pygments/lexers/_tsql_builtins.py +1004 -0
  623. wandb/vendor/pygments/lexers/_vim_builtins.py +1939 -0
  624. wandb/vendor/pygments/lexers/actionscript.py +240 -0
  625. wandb/vendor/pygments/lexers/agile.py +24 -0
  626. wandb/vendor/pygments/lexers/algebra.py +221 -0
  627. wandb/vendor/pygments/lexers/ambient.py +76 -0
  628. wandb/vendor/pygments/lexers/ampl.py +87 -0
  629. wandb/vendor/pygments/lexers/apl.py +101 -0
  630. wandb/vendor/pygments/lexers/archetype.py +318 -0
  631. wandb/vendor/pygments/lexers/asm.py +641 -0
  632. wandb/vendor/pygments/lexers/automation.py +374 -0
  633. wandb/vendor/pygments/lexers/basic.py +500 -0
  634. wandb/vendor/pygments/lexers/bibtex.py +160 -0
  635. wandb/vendor/pygments/lexers/business.py +612 -0
  636. wandb/vendor/pygments/lexers/c_cpp.py +252 -0
  637. wandb/vendor/pygments/lexers/c_like.py +541 -0
  638. wandb/vendor/pygments/lexers/capnproto.py +78 -0
  639. wandb/vendor/pygments/lexers/chapel.py +102 -0
  640. wandb/vendor/pygments/lexers/clean.py +288 -0
  641. wandb/vendor/pygments/lexers/compiled.py +34 -0
  642. wandb/vendor/pygments/lexers/configs.py +833 -0
  643. wandb/vendor/pygments/lexers/console.py +114 -0
  644. wandb/vendor/pygments/lexers/crystal.py +393 -0
  645. wandb/vendor/pygments/lexers/csound.py +366 -0
  646. wandb/vendor/pygments/lexers/css.py +689 -0
  647. wandb/vendor/pygments/lexers/d.py +251 -0
  648. wandb/vendor/pygments/lexers/dalvik.py +125 -0
  649. wandb/vendor/pygments/lexers/data.py +555 -0
  650. wandb/vendor/pygments/lexers/diff.py +165 -0
  651. wandb/vendor/pygments/lexers/dotnet.py +691 -0
  652. wandb/vendor/pygments/lexers/dsls.py +878 -0
  653. wandb/vendor/pygments/lexers/dylan.py +289 -0
  654. wandb/vendor/pygments/lexers/ecl.py +125 -0
  655. wandb/vendor/pygments/lexers/eiffel.py +65 -0
  656. wandb/vendor/pygments/lexers/elm.py +121 -0
  657. wandb/vendor/pygments/lexers/erlang.py +533 -0
  658. wandb/vendor/pygments/lexers/esoteric.py +277 -0
  659. wandb/vendor/pygments/lexers/ezhil.py +69 -0
  660. wandb/vendor/pygments/lexers/factor.py +344 -0
  661. wandb/vendor/pygments/lexers/fantom.py +250 -0
  662. wandb/vendor/pygments/lexers/felix.py +273 -0
  663. wandb/vendor/pygments/lexers/forth.py +177 -0
  664. wandb/vendor/pygments/lexers/fortran.py +205 -0
  665. wandb/vendor/pygments/lexers/foxpro.py +428 -0
  666. wandb/vendor/pygments/lexers/functional.py +21 -0
  667. wandb/vendor/pygments/lexers/go.py +101 -0
  668. wandb/vendor/pygments/lexers/grammar_notation.py +213 -0
  669. wandb/vendor/pygments/lexers/graph.py +80 -0
  670. wandb/vendor/pygments/lexers/graphics.py +553 -0
  671. wandb/vendor/pygments/lexers/haskell.py +843 -0
  672. wandb/vendor/pygments/lexers/haxe.py +936 -0
  673. wandb/vendor/pygments/lexers/hdl.py +382 -0
  674. wandb/vendor/pygments/lexers/hexdump.py +103 -0
  675. wandb/vendor/pygments/lexers/html.py +602 -0
  676. wandb/vendor/pygments/lexers/idl.py +270 -0
  677. wandb/vendor/pygments/lexers/igor.py +288 -0
  678. wandb/vendor/pygments/lexers/inferno.py +96 -0
  679. wandb/vendor/pygments/lexers/installers.py +322 -0
  680. wandb/vendor/pygments/lexers/int_fiction.py +1343 -0
  681. wandb/vendor/pygments/lexers/iolang.py +63 -0
  682. wandb/vendor/pygments/lexers/j.py +146 -0
  683. wandb/vendor/pygments/lexers/javascript.py +1525 -0
  684. wandb/vendor/pygments/lexers/julia.py +333 -0
  685. wandb/vendor/pygments/lexers/jvm.py +1573 -0
  686. wandb/vendor/pygments/lexers/lisp.py +2621 -0
  687. wandb/vendor/pygments/lexers/make.py +202 -0
  688. wandb/vendor/pygments/lexers/markup.py +595 -0
  689. wandb/vendor/pygments/lexers/math.py +21 -0
  690. wandb/vendor/pygments/lexers/matlab.py +663 -0
  691. wandb/vendor/pygments/lexers/ml.py +769 -0
  692. wandb/vendor/pygments/lexers/modeling.py +358 -0
  693. wandb/vendor/pygments/lexers/modula2.py +1561 -0
  694. wandb/vendor/pygments/lexers/monte.py +204 -0
  695. wandb/vendor/pygments/lexers/ncl.py +894 -0
  696. wandb/vendor/pygments/lexers/nimrod.py +159 -0
  697. wandb/vendor/pygments/lexers/nit.py +64 -0
  698. wandb/vendor/pygments/lexers/nix.py +136 -0
  699. wandb/vendor/pygments/lexers/oberon.py +105 -0
  700. wandb/vendor/pygments/lexers/objective.py +504 -0
  701. wandb/vendor/pygments/lexers/ooc.py +85 -0
  702. wandb/vendor/pygments/lexers/other.py +41 -0
  703. wandb/vendor/pygments/lexers/parasail.py +79 -0
  704. wandb/vendor/pygments/lexers/parsers.py +835 -0
  705. wandb/vendor/pygments/lexers/pascal.py +644 -0
  706. wandb/vendor/pygments/lexers/pawn.py +199 -0
  707. wandb/vendor/pygments/lexers/perl.py +620 -0
  708. wandb/vendor/pygments/lexers/php.py +267 -0
  709. wandb/vendor/pygments/lexers/praat.py +294 -0
  710. wandb/vendor/pygments/lexers/prolog.py +306 -0
  711. wandb/vendor/pygments/lexers/python.py +939 -0
  712. wandb/vendor/pygments/lexers/qvt.py +152 -0
  713. wandb/vendor/pygments/lexers/r.py +453 -0
  714. wandb/vendor/pygments/lexers/rdf.py +270 -0
  715. wandb/vendor/pygments/lexers/rebol.py +431 -0
  716. wandb/vendor/pygments/lexers/resource.py +85 -0
  717. wandb/vendor/pygments/lexers/rnc.py +67 -0
  718. wandb/vendor/pygments/lexers/roboconf.py +82 -0
  719. wandb/vendor/pygments/lexers/robotframework.py +560 -0
  720. wandb/vendor/pygments/lexers/ruby.py +519 -0
  721. wandb/vendor/pygments/lexers/rust.py +220 -0
  722. wandb/vendor/pygments/lexers/sas.py +228 -0
  723. wandb/vendor/pygments/lexers/scripting.py +1222 -0
  724. wandb/vendor/pygments/lexers/shell.py +794 -0
  725. wandb/vendor/pygments/lexers/smalltalk.py +195 -0
  726. wandb/vendor/pygments/lexers/smv.py +79 -0
  727. wandb/vendor/pygments/lexers/snobol.py +83 -0
  728. wandb/vendor/pygments/lexers/special.py +103 -0
  729. wandb/vendor/pygments/lexers/sql.py +681 -0
  730. wandb/vendor/pygments/lexers/stata.py +108 -0
  731. wandb/vendor/pygments/lexers/supercollider.py +90 -0
  732. wandb/vendor/pygments/lexers/tcl.py +145 -0
  733. wandb/vendor/pygments/lexers/templates.py +2283 -0
  734. wandb/vendor/pygments/lexers/testing.py +207 -0
  735. wandb/vendor/pygments/lexers/text.py +25 -0
  736. wandb/vendor/pygments/lexers/textedit.py +169 -0
  737. wandb/vendor/pygments/lexers/textfmts.py +297 -0
  738. wandb/vendor/pygments/lexers/theorem.py +458 -0
  739. wandb/vendor/pygments/lexers/trafficscript.py +54 -0
  740. wandb/vendor/pygments/lexers/typoscript.py +226 -0
  741. wandb/vendor/pygments/lexers/urbi.py +133 -0
  742. wandb/vendor/pygments/lexers/varnish.py +190 -0
  743. wandb/vendor/pygments/lexers/verification.py +111 -0
  744. wandb/vendor/pygments/lexers/web.py +24 -0
  745. wandb/vendor/pygments/lexers/webmisc.py +988 -0
  746. wandb/vendor/pygments/lexers/whiley.py +116 -0
  747. wandb/vendor/pygments/lexers/x10.py +69 -0
  748. wandb/vendor/pygments/modeline.py +44 -0
  749. wandb/vendor/pygments/plugin.py +68 -0
  750. wandb/vendor/pygments/regexopt.py +92 -0
  751. wandb/vendor/pygments/scanner.py +105 -0
  752. wandb/vendor/pygments/sphinxext.py +158 -0
  753. wandb/vendor/pygments/style.py +155 -0
  754. wandb/vendor/pygments/styles/__init__.py +80 -0
  755. wandb/vendor/pygments/styles/abap.py +29 -0
  756. wandb/vendor/pygments/styles/algol.py +63 -0
  757. wandb/vendor/pygments/styles/algol_nu.py +63 -0
  758. wandb/vendor/pygments/styles/arduino.py +98 -0
  759. wandb/vendor/pygments/styles/autumn.py +65 -0
  760. wandb/vendor/pygments/styles/borland.py +51 -0
  761. wandb/vendor/pygments/styles/bw.py +49 -0
  762. wandb/vendor/pygments/styles/colorful.py +81 -0
  763. wandb/vendor/pygments/styles/default.py +73 -0
  764. wandb/vendor/pygments/styles/emacs.py +72 -0
  765. wandb/vendor/pygments/styles/friendly.py +72 -0
  766. wandb/vendor/pygments/styles/fruity.py +42 -0
  767. wandb/vendor/pygments/styles/igor.py +29 -0
  768. wandb/vendor/pygments/styles/lovelace.py +97 -0
  769. wandb/vendor/pygments/styles/manni.py +75 -0
  770. wandb/vendor/pygments/styles/monokai.py +106 -0
  771. wandb/vendor/pygments/styles/murphy.py +80 -0
  772. wandb/vendor/pygments/styles/native.py +65 -0
  773. wandb/vendor/pygments/styles/paraiso_dark.py +125 -0
  774. wandb/vendor/pygments/styles/paraiso_light.py +125 -0
  775. wandb/vendor/pygments/styles/pastie.py +75 -0
  776. wandb/vendor/pygments/styles/perldoc.py +69 -0
  777. wandb/vendor/pygments/styles/rainbow_dash.py +89 -0
  778. wandb/vendor/pygments/styles/rrt.py +33 -0
  779. wandb/vendor/pygments/styles/sas.py +44 -0
  780. wandb/vendor/pygments/styles/stata.py +40 -0
  781. wandb/vendor/pygments/styles/tango.py +141 -0
  782. wandb/vendor/pygments/styles/trac.py +63 -0
  783. wandb/vendor/pygments/styles/vim.py +63 -0
  784. wandb/vendor/pygments/styles/vs.py +38 -0
  785. wandb/vendor/pygments/styles/xcode.py +51 -0
  786. wandb/vendor/pygments/token.py +213 -0
  787. wandb/vendor/pygments/unistring.py +217 -0
  788. wandb/vendor/pygments/util.py +388 -0
  789. wandb/vendor/pynvml/__init__.py +0 -0
  790. wandb/vendor/pynvml/pynvml.py +4779 -0
  791. wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py +17 -0
  792. wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py +615 -0
  793. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py +98 -0
  794. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py +369 -0
  795. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py +172 -0
  796. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py +239 -0
  797. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py +218 -0
  798. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py +81 -0
  799. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py +575 -0
  800. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py +730 -0
  801. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py +145 -0
  802. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py +133 -0
  803. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py +348 -0
  804. wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py +265 -0
  805. wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py +174 -0
  806. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py +151 -0
  807. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py +249 -0
  808. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py +29 -0
  809. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py +198 -0
  810. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py +88 -0
  811. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py +293 -0
  812. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py +157 -0
  813. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py +41 -0
  814. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py +40 -0
  815. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py +57 -0
  816. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py +64 -0
  817. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py +123 -0
  818. wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py +28 -0
  819. wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py +577 -0
  820. wandb/wandb_agent.py +588 -0
  821. wandb/wandb_controller.py +721 -0
  822. wandb/wandb_run.py +9 -0
  823. wandb-0.18.2.dist-info/METADATA +213 -0
  824. wandb-0.18.2.dist-info/RECORD +827 -0
  825. wandb-0.18.2.dist-info/WHEEL +5 -0
  826. wandb-0.18.2.dist-info/entry_points.txt +3 -0
  827. wandb-0.18.2.dist-info/licenses/LICENSE +21 -0
wandb/util.py ADDED
@@ -0,0 +1,1956 @@
1
+ import colorsys
2
+ import contextlib
3
+ import dataclasses
4
+ import enum
5
+ import functools
6
+ import gzip
7
+ import importlib
8
+ import importlib.util
9
+ import itertools
10
+ import json
11
+ import logging
12
+ import math
13
+ import numbers
14
+ import os
15
+ import pathlib
16
+ import platform
17
+ import queue
18
+ import random
19
+ import re
20
+ import secrets
21
+ import shlex
22
+ import socket
23
+ import string
24
+ import sys
25
+ import tarfile
26
+ import tempfile
27
+ import threading
28
+ import time
29
+ import types
30
+ import urllib
31
+ from dataclasses import asdict, is_dataclass
32
+ from datetime import date, datetime, timedelta
33
+ from importlib import import_module
34
+ from sys import getsizeof
35
+ from types import ModuleType
36
+ from typing import (
37
+ IO,
38
+ TYPE_CHECKING,
39
+ Any,
40
+ Callable,
41
+ Dict,
42
+ Generator,
43
+ Iterable,
44
+ List,
45
+ Mapping,
46
+ Optional,
47
+ Sequence,
48
+ TextIO,
49
+ Tuple,
50
+ TypeVar,
51
+ Union,
52
+ )
53
+
54
+ import requests
55
+ import yaml
56
+
57
+ import wandb
58
+ import wandb.env
59
+ from wandb.errors import (
60
+ AuthenticationError,
61
+ CommError,
62
+ UsageError,
63
+ WandbCoreNotAvailableError,
64
+ term,
65
+ )
66
+ from wandb.sdk.internal.thread_local_settings import _thread_local_api_settings
67
+ from wandb.sdk.lib import filesystem, runid
68
+ from wandb.sdk.lib.json_util import dump, dumps
69
+ from wandb.sdk.lib.paths import FilePathStr, StrPath
70
+
71
+ if TYPE_CHECKING:
72
+ import packaging.version # type: ignore[import-not-found]
73
+
74
+ import wandb.sdk.internal.settings_static
75
+ import wandb.sdk.wandb_settings
76
+ from wandb.sdk.artifacts.artifact import Artifact
77
+
78
+ CheckRetryFnType = Callable[[Exception], Union[bool, timedelta]]
79
+ T = TypeVar("T")
80
+
81
+
82
+ logger = logging.getLogger(__name__)
83
+ _not_importable = set()
84
+
85
+ MAX_LINE_BYTES = (10 << 20) - (100 << 10) # imposed by back end
86
+ IS_GIT = os.path.exists(os.path.join(os.path.dirname(__file__), "..", ".git"))
87
+ RE_WINFNAMES = re.compile(r'[<>:"\\?*]')
88
+
89
+ # From https://docs.docker.com/engine/reference/commandline/tag/
90
+ # "Name components may contain lowercase letters, digits and separators.
91
+ # A separator is defined as a period, one or two underscores, or one or more dashes.
92
+ # A name component may not start or end with a separator."
93
+ DOCKER_IMAGE_NAME_SEPARATOR = "(?:__|[._]|[-]+)"
94
+ RE_DOCKER_IMAGE_NAME_SEPARATOR_START = re.compile("^" + DOCKER_IMAGE_NAME_SEPARATOR)
95
+ RE_DOCKER_IMAGE_NAME_SEPARATOR_END = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "$")
96
+ RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT = re.compile(DOCKER_IMAGE_NAME_SEPARATOR + "{2,}")
97
+ RE_DOCKER_IMAGE_NAME_CHARS = re.compile(r"[^a-z0-9._\-]")
98
+
99
+ # these match the environments for gorilla
100
+ if IS_GIT:
101
+ SENTRY_ENV = "development"
102
+ else:
103
+ SENTRY_ENV = "production"
104
+
105
+
106
+ PLATFORM_WINDOWS = "windows"
107
+ PLATFORM_LINUX = "linux"
108
+ PLATFORM_BSD = "bsd"
109
+ PLATFORM_DARWIN = "darwin"
110
+ PLATFORM_UNKNOWN = "unknown"
111
+
112
+ LAUNCH_JOB_ARTIFACT_SLOT_NAME = "_wandb_job"
113
+
114
+
115
+ def get_platform_name() -> str:
116
+ if sys.platform.startswith("win"):
117
+ return PLATFORM_WINDOWS
118
+ elif sys.platform.startswith("darwin"):
119
+ return PLATFORM_DARWIN
120
+ elif sys.platform.startswith("linux"):
121
+ return PLATFORM_LINUX
122
+ elif sys.platform.startswith(
123
+ (
124
+ "dragonfly",
125
+ "freebsd",
126
+ "netbsd",
127
+ "openbsd",
128
+ )
129
+ ):
130
+ return PLATFORM_BSD
131
+ else:
132
+ return PLATFORM_UNKNOWN
133
+
134
+
135
+ POW_10_BYTES = [
136
+ ("B", 10**0),
137
+ ("KB", 10**3),
138
+ ("MB", 10**6),
139
+ ("GB", 10**9),
140
+ ("TB", 10**12),
141
+ ("PB", 10**15),
142
+ ("EB", 10**18),
143
+ ]
144
+
145
+ POW_2_BYTES = [
146
+ ("B", 2**0),
147
+ ("KiB", 2**10),
148
+ ("MiB", 2**20),
149
+ ("GiB", 2**30),
150
+ ("TiB", 2**40),
151
+ ("PiB", 2**50),
152
+ ("EiB", 2**60),
153
+ ]
154
+
155
+
156
+ def vendor_setup() -> Callable:
157
+ """Create a function that restores user paths after vendor imports.
158
+
159
+ This enables us to use the vendor directory for packages we don't depend on. Call
160
+ the returned function after imports are complete. If you don't you may modify the
161
+ user's path which is never good.
162
+
163
+ Usage:
164
+
165
+ ```python
166
+ reset_path = vendor_setup()
167
+ # do any vendor imports...
168
+ reset_path()
169
+ ```
170
+ """
171
+ original_path = [directory for directory in sys.path]
172
+
173
+ def reset_import_path() -> None:
174
+ sys.path = original_path
175
+
176
+ parent_dir = os.path.abspath(os.path.dirname(__file__))
177
+ vendor_dir = os.path.join(parent_dir, "vendor")
178
+ vendor_packages = (
179
+ "gql-0.2.0",
180
+ "graphql-core-1.1",
181
+ "watchdog_0_9_0",
182
+ "promise-2.3.0",
183
+ )
184
+ package_dirs = [os.path.join(vendor_dir, p) for p in vendor_packages]
185
+ for p in [vendor_dir] + package_dirs:
186
+ if p not in sys.path:
187
+ sys.path.insert(1, p)
188
+
189
+ return reset_import_path
190
+
191
+
192
+ def vendor_import(name: str) -> Any:
193
+ reset_path = vendor_setup()
194
+ module = import_module(name)
195
+ reset_path()
196
+ return module
197
+
198
+
199
+ class LazyModuleState:
200
+ def __init__(self, module: types.ModuleType) -> None:
201
+ self.module = module
202
+ self.load_started = False
203
+ self.lock = threading.RLock()
204
+
205
+ def load(self) -> None:
206
+ with self.lock:
207
+ if self.load_started:
208
+ return
209
+ self.load_started = True
210
+ assert self.module.__spec__ is not None
211
+ assert self.module.__spec__.loader is not None
212
+ self.module.__spec__.loader.exec_module(self.module)
213
+ self.module.__class__ = types.ModuleType
214
+
215
+
216
+ class LazyModule(types.ModuleType):
217
+ def __getattribute__(self, name: str) -> Any:
218
+ state = object.__getattribute__(self, "__lazy_module_state__")
219
+ state.load()
220
+ return object.__getattribute__(self, name)
221
+
222
+ def __setattr__(self, name: str, value: Any) -> None:
223
+ state = object.__getattribute__(self, "__lazy_module_state__")
224
+ state.load()
225
+ object.__setattr__(self, name, value)
226
+
227
+ def __delattr__(self, name: str) -> None:
228
+ state = object.__getattribute__(self, "__lazy_module_state__")
229
+ state.load()
230
+ object.__delattr__(self, name)
231
+
232
+
233
+ def import_module_lazy(name: str) -> types.ModuleType:
234
+ """Import a module lazily, only when it is used.
235
+
236
+ Inspired by importlib.util.LazyLoader, but improved so that the module loading is
237
+ thread-safe. Circular dependency between modules can lead to a deadlock if the two
238
+ modules are loaded from different threads.
239
+
240
+ :param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
241
+ """
242
+ try:
243
+ return sys.modules[name]
244
+ except KeyError:
245
+ spec = importlib.util.find_spec(name)
246
+ if spec is None:
247
+ raise ModuleNotFoundError
248
+ module = importlib.util.module_from_spec(spec)
249
+ module.__lazy_module_state__ = LazyModuleState(module) # type: ignore
250
+ module.__class__ = LazyModule
251
+ sys.modules[name] = module
252
+ return module
253
+
254
+
255
+ def get_module(
256
+ name: str,
257
+ required: Optional[Union[str, bool]] = None,
258
+ lazy: bool = True,
259
+ ) -> Any:
260
+ """Return module or None. Absolute import is required.
261
+
262
+ :param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
263
+ :param (str) required: A string to raise a ValueError if missing
264
+ :param (bool) lazy: If True, return a lazy loader for the module.
265
+ :return: (module|None) If import succeeds, the module will be returned.
266
+ """
267
+ if name not in _not_importable:
268
+ try:
269
+ if not lazy:
270
+ return import_module(name)
271
+ else:
272
+ return import_module_lazy(name)
273
+ except Exception:
274
+ _not_importable.add(name)
275
+ msg = f"Error importing optional module {name}"
276
+ if required:
277
+ logger.exception(msg)
278
+ if required and name in _not_importable:
279
+ raise wandb.Error(required)
280
+
281
+
282
+ def get_optional_module(name) -> Optional["importlib.ModuleInterface"]: # type: ignore
283
+ return get_module(name)
284
+
285
+
286
+ np = get_module("numpy")
287
+
288
+ pd_available = False
289
+ pandas_spec = importlib.util.find_spec("pandas")
290
+ if pandas_spec is not None:
291
+ pd_available = True
292
+
293
+ # TODO: Revisit these limits
294
+ VALUE_BYTES_LIMIT = 100000
295
+
296
+
297
+ def app_url(api_url: str) -> str:
298
+ """Return the frontend app url without a trailing slash."""
299
+ # TODO: move me to settings
300
+ app_url = wandb.env.get_app_url()
301
+ if app_url is not None:
302
+ return str(app_url.strip("/"))
303
+ if "://api.wandb.test" in api_url:
304
+ # dev mode
305
+ return api_url.replace("://api.", "://app.").strip("/")
306
+ elif "://api.wandb." in api_url:
307
+ # cloud
308
+ return api_url.replace("://api.", "://").strip("/")
309
+ elif "://api." in api_url:
310
+ # onprem cloud
311
+ return api_url.replace("://api.", "://app.").strip("/")
312
+ # wandb/local
313
+ return api_url
314
+
315
+
316
+ def get_full_typename(o: Any) -> Any:
317
+ """Determine types based on type names.
318
+
319
+ Avoids needing to to import (and therefore depend on) PyTorch, TensorFlow, etc.
320
+ """
321
+ instance_name = o.__class__.__module__ + "." + o.__class__.__name__
322
+ if instance_name in ["builtins.module", "__builtin__.module"]:
323
+ return o.__name__
324
+ else:
325
+ return instance_name
326
+
327
+
328
+ def get_h5_typename(o: Any) -> Any:
329
+ typename = get_full_typename(o)
330
+ if is_tf_tensor_typename(typename):
331
+ return "tensorflow.Tensor"
332
+ elif is_pytorch_tensor_typename(typename):
333
+ return "torch.Tensor"
334
+ else:
335
+ return o.__class__.__module__.split(".")[0] + "." + o.__class__.__name__
336
+
337
+
338
+ def is_uri(string: str) -> bool:
339
+ parsed_uri = urllib.parse.urlparse(string)
340
+ return len(parsed_uri.scheme) > 0
341
+
342
+
343
+ def local_file_uri_to_path(uri: str) -> str:
344
+ """Convert URI to local filesystem path.
345
+
346
+ No-op if the uri does not have the expected scheme.
347
+ """
348
+ path = urllib.parse.urlparse(uri).path if uri.startswith("file:") else uri
349
+ return urllib.request.url2pathname(path)
350
+
351
+
352
+ def get_local_path_or_none(path_or_uri: str) -> Optional[str]:
353
+ """Return path if local, None otherwise.
354
+
355
+ Return None if the argument is a local path (not a scheme or file:///). Otherwise
356
+ return `path_or_uri`.
357
+ """
358
+ parsed_uri = urllib.parse.urlparse(path_or_uri)
359
+ if (
360
+ len(parsed_uri.scheme) == 0
361
+ or parsed_uri.scheme == "file"
362
+ and len(parsed_uri.netloc) == 0
363
+ ):
364
+ return local_file_uri_to_path(path_or_uri)
365
+ else:
366
+ return None
367
+
368
+
369
+ def make_tarfile(
370
+ output_filename: str,
371
+ source_dir: str,
372
+ archive_name: str,
373
+ custom_filter: Optional[Callable] = None,
374
+ ) -> None:
375
+ # Helper for filtering out modification timestamps
376
+ def _filter_timestamps(tar_info: "tarfile.TarInfo") -> Optional["tarfile.TarInfo"]:
377
+ tar_info.mtime = 0
378
+ return tar_info if custom_filter is None else custom_filter(tar_info)
379
+
380
+ descriptor, unzipped_filename = tempfile.mkstemp()
381
+ try:
382
+ with tarfile.open(unzipped_filename, "w") as tar:
383
+ tar.add(source_dir, arcname=archive_name, filter=_filter_timestamps)
384
+ # When gzipping the tar, don't include the tar's filename or modification time in the
385
+ # zipped archive (see https://docs.python.org/3/library/gzip.html#gzip.GzipFile)
386
+ with gzip.GzipFile(
387
+ filename="", fileobj=open(output_filename, "wb"), mode="wb", mtime=0
388
+ ) as gzipped_tar, open(unzipped_filename, "rb") as tar_file:
389
+ gzipped_tar.write(tar_file.read())
390
+ finally:
391
+ os.close(descriptor)
392
+ os.remove(unzipped_filename)
393
+
394
+
395
+ def is_tf_tensor(obj: Any) -> bool:
396
+ import tensorflow # type: ignore
397
+
398
+ return isinstance(obj, tensorflow.Tensor)
399
+
400
+
401
+ def is_tf_tensor_typename(typename: str) -> bool:
402
+ return typename.startswith("tensorflow.") and (
403
+ "Tensor" in typename or "Variable" in typename
404
+ )
405
+
406
+
407
+ def is_tf_eager_tensor_typename(typename: str) -> bool:
408
+ return typename.startswith("tensorflow.") and ("EagerTensor" in typename)
409
+
410
+
411
+ def is_pytorch_tensor(obj: Any) -> bool:
412
+ import torch # type: ignore
413
+
414
+ return isinstance(obj, torch.Tensor)
415
+
416
+
417
+ def is_pytorch_tensor_typename(typename: str) -> bool:
418
+ return typename.startswith("torch.") and (
419
+ "Tensor" in typename or "Variable" in typename
420
+ )
421
+
422
+
423
+ def is_jax_tensor_typename(typename: str) -> bool:
424
+ return typename.startswith("jaxlib.") and "Array" in typename
425
+
426
+
427
+ def get_jax_tensor(obj: Any) -> Optional[Any]:
428
+ import jax # type: ignore
429
+
430
+ return jax.device_get(obj)
431
+
432
+
433
+ def is_fastai_tensor_typename(typename: str) -> bool:
434
+ return typename.startswith("fastai.") and ("Tensor" in typename)
435
+
436
+
437
+ def is_pandas_data_frame_typename(typename: str) -> bool:
438
+ return typename.startswith("pandas.") and "DataFrame" in typename
439
+
440
+
441
+ def is_matplotlib_typename(typename: str) -> bool:
442
+ return typename.startswith("matplotlib.")
443
+
444
+
445
+ def is_plotly_typename(typename: str) -> bool:
446
+ return typename.startswith("plotly.")
447
+
448
+
449
+ def is_plotly_figure_typename(typename: str) -> bool:
450
+ return typename.startswith("plotly.") and typename.endswith(".Figure")
451
+
452
+
453
+ def is_numpy_array(obj: Any) -> bool:
454
+ return np and isinstance(obj, np.ndarray)
455
+
456
+
457
+ def is_pandas_data_frame(obj: Any) -> bool:
458
+ if pd_available:
459
+ import pandas as pd
460
+
461
+ return isinstance(obj, pd.DataFrame)
462
+ else:
463
+ return is_pandas_data_frame_typename(get_full_typename(obj))
464
+
465
+
466
+ def ensure_matplotlib_figure(obj: Any) -> Any:
467
+ """Extract the current figure from a matplotlib object.
468
+
469
+ Return the object itself if it's a figure.
470
+ Raises ValueError if the object can't be converted.
471
+ """
472
+ import matplotlib # type: ignore
473
+ from matplotlib.figure import Figure # type: ignore
474
+
475
+ # there are combinations of plotly and matplotlib versions that don't work well together,
476
+ # this patches matplotlib to add a removed method that plotly assumes exists
477
+ from matplotlib.spines import Spine # type: ignore
478
+
479
+ def is_frame_like(self: Any) -> bool:
480
+ """Return True if directly on axes frame.
481
+
482
+ This is useful for determining if a spine is the edge of an
483
+ old style MPL plot. If so, this function will return True.
484
+ """
485
+ position = self._position or ("outward", 0.0)
486
+ if isinstance(position, str):
487
+ if position == "center":
488
+ position = ("axes", 0.5)
489
+ elif position == "zero":
490
+ position = ("data", 0)
491
+ if len(position) != 2:
492
+ raise ValueError("position should be 2-tuple")
493
+ position_type, amount = position # type: ignore
494
+ if position_type == "outward" and amount == 0:
495
+ return True
496
+ else:
497
+ return False
498
+
499
+ Spine.is_frame_like = is_frame_like
500
+
501
+ if obj == matplotlib.pyplot:
502
+ obj = obj.gcf()
503
+ elif not isinstance(obj, Figure):
504
+ if hasattr(obj, "figure"):
505
+ obj = obj.figure
506
+ # Some matplotlib objects have a figure function
507
+ if not isinstance(obj, Figure):
508
+ raise ValueError(
509
+ "Only matplotlib.pyplot or matplotlib.pyplot.Figure objects are accepted."
510
+ )
511
+ return obj
512
+
513
+
514
+ def matplotlib_to_plotly(obj: Any) -> Any:
515
+ obj = ensure_matplotlib_figure(obj)
516
+ tools = get_module(
517
+ "plotly.tools",
518
+ required=(
519
+ "plotly is required to log interactive plots, install with: "
520
+ "`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
521
+ ),
522
+ )
523
+ return tools.mpl_to_plotly(obj)
524
+
525
+
526
+ def matplotlib_contains_images(obj: Any) -> bool:
527
+ obj = ensure_matplotlib_figure(obj)
528
+ return any(len(ax.images) > 0 for ax in obj.axes)
529
+
530
+
531
+ def _numpy_generic_convert(obj: Any) -> Any:
532
+ obj = obj.item()
533
+ if isinstance(obj, float) and math.isnan(obj):
534
+ obj = None
535
+ elif isinstance(obj, np.generic) and (
536
+ obj.dtype.kind == "f" or obj.dtype == "bfloat16"
537
+ ):
538
+ # obj is a numpy float with precision greater than that of native python float
539
+ # (i.e., float96 or float128) or it is of custom type such as bfloat16.
540
+ # in these cases, obj.item() does not return a native
541
+ # python float (in the first case - to avoid loss of precision,
542
+ # so we need to explicitly cast this down to a 64bit float)
543
+ obj = float(obj)
544
+ return obj
545
+
546
+
547
+ def _sanitize_numpy_keys(
548
+ d: Dict,
549
+ visited: Optional[Dict[int, Dict]] = None,
550
+ ) -> Tuple[Dict, bool]:
551
+ """Returns a dictionary where all NumPy keys are converted.
552
+
553
+ Args:
554
+ d: The dictionary to sanitize.
555
+
556
+ Returns:
557
+ A sanitized dictionary, and a boolean indicating whether anything was
558
+ changed.
559
+ """
560
+ out: Dict[Any, Any] = dict()
561
+ converted = False
562
+
563
+ # Work with recursive dictionaries: if a dictionary has already been
564
+ # converted, reuse its converted value to retain the recursive structure
565
+ # of the input.
566
+ if visited is None:
567
+ visited = {id(d): out}
568
+ elif id(d) in visited:
569
+ return visited[id(d)], False
570
+ visited[id(d)] = out
571
+
572
+ for key, value in d.items():
573
+ if isinstance(value, dict):
574
+ value, converted_value = _sanitize_numpy_keys(value, visited)
575
+ converted |= converted_value
576
+ if isinstance(key, np.generic):
577
+ key = _numpy_generic_convert(key)
578
+ converted = True
579
+ out[key] = value
580
+
581
+ return out, converted
582
+
583
+
584
+ def json_friendly( # noqa: C901
585
+ obj: Any,
586
+ ) -> Union[Tuple[Any, bool], Tuple[Union[None, str, float], bool]]:
587
+ """Convert an object into something that's more becoming of JSON."""
588
+ converted = True
589
+ typename = get_full_typename(obj)
590
+
591
+ if is_tf_eager_tensor_typename(typename):
592
+ obj = obj.numpy()
593
+ elif is_tf_tensor_typename(typename):
594
+ try:
595
+ obj = obj.eval()
596
+ except RuntimeError:
597
+ obj = obj.numpy()
598
+ elif is_pytorch_tensor_typename(typename) or is_fastai_tensor_typename(typename):
599
+ try:
600
+ if obj.requires_grad:
601
+ obj = obj.detach()
602
+ except AttributeError:
603
+ pass # before 0.4 is only present on variables
604
+
605
+ try:
606
+ obj = obj.data
607
+ except RuntimeError:
608
+ pass # happens for Tensors before 0.4
609
+
610
+ if obj.size():
611
+ obj = obj.cpu().detach().numpy()
612
+ else:
613
+ return obj.item(), True
614
+ elif is_jax_tensor_typename(typename):
615
+ obj = get_jax_tensor(obj)
616
+
617
+ if is_numpy_array(obj):
618
+ if obj.size == 1:
619
+ obj = obj.flatten()[0]
620
+ elif obj.size <= 32:
621
+ obj = obj.tolist()
622
+ elif np and isinstance(obj, np.generic):
623
+ obj = _numpy_generic_convert(obj)
624
+ elif isinstance(obj, bytes):
625
+ obj = obj.decode("utf-8")
626
+ elif isinstance(obj, (datetime, date)):
627
+ obj = obj.isoformat()
628
+ elif callable(obj):
629
+ obj = (
630
+ f"{obj.__module__}.{obj.__qualname__}"
631
+ if hasattr(obj, "__qualname__") and hasattr(obj, "__module__")
632
+ else str(obj)
633
+ )
634
+ elif isinstance(obj, float) and math.isnan(obj):
635
+ obj = None
636
+ elif isinstance(obj, dict) and np:
637
+ obj, converted = _sanitize_numpy_keys(obj)
638
+ elif isinstance(obj, set):
639
+ # set is not json serializable, so we convert it to tuple
640
+ obj = tuple(obj)
641
+ elif isinstance(obj, enum.Enum):
642
+ obj = obj.name
643
+ else:
644
+ converted = False
645
+ if getsizeof(obj) > VALUE_BYTES_LIMIT:
646
+ wandb.termwarn(
647
+ "Serializing object of type {} that is {} bytes".format(
648
+ type(obj).__name__, getsizeof(obj)
649
+ )
650
+ )
651
+ return obj, converted
652
+
653
+
654
+ def json_friendly_val(val: Any) -> Any:
655
+ """Make any value (including dict, slice, sequence, dataclass) JSON friendly."""
656
+ converted: Union[dict, list]
657
+ if isinstance(val, dict):
658
+ converted = {}
659
+ for key, value in val.items():
660
+ converted[key] = json_friendly_val(value)
661
+ return converted
662
+ if isinstance(val, slice):
663
+ converted = dict(
664
+ slice_start=val.start, slice_step=val.step, slice_stop=val.stop
665
+ )
666
+ return converted
667
+ val, _ = json_friendly(val)
668
+ if isinstance(val, Sequence) and not isinstance(val, str):
669
+ converted = []
670
+ for value in val:
671
+ converted.append(json_friendly_val(value))
672
+ return converted
673
+ if is_dataclass(val) and not isinstance(val, type):
674
+ converted = asdict(val)
675
+ return converted
676
+ else:
677
+ if val.__class__.__module__ not in ("builtins", "__builtin__"):
678
+ val = str(val)
679
+ return val
680
+
681
+
682
+ def alias_is_version_index(alias: str) -> bool:
683
+ return len(alias) >= 2 and alias[0] == "v" and alias[1:].isnumeric()
684
+
685
+
686
+ def convert_plots(obj: Any) -> Any:
687
+ if is_matplotlib_typename(get_full_typename(obj)):
688
+ tools = get_module(
689
+ "plotly.tools",
690
+ required=(
691
+ "plotly is required to log interactive plots, install with: "
692
+ "`pip install plotly` or convert the plot to an image with `wandb.Image(plt)`"
693
+ ),
694
+ )
695
+ obj = tools.mpl_to_plotly(obj)
696
+
697
+ if is_plotly_typename(get_full_typename(obj)):
698
+ return {"_type": "plotly", "plot": obj.to_plotly_json()}
699
+ else:
700
+ return obj
701
+
702
+
703
+ def maybe_compress_history(obj: Any) -> Tuple[Any, bool]:
704
+ if np and isinstance(obj, np.ndarray) and obj.size > 32:
705
+ return wandb.Histogram(obj, num_bins=32).to_json(), True
706
+ else:
707
+ return obj, False
708
+
709
+
710
+ def maybe_compress_summary(obj: Any, h5_typename: str) -> Tuple[Any, bool]:
711
+ if np and isinstance(obj, np.ndarray) and obj.size > 32:
712
+ return (
713
+ {
714
+ "_type": h5_typename, # may not be ndarray
715
+ "var": np.var(obj).item(),
716
+ "mean": np.mean(obj).item(),
717
+ "min": np.amin(obj).item(),
718
+ "max": np.amax(obj).item(),
719
+ "10%": np.percentile(obj, 10),
720
+ "25%": np.percentile(obj, 25),
721
+ "75%": np.percentile(obj, 75),
722
+ "90%": np.percentile(obj, 90),
723
+ "size": obj.size,
724
+ },
725
+ True,
726
+ )
727
+ else:
728
+ return obj, False
729
+
730
+
731
+ def launch_browser(attempt_launch_browser: bool = True) -> bool:
732
+ """Decide if we should launch a browser."""
733
+ _display_variables = ["DISPLAY", "WAYLAND_DISPLAY", "MIR_SOCKET"]
734
+ _webbrowser_names_blocklist = ["www-browser", "lynx", "links", "elinks", "w3m"]
735
+
736
+ import webbrowser
737
+
738
+ launch_browser = attempt_launch_browser
739
+ if launch_browser:
740
+ if "linux" in sys.platform and not any(
741
+ os.getenv(var) for var in _display_variables
742
+ ):
743
+ launch_browser = False
744
+ try:
745
+ browser = webbrowser.get()
746
+ if hasattr(browser, "name") and browser.name in _webbrowser_names_blocklist:
747
+ launch_browser = False
748
+ except webbrowser.Error:
749
+ launch_browser = False
750
+
751
+ return launch_browser
752
+
753
+
754
+ def generate_id(length: int = 8) -> str:
755
+ # Do not use this; use wandb.sdk.lib.runid.generate_id instead.
756
+ # This is kept only for legacy code.
757
+ return runid.generate_id(length)
758
+
759
+
760
+ def parse_tfjob_config() -> Any:
761
+ """Attempt to parse TFJob config, returning False if it can't find it."""
762
+ if os.getenv("TF_CONFIG"):
763
+ try:
764
+ return json.loads(os.environ["TF_CONFIG"])
765
+ except ValueError:
766
+ return False
767
+ else:
768
+ return False
769
+
770
+
771
+ class WandBJSONEncoder(json.JSONEncoder):
772
+ """A JSON Encoder that handles some extra types."""
773
+
774
+ def default(self, obj: Any) -> Any:
775
+ if hasattr(obj, "json_encode"):
776
+ return obj.json_encode()
777
+ # if hasattr(obj, 'to_json'):
778
+ # return obj.to_json()
779
+ tmp_obj, converted = json_friendly(obj)
780
+ if converted:
781
+ return tmp_obj
782
+ return json.JSONEncoder.default(self, obj)
783
+
784
+
785
+ class WandBJSONEncoderOld(json.JSONEncoder):
786
+ """A JSON Encoder that handles some extra types."""
787
+
788
+ def default(self, obj: Any) -> Any:
789
+ tmp_obj, converted = json_friendly(obj)
790
+ tmp_obj, compressed = maybe_compress_summary(tmp_obj, get_h5_typename(obj))
791
+ if converted:
792
+ return tmp_obj
793
+ return json.JSONEncoder.default(self, tmp_obj)
794
+
795
+
796
+ class WandBHistoryJSONEncoder(json.JSONEncoder):
797
+ """A JSON Encoder that handles some extra types.
798
+
799
+ This encoder turns numpy like objects with a size > 32 into histograms.
800
+ """
801
+
802
+ def default(self, obj: Any) -> Any:
803
+ obj, converted = json_friendly(obj)
804
+ obj, compressed = maybe_compress_history(obj)
805
+ if converted:
806
+ return obj
807
+ return json.JSONEncoder.default(self, obj)
808
+
809
+
810
+ class JSONEncoderUncompressed(json.JSONEncoder):
811
+ """A JSON Encoder that handles some extra types.
812
+
813
+ This encoder turns numpy like objects with a size > 32 into histograms.
814
+ """
815
+
816
+ def default(self, obj: Any) -> Any:
817
+ if is_numpy_array(obj):
818
+ return obj.tolist()
819
+ elif np and isinstance(obj, np.generic):
820
+ obj = obj.item()
821
+ return json.JSONEncoder.default(self, obj)
822
+
823
+
824
+ def json_dump_safer(obj: Any, fp: IO[str], **kwargs: Any) -> None:
825
+ """Convert obj to json, with some extra encodable types."""
826
+ return dump(obj, fp, cls=WandBJSONEncoder, **kwargs)
827
+
828
+
829
+ def json_dumps_safer(obj: Any, **kwargs: Any) -> str:
830
+ """Convert obj to json, with some extra encodable types."""
831
+ return dumps(obj, cls=WandBJSONEncoder, **kwargs)
832
+
833
+
834
+ # This is used for dumping raw json into files
835
+ def json_dump_uncompressed(obj: Any, fp: IO[str], **kwargs: Any) -> None:
836
+ """Convert obj to json, with some extra encodable types."""
837
+ return dump(obj, fp, cls=JSONEncoderUncompressed, **kwargs)
838
+
839
+
840
+ def json_dumps_safer_history(obj: Any, **kwargs: Any) -> str:
841
+ """Convert obj to json, with some extra encodable types, including histograms."""
842
+ return dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs)
843
+
844
+
845
+ def make_json_if_not_number(
846
+ v: Union[int, float, str, Mapping, Sequence],
847
+ ) -> Union[int, float, str]:
848
+ """If v is not a basic type convert it to json."""
849
+ if isinstance(v, (float, int)):
850
+ return v
851
+ return json_dumps_safer(v)
852
+
853
+
854
+ def make_safe_for_json(obj: Any) -> Any:
855
+ """Replace invalid json floats with strings. Also converts to lists and dicts."""
856
+ if isinstance(obj, Mapping):
857
+ return {k: make_safe_for_json(v) for k, v in obj.items()}
858
+ elif isinstance(obj, str):
859
+ # str's are Sequence, so we need to short-circuit
860
+ return obj
861
+ elif isinstance(obj, Sequence):
862
+ return [make_safe_for_json(v) for v in obj]
863
+ elif isinstance(obj, float):
864
+ # W&B backend and UI handle these strings
865
+ if obj != obj: # standard way to check for NaN
866
+ return "NaN"
867
+ elif obj == float("+inf"):
868
+ return "Infinity"
869
+ elif obj == float("-inf"):
870
+ return "-Infinity"
871
+ return obj
872
+
873
+
874
+ def no_retry_4xx(e: Exception) -> bool:
875
+ if not isinstance(e, requests.HTTPError):
876
+ return True
877
+ assert e.response is not None
878
+ if not (400 <= e.response.status_code < 500) or e.response.status_code == 429:
879
+ return True
880
+ body = json.loads(e.response.content)
881
+ raise UsageError(body["errors"][0]["message"])
882
+
883
+
884
+ def no_retry_auth(e: Any) -> bool:
885
+ if hasattr(e, "exception"):
886
+ e = e.exception
887
+ if not isinstance(e, requests.HTTPError):
888
+ return True
889
+ if e.response is None:
890
+ return True
891
+ # Don't retry bad request errors; raise immediately
892
+ if e.response.status_code in (400, 409):
893
+ return False
894
+ # Retry all non-forbidden/unauthorized/not-found errors.
895
+ if e.response.status_code not in (401, 403, 404):
896
+ return True
897
+ # Crash w/message on forbidden/unauthorized errors.
898
+ if e.response.status_code == 401:
899
+ raise AuthenticationError(
900
+ "The API key you provided is either invalid or missing. "
901
+ f"If the `{wandb.env.API_KEY}` environment variable is set, make sure it is correct. "
902
+ "Otherwise, to resolve this issue, you may try running the 'wandb login --relogin' command. "
903
+ "If you are using a local server, make sure that you're using the correct hostname. "
904
+ "If you're not sure, you can try logging in again using the 'wandb login --relogin --host [hostname]' command."
905
+ f"(Error {e.response.status_code}: {e.response.reason})"
906
+ )
907
+ elif wandb.run:
908
+ raise CommError(f"Permission denied to access {wandb.run.path}")
909
+ else:
910
+ raise CommError(
911
+ "It appears that you do not have permission to access the requested resource. "
912
+ "Please reach out to the project owner to grant you access. "
913
+ "If you have the correct permissions, verify that there are no issues with your networking setup."
914
+ f"(Error {e.response.status_code}: {e.response.reason})"
915
+ )
916
+
917
+
918
+ def check_retry_conflict(e: Any) -> Optional[bool]:
919
+ """Check if the exception is a conflict type so it can be retried.
920
+
921
+ Returns:
922
+ True - Should retry this operation
923
+ False - Should not retry this operation
924
+ None - No decision, let someone else decide
925
+ """
926
+ if hasattr(e, "exception"):
927
+ e = e.exception
928
+ if isinstance(e, requests.HTTPError) and e.response is not None:
929
+ if e.response.status_code == 409:
930
+ return True
931
+ return None
932
+
933
+
934
+ def check_retry_conflict_or_gone(e: Any) -> Optional[bool]:
935
+ """Check if the exception is a conflict or gone type, so it can be retried or not.
936
+
937
+ Returns:
938
+ True - Should retry this operation
939
+ False - Should not retry this operation
940
+ None - No decision, let someone else decide
941
+ """
942
+ if hasattr(e, "exception"):
943
+ e = e.exception
944
+ if isinstance(e, requests.HTTPError) and e.response is not None:
945
+ if e.response.status_code == 409:
946
+ return True
947
+ if e.response.status_code == 410:
948
+ return False
949
+ return None
950
+
951
+
952
+ def make_check_retry_fn(
953
+ fallback_retry_fn: CheckRetryFnType,
954
+ check_fn: Callable[[Exception], Optional[bool]],
955
+ check_timedelta: Optional[timedelta] = None,
956
+ ) -> CheckRetryFnType:
957
+ """Return a check_retry_fn which can be used by lib.Retry().
958
+
959
+ Arguments:
960
+ fallback_fn: Use this function if check_fn didn't decide if a retry should happen.
961
+ check_fn: Function which returns bool if retry should happen or None if unsure.
962
+ check_timedelta: Optional retry timeout if we check_fn matches the exception
963
+ """
964
+
965
+ def check_retry_fn(e: Exception) -> Union[bool, timedelta]:
966
+ check = check_fn(e)
967
+ if check is None:
968
+ return fallback_retry_fn(e)
969
+ if check is False:
970
+ return False
971
+ if check_timedelta:
972
+ return check_timedelta
973
+ return True
974
+
975
+ return check_retry_fn
976
+
977
+
978
+ def find_runner(program: str) -> Union[None, list, List[str]]:
979
+ """Return a command that will run program.
980
+
981
+ Arguments:
982
+ program: The string name of the program to try to run.
983
+
984
+ Returns:
985
+ commandline list of strings to run the program (eg. with subprocess.call()) or None
986
+ """
987
+ if os.path.isfile(program) and not os.access(program, os.X_OK):
988
+ # program is a path to a non-executable file
989
+ try:
990
+ opened = open(program)
991
+ except OSError: # PermissionError doesn't exist in 2.7
992
+ return None
993
+ first_line = opened.readline().strip()
994
+ if first_line.startswith("#!"):
995
+ return shlex.split(first_line[2:])
996
+ if program.endswith(".py"):
997
+ return [sys.executable]
998
+ return None
999
+
1000
+
1001
+ def downsample(values: Sequence, target_length: int) -> list:
1002
+ """Downsample 1d values to target_length, including start and end.
1003
+
1004
+ Algorithm just rounds index down.
1005
+
1006
+ Values can be any sequence, including a generator.
1007
+ """
1008
+ if not target_length > 1:
1009
+ raise UsageError("target_length must be > 1")
1010
+ values = list(values)
1011
+ if len(values) < target_length:
1012
+ return values
1013
+ ratio = float(len(values) - 1) / (target_length - 1)
1014
+ result = []
1015
+ for i in range(target_length):
1016
+ result.append(values[int(i * ratio)])
1017
+ return result
1018
+
1019
+
1020
+ def has_num(dictionary: Mapping, key: Any) -> bool:
1021
+ return key in dictionary and isinstance(dictionary[key], numbers.Number)
1022
+
1023
+
1024
+ def get_log_file_path() -> str:
1025
+ """Log file path used in error messages.
1026
+
1027
+ It would probably be better if this pointed to a log file in a
1028
+ run directory.
1029
+ """
1030
+ # TODO(jhr, cvp): refactor
1031
+ if wandb.run is not None:
1032
+ return wandb.run._settings.log_internal
1033
+ return os.path.join("wandb", "debug-internal.log")
1034
+
1035
+
1036
+ def docker_image_regex(image: str) -> Any:
1037
+ """Regex match for valid docker image names."""
1038
+ if image:
1039
+ return re.match(
1040
+ r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-zA-Z0-9-]{1,63}(?<!-))*(?::[0-9]{1,5})?/)?((?![._-])(?:[a-z0-9._-]*)(?<![._-])(?:/(?![._-])[a-z0-9._-]*(?<![._-]))*)(?::(?![.-])[a-zA-Z0-9_.-]{1,128})?$",
1041
+ image,
1042
+ )
1043
+ return None
1044
+
1045
+
1046
+ def image_from_docker_args(args: List[str]) -> Optional[str]:
1047
+ """Scan docker run args and attempt to find the most likely docker image argument.
1048
+
1049
+ It excludes any arguments that start with a dash, and the argument after it if it
1050
+ isn't a boolean switch. This can be improved, we currently fallback gracefully when
1051
+ this fails.
1052
+ """
1053
+ bool_args = [
1054
+ "-t",
1055
+ "--tty",
1056
+ "--rm",
1057
+ "--privileged",
1058
+ "--oom-kill-disable",
1059
+ "--no-healthcheck",
1060
+ "-i",
1061
+ "--interactive",
1062
+ "--init",
1063
+ "--help",
1064
+ "--detach",
1065
+ "-d",
1066
+ "--sig-proxy",
1067
+ "-it",
1068
+ "-itd",
1069
+ ]
1070
+ last_flag = -2
1071
+ last_arg = ""
1072
+ possible_images = []
1073
+ if len(args) > 0 and args[0] == "run":
1074
+ args.pop(0)
1075
+ for i, arg in enumerate(args):
1076
+ if arg.startswith("-"):
1077
+ last_flag = i
1078
+ last_arg = arg
1079
+ elif "@sha256:" in arg:
1080
+ # Because our regex doesn't match digests
1081
+ possible_images.append(arg)
1082
+ elif docker_image_regex(arg):
1083
+ if last_flag == i - 2:
1084
+ possible_images.append(arg)
1085
+ elif "=" in last_arg:
1086
+ possible_images.append(arg)
1087
+ elif last_arg in bool_args and last_flag == i - 1:
1088
+ possible_images.append(arg)
1089
+ most_likely = None
1090
+ for img in possible_images:
1091
+ if ":" in img or "@" in img or "/" in img:
1092
+ most_likely = img
1093
+ break
1094
+ if most_likely is None and len(possible_images) > 0:
1095
+ most_likely = possible_images[0]
1096
+ return most_likely
1097
+
1098
+
1099
+ def load_yaml(file: Any) -> Any:
1100
+ return yaml.safe_load(file)
1101
+
1102
+
1103
+ def image_id_from_k8s() -> Optional[str]:
1104
+ """Ping the k8s metadata service for the image id.
1105
+
1106
+ Specify the KUBERNETES_NAMESPACE environment variable if your pods are not in the
1107
+ default namespace:
1108
+
1109
+ - name: KUBERNETES_NAMESPACE valueFrom:
1110
+ fieldRef:
1111
+ fieldPath: metadata.namespace
1112
+ """
1113
+ token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token"
1114
+
1115
+ if not os.path.exists(token_path):
1116
+ return None
1117
+
1118
+ try:
1119
+ with open(token_path) as token_file:
1120
+ token = token_file.read()
1121
+ except FileNotFoundError:
1122
+ logger.warning(f"Token file not found at {token_path}.")
1123
+ return None
1124
+ except PermissionError as e:
1125
+ current_uid = os.getuid()
1126
+ warning = (
1127
+ f"Unable to read the token file at {token_path} due to permission error ({e})."
1128
+ f"The current user id is {current_uid}. "
1129
+ "Consider changing the securityContext to run the container as the current user."
1130
+ )
1131
+ logger.warning(warning)
1132
+ wandb.termwarn(warning)
1133
+ return None
1134
+
1135
+ if not token:
1136
+ return None
1137
+
1138
+ k8s_server = "https://{}:{}/api/v1/namespaces/{}/pods/{}".format(
1139
+ os.getenv("KUBERNETES_SERVICE_HOST"),
1140
+ os.getenv("KUBERNETES_PORT_443_TCP_PORT"),
1141
+ os.getenv("KUBERNETES_NAMESPACE", "default"),
1142
+ os.getenv("HOSTNAME"),
1143
+ )
1144
+ try:
1145
+ res = requests.get(
1146
+ k8s_server,
1147
+ verify="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
1148
+ timeout=3,
1149
+ headers={"Authorization": f"Bearer {token}"},
1150
+ )
1151
+ res.raise_for_status()
1152
+ except requests.RequestException:
1153
+ return None
1154
+ try:
1155
+ return str( # noqa: B005
1156
+ res.json()["status"]["containerStatuses"][0]["imageID"]
1157
+ ).strip("docker-pullable://")
1158
+ except (ValueError, KeyError, IndexError):
1159
+ logger.exception("Error checking kubernetes for image id")
1160
+ return None
1161
+
1162
+
1163
+ def async_call(
1164
+ target: Callable, timeout: Optional[Union[int, float]] = None
1165
+ ) -> Callable:
1166
+ """Wrap a method to run in the background with an optional timeout.
1167
+
1168
+ Returns a new method that will call the original with any args, waiting for upto
1169
+ timeout seconds. This new method blocks on the original and returns the result or
1170
+ None if timeout was reached, along with the thread. You can check thread.is_alive()
1171
+ to determine if a timeout was reached. If an exception is thrown in the thread, we
1172
+ reraise it.
1173
+ """
1174
+ q: queue.Queue = queue.Queue()
1175
+
1176
+ def wrapped_target(q: "queue.Queue", *args: Any, **kwargs: Any) -> Any:
1177
+ try:
1178
+ q.put(target(*args, **kwargs))
1179
+ except Exception as e:
1180
+ q.put(e)
1181
+
1182
+ def wrapper(
1183
+ *args: Any, **kwargs: Any
1184
+ ) -> Union[Tuple[Exception, "threading.Thread"], Tuple[None, "threading.Thread"]]:
1185
+ thread = threading.Thread(
1186
+ target=wrapped_target, args=(q,) + args, kwargs=kwargs
1187
+ )
1188
+ thread.daemon = True
1189
+ thread.start()
1190
+ try:
1191
+ result = q.get(True, timeout)
1192
+ if isinstance(result, Exception):
1193
+ raise result.with_traceback(sys.exc_info()[2])
1194
+ return result, thread
1195
+ except queue.Empty:
1196
+ return None, thread
1197
+
1198
+ return wrapper
1199
+
1200
+
1201
+ def read_many_from_queue(
1202
+ q: "queue.Queue", max_items: int, queue_timeout: Union[int, float]
1203
+ ) -> list:
1204
+ try:
1205
+ item = q.get(True, queue_timeout)
1206
+ except queue.Empty:
1207
+ return []
1208
+ items = [item]
1209
+ for _ in range(max_items):
1210
+ try:
1211
+ item = q.get_nowait()
1212
+ except queue.Empty:
1213
+ return items
1214
+ items.append(item)
1215
+ return items
1216
+
1217
+
1218
+ def stopwatch_now() -> float:
1219
+ """Get a time value for interval comparisons.
1220
+
1221
+ When possible it is a monotonic clock to prevent backwards time issues.
1222
+ """
1223
+ return time.monotonic()
1224
+
1225
+
1226
+ def class_colors(class_count: int) -> List[List[int]]:
1227
+ # make class 0 black, and the rest equally spaced fully saturated hues
1228
+ return [[0, 0, 0]] + [
1229
+ colorsys.hsv_to_rgb(i / (class_count - 1.0), 1.0, 1.0) # type: ignore
1230
+ for i in range(class_count - 1)
1231
+ ]
1232
+
1233
+
1234
+ def _prompt_choice(
1235
+ input_timeout: Union[int, float, None] = None,
1236
+ jupyter: bool = False,
1237
+ ) -> str:
1238
+ input_fn: Callable = input
1239
+ prompt = term.LOG_STRING
1240
+ if input_timeout is not None:
1241
+ # delayed import to mitigate risk of timed_input complexity
1242
+ from wandb.sdk.lib import timed_input
1243
+
1244
+ input_fn = functools.partial(timed_input.timed_input, timeout=input_timeout)
1245
+ # timed_input doesn't handle enhanced prompts
1246
+ if platform.system() == "Windows":
1247
+ prompt = "wandb"
1248
+
1249
+ text = f"{prompt}: Enter your choice: "
1250
+ if input_fn == input:
1251
+ choice = input_fn(text)
1252
+ else:
1253
+ choice = input_fn(text, jupyter=jupyter)
1254
+ return choice # type: ignore
1255
+
1256
+
1257
+ def prompt_choices(
1258
+ choices: Sequence[str],
1259
+ input_timeout: Union[int, float, None] = None,
1260
+ jupyter: bool = False,
1261
+ ) -> str:
1262
+ """Allow a user to choose from a list of options."""
1263
+ for i, choice in enumerate(choices):
1264
+ wandb.termlog(f"({i+1}) {choice}")
1265
+
1266
+ idx = -1
1267
+ while idx < 0 or idx > len(choices) - 1:
1268
+ choice = _prompt_choice(input_timeout=input_timeout, jupyter=jupyter)
1269
+ if not choice:
1270
+ continue
1271
+ idx = -1
1272
+ try:
1273
+ idx = int(choice) - 1
1274
+ except ValueError:
1275
+ pass
1276
+ if idx < 0 or idx > len(choices) - 1:
1277
+ wandb.termwarn("Invalid choice")
1278
+ result = choices[idx]
1279
+ wandb.termlog(f"You chose {result!r}")
1280
+ return result
1281
+
1282
+
1283
+ def guess_data_type(shape: Sequence[int], risky: bool = False) -> Optional[str]:
1284
+ """Infer the type of data based on the shape of the tensors.
1285
+
1286
+ Arguments:
1287
+ shape (Sequence[int]): The shape of the data
1288
+ risky(bool): some guesses are more likely to be wrong.
1289
+ """
1290
+ # (samples,) or (samples,logits)
1291
+ if len(shape) in (1, 2):
1292
+ return "label"
1293
+ # Assume image mask like fashion mnist: (no color channel)
1294
+ # This is risky because RNNs often have 3 dim tensors: batch, time, channels
1295
+ if risky and len(shape) == 3:
1296
+ return "image"
1297
+ if len(shape) == 4:
1298
+ if shape[-1] in (1, 3, 4):
1299
+ # (samples, height, width, Y \ RGB \ RGBA)
1300
+ return "image"
1301
+ else:
1302
+ # (samples, height, width, logits)
1303
+ return "segmentation_mask"
1304
+ return None
1305
+
1306
+
1307
+ def download_file_from_url(
1308
+ dest_path: str, source_url: str, api_key: Optional[str] = None
1309
+ ) -> None:
1310
+ auth = None
1311
+ if not _thread_local_api_settings.cookies:
1312
+ auth = ("api", api_key or "")
1313
+ response = requests.get(
1314
+ source_url,
1315
+ auth=auth,
1316
+ headers=_thread_local_api_settings.headers,
1317
+ cookies=_thread_local_api_settings.cookies,
1318
+ stream=True,
1319
+ timeout=5,
1320
+ )
1321
+ response.raise_for_status()
1322
+
1323
+ if os.sep in dest_path:
1324
+ filesystem.mkdir_exists_ok(os.path.dirname(dest_path))
1325
+ with fsync_open(dest_path, "wb") as file:
1326
+ for data in response.iter_content(chunk_size=1024):
1327
+ file.write(data)
1328
+
1329
+
1330
+ def download_file_into_memory(source_url: str, api_key: Optional[str] = None) -> bytes:
1331
+ auth = None
1332
+ if not _thread_local_api_settings.cookies:
1333
+ auth = ("api", api_key or "")
1334
+ response = requests.get(
1335
+ source_url,
1336
+ auth=auth,
1337
+ headers=_thread_local_api_settings.headers,
1338
+ cookies=_thread_local_api_settings.cookies,
1339
+ stream=True,
1340
+ timeout=5,
1341
+ )
1342
+ response.raise_for_status()
1343
+ return response.content
1344
+
1345
+
1346
+ def isatty(ob: IO) -> bool:
1347
+ return hasattr(ob, "isatty") and ob.isatty()
1348
+
1349
+
1350
+ def to_human_size(size: int, units: Optional[List[Tuple[str, Any]]] = None) -> str:
1351
+ units = units or POW_10_BYTES
1352
+ unit, value = units[0]
1353
+ factor = round(float(size) / value, 1)
1354
+ return (
1355
+ f"{factor}{unit}"
1356
+ if factor < 1024 or len(units) == 1
1357
+ else to_human_size(size, units[1:])
1358
+ )
1359
+
1360
+
1361
+ def from_human_size(size: str, units: Optional[List[Tuple[str, Any]]] = None) -> int:
1362
+ units = units or POW_10_BYTES
1363
+ units_dict = {unit.upper(): value for (unit, value) in units}
1364
+ regex = re.compile(
1365
+ r"(\d+\.?\d*)\s*({})?".format("|".join(units_dict.keys())), re.IGNORECASE
1366
+ )
1367
+ match = re.match(regex, size)
1368
+ if not match:
1369
+ raise ValueError("size must be of the form `10`, `10B` or `10 B`.")
1370
+ factor, unit = (
1371
+ float(match.group(1)),
1372
+ units_dict[match.group(2).upper()] if match.group(2) else 1,
1373
+ )
1374
+ return int(factor * unit)
1375
+
1376
+
1377
+ def auto_project_name(program: Optional[str]) -> str:
1378
+ # if we're in git, set project name to git repo name + relative path within repo
1379
+ from wandb.sdk.lib.gitlib import GitRepo
1380
+
1381
+ root_dir = GitRepo().root_dir
1382
+ if root_dir is None:
1383
+ return "uncategorized"
1384
+ # On windows, GitRepo returns paths in unix style, but os.path is windows
1385
+ # style. Coerce here.
1386
+ root_dir = to_native_slash_path(root_dir)
1387
+ repo_name = os.path.basename(root_dir)
1388
+ if program is None:
1389
+ return str(repo_name)
1390
+ if not os.path.isabs(program):
1391
+ program = os.path.join(os.curdir, program)
1392
+ prog_dir = os.path.dirname(os.path.abspath(program))
1393
+ if not prog_dir.startswith(root_dir):
1394
+ return str(repo_name)
1395
+ project = repo_name
1396
+ sub_path = os.path.relpath(prog_dir, root_dir)
1397
+ if sub_path != ".":
1398
+ project += "-" + sub_path
1399
+ return str(project.replace(os.sep, "_"))
1400
+
1401
+
1402
+ # TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
1403
+ def to_forward_slash_path(path: str) -> str:
1404
+ if platform.system() == "Windows":
1405
+ path = path.replace("\\", "/")
1406
+ return path
1407
+
1408
+
1409
+ # TODO(hugh): Deprecate version here and use wandb/sdk/lib/paths.py
1410
+ def to_native_slash_path(path: str) -> FilePathStr:
1411
+ return FilePathStr(path.replace("/", os.sep))
1412
+
1413
+
1414
+ def check_and_warn_old(files: List[str]) -> bool:
1415
+ if "wandb-metadata.json" in files:
1416
+ wandb.termwarn("These runs were logged with a previous version of wandb.")
1417
+ wandb.termwarn(
1418
+ "Run pip install wandb<0.10.0 to get the old library and sync your runs."
1419
+ )
1420
+ return True
1421
+ return False
1422
+
1423
+
1424
+ class ImportMetaHook:
1425
+ def __init__(self) -> None:
1426
+ self.modules: Dict[str, ModuleType] = dict()
1427
+ self.on_import: Dict[str, list] = dict()
1428
+
1429
+ def add(self, fullname: str, on_import: Callable) -> None:
1430
+ self.on_import.setdefault(fullname, []).append(on_import)
1431
+
1432
+ def install(self) -> None:
1433
+ sys.meta_path.insert(0, self) # type: ignore
1434
+
1435
+ def uninstall(self) -> None:
1436
+ sys.meta_path.remove(self) # type: ignore
1437
+
1438
+ def find_module(
1439
+ self, fullname: str, path: Optional[str] = None
1440
+ ) -> Optional["ImportMetaHook"]:
1441
+ if fullname in self.on_import:
1442
+ return self
1443
+ return None
1444
+
1445
+ def load_module(self, fullname: str) -> ModuleType:
1446
+ self.uninstall()
1447
+ mod = importlib.import_module(fullname)
1448
+ self.install()
1449
+ self.modules[fullname] = mod
1450
+ on_imports = self.on_import.get(fullname)
1451
+ if on_imports:
1452
+ for f in on_imports:
1453
+ f()
1454
+ return mod
1455
+
1456
+ def get_modules(self) -> Tuple[str, ...]:
1457
+ return tuple(self.modules)
1458
+
1459
+ def get_module(self, module: str) -> ModuleType:
1460
+ return self.modules[module]
1461
+
1462
+
1463
+ _import_hook: Optional[ImportMetaHook] = None
1464
+
1465
+
1466
+ def add_import_hook(fullname: str, on_import: Callable) -> None:
1467
+ global _import_hook
1468
+ if _import_hook is None:
1469
+ _import_hook = ImportMetaHook()
1470
+ _import_hook.install()
1471
+ _import_hook.add(fullname, on_import)
1472
+
1473
+
1474
+ def host_from_path(path: Optional[str]) -> str:
1475
+ """Return the host of the path."""
1476
+ url = urllib.parse.urlparse(path)
1477
+ return str(url.netloc)
1478
+
1479
+
1480
+ def uri_from_path(path: Optional[str]) -> str:
1481
+ """Return the URI of the path."""
1482
+ url = urllib.parse.urlparse(path)
1483
+ uri = url.path if url.path[0] != "/" else url.path[1:]
1484
+ return str(uri)
1485
+
1486
+
1487
+ def is_unicode_safe(stream: TextIO) -> bool:
1488
+ """Return True if the stream supports UTF-8."""
1489
+ encoding = getattr(stream, "encoding", None)
1490
+ return encoding.lower() in {"utf-8", "utf_8"} if encoding else False
1491
+
1492
+
1493
+ def _has_internet() -> bool:
1494
+ """Attempt to open a DNS connection to Googles root servers."""
1495
+ try:
1496
+ s = socket.create_connection(("8.8.8.8", 53), 0.5)
1497
+ s.close()
1498
+ return True
1499
+ except OSError:
1500
+ return False
1501
+
1502
+
1503
+ def rand_alphanumeric(
1504
+ length: int = 8, rand: Optional[Union[ModuleType, random.Random]] = None
1505
+ ) -> str:
1506
+ wandb.termerror("rand_alphanumeric is deprecated, use 'secrets.token_hex'")
1507
+ rand = rand or random
1508
+ return "".join(rand.choice("0123456789ABCDEF") for _ in range(length))
1509
+
1510
+
1511
+ @contextlib.contextmanager
1512
+ def fsync_open(
1513
+ path: StrPath, mode: str = "w", encoding: Optional[str] = None
1514
+ ) -> Generator[IO[Any], None, None]:
1515
+ """Open a path for I/O and guarantee that the file is flushed and synced."""
1516
+ with open(path, mode, encoding=encoding) as f:
1517
+ yield f
1518
+
1519
+ f.flush()
1520
+ os.fsync(f.fileno())
1521
+
1522
+
1523
+ def _is_kaggle() -> bool:
1524
+ return (
1525
+ os.getenv("KAGGLE_KERNEL_RUN_TYPE") is not None
1526
+ or "kaggle_environments" in sys.modules
1527
+ )
1528
+
1529
+
1530
+ def _is_likely_kaggle() -> bool:
1531
+ # Telemetry to mark first runs from Kagglers.
1532
+ return (
1533
+ _is_kaggle()
1534
+ or os.path.exists(
1535
+ os.path.expanduser(os.path.join("~", ".kaggle", "kaggle.json"))
1536
+ )
1537
+ or "kaggle" in sys.modules
1538
+ )
1539
+
1540
+
1541
+ def _is_databricks() -> bool:
1542
+ # check if we are running inside a databricks notebook by
1543
+ # inspecting sys.modules, searching for dbutils and verifying that
1544
+ # it has the appropriate structure
1545
+
1546
+ if "dbutils" in sys.modules:
1547
+ dbutils = sys.modules["dbutils"]
1548
+ if hasattr(dbutils, "shell"):
1549
+ shell = dbutils.shell
1550
+ if hasattr(shell, "sc"):
1551
+ sc = shell.sc
1552
+ if hasattr(sc, "appName"):
1553
+ return bool(sc.appName == "Databricks Shell")
1554
+ return False
1555
+
1556
+
1557
+ def _is_py_requirements_or_dockerfile(path: str) -> bool:
1558
+ file = os.path.basename(path)
1559
+ return (
1560
+ file.endswith(".py")
1561
+ or file.startswith("Dockerfile")
1562
+ or file == "requirements.txt"
1563
+ )
1564
+
1565
+
1566
+ def check_windows_valid_filename(path: Union[int, str]) -> bool:
1567
+ return not bool(re.search(RE_WINFNAMES, path)) # type: ignore
1568
+
1569
+
1570
+ def artifact_to_json(artifact: "Artifact") -> Dict[str, Any]:
1571
+ return {
1572
+ "_type": "artifactVersion",
1573
+ "_version": "v0",
1574
+ "id": artifact.id,
1575
+ "version": artifact.source_version,
1576
+ "sequenceName": artifact.source_name.split(":")[0],
1577
+ "usedAs": artifact.use_as,
1578
+ }
1579
+
1580
+
1581
+ def check_dict_contains_nested_artifact(d: dict, nested: bool = False) -> bool:
1582
+ for item in d.values():
1583
+ if isinstance(item, dict):
1584
+ contains_artifacts = check_dict_contains_nested_artifact(item, True)
1585
+ if contains_artifacts:
1586
+ return True
1587
+ elif (isinstance(item, wandb.Artifact) or _is_artifact_string(item)) and nested:
1588
+ return True
1589
+ return False
1590
+
1591
+
1592
+ def load_json_yaml_dict(config: str) -> Any:
1593
+ ext = os.path.splitext(config)[-1]
1594
+ if ext == ".json":
1595
+ with open(config) as f:
1596
+ return json.load(f)
1597
+ elif ext == ".yaml":
1598
+ with open(config) as f:
1599
+ return yaml.safe_load(f)
1600
+ else:
1601
+ try:
1602
+ return json.loads(config)
1603
+ except ValueError:
1604
+ return None
1605
+
1606
+
1607
+ def _parse_entity_project_item(path: str) -> tuple:
1608
+ """Parse paths with the following formats: {item}, {project}/{item}, & {entity}/{project}/{item}.
1609
+
1610
+ Args:
1611
+ path: `str`, input path; must be between 0 and 3 in length.
1612
+
1613
+ Returns:
1614
+ tuple of length 3 - (item, project, entity)
1615
+
1616
+ Example:
1617
+ alias, project, entity = _parse_entity_project_item("myproj/mymodel:best")
1618
+
1619
+ assert entity == ""
1620
+ assert project == "myproj"
1621
+ assert alias == "mymodel:best"
1622
+
1623
+ """
1624
+ words = path.split("/")
1625
+ if len(words) > 3:
1626
+ raise ValueError(
1627
+ "Invalid path: must be str the form {item}, {project}/{item}, or {entity}/{project}/{item}"
1628
+ )
1629
+ padded_words = [""] * (3 - len(words)) + words
1630
+ return tuple(reversed(padded_words))
1631
+
1632
+
1633
+ def _resolve_aliases(aliases: Optional[Union[str, Iterable[str]]]) -> List[str]:
1634
+ """Add the 'latest' alias and ensure that all aliases are unique.
1635
+
1636
+ Takes in `aliases` which can be None, str, or List[str] and returns List[str].
1637
+ Ensures that "latest" is always present in the returned list.
1638
+
1639
+ Args:
1640
+ aliases: `Optional[Union[str, List[str]]]`
1641
+
1642
+ Returns:
1643
+ List[str], with "latest" always present.
1644
+
1645
+ Usage:
1646
+
1647
+ ```python
1648
+ aliases = _resolve_aliases(["best", "dev"])
1649
+ assert aliases == ["best", "dev", "latest"]
1650
+
1651
+ aliases = _resolve_aliases("boom")
1652
+ assert aliases == ["boom", "latest"]
1653
+ ```
1654
+ """
1655
+ aliases = aliases or ["latest"]
1656
+
1657
+ if isinstance(aliases, str):
1658
+ aliases = [aliases]
1659
+
1660
+ try:
1661
+ return list(set(aliases) | {"latest"})
1662
+ except TypeError as exc:
1663
+ raise ValueError("`aliases` must be Iterable or None") from exc
1664
+
1665
+
1666
+ def _is_artifact_object(v: Any) -> bool:
1667
+ return isinstance(v, wandb.Artifact)
1668
+
1669
+
1670
+ def _is_artifact_string(v: Any) -> bool:
1671
+ return isinstance(v, str) and v.startswith("wandb-artifact://")
1672
+
1673
+
1674
+ def _is_artifact_version_weave_dict(v: Any) -> bool:
1675
+ return isinstance(v, dict) and v.get("_type") == "artifactVersion"
1676
+
1677
+
1678
+ def _is_artifact_representation(v: Any) -> bool:
1679
+ return (
1680
+ _is_artifact_object(v)
1681
+ or _is_artifact_string(v)
1682
+ or _is_artifact_version_weave_dict(v)
1683
+ )
1684
+
1685
+
1686
+ def parse_artifact_string(v: str) -> Tuple[str, Optional[str], bool]:
1687
+ if not v.startswith("wandb-artifact://"):
1688
+ raise ValueError(f"Invalid artifact string: {v}")
1689
+ parsed_v = v[len("wandb-artifact://") :]
1690
+ base_uri = None
1691
+ url_info = urllib.parse.urlparse(parsed_v)
1692
+ if url_info.scheme != "":
1693
+ base_uri = f"{url_info.scheme}://{url_info.netloc}"
1694
+ parts = url_info.path.split("/")[1:]
1695
+ else:
1696
+ parts = parsed_v.split("/")
1697
+ if parts[0] == "_id":
1698
+ # for now can't fetch paths but this will be supported in the future
1699
+ # when we allow passing typed media objects, this can be extended
1700
+ # to include paths
1701
+ return parts[1], base_uri, True
1702
+
1703
+ if len(parts) < 3:
1704
+ raise ValueError(f"Invalid artifact string: {v}")
1705
+
1706
+ # for now can't fetch paths but this will be supported in the future
1707
+ # when we allow passing typed media objects, this can be extended
1708
+ # to include paths
1709
+ entity, project, name_and_alias_or_version = parts[:3]
1710
+ return f"{entity}/{project}/{name_and_alias_or_version}", base_uri, False
1711
+
1712
+
1713
+ def _get_max_cli_version() -> Union[str, None]:
1714
+ max_cli_version = wandb.api.max_cli_version()
1715
+ return str(max_cli_version) if max_cli_version is not None else None
1716
+
1717
+
1718
+ def _is_offline() -> bool:
1719
+ return ( # type: ignore[no-any-return]
1720
+ wandb.run is not None and wandb.run.settings._offline
1721
+ ) or wandb.setup().settings._offline # type: ignore
1722
+
1723
+
1724
+ def ensure_text(
1725
+ string: Union[str, bytes], encoding: str = "utf-8", errors: str = "strict"
1726
+ ) -> str:
1727
+ """Coerce s to str."""
1728
+ if isinstance(string, bytes):
1729
+ return string.decode(encoding, errors)
1730
+ elif isinstance(string, str):
1731
+ return string
1732
+ else:
1733
+ raise TypeError(f"not expecting type {type(string)!r}")
1734
+
1735
+
1736
+ def make_artifact_name_safe(name: str) -> str:
1737
+ """Make an artifact name safe for use in artifacts."""
1738
+ # artifact names may only contain alphanumeric characters, dashes, underscores, and dots.
1739
+ cleaned = re.sub(r"[^a-zA-Z0-9_\-.]", "_", name)
1740
+ if len(cleaned) <= 128:
1741
+ return cleaned
1742
+ # truncate with dots in the middle using regex
1743
+ return re.sub(r"(^.{63}).*(.{63}$)", r"\g<1>..\g<2>", cleaned)
1744
+
1745
+
1746
+ def make_docker_image_name_safe(name: str) -> str:
1747
+ """Make a docker image name safe for use in artifacts."""
1748
+ safe_chars = RE_DOCKER_IMAGE_NAME_CHARS.sub("__", name.lower())
1749
+ deduped = RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT.sub("__", safe_chars)
1750
+ trimmed_start = RE_DOCKER_IMAGE_NAME_SEPARATOR_START.sub("", deduped)
1751
+ trimmed = RE_DOCKER_IMAGE_NAME_SEPARATOR_END.sub("", trimmed_start)
1752
+ return trimmed if trimmed else "image"
1753
+
1754
+
1755
+ def merge_dicts(
1756
+ source: Dict[str, Any],
1757
+ destination: Dict[str, Any],
1758
+ ) -> Dict[str, Any]:
1759
+ """Recursively merge two dictionaries.
1760
+
1761
+ This mutates the destination and its nested dictionaries and lists.
1762
+
1763
+ Instances of `dict` are recursively merged and instances of `list`
1764
+ are appended to the destination. If the destination type is not
1765
+ `dict` or `list`, respectively, the key is overwritten with the
1766
+ source value.
1767
+
1768
+ For all other types, the source value overwrites the destination value.
1769
+ """
1770
+ for key, value in source.items():
1771
+ if isinstance(value, dict):
1772
+ node = destination.get(key)
1773
+ if isinstance(node, dict):
1774
+ merge_dicts(value, node)
1775
+ else:
1776
+ destination[key] = value
1777
+
1778
+ elif isinstance(value, list):
1779
+ dest_value = destination.get(key)
1780
+ if isinstance(dest_value, list):
1781
+ dest_value.extend(value)
1782
+ else:
1783
+ destination[key] = value
1784
+
1785
+ else:
1786
+ destination[key] = value
1787
+
1788
+ return destination
1789
+
1790
+
1791
+ def coalesce(*arg: Any) -> Any:
1792
+ """Return the first non-none value in the list of arguments.
1793
+
1794
+ Similar to ?? in C#.
1795
+ """
1796
+ return next((a for a in arg if a is not None), None)
1797
+
1798
+
1799
+ def recursive_cast_dictlike_to_dict(d: Dict[str, Any]) -> Dict[str, Any]:
1800
+ for k, v in d.items():
1801
+ if isinstance(v, dict):
1802
+ recursive_cast_dictlike_to_dict(v)
1803
+ elif hasattr(v, "keys"):
1804
+ d[k] = dict(v)
1805
+ recursive_cast_dictlike_to_dict(d[k])
1806
+ return d
1807
+
1808
+
1809
+ def remove_keys_with_none_values(
1810
+ d: Union[Dict[str, Any], Any],
1811
+ ) -> Union[Dict[str, Any], Any]:
1812
+ # otherwise iterrows will create a bunch of ugly charts
1813
+ if not isinstance(d, dict):
1814
+ return d
1815
+
1816
+ if isinstance(d, dict):
1817
+ new_dict = {}
1818
+ for k, v in d.items():
1819
+ new_v = remove_keys_with_none_values(v)
1820
+ if new_v is not None and not (isinstance(new_v, dict) and len(new_v) == 0):
1821
+ new_dict[k] = new_v
1822
+ return new_dict if new_dict else None
1823
+
1824
+
1825
+ def batched(n: int, iterable: Iterable[T]) -> Generator[List[T], None, None]:
1826
+ i = iter(iterable)
1827
+ batch = list(itertools.islice(i, n))
1828
+ while batch:
1829
+ yield batch
1830
+ batch = list(itertools.islice(i, n))
1831
+
1832
+
1833
+ def random_string(length: int = 12) -> str:
1834
+ """Generate a random string of a given length.
1835
+
1836
+ :param length: Length of the string to generate.
1837
+ :return: Random string.
1838
+ """
1839
+ return "".join(
1840
+ secrets.choice(string.ascii_lowercase + string.digits) for _ in range(length)
1841
+ )
1842
+
1843
+
1844
+ def sample_with_exponential_decay_weights(
1845
+ xs: Union[Iterable, Iterable[Iterable]],
1846
+ ys: Iterable[Iterable],
1847
+ keys: Optional[Iterable] = None,
1848
+ sample_size: int = 1500,
1849
+ ) -> Tuple[List, List, Optional[List]]:
1850
+ """Sample from a list of lists with weights that decay exponentially.
1851
+
1852
+ May be used with the wandb.plot.line_series function.
1853
+ """
1854
+ xs_array = np.array(xs)
1855
+ ys_array = np.array(ys)
1856
+ keys_array = np.array(keys) if keys else None
1857
+ weights = np.exp(-np.arange(len(xs_array)) / len(xs_array))
1858
+ weights /= np.sum(weights)
1859
+ sampled_indices = np.random.choice(len(xs_array), size=sample_size, p=weights)
1860
+ sampled_xs = xs_array[sampled_indices].tolist()
1861
+ sampled_ys = ys_array[sampled_indices].tolist()
1862
+ sampled_keys = keys_array[sampled_indices].tolist() if keys else None
1863
+
1864
+ return sampled_xs, sampled_ys, sampled_keys
1865
+
1866
+
1867
+ @dataclasses.dataclass(frozen=True)
1868
+ class InstalledDistribution:
1869
+ """An installed distribution.
1870
+
1871
+ Attributes:
1872
+ key: The distribution name as it would be imported.
1873
+ version: The distribution's version string.
1874
+ """
1875
+
1876
+ key: str
1877
+ version: str
1878
+
1879
+
1880
+ def working_set() -> Iterable[InstalledDistribution]:
1881
+ """Return the working set of installed distributions.
1882
+
1883
+ Uses importlib.metadata in Python versions above 3.7, and importlib_metadata otherwise.
1884
+ """
1885
+ try:
1886
+ from importlib.metadata import distributions
1887
+ except ImportError:
1888
+ from importlib_metadata import distributions # type: ignore
1889
+
1890
+ for d in distributions():
1891
+ try:
1892
+ # In some distributions, the "Name" attribute may not be present,
1893
+ # which can raise a KeyError. To handle this, we catch the exception
1894
+ # and skip those distributions.
1895
+ # For additional context, see: https://github.com/python/importlib_metadata/issues/371.
1896
+ yield InstalledDistribution(key=d.metadata["Name"], version=d.version)
1897
+ except KeyError:
1898
+ pass
1899
+
1900
+
1901
+ def parse_version(version: str) -> "packaging.version.Version":
1902
+ """Parse a version string into a version object.
1903
+
1904
+ This function is a wrapper around the `packaging.version.parse` function, which
1905
+ is used to parse version strings into version objects. If the `packaging` library
1906
+ is not installed, it falls back to the `pkg_resources` library.
1907
+ """
1908
+ try:
1909
+ from packaging.version import parse as parse_version # type: ignore
1910
+ except ImportError:
1911
+ from pkg_resources import parse_version # type: ignore[assignment]
1912
+
1913
+ return parse_version(version)
1914
+
1915
+
1916
+ def get_core_path() -> str:
1917
+ """Returns the path to the wandb-core binary.
1918
+
1919
+ The path can be set explicitly via the _WANDB_CORE_PATH environment
1920
+ variable. Otherwise, the path to the binary in the current package
1921
+ is returned.
1922
+
1923
+ Returns:
1924
+ str: The path to the wandb-core package.
1925
+
1926
+ Raises:
1927
+ WandbCoreNotAvailableError: If wandb-core was not built for the current system.
1928
+ """
1929
+ # NOTE: Environment variable _WANDB_CORE_PATH is a temporary development feature
1930
+ # to assist in running the core service from a live development directory.
1931
+ path_from_env: str = os.environ.get("_WANDB_CORE_PATH", "")
1932
+ if path_from_env:
1933
+ wandb.termwarn(
1934
+ f"Using wandb-core from path `_WANDB_CORE_PATH={path_from_env}`. "
1935
+ "This is a development feature and may not work as expected."
1936
+ )
1937
+ return path_from_env
1938
+
1939
+ bin_path = pathlib.Path(__file__).parent / "bin" / "wandb-core"
1940
+ if not bin_path.exists():
1941
+ raise WandbCoreNotAvailableError(
1942
+ f"Looks like wandb-core is not compiled for your system ({platform.platform()}):"
1943
+ " Please contact support at support@wandb.com to request `wandb-core`"
1944
+ " support for your system."
1945
+ )
1946
+
1947
+ return str(bin_path)
1948
+
1949
+
1950
+ class NonOctalStringDumper(yaml.Dumper):
1951
+ """Prevents strings containing non-octal values like "008" and "009" from being converted to numbers in in the yaml string saved as the sweep config."""
1952
+
1953
+ def represent_scalar(self, tag, value, style=None):
1954
+ if tag == "tag:yaml.org,2002:str" and value.startswith("0") and len(value) > 1:
1955
+ return super().represent_scalar(tag, value, style="'")
1956
+ return super().represent_scalar(tag, value, style)