wandb 0.19.1__py3-none-musllinux_1_2_aarch64.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (822) hide show
  1. package_readme.md +97 -0
  2. wandb/__init__.py +246 -0
  3. wandb/__init__.pyi +1197 -0
  4. wandb/__main__.py +3 -0
  5. wandb/_globals.py +19 -0
  6. wandb/agents/__init__.py +0 -0
  7. wandb/agents/pyagent.py +363 -0
  8. wandb/analytics/__init__.py +3 -0
  9. wandb/analytics/sentry.py +263 -0
  10. wandb/apis/__init__.py +48 -0
  11. wandb/apis/attrs.py +51 -0
  12. wandb/apis/importers/__init__.py +1 -0
  13. wandb/apis/importers/internals/internal.py +385 -0
  14. wandb/apis/importers/internals/protocols.py +103 -0
  15. wandb/apis/importers/internals/util.py +78 -0
  16. wandb/apis/importers/mlflow.py +254 -0
  17. wandb/apis/importers/validation.py +108 -0
  18. wandb/apis/importers/wandb.py +1603 -0
  19. wandb/apis/internal.py +232 -0
  20. wandb/apis/normalize.py +73 -0
  21. wandb/apis/paginator.py +81 -0
  22. wandb/apis/public/__init__.py +34 -0
  23. wandb/apis/public/api.py +1387 -0
  24. wandb/apis/public/artifacts.py +1095 -0
  25. wandb/apis/public/const.py +4 -0
  26. wandb/apis/public/files.py +263 -0
  27. wandb/apis/public/history.py +149 -0
  28. wandb/apis/public/jobs.py +653 -0
  29. wandb/apis/public/projects.py +154 -0
  30. wandb/apis/public/query_generator.py +166 -0
  31. wandb/apis/public/reports.py +458 -0
  32. wandb/apis/public/runs.py +1012 -0
  33. wandb/apis/public/sweeps.py +240 -0
  34. wandb/apis/public/teams.py +198 -0
  35. wandb/apis/public/users.py +136 -0
  36. wandb/apis/public/utils.py +68 -0
  37. wandb/apis/reports/__init__.py +1 -0
  38. wandb/apis/reports/v1/__init__.py +8 -0
  39. wandb/apis/reports/v2/__init__.py +8 -0
  40. wandb/apis/workspaces/__init__.py +8 -0
  41. wandb/beta/workflows.py +288 -0
  42. wandb/bin/gpu_stats +0 -0
  43. wandb/bin/wandb-core +0 -0
  44. wandb/cli/__init__.py +0 -0
  45. wandb/cli/beta.py +178 -0
  46. wandb/cli/cli.py +2812 -0
  47. wandb/data_types.py +66 -0
  48. wandb/docker/__init__.py +343 -0
  49. wandb/docker/auth.py +435 -0
  50. wandb/docker/wandb-entrypoint.sh +33 -0
  51. wandb/docker/www_authenticate.py +94 -0
  52. wandb/env.py +513 -0
  53. wandb/errors/__init__.py +17 -0
  54. wandb/errors/errors.py +37 -0
  55. wandb/errors/links.py +73 -0
  56. wandb/errors/term.py +415 -0
  57. wandb/errors/util.py +57 -0
  58. wandb/errors/warnings.py +2 -0
  59. wandb/filesync/__init__.py +0 -0
  60. wandb/filesync/dir_watcher.py +403 -0
  61. wandb/filesync/stats.py +100 -0
  62. wandb/filesync/step_checksum.py +142 -0
  63. wandb/filesync/step_prepare.py +179 -0
  64. wandb/filesync/step_upload.py +287 -0
  65. wandb/filesync/upload_job.py +142 -0
  66. wandb/integration/__init__.py +0 -0
  67. wandb/integration/catboost/__init__.py +5 -0
  68. wandb/integration/catboost/catboost.py +178 -0
  69. wandb/integration/cohere/__init__.py +3 -0
  70. wandb/integration/cohere/cohere.py +21 -0
  71. wandb/integration/cohere/resolver.py +347 -0
  72. wandb/integration/diffusers/__init__.py +3 -0
  73. wandb/integration/diffusers/autologger.py +76 -0
  74. wandb/integration/diffusers/pipeline_resolver.py +50 -0
  75. wandb/integration/diffusers/resolvers/__init__.py +9 -0
  76. wandb/integration/diffusers/resolvers/multimodal.py +882 -0
  77. wandb/integration/diffusers/resolvers/utils.py +102 -0
  78. wandb/integration/fastai/__init__.py +245 -0
  79. wandb/integration/gym/__init__.py +99 -0
  80. wandb/integration/huggingface/__init__.py +3 -0
  81. wandb/integration/huggingface/huggingface.py +18 -0
  82. wandb/integration/huggingface/resolver.py +213 -0
  83. wandb/integration/keras/__init__.py +11 -0
  84. wandb/integration/keras/callbacks/__init__.py +5 -0
  85. wandb/integration/keras/callbacks/metrics_logger.py +129 -0
  86. wandb/integration/keras/callbacks/model_checkpoint.py +188 -0
  87. wandb/integration/keras/callbacks/tables_builder.py +228 -0
  88. wandb/integration/keras/keras.py +1089 -0
  89. wandb/integration/kfp/__init__.py +6 -0
  90. wandb/integration/kfp/helpers.py +28 -0
  91. wandb/integration/kfp/kfp_patch.py +334 -0
  92. wandb/integration/kfp/wandb_logging.py +182 -0
  93. wandb/integration/langchain/__init__.py +3 -0
  94. wandb/integration/langchain/wandb_tracer.py +48 -0
  95. wandb/integration/lightgbm/__init__.py +239 -0
  96. wandb/integration/lightning/__init__.py +0 -0
  97. wandb/integration/lightning/fabric/__init__.py +3 -0
  98. wandb/integration/lightning/fabric/logger.py +764 -0
  99. wandb/integration/metaflow/__init__.py +3 -0
  100. wandb/integration/metaflow/metaflow.py +383 -0
  101. wandb/integration/openai/__init__.py +3 -0
  102. wandb/integration/openai/fine_tuning.py +480 -0
  103. wandb/integration/openai/openai.py +22 -0
  104. wandb/integration/openai/resolver.py +240 -0
  105. wandb/integration/prodigy/__init__.py +3 -0
  106. wandb/integration/prodigy/prodigy.py +299 -0
  107. wandb/integration/sacred/__init__.py +117 -0
  108. wandb/integration/sagemaker/__init__.py +12 -0
  109. wandb/integration/sagemaker/auth.py +28 -0
  110. wandb/integration/sagemaker/config.py +49 -0
  111. wandb/integration/sagemaker/files.py +3 -0
  112. wandb/integration/sagemaker/resources.py +34 -0
  113. wandb/integration/sb3/__init__.py +3 -0
  114. wandb/integration/sb3/sb3.py +147 -0
  115. wandb/integration/sklearn/__init__.py +37 -0
  116. wandb/integration/sklearn/calculate/__init__.py +32 -0
  117. wandb/integration/sklearn/calculate/calibration_curves.py +125 -0
  118. wandb/integration/sklearn/calculate/class_proportions.py +68 -0
  119. wandb/integration/sklearn/calculate/confusion_matrix.py +93 -0
  120. wandb/integration/sklearn/calculate/decision_boundaries.py +40 -0
  121. wandb/integration/sklearn/calculate/elbow_curve.py +55 -0
  122. wandb/integration/sklearn/calculate/feature_importances.py +67 -0
  123. wandb/integration/sklearn/calculate/learning_curve.py +64 -0
  124. wandb/integration/sklearn/calculate/outlier_candidates.py +69 -0
  125. wandb/integration/sklearn/calculate/residuals.py +86 -0
  126. wandb/integration/sklearn/calculate/silhouette.py +118 -0
  127. wandb/integration/sklearn/calculate/summary_metrics.py +62 -0
  128. wandb/integration/sklearn/plot/__init__.py +35 -0
  129. wandb/integration/sklearn/plot/classifier.py +329 -0
  130. wandb/integration/sklearn/plot/clusterer.py +146 -0
  131. wandb/integration/sklearn/plot/regressor.py +121 -0
  132. wandb/integration/sklearn/plot/shared.py +91 -0
  133. wandb/integration/sklearn/utils.py +183 -0
  134. wandb/integration/tensorboard/__init__.py +10 -0
  135. wandb/integration/tensorboard/log.py +354 -0
  136. wandb/integration/tensorboard/monkeypatch.py +186 -0
  137. wandb/integration/tensorflow/__init__.py +5 -0
  138. wandb/integration/tensorflow/estimator_hook.py +54 -0
  139. wandb/integration/torch/__init__.py +0 -0
  140. wandb/integration/torch/wandb_torch.py +554 -0
  141. wandb/integration/ultralytics/__init__.py +11 -0
  142. wandb/integration/ultralytics/bbox_utils.py +215 -0
  143. wandb/integration/ultralytics/callback.py +524 -0
  144. wandb/integration/ultralytics/classification_utils.py +83 -0
  145. wandb/integration/ultralytics/mask_utils.py +202 -0
  146. wandb/integration/ultralytics/pose_utils.py +103 -0
  147. wandb/integration/xgboost/__init__.py +11 -0
  148. wandb/integration/xgboost/xgboost.py +189 -0
  149. wandb/integration/yolov8/__init__.py +0 -0
  150. wandb/integration/yolov8/yolov8.py +284 -0
  151. wandb/jupyter.py +513 -0
  152. wandb/mpmain/__init__.py +0 -0
  153. wandb/mpmain/__main__.py +1 -0
  154. wandb/old/__init__.py +0 -0
  155. wandb/old/core.py +53 -0
  156. wandb/old/settings.py +173 -0
  157. wandb/old/summary.py +440 -0
  158. wandb/plot/__init__.py +28 -0
  159. wandb/plot/bar.py +70 -0
  160. wandb/plot/confusion_matrix.py +181 -0
  161. wandb/plot/custom_chart.py +124 -0
  162. wandb/plot/histogram.py +65 -0
  163. wandb/plot/line.py +74 -0
  164. wandb/plot/line_series.py +176 -0
  165. wandb/plot/pr_curve.py +185 -0
  166. wandb/plot/roc_curve.py +163 -0
  167. wandb/plot/scatter.py +66 -0
  168. wandb/plot/utils.py +183 -0
  169. wandb/plot/viz.py +41 -0
  170. wandb/proto/__init__.py +0 -0
  171. wandb/proto/v3/__init__.py +0 -0
  172. wandb/proto/v3/wandb_base_pb2.py +55 -0
  173. wandb/proto/v3/wandb_internal_pb2.py +1658 -0
  174. wandb/proto/v3/wandb_server_pb2.py +228 -0
  175. wandb/proto/v3/wandb_settings_pb2.py +122 -0
  176. wandb/proto/v3/wandb_telemetry_pb2.py +106 -0
  177. wandb/proto/v4/__init__.py +0 -0
  178. wandb/proto/v4/wandb_base_pb2.py +30 -0
  179. wandb/proto/v4/wandb_internal_pb2.py +370 -0
  180. wandb/proto/v4/wandb_server_pb2.py +67 -0
  181. wandb/proto/v4/wandb_settings_pb2.py +47 -0
  182. wandb/proto/v4/wandb_telemetry_pb2.py +41 -0
  183. wandb/proto/v5/wandb_base_pb2.py +31 -0
  184. wandb/proto/v5/wandb_internal_pb2.py +371 -0
  185. wandb/proto/v5/wandb_server_pb2.py +68 -0
  186. wandb/proto/v5/wandb_settings_pb2.py +48 -0
  187. wandb/proto/v5/wandb_telemetry_pb2.py +42 -0
  188. wandb/proto/wandb_base_pb2.py +10 -0
  189. wandb/proto/wandb_deprecated.py +45 -0
  190. wandb/proto/wandb_generate_deprecated.py +30 -0
  191. wandb/proto/wandb_generate_proto.py +49 -0
  192. wandb/proto/wandb_internal_pb2.py +16 -0
  193. wandb/proto/wandb_server_pb2.py +10 -0
  194. wandb/proto/wandb_settings_pb2.py +10 -0
  195. wandb/proto/wandb_telemetry_pb2.py +10 -0
  196. wandb/py.typed +0 -0
  197. wandb/sdk/__init__.py +37 -0
  198. wandb/sdk/artifacts/__init__.py +0 -0
  199. wandb/sdk/artifacts/_validators.py +121 -0
  200. wandb/sdk/artifacts/artifact.py +2364 -0
  201. wandb/sdk/artifacts/artifact_download_logger.py +43 -0
  202. wandb/sdk/artifacts/artifact_file_cache.py +249 -0
  203. wandb/sdk/artifacts/artifact_instance_cache.py +17 -0
  204. wandb/sdk/artifacts/artifact_manifest.py +75 -0
  205. wandb/sdk/artifacts/artifact_manifest_entry.py +249 -0
  206. wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
  207. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +92 -0
  208. wandb/sdk/artifacts/artifact_saver.py +265 -0
  209. wandb/sdk/artifacts/artifact_state.py +11 -0
  210. wandb/sdk/artifacts/artifact_ttl.py +7 -0
  211. wandb/sdk/artifacts/exceptions.py +57 -0
  212. wandb/sdk/artifacts/staging.py +25 -0
  213. wandb/sdk/artifacts/storage_handler.py +62 -0
  214. wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
  215. wandb/sdk/artifacts/storage_handlers/azure_handler.py +213 -0
  216. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
  217. wandb/sdk/artifacts/storage_handlers/http_handler.py +114 -0
  218. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +139 -0
  219. wandb/sdk/artifacts/storage_handlers/multi_handler.py +56 -0
  220. wandb/sdk/artifacts/storage_handlers/s3_handler.py +298 -0
  221. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +72 -0
  222. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +135 -0
  223. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +74 -0
  224. wandb/sdk/artifacts/storage_layout.py +6 -0
  225. wandb/sdk/artifacts/storage_policies/__init__.py +4 -0
  226. wandb/sdk/artifacts/storage_policies/register.py +1 -0
  227. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +378 -0
  228. wandb/sdk/artifacts/storage_policy.py +72 -0
  229. wandb/sdk/backend/__init__.py +0 -0
  230. wandb/sdk/backend/backend.py +221 -0
  231. wandb/sdk/data_types/__init__.py +0 -0
  232. wandb/sdk/data_types/_dtypes.py +918 -0
  233. wandb/sdk/data_types/_private.py +10 -0
  234. wandb/sdk/data_types/audio.py +165 -0
  235. wandb/sdk/data_types/base_types/__init__.py +0 -0
  236. wandb/sdk/data_types/base_types/json_metadata.py +55 -0
  237. wandb/sdk/data_types/base_types/media.py +376 -0
  238. wandb/sdk/data_types/base_types/wb_value.py +282 -0
  239. wandb/sdk/data_types/bokeh.py +70 -0
  240. wandb/sdk/data_types/graph.py +405 -0
  241. wandb/sdk/data_types/helper_types/__init__.py +0 -0
  242. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +305 -0
  243. wandb/sdk/data_types/helper_types/classes.py +159 -0
  244. wandb/sdk/data_types/helper_types/image_mask.py +241 -0
  245. wandb/sdk/data_types/histogram.py +94 -0
  246. wandb/sdk/data_types/html.py +115 -0
  247. wandb/sdk/data_types/image.py +847 -0
  248. wandb/sdk/data_types/molecule.py +241 -0
  249. wandb/sdk/data_types/object_3d.py +470 -0
  250. wandb/sdk/data_types/plotly.py +82 -0
  251. wandb/sdk/data_types/saved_model.py +445 -0
  252. wandb/sdk/data_types/table.py +1204 -0
  253. wandb/sdk/data_types/trace_tree.py +438 -0
  254. wandb/sdk/data_types/utils.py +228 -0
  255. wandb/sdk/data_types/video.py +268 -0
  256. wandb/sdk/integration_utils/__init__.py +0 -0
  257. wandb/sdk/integration_utils/auto_logging.py +232 -0
  258. wandb/sdk/integration_utils/data_logging.py +475 -0
  259. wandb/sdk/interface/__init__.py +0 -0
  260. wandb/sdk/interface/constants.py +4 -0
  261. wandb/sdk/interface/interface.py +1010 -0
  262. wandb/sdk/interface/interface_queue.py +53 -0
  263. wandb/sdk/interface/interface_relay.py +53 -0
  264. wandb/sdk/interface/interface_shared.py +546 -0
  265. wandb/sdk/interface/interface_sock.py +61 -0
  266. wandb/sdk/interface/message_future.py +27 -0
  267. wandb/sdk/interface/message_future_poll.py +50 -0
  268. wandb/sdk/interface/router.py +115 -0
  269. wandb/sdk/interface/router_queue.py +41 -0
  270. wandb/sdk/interface/router_relay.py +37 -0
  271. wandb/sdk/interface/router_sock.py +36 -0
  272. wandb/sdk/interface/summary_record.py +67 -0
  273. wandb/sdk/internal/__init__.py +0 -0
  274. wandb/sdk/internal/context.py +89 -0
  275. wandb/sdk/internal/datastore.py +297 -0
  276. wandb/sdk/internal/file_pusher.py +181 -0
  277. wandb/sdk/internal/file_stream.py +695 -0
  278. wandb/sdk/internal/flow_control.py +263 -0
  279. wandb/sdk/internal/handler.py +905 -0
  280. wandb/sdk/internal/internal.py +403 -0
  281. wandb/sdk/internal/internal_api.py +4587 -0
  282. wandb/sdk/internal/internal_util.py +97 -0
  283. wandb/sdk/internal/job_builder.py +638 -0
  284. wandb/sdk/internal/profiler.py +78 -0
  285. wandb/sdk/internal/progress.py +79 -0
  286. wandb/sdk/internal/run.py +25 -0
  287. wandb/sdk/internal/sample.py +70 -0
  288. wandb/sdk/internal/sender.py +1696 -0
  289. wandb/sdk/internal/sender_config.py +197 -0
  290. wandb/sdk/internal/settings_static.py +97 -0
  291. wandb/sdk/internal/system/__init__.py +0 -0
  292. wandb/sdk/internal/system/assets/__init__.py +25 -0
  293. wandb/sdk/internal/system/assets/aggregators.py +37 -0
  294. wandb/sdk/internal/system/assets/asset_registry.py +20 -0
  295. wandb/sdk/internal/system/assets/cpu.py +163 -0
  296. wandb/sdk/internal/system/assets/disk.py +210 -0
  297. wandb/sdk/internal/system/assets/gpu.py +416 -0
  298. wandb/sdk/internal/system/assets/gpu_amd.py +233 -0
  299. wandb/sdk/internal/system/assets/interfaces.py +205 -0
  300. wandb/sdk/internal/system/assets/ipu.py +177 -0
  301. wandb/sdk/internal/system/assets/memory.py +166 -0
  302. wandb/sdk/internal/system/assets/network.py +125 -0
  303. wandb/sdk/internal/system/assets/open_metrics.py +293 -0
  304. wandb/sdk/internal/system/assets/tpu.py +154 -0
  305. wandb/sdk/internal/system/assets/trainium.py +393 -0
  306. wandb/sdk/internal/system/env_probe_helpers.py +13 -0
  307. wandb/sdk/internal/system/system_info.py +250 -0
  308. wandb/sdk/internal/system/system_monitor.py +222 -0
  309. wandb/sdk/internal/tb_watcher.py +519 -0
  310. wandb/sdk/internal/thread_local_settings.py +18 -0
  311. wandb/sdk/internal/writer.py +204 -0
  312. wandb/sdk/launch/__init__.py +15 -0
  313. wandb/sdk/launch/_launch.py +331 -0
  314. wandb/sdk/launch/_launch_add.py +255 -0
  315. wandb/sdk/launch/_project_spec.py +566 -0
  316. wandb/sdk/launch/agent/__init__.py +5 -0
  317. wandb/sdk/launch/agent/agent.py +924 -0
  318. wandb/sdk/launch/agent/config.py +296 -0
  319. wandb/sdk/launch/agent/job_status_tracker.py +53 -0
  320. wandb/sdk/launch/agent/run_queue_item_file_saver.py +39 -0
  321. wandb/sdk/launch/builder/__init__.py +0 -0
  322. wandb/sdk/launch/builder/abstract.py +156 -0
  323. wandb/sdk/launch/builder/build.py +297 -0
  324. wandb/sdk/launch/builder/context_manager.py +235 -0
  325. wandb/sdk/launch/builder/docker_builder.py +177 -0
  326. wandb/sdk/launch/builder/kaniko_builder.py +594 -0
  327. wandb/sdk/launch/builder/noop.py +58 -0
  328. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +188 -0
  329. wandb/sdk/launch/builder/templates/dockerfile.py +92 -0
  330. wandb/sdk/launch/create_job.py +528 -0
  331. wandb/sdk/launch/environment/abstract.py +29 -0
  332. wandb/sdk/launch/environment/aws_environment.py +322 -0
  333. wandb/sdk/launch/environment/azure_environment.py +105 -0
  334. wandb/sdk/launch/environment/gcp_environment.py +335 -0
  335. wandb/sdk/launch/environment/local_environment.py +65 -0
  336. wandb/sdk/launch/errors.py +13 -0
  337. wandb/sdk/launch/git_reference.py +109 -0
  338. wandb/sdk/launch/inputs/files.py +148 -0
  339. wandb/sdk/launch/inputs/internal.py +315 -0
  340. wandb/sdk/launch/inputs/manage.py +113 -0
  341. wandb/sdk/launch/inputs/schema.py +39 -0
  342. wandb/sdk/launch/loader.py +249 -0
  343. wandb/sdk/launch/registry/abstract.py +48 -0
  344. wandb/sdk/launch/registry/anon.py +29 -0
  345. wandb/sdk/launch/registry/azure_container_registry.py +124 -0
  346. wandb/sdk/launch/registry/elastic_container_registry.py +192 -0
  347. wandb/sdk/launch/registry/google_artifact_registry.py +219 -0
  348. wandb/sdk/launch/registry/local_registry.py +65 -0
  349. wandb/sdk/launch/runner/__init__.py +0 -0
  350. wandb/sdk/launch/runner/abstract.py +185 -0
  351. wandb/sdk/launch/runner/kubernetes_monitor.py +472 -0
  352. wandb/sdk/launch/runner/kubernetes_runner.py +963 -0
  353. wandb/sdk/launch/runner/local_container.py +301 -0
  354. wandb/sdk/launch/runner/local_process.py +78 -0
  355. wandb/sdk/launch/runner/sagemaker_runner.py +426 -0
  356. wandb/sdk/launch/runner/vertex_runner.py +230 -0
  357. wandb/sdk/launch/sweeps/__init__.py +37 -0
  358. wandb/sdk/launch/sweeps/scheduler.py +740 -0
  359. wandb/sdk/launch/sweeps/scheduler_sweep.py +90 -0
  360. wandb/sdk/launch/sweeps/utils.py +316 -0
  361. wandb/sdk/launch/utils.py +747 -0
  362. wandb/sdk/launch/wandb_reference.py +138 -0
  363. wandb/sdk/lib/__init__.py +5 -0
  364. wandb/sdk/lib/apikey.py +269 -0
  365. wandb/sdk/lib/capped_dict.py +26 -0
  366. wandb/sdk/lib/config_util.py +101 -0
  367. wandb/sdk/lib/credentials.py +141 -0
  368. wandb/sdk/lib/deprecate.py +42 -0
  369. wandb/sdk/lib/disabled.py +29 -0
  370. wandb/sdk/lib/exit_hooks.py +54 -0
  371. wandb/sdk/lib/file_stream_utils.py +118 -0
  372. wandb/sdk/lib/filenames.py +64 -0
  373. wandb/sdk/lib/filesystem.py +372 -0
  374. wandb/sdk/lib/fsm.py +180 -0
  375. wandb/sdk/lib/gitlib.py +239 -0
  376. wandb/sdk/lib/gql_request.py +65 -0
  377. wandb/sdk/lib/handler_util.py +21 -0
  378. wandb/sdk/lib/hashutil.py +84 -0
  379. wandb/sdk/lib/import_hooks.py +275 -0
  380. wandb/sdk/lib/ipython.py +126 -0
  381. wandb/sdk/lib/json_util.py +80 -0
  382. wandb/sdk/lib/lazyloader.py +63 -0
  383. wandb/sdk/lib/mailbox.py +456 -0
  384. wandb/sdk/lib/module.py +78 -0
  385. wandb/sdk/lib/paths.py +106 -0
  386. wandb/sdk/lib/preinit.py +42 -0
  387. wandb/sdk/lib/printer.py +548 -0
  388. wandb/sdk/lib/progress.py +279 -0
  389. wandb/sdk/lib/proto_util.py +90 -0
  390. wandb/sdk/lib/redirect.py +845 -0
  391. wandb/sdk/lib/retry.py +289 -0
  392. wandb/sdk/lib/run_moment.py +72 -0
  393. wandb/sdk/lib/runid.py +12 -0
  394. wandb/sdk/lib/server.py +38 -0
  395. wandb/sdk/lib/service_connection.py +216 -0
  396. wandb/sdk/lib/service_token.py +94 -0
  397. wandb/sdk/lib/sock_client.py +290 -0
  398. wandb/sdk/lib/sparkline.py +44 -0
  399. wandb/sdk/lib/telemetry.py +100 -0
  400. wandb/sdk/lib/timed_input.py +133 -0
  401. wandb/sdk/lib/timer.py +19 -0
  402. wandb/sdk/service/__init__.py +0 -0
  403. wandb/sdk/service/_startup_debug.py +22 -0
  404. wandb/sdk/service/port_file.py +53 -0
  405. wandb/sdk/service/server.py +107 -0
  406. wandb/sdk/service/server_sock.py +274 -0
  407. wandb/sdk/service/service.py +242 -0
  408. wandb/sdk/service/streams.py +425 -0
  409. wandb/sdk/verify/__init__.py +0 -0
  410. wandb/sdk/verify/verify.py +501 -0
  411. wandb/sdk/wandb_alerts.py +12 -0
  412. wandb/sdk/wandb_config.py +322 -0
  413. wandb/sdk/wandb_helper.py +54 -0
  414. wandb/sdk/wandb_init.py +1313 -0
  415. wandb/sdk/wandb_login.py +339 -0
  416. wandb/sdk/wandb_metric.py +110 -0
  417. wandb/sdk/wandb_require.py +94 -0
  418. wandb/sdk/wandb_require_helpers.py +44 -0
  419. wandb/sdk/wandb_run.py +4066 -0
  420. wandb/sdk/wandb_settings.py +1309 -0
  421. wandb/sdk/wandb_setup.py +402 -0
  422. wandb/sdk/wandb_summary.py +150 -0
  423. wandb/sdk/wandb_sweep.py +119 -0
  424. wandb/sdk/wandb_sync.py +82 -0
  425. wandb/sdk/wandb_watch.py +150 -0
  426. wandb/sklearn.py +35 -0
  427. wandb/sync/__init__.py +3 -0
  428. wandb/sync/sync.py +442 -0
  429. wandb/trigger.py +29 -0
  430. wandb/util.py +1955 -0
  431. wandb/vendor/__init__.py +0 -0
  432. wandb/vendor/gql-0.2.0/setup.py +40 -0
  433. wandb/vendor/gql-0.2.0/tests/__init__.py +0 -0
  434. wandb/vendor/gql-0.2.0/tests/starwars/__init__.py +0 -0
  435. wandb/vendor/gql-0.2.0/tests/starwars/fixtures.py +96 -0
  436. wandb/vendor/gql-0.2.0/tests/starwars/schema.py +146 -0
  437. wandb/vendor/gql-0.2.0/tests/starwars/test_dsl.py +293 -0
  438. wandb/vendor/gql-0.2.0/tests/starwars/test_query.py +355 -0
  439. wandb/vendor/gql-0.2.0/tests/starwars/test_validation.py +171 -0
  440. wandb/vendor/gql-0.2.0/tests/test_client.py +31 -0
  441. wandb/vendor/gql-0.2.0/tests/test_transport.py +89 -0
  442. wandb/vendor/gql-0.2.0/wandb_gql/__init__.py +4 -0
  443. wandb/vendor/gql-0.2.0/wandb_gql/client.py +75 -0
  444. wandb/vendor/gql-0.2.0/wandb_gql/dsl.py +152 -0
  445. wandb/vendor/gql-0.2.0/wandb_gql/gql.py +10 -0
  446. wandb/vendor/gql-0.2.0/wandb_gql/transport/__init__.py +0 -0
  447. wandb/vendor/gql-0.2.0/wandb_gql/transport/http.py +6 -0
  448. wandb/vendor/gql-0.2.0/wandb_gql/transport/local_schema.py +15 -0
  449. wandb/vendor/gql-0.2.0/wandb_gql/transport/requests.py +46 -0
  450. wandb/vendor/gql-0.2.0/wandb_gql/utils.py +21 -0
  451. wandb/vendor/graphql-core-1.1/setup.py +86 -0
  452. wandb/vendor/graphql-core-1.1/wandb_graphql/__init__.py +287 -0
  453. wandb/vendor/graphql-core-1.1/wandb_graphql/error/__init__.py +6 -0
  454. wandb/vendor/graphql-core-1.1/wandb_graphql/error/base.py +42 -0
  455. wandb/vendor/graphql-core-1.1/wandb_graphql/error/format_error.py +11 -0
  456. wandb/vendor/graphql-core-1.1/wandb_graphql/error/located_error.py +29 -0
  457. wandb/vendor/graphql-core-1.1/wandb_graphql/error/syntax_error.py +36 -0
  458. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/__init__.py +26 -0
  459. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/base.py +311 -0
  460. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executor.py +398 -0
  461. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/__init__.py +0 -0
  462. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/asyncio.py +53 -0
  463. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/gevent.py +22 -0
  464. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/process.py +32 -0
  465. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/sync.py +7 -0
  466. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/thread.py +35 -0
  467. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/executors/utils.py +6 -0
  468. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/__init__.py +0 -0
  469. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/executor.py +66 -0
  470. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/fragment.py +252 -0
  471. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/resolver.py +151 -0
  472. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/experimental/utils.py +7 -0
  473. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/middleware.py +57 -0
  474. wandb/vendor/graphql-core-1.1/wandb_graphql/execution/values.py +145 -0
  475. wandb/vendor/graphql-core-1.1/wandb_graphql/graphql.py +60 -0
  476. wandb/vendor/graphql-core-1.1/wandb_graphql/language/__init__.py +0 -0
  477. wandb/vendor/graphql-core-1.1/wandb_graphql/language/ast.py +1349 -0
  478. wandb/vendor/graphql-core-1.1/wandb_graphql/language/base.py +19 -0
  479. wandb/vendor/graphql-core-1.1/wandb_graphql/language/lexer.py +435 -0
  480. wandb/vendor/graphql-core-1.1/wandb_graphql/language/location.py +30 -0
  481. wandb/vendor/graphql-core-1.1/wandb_graphql/language/parser.py +779 -0
  482. wandb/vendor/graphql-core-1.1/wandb_graphql/language/printer.py +193 -0
  483. wandb/vendor/graphql-core-1.1/wandb_graphql/language/source.py +18 -0
  484. wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor.py +222 -0
  485. wandb/vendor/graphql-core-1.1/wandb_graphql/language/visitor_meta.py +82 -0
  486. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/__init__.py +0 -0
  487. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/cached_property.py +17 -0
  488. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/contain_subset.py +28 -0
  489. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/default_ordered_dict.py +40 -0
  490. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/ordereddict.py +8 -0
  491. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/pair_set.py +43 -0
  492. wandb/vendor/graphql-core-1.1/wandb_graphql/pyutils/version.py +78 -0
  493. wandb/vendor/graphql-core-1.1/wandb_graphql/type/__init__.py +67 -0
  494. wandb/vendor/graphql-core-1.1/wandb_graphql/type/definition.py +619 -0
  495. wandb/vendor/graphql-core-1.1/wandb_graphql/type/directives.py +132 -0
  496. wandb/vendor/graphql-core-1.1/wandb_graphql/type/introspection.py +440 -0
  497. wandb/vendor/graphql-core-1.1/wandb_graphql/type/scalars.py +131 -0
  498. wandb/vendor/graphql-core-1.1/wandb_graphql/type/schema.py +100 -0
  499. wandb/vendor/graphql-core-1.1/wandb_graphql/type/typemap.py +145 -0
  500. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/__init__.py +0 -0
  501. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/assert_valid_name.py +9 -0
  502. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_from_value.py +65 -0
  503. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_code.py +49 -0
  504. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/ast_to_dict.py +24 -0
  505. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/base.py +75 -0
  506. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_ast_schema.py +291 -0
  507. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/build_client_schema.py +250 -0
  508. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/concat_ast.py +9 -0
  509. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/extend_schema.py +357 -0
  510. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_field_def.py +27 -0
  511. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/get_operation_ast.py +21 -0
  512. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/introspection_query.py +90 -0
  513. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_literal_value.py +67 -0
  514. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/is_valid_value.py +66 -0
  515. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/quoted_or_list.py +21 -0
  516. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/schema_printer.py +168 -0
  517. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/suggestion_list.py +56 -0
  518. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_comparators.py +69 -0
  519. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_from_ast.py +21 -0
  520. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/type_info.py +149 -0
  521. wandb/vendor/graphql-core-1.1/wandb_graphql/utils/value_from_ast.py +69 -0
  522. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/__init__.py +4 -0
  523. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/__init__.py +79 -0
  524. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/arguments_of_correct_type.py +24 -0
  525. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/base.py +8 -0
  526. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/default_values_of_correct_type.py +44 -0
  527. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fields_on_correct_type.py +113 -0
  528. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/fragments_on_composite_types.py +33 -0
  529. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_argument_names.py +70 -0
  530. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_directives.py +97 -0
  531. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_fragment_names.py +19 -0
  532. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/known_type_names.py +43 -0
  533. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/lone_anonymous_operation.py +23 -0
  534. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_fragment_cycles.py +59 -0
  535. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_undefined_variables.py +36 -0
  536. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_fragments.py +38 -0
  537. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/no_unused_variables.py +37 -0
  538. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/overlapping_fields_can_be_merged.py +529 -0
  539. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/possible_fragment_spreads.py +44 -0
  540. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/provided_non_null_arguments.py +46 -0
  541. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/scalar_leafs.py +33 -0
  542. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_argument_names.py +32 -0
  543. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_fragment_names.py +28 -0
  544. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_input_field_names.py +33 -0
  545. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_operation_names.py +31 -0
  546. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/unique_variable_names.py +27 -0
  547. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_are_input_types.py +21 -0
  548. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/rules/variables_in_allowed_position.py +53 -0
  549. wandb/vendor/graphql-core-1.1/wandb_graphql/validation/validation.py +158 -0
  550. wandb/vendor/promise-2.3.0/conftest.py +30 -0
  551. wandb/vendor/promise-2.3.0/setup.py +64 -0
  552. wandb/vendor/promise-2.3.0/tests/__init__.py +0 -0
  553. wandb/vendor/promise-2.3.0/tests/conftest.py +8 -0
  554. wandb/vendor/promise-2.3.0/tests/test_awaitable.py +32 -0
  555. wandb/vendor/promise-2.3.0/tests/test_awaitable_35.py +47 -0
  556. wandb/vendor/promise-2.3.0/tests/test_benchmark.py +116 -0
  557. wandb/vendor/promise-2.3.0/tests/test_complex_threads.py +23 -0
  558. wandb/vendor/promise-2.3.0/tests/test_dataloader.py +452 -0
  559. wandb/vendor/promise-2.3.0/tests/test_dataloader_awaitable_35.py +99 -0
  560. wandb/vendor/promise-2.3.0/tests/test_dataloader_extra.py +65 -0
  561. wandb/vendor/promise-2.3.0/tests/test_extra.py +670 -0
  562. wandb/vendor/promise-2.3.0/tests/test_issues.py +132 -0
  563. wandb/vendor/promise-2.3.0/tests/test_promise_list.py +70 -0
  564. wandb/vendor/promise-2.3.0/tests/test_spec.py +584 -0
  565. wandb/vendor/promise-2.3.0/tests/test_thread_safety.py +115 -0
  566. wandb/vendor/promise-2.3.0/tests/utils.py +3 -0
  567. wandb/vendor/promise-2.3.0/wandb_promise/__init__.py +38 -0
  568. wandb/vendor/promise-2.3.0/wandb_promise/async_.py +135 -0
  569. wandb/vendor/promise-2.3.0/wandb_promise/compat.py +32 -0
  570. wandb/vendor/promise-2.3.0/wandb_promise/dataloader.py +326 -0
  571. wandb/vendor/promise-2.3.0/wandb_promise/iterate_promise.py +12 -0
  572. wandb/vendor/promise-2.3.0/wandb_promise/promise.py +848 -0
  573. wandb/vendor/promise-2.3.0/wandb_promise/promise_list.py +151 -0
  574. wandb/vendor/promise-2.3.0/wandb_promise/pyutils/__init__.py +0 -0
  575. wandb/vendor/promise-2.3.0/wandb_promise/pyutils/version.py +83 -0
  576. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/__init__.py +0 -0
  577. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/asyncio.py +22 -0
  578. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/gevent.py +21 -0
  579. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/immediate.py +27 -0
  580. wandb/vendor/promise-2.3.0/wandb_promise/schedulers/thread.py +18 -0
  581. wandb/vendor/promise-2.3.0/wandb_promise/utils.py +56 -0
  582. wandb/vendor/pygments/__init__.py +90 -0
  583. wandb/vendor/pygments/cmdline.py +568 -0
  584. wandb/vendor/pygments/console.py +74 -0
  585. wandb/vendor/pygments/filter.py +74 -0
  586. wandb/vendor/pygments/filters/__init__.py +350 -0
  587. wandb/vendor/pygments/formatter.py +95 -0
  588. wandb/vendor/pygments/formatters/__init__.py +153 -0
  589. wandb/vendor/pygments/formatters/_mapping.py +85 -0
  590. wandb/vendor/pygments/formatters/bbcode.py +109 -0
  591. wandb/vendor/pygments/formatters/html.py +851 -0
  592. wandb/vendor/pygments/formatters/img.py +600 -0
  593. wandb/vendor/pygments/formatters/irc.py +182 -0
  594. wandb/vendor/pygments/formatters/latex.py +482 -0
  595. wandb/vendor/pygments/formatters/other.py +160 -0
  596. wandb/vendor/pygments/formatters/rtf.py +147 -0
  597. wandb/vendor/pygments/formatters/svg.py +153 -0
  598. wandb/vendor/pygments/formatters/terminal.py +136 -0
  599. wandb/vendor/pygments/formatters/terminal256.py +309 -0
  600. wandb/vendor/pygments/lexer.py +871 -0
  601. wandb/vendor/pygments/lexers/__init__.py +329 -0
  602. wandb/vendor/pygments/lexers/_asy_builtins.py +1645 -0
  603. wandb/vendor/pygments/lexers/_cl_builtins.py +232 -0
  604. wandb/vendor/pygments/lexers/_cocoa_builtins.py +72 -0
  605. wandb/vendor/pygments/lexers/_csound_builtins.py +1346 -0
  606. wandb/vendor/pygments/lexers/_lasso_builtins.py +5327 -0
  607. wandb/vendor/pygments/lexers/_lua_builtins.py +295 -0
  608. wandb/vendor/pygments/lexers/_mapping.py +500 -0
  609. wandb/vendor/pygments/lexers/_mql_builtins.py +1172 -0
  610. wandb/vendor/pygments/lexers/_openedge_builtins.py +2547 -0
  611. wandb/vendor/pygments/lexers/_php_builtins.py +4756 -0
  612. wandb/vendor/pygments/lexers/_postgres_builtins.py +621 -0
  613. wandb/vendor/pygments/lexers/_scilab_builtins.py +3094 -0
  614. wandb/vendor/pygments/lexers/_sourcemod_builtins.py +1163 -0
  615. wandb/vendor/pygments/lexers/_stan_builtins.py +532 -0
  616. wandb/vendor/pygments/lexers/_stata_builtins.py +419 -0
  617. wandb/vendor/pygments/lexers/_tsql_builtins.py +1004 -0
  618. wandb/vendor/pygments/lexers/_vim_builtins.py +1939 -0
  619. wandb/vendor/pygments/lexers/actionscript.py +240 -0
  620. wandb/vendor/pygments/lexers/agile.py +24 -0
  621. wandb/vendor/pygments/lexers/algebra.py +221 -0
  622. wandb/vendor/pygments/lexers/ambient.py +76 -0
  623. wandb/vendor/pygments/lexers/ampl.py +87 -0
  624. wandb/vendor/pygments/lexers/apl.py +101 -0
  625. wandb/vendor/pygments/lexers/archetype.py +318 -0
  626. wandb/vendor/pygments/lexers/asm.py +641 -0
  627. wandb/vendor/pygments/lexers/automation.py +374 -0
  628. wandb/vendor/pygments/lexers/basic.py +500 -0
  629. wandb/vendor/pygments/lexers/bibtex.py +160 -0
  630. wandb/vendor/pygments/lexers/business.py +612 -0
  631. wandb/vendor/pygments/lexers/c_cpp.py +252 -0
  632. wandb/vendor/pygments/lexers/c_like.py +541 -0
  633. wandb/vendor/pygments/lexers/capnproto.py +78 -0
  634. wandb/vendor/pygments/lexers/chapel.py +102 -0
  635. wandb/vendor/pygments/lexers/clean.py +288 -0
  636. wandb/vendor/pygments/lexers/compiled.py +34 -0
  637. wandb/vendor/pygments/lexers/configs.py +833 -0
  638. wandb/vendor/pygments/lexers/console.py +114 -0
  639. wandb/vendor/pygments/lexers/crystal.py +393 -0
  640. wandb/vendor/pygments/lexers/csound.py +366 -0
  641. wandb/vendor/pygments/lexers/css.py +689 -0
  642. wandb/vendor/pygments/lexers/d.py +251 -0
  643. wandb/vendor/pygments/lexers/dalvik.py +125 -0
  644. wandb/vendor/pygments/lexers/data.py +555 -0
  645. wandb/vendor/pygments/lexers/diff.py +165 -0
  646. wandb/vendor/pygments/lexers/dotnet.py +691 -0
  647. wandb/vendor/pygments/lexers/dsls.py +878 -0
  648. wandb/vendor/pygments/lexers/dylan.py +289 -0
  649. wandb/vendor/pygments/lexers/ecl.py +125 -0
  650. wandb/vendor/pygments/lexers/eiffel.py +65 -0
  651. wandb/vendor/pygments/lexers/elm.py +121 -0
  652. wandb/vendor/pygments/lexers/erlang.py +533 -0
  653. wandb/vendor/pygments/lexers/esoteric.py +277 -0
  654. wandb/vendor/pygments/lexers/ezhil.py +69 -0
  655. wandb/vendor/pygments/lexers/factor.py +344 -0
  656. wandb/vendor/pygments/lexers/fantom.py +250 -0
  657. wandb/vendor/pygments/lexers/felix.py +273 -0
  658. wandb/vendor/pygments/lexers/forth.py +177 -0
  659. wandb/vendor/pygments/lexers/fortran.py +205 -0
  660. wandb/vendor/pygments/lexers/foxpro.py +428 -0
  661. wandb/vendor/pygments/lexers/functional.py +21 -0
  662. wandb/vendor/pygments/lexers/go.py +101 -0
  663. wandb/vendor/pygments/lexers/grammar_notation.py +213 -0
  664. wandb/vendor/pygments/lexers/graph.py +80 -0
  665. wandb/vendor/pygments/lexers/graphics.py +553 -0
  666. wandb/vendor/pygments/lexers/haskell.py +843 -0
  667. wandb/vendor/pygments/lexers/haxe.py +936 -0
  668. wandb/vendor/pygments/lexers/hdl.py +382 -0
  669. wandb/vendor/pygments/lexers/hexdump.py +103 -0
  670. wandb/vendor/pygments/lexers/html.py +602 -0
  671. wandb/vendor/pygments/lexers/idl.py +270 -0
  672. wandb/vendor/pygments/lexers/igor.py +288 -0
  673. wandb/vendor/pygments/lexers/inferno.py +96 -0
  674. wandb/vendor/pygments/lexers/installers.py +322 -0
  675. wandb/vendor/pygments/lexers/int_fiction.py +1343 -0
  676. wandb/vendor/pygments/lexers/iolang.py +63 -0
  677. wandb/vendor/pygments/lexers/j.py +146 -0
  678. wandb/vendor/pygments/lexers/javascript.py +1525 -0
  679. wandb/vendor/pygments/lexers/julia.py +333 -0
  680. wandb/vendor/pygments/lexers/jvm.py +1573 -0
  681. wandb/vendor/pygments/lexers/lisp.py +2621 -0
  682. wandb/vendor/pygments/lexers/make.py +202 -0
  683. wandb/vendor/pygments/lexers/markup.py +595 -0
  684. wandb/vendor/pygments/lexers/math.py +21 -0
  685. wandb/vendor/pygments/lexers/matlab.py +663 -0
  686. wandb/vendor/pygments/lexers/ml.py +769 -0
  687. wandb/vendor/pygments/lexers/modeling.py +358 -0
  688. wandb/vendor/pygments/lexers/modula2.py +1561 -0
  689. wandb/vendor/pygments/lexers/monte.py +204 -0
  690. wandb/vendor/pygments/lexers/ncl.py +894 -0
  691. wandb/vendor/pygments/lexers/nimrod.py +159 -0
  692. wandb/vendor/pygments/lexers/nit.py +64 -0
  693. wandb/vendor/pygments/lexers/nix.py +136 -0
  694. wandb/vendor/pygments/lexers/oberon.py +105 -0
  695. wandb/vendor/pygments/lexers/objective.py +504 -0
  696. wandb/vendor/pygments/lexers/ooc.py +85 -0
  697. wandb/vendor/pygments/lexers/other.py +41 -0
  698. wandb/vendor/pygments/lexers/parasail.py +79 -0
  699. wandb/vendor/pygments/lexers/parsers.py +835 -0
  700. wandb/vendor/pygments/lexers/pascal.py +644 -0
  701. wandb/vendor/pygments/lexers/pawn.py +199 -0
  702. wandb/vendor/pygments/lexers/perl.py +620 -0
  703. wandb/vendor/pygments/lexers/php.py +267 -0
  704. wandb/vendor/pygments/lexers/praat.py +294 -0
  705. wandb/vendor/pygments/lexers/prolog.py +306 -0
  706. wandb/vendor/pygments/lexers/python.py +939 -0
  707. wandb/vendor/pygments/lexers/qvt.py +152 -0
  708. wandb/vendor/pygments/lexers/r.py +453 -0
  709. wandb/vendor/pygments/lexers/rdf.py +270 -0
  710. wandb/vendor/pygments/lexers/rebol.py +431 -0
  711. wandb/vendor/pygments/lexers/resource.py +85 -0
  712. wandb/vendor/pygments/lexers/rnc.py +67 -0
  713. wandb/vendor/pygments/lexers/roboconf.py +82 -0
  714. wandb/vendor/pygments/lexers/robotframework.py +560 -0
  715. wandb/vendor/pygments/lexers/ruby.py +519 -0
  716. wandb/vendor/pygments/lexers/rust.py +220 -0
  717. wandb/vendor/pygments/lexers/sas.py +228 -0
  718. wandb/vendor/pygments/lexers/scripting.py +1222 -0
  719. wandb/vendor/pygments/lexers/shell.py +794 -0
  720. wandb/vendor/pygments/lexers/smalltalk.py +195 -0
  721. wandb/vendor/pygments/lexers/smv.py +79 -0
  722. wandb/vendor/pygments/lexers/snobol.py +83 -0
  723. wandb/vendor/pygments/lexers/special.py +103 -0
  724. wandb/vendor/pygments/lexers/sql.py +681 -0
  725. wandb/vendor/pygments/lexers/stata.py +108 -0
  726. wandb/vendor/pygments/lexers/supercollider.py +90 -0
  727. wandb/vendor/pygments/lexers/tcl.py +145 -0
  728. wandb/vendor/pygments/lexers/templates.py +2283 -0
  729. wandb/vendor/pygments/lexers/testing.py +207 -0
  730. wandb/vendor/pygments/lexers/text.py +25 -0
  731. wandb/vendor/pygments/lexers/textedit.py +169 -0
  732. wandb/vendor/pygments/lexers/textfmts.py +297 -0
  733. wandb/vendor/pygments/lexers/theorem.py +458 -0
  734. wandb/vendor/pygments/lexers/trafficscript.py +54 -0
  735. wandb/vendor/pygments/lexers/typoscript.py +226 -0
  736. wandb/vendor/pygments/lexers/urbi.py +133 -0
  737. wandb/vendor/pygments/lexers/varnish.py +190 -0
  738. wandb/vendor/pygments/lexers/verification.py +111 -0
  739. wandb/vendor/pygments/lexers/web.py +24 -0
  740. wandb/vendor/pygments/lexers/webmisc.py +988 -0
  741. wandb/vendor/pygments/lexers/whiley.py +116 -0
  742. wandb/vendor/pygments/lexers/x10.py +69 -0
  743. wandb/vendor/pygments/modeline.py +44 -0
  744. wandb/vendor/pygments/plugin.py +68 -0
  745. wandb/vendor/pygments/regexopt.py +92 -0
  746. wandb/vendor/pygments/scanner.py +105 -0
  747. wandb/vendor/pygments/sphinxext.py +158 -0
  748. wandb/vendor/pygments/style.py +155 -0
  749. wandb/vendor/pygments/styles/__init__.py +80 -0
  750. wandb/vendor/pygments/styles/abap.py +29 -0
  751. wandb/vendor/pygments/styles/algol.py +63 -0
  752. wandb/vendor/pygments/styles/algol_nu.py +63 -0
  753. wandb/vendor/pygments/styles/arduino.py +98 -0
  754. wandb/vendor/pygments/styles/autumn.py +65 -0
  755. wandb/vendor/pygments/styles/borland.py +51 -0
  756. wandb/vendor/pygments/styles/bw.py +49 -0
  757. wandb/vendor/pygments/styles/colorful.py +81 -0
  758. wandb/vendor/pygments/styles/default.py +73 -0
  759. wandb/vendor/pygments/styles/emacs.py +72 -0
  760. wandb/vendor/pygments/styles/friendly.py +72 -0
  761. wandb/vendor/pygments/styles/fruity.py +42 -0
  762. wandb/vendor/pygments/styles/igor.py +29 -0
  763. wandb/vendor/pygments/styles/lovelace.py +97 -0
  764. wandb/vendor/pygments/styles/manni.py +75 -0
  765. wandb/vendor/pygments/styles/monokai.py +106 -0
  766. wandb/vendor/pygments/styles/murphy.py +80 -0
  767. wandb/vendor/pygments/styles/native.py +65 -0
  768. wandb/vendor/pygments/styles/paraiso_dark.py +125 -0
  769. wandb/vendor/pygments/styles/paraiso_light.py +125 -0
  770. wandb/vendor/pygments/styles/pastie.py +75 -0
  771. wandb/vendor/pygments/styles/perldoc.py +69 -0
  772. wandb/vendor/pygments/styles/rainbow_dash.py +89 -0
  773. wandb/vendor/pygments/styles/rrt.py +33 -0
  774. wandb/vendor/pygments/styles/sas.py +44 -0
  775. wandb/vendor/pygments/styles/stata.py +40 -0
  776. wandb/vendor/pygments/styles/tango.py +141 -0
  777. wandb/vendor/pygments/styles/trac.py +63 -0
  778. wandb/vendor/pygments/styles/vim.py +63 -0
  779. wandb/vendor/pygments/styles/vs.py +38 -0
  780. wandb/vendor/pygments/styles/xcode.py +51 -0
  781. wandb/vendor/pygments/token.py +213 -0
  782. wandb/vendor/pygments/unistring.py +217 -0
  783. wandb/vendor/pygments/util.py +388 -0
  784. wandb/vendor/pynvml/__init__.py +0 -0
  785. wandb/vendor/pynvml/pynvml.py +4779 -0
  786. wandb/vendor/watchdog_0_9_0/wandb_watchdog/__init__.py +17 -0
  787. wandb/vendor/watchdog_0_9_0/wandb_watchdog/events.py +615 -0
  788. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/__init__.py +98 -0
  789. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/api.py +369 -0
  790. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents.py +172 -0
  791. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/fsevents2.py +239 -0
  792. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify.py +218 -0
  793. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_buffer.py +81 -0
  794. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/inotify_c.py +575 -0
  795. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/kqueue.py +730 -0
  796. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/polling.py +145 -0
  797. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/read_directory_changes.py +133 -0
  798. wandb/vendor/watchdog_0_9_0/wandb_watchdog/observers/winapi.py +348 -0
  799. wandb/vendor/watchdog_0_9_0/wandb_watchdog/patterns.py +265 -0
  800. wandb/vendor/watchdog_0_9_0/wandb_watchdog/tricks/__init__.py +174 -0
  801. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/__init__.py +151 -0
  802. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/bricks.py +249 -0
  803. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/compat.py +29 -0
  804. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/decorators.py +198 -0
  805. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/delayed_queue.py +88 -0
  806. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/dirsnapshot.py +293 -0
  807. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/echo.py +157 -0
  808. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/event_backport.py +41 -0
  809. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/importlib2.py +40 -0
  810. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/platform.py +57 -0
  811. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/unicode_paths.py +64 -0
  812. wandb/vendor/watchdog_0_9_0/wandb_watchdog/utils/win32stat.py +123 -0
  813. wandb/vendor/watchdog_0_9_0/wandb_watchdog/version.py +28 -0
  814. wandb/vendor/watchdog_0_9_0/wandb_watchdog/watchmedo.py +577 -0
  815. wandb/wandb_agent.py +588 -0
  816. wandb/wandb_controller.py +719 -0
  817. wandb/wandb_run.py +9 -0
  818. wandb-0.19.1.dist-info/METADATA +223 -0
  819. wandb-0.19.1.dist-info/RECORD +822 -0
  820. wandb-0.19.1.dist-info/WHEEL +5 -0
  821. wandb-0.19.1.dist-info/entry_points.txt +3 -0
  822. wandb-0.19.1.dist-info/licenses/LICENSE +21 -0
wandb/sdk/wandb_run.py ADDED
@@ -0,0 +1,4066 @@
1
+ from __future__ import annotations
2
+
3
+ import _thread as thread
4
+ import atexit
5
+ import functools
6
+ import glob
7
+ import json
8
+ import logging
9
+ import numbers
10
+ import os
11
+ import pathlib
12
+ import re
13
+ import sys
14
+ import threading
15
+ import time
16
+ import traceback
17
+ import warnings
18
+ from collections.abc import Mapping
19
+ from dataclasses import dataclass, field
20
+ from datetime import datetime, timedelta, timezone
21
+ from enum import IntEnum
22
+ from types import TracebackType
23
+ from typing import TYPE_CHECKING, Any, Callable, Literal, NamedTuple, Sequence, TextIO
24
+
25
+ import requests
26
+
27
+ import wandb
28
+ import wandb.env
29
+ import wandb.util
30
+ from wandb import trigger
31
+ from wandb._globals import _datatypes_set_callback
32
+ from wandb.apis import internal, public
33
+ from wandb.apis.internal import Api
34
+ from wandb.apis.public import Api as PublicApi
35
+ from wandb.errors import CommError, UnsupportedError, UsageError
36
+ from wandb.errors.links import url_registry
37
+ from wandb.integration.torch import wandb_torch
38
+ from wandb.plot import CustomChart, Visualize
39
+ from wandb.proto.wandb_internal_pb2 import (
40
+ MetricRecord,
41
+ PollExitResponse,
42
+ Result,
43
+ RunRecord,
44
+ )
45
+ from wandb.sdk.artifacts.artifact import Artifact
46
+ from wandb.sdk.internal import job_builder
47
+ from wandb.sdk.lib.import_hooks import (
48
+ register_post_import_hook,
49
+ unregister_post_import_hook,
50
+ )
51
+ from wandb.sdk.lib.paths import FilePathStr, LogicalPath, StrPath
52
+ from wandb.util import (
53
+ _is_artifact_object,
54
+ _is_artifact_string,
55
+ _is_artifact_version_weave_dict,
56
+ _is_py_requirements_or_dockerfile,
57
+ _resolve_aliases,
58
+ add_import_hook,
59
+ parse_artifact_string,
60
+ )
61
+
62
+ from . import wandb_config, wandb_metric, wandb_summary
63
+ from .artifacts._validators import (
64
+ is_artifact_registry_project,
65
+ validate_aliases,
66
+ validate_tags,
67
+ )
68
+ from .data_types._dtypes import TypeRegistry
69
+ from .interface.interface import FilesDict, GlobStr, InterfaceBase, PolicyName
70
+ from .interface.summary_record import SummaryRecord
71
+ from .lib import (
72
+ config_util,
73
+ deprecate,
74
+ filenames,
75
+ filesystem,
76
+ ipython,
77
+ module,
78
+ printer,
79
+ progress,
80
+ proto_util,
81
+ redirect,
82
+ telemetry,
83
+ )
84
+ from .lib.exit_hooks import ExitHooks
85
+ from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
86
+ from .wandb_alerts import AlertLevel
87
+ from .wandb_settings import Settings
88
+ from .wandb_setup import _WandbSetup
89
+
90
+ if TYPE_CHECKING:
91
+ from typing import TypedDict
92
+
93
+ import torch # type: ignore [import-not-found]
94
+
95
+ import wandb.apis.public
96
+ import wandb.sdk.backend.backend
97
+ import wandb.sdk.interface.interface_queue
98
+ from wandb.proto.wandb_internal_pb2 import (
99
+ GetSummaryResponse,
100
+ InternalMessagesResponse,
101
+ SampledHistoryResponse,
102
+ )
103
+
104
+ class GitSourceDict(TypedDict):
105
+ remote: str
106
+ commit: str
107
+ entrypoint: list[str]
108
+ args: Sequence[str]
109
+
110
+ class ArtifactSourceDict(TypedDict):
111
+ artifact: str
112
+ entrypoint: list[str]
113
+ args: Sequence[str]
114
+
115
+ class ImageSourceDict(TypedDict):
116
+ image: str
117
+ args: Sequence[str]
118
+
119
+ class JobSourceDict(TypedDict, total=False):
120
+ _version: str
121
+ source_type: str
122
+ source: GitSourceDict | ArtifactSourceDict | ImageSourceDict
123
+ input_types: dict[str, Any]
124
+ output_types: dict[str, Any]
125
+ runtime: str | None
126
+
127
+
128
+ logger = logging.getLogger("wandb")
129
+ EXIT_TIMEOUT = 60
130
+ RE_LABEL = re.compile(r"[a-zA-Z0-9_-]+$")
131
+
132
+
133
+ class TeardownStage(IntEnum):
134
+ EARLY = 1
135
+ LATE = 2
136
+
137
+
138
+ class TeardownHook(NamedTuple):
139
+ call: Callable[[], None]
140
+ stage: TeardownStage
141
+
142
+
143
+ class RunStatusChecker:
144
+ """Periodically polls the background process for relevant updates.
145
+
146
+ - check if the user has requested a stop.
147
+ - check the network status.
148
+ - check the run sync status.
149
+ """
150
+
151
+ _stop_status_lock: threading.Lock
152
+ _stop_status_handle: MailboxHandle | None
153
+ _network_status_lock: threading.Lock
154
+ _network_status_handle: MailboxHandle | None
155
+ _internal_messages_lock: threading.Lock
156
+ _internal_messages_handle: MailboxHandle | None
157
+
158
+ def __init__(
159
+ self,
160
+ interface: InterfaceBase,
161
+ stop_polling_interval: int = 15,
162
+ retry_polling_interval: int = 5,
163
+ internal_messages_polling_interval: int = 10,
164
+ ) -> None:
165
+ self._interface = interface
166
+ self._stop_polling_interval = stop_polling_interval
167
+ self._retry_polling_interval = retry_polling_interval
168
+ self._internal_messages_polling_interval = internal_messages_polling_interval
169
+
170
+ self._join_event = threading.Event()
171
+
172
+ self._stop_status_lock = threading.Lock()
173
+ self._stop_status_handle = None
174
+ self._stop_thread = threading.Thread(
175
+ target=self.check_stop_status,
176
+ name="ChkStopThr",
177
+ daemon=True,
178
+ )
179
+
180
+ self._network_status_lock = threading.Lock()
181
+ self._network_status_handle = None
182
+ self._network_status_thread = threading.Thread(
183
+ target=self.check_network_status,
184
+ name="NetStatThr",
185
+ daemon=True,
186
+ )
187
+
188
+ self._internal_messages_lock = threading.Lock()
189
+ self._internal_messages_handle = None
190
+ self._internal_messages_thread = threading.Thread(
191
+ target=self.check_internal_messages,
192
+ name="IntMsgThr",
193
+ daemon=True,
194
+ )
195
+
196
+ def start(self) -> None:
197
+ self._stop_thread.start()
198
+ self._network_status_thread.start()
199
+ self._internal_messages_thread.start()
200
+
201
+ @staticmethod
202
+ def _abandon_status_check(
203
+ lock: threading.Lock,
204
+ handle: MailboxHandle | None,
205
+ ):
206
+ with lock:
207
+ if handle:
208
+ handle.abandon()
209
+
210
+ def _loop_check_status(
211
+ self,
212
+ *,
213
+ lock: threading.Lock,
214
+ set_handle: Any,
215
+ timeout: int,
216
+ request: Any,
217
+ process: Any,
218
+ ) -> None:
219
+ local_handle: MailboxHandle | None = None
220
+ join_requested = False
221
+ while not join_requested:
222
+ time_probe = time.monotonic()
223
+ if not local_handle:
224
+ local_handle = request()
225
+ assert local_handle
226
+
227
+ with lock:
228
+ if self._join_event.is_set():
229
+ break
230
+ set_handle(local_handle)
231
+ try:
232
+ result = local_handle.wait(timeout=timeout, release=False)
233
+ except MailboxError:
234
+ # background threads are oportunistically getting results
235
+ # from the internal process but the internal process could
236
+ # be shutdown at any time. In this case assume that the
237
+ # thread should exit silently. This is possible
238
+ # because we do not have an atexit handler for the user
239
+ # process which quiesces active threads.
240
+ break
241
+ with lock:
242
+ set_handle(None)
243
+
244
+ if result:
245
+ process(result)
246
+ # if request finished, clear the handle to send on the next interval
247
+ local_handle.abandon()
248
+ local_handle = None
249
+
250
+ time_elapsed = time.monotonic() - time_probe
251
+ wait_time = max(timeout - time_elapsed, 0)
252
+ join_requested = self._join_event.wait(timeout=wait_time)
253
+
254
+ def check_network_status(self) -> None:
255
+ def _process_network_status(result: Result) -> None:
256
+ network_status = result.response.network_status_response
257
+ for hr in network_status.network_responses:
258
+ if (
259
+ hr.http_status_code == 200 or hr.http_status_code == 0
260
+ ): # we use 0 for non-http errors (eg wandb errors)
261
+ wandb.termlog(f"{hr.http_response_text}")
262
+ else:
263
+ wandb.termlog(
264
+ "{} encountered ({}), retrying request".format(
265
+ hr.http_status_code, hr.http_response_text.rstrip()
266
+ )
267
+ )
268
+
269
+ try:
270
+ self._loop_check_status(
271
+ lock=self._network_status_lock,
272
+ set_handle=lambda x: setattr(self, "_network_status_handle", x),
273
+ timeout=self._retry_polling_interval,
274
+ request=self._interface.deliver_network_status,
275
+ process=_process_network_status,
276
+ )
277
+ except BrokenPipeError:
278
+ self._abandon_status_check(
279
+ self._network_status_lock,
280
+ self._network_status_handle,
281
+ )
282
+
283
+ def check_stop_status(self) -> None:
284
+ def _process_stop_status(result: Result) -> None:
285
+ stop_status = result.response.stop_status_response
286
+ if stop_status.run_should_stop:
287
+ # TODO(frz): This check is required
288
+ # until WB-3606 is resolved on server side.
289
+ if not wandb.agents.pyagent.is_running(): # type: ignore
290
+ thread.interrupt_main()
291
+ return
292
+
293
+ try:
294
+ self._loop_check_status(
295
+ lock=self._stop_status_lock,
296
+ set_handle=lambda x: setattr(self, "_stop_status_handle", x),
297
+ timeout=self._stop_polling_interval,
298
+ request=self._interface.deliver_stop_status,
299
+ process=_process_stop_status,
300
+ )
301
+ except BrokenPipeError:
302
+ self._abandon_status_check(
303
+ self._stop_status_lock,
304
+ self._stop_status_handle,
305
+ )
306
+
307
+ def check_internal_messages(self) -> None:
308
+ def _process_internal_messages(result: Result) -> None:
309
+ internal_messages = result.response.internal_messages_response
310
+ for msg in internal_messages.messages.warning:
311
+ wandb.termwarn(msg)
312
+
313
+ try:
314
+ self._loop_check_status(
315
+ lock=self._internal_messages_lock,
316
+ set_handle=lambda x: setattr(self, "_internal_messages_handle", x),
317
+ timeout=self._internal_messages_polling_interval,
318
+ request=self._interface.deliver_internal_messages,
319
+ process=_process_internal_messages,
320
+ )
321
+ except BrokenPipeError:
322
+ self._abandon_status_check(
323
+ self._internal_messages_lock,
324
+ self._internal_messages_handle,
325
+ )
326
+
327
+ def stop(self) -> None:
328
+ self._join_event.set()
329
+ self._abandon_status_check(
330
+ self._stop_status_lock,
331
+ self._stop_status_handle,
332
+ )
333
+ self._abandon_status_check(
334
+ self._network_status_lock,
335
+ self._network_status_handle,
336
+ )
337
+ self._abandon_status_check(
338
+ self._internal_messages_lock,
339
+ self._internal_messages_handle,
340
+ )
341
+
342
+ def join(self) -> None:
343
+ self.stop()
344
+ self._stop_thread.join()
345
+ self._network_status_thread.join()
346
+ self._internal_messages_thread.join()
347
+
348
+
349
+ class _run_decorator: # noqa: N801
350
+ _is_attaching: str = ""
351
+
352
+ class Dummy: ...
353
+
354
+ @classmethod
355
+ def _attach(cls, func: Callable) -> Callable:
356
+ @functools.wraps(func)
357
+ def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
358
+ # * `_attach_id` is only assigned in service hence for all non-service cases
359
+ # it will be a passthrough.
360
+ # * `_attach_pid` is only assigned in _init (using _attach_pid guarantees single attach):
361
+ # - for non-fork case the object is shared through pickling so will be None.
362
+ # - for fork case the new process share mem space hence the value would be of parent process.
363
+ if (
364
+ getattr(self, "_attach_id", None)
365
+ and getattr(self, "_attach_pid", None) != os.getpid()
366
+ ):
367
+ if cls._is_attaching:
368
+ message = (
369
+ f"Trying to attach `{func.__name__}` "
370
+ f"while in the middle of attaching `{cls._is_attaching}`"
371
+ )
372
+ raise RuntimeError(message)
373
+ cls._is_attaching = func.__name__
374
+ try:
375
+ wandb._attach(run=self) # type: ignore
376
+ except Exception as e:
377
+ # In case the attach fails we will raise the exception that caused the issue.
378
+ # This exception should be caught and fail the execution of the program.
379
+ cls._is_attaching = ""
380
+ raise e
381
+ cls._is_attaching = ""
382
+ return func(self, *args, **kwargs)
383
+
384
+ return wrapper
385
+
386
+ @classmethod
387
+ def _noop_on_finish(cls, message: str = "", only_warn: bool = False) -> Callable:
388
+ def decorator_fn(func: Callable) -> Callable:
389
+ @functools.wraps(func)
390
+ def wrapper_fn(self: type[Run], *args: Any, **kwargs: Any) -> Any:
391
+ if not getattr(self, "_is_finished", False):
392
+ return func(self, *args, **kwargs)
393
+
394
+ default_message = (
395
+ f"Run ({self.id}) is finished. The call to `{func.__name__}` will be ignored. "
396
+ f"Please make sure that you are using an active run."
397
+ )
398
+ resolved_message = message or default_message
399
+ if only_warn:
400
+ warnings.warn(resolved_message, UserWarning, stacklevel=2)
401
+ else:
402
+ raise UsageError(resolved_message)
403
+
404
+ return wrapper_fn
405
+
406
+ return decorator_fn
407
+
408
+ @classmethod
409
+ def _noop(cls, func: Callable) -> Callable:
410
+ @functools.wraps(func)
411
+ def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
412
+ # `_attach_id` is only assigned in service hence for all service cases
413
+ # it will be a passthrough. We don't pickle non-service so again a way
414
+ # to see that we are in non-service case
415
+ if getattr(self, "_attach_id", None) is None:
416
+ # `_init_pid` is only assigned in __init__ (this will be constant check for mp):
417
+ # - for non-fork case the object is shared through pickling,
418
+ # and we don't pickle non-service so will be None
419
+ # - for fork case the new process share mem space hence the value would be of parent process.
420
+ _init_pid = getattr(self, "_init_pid", None)
421
+ if _init_pid != os.getpid():
422
+ message = (
423
+ f"`{func.__name__}` ignored (called from pid={os.getpid()}, "
424
+ f"`init` called from pid={_init_pid}). "
425
+ f"See: {url_registry.url('multiprocess')}"
426
+ )
427
+ # - if this process was pickled in non-service case,
428
+ # we ignore the attributes (since pickle is not supported)
429
+ # - for fork case will use the settings of the parent process
430
+ # - only point of inconsistent behavior from forked and non-forked cases
431
+ settings = getattr(self, "_settings", None)
432
+ if settings and settings.strict:
433
+ wandb.termerror(message, repeat=False)
434
+ raise UnsupportedError(
435
+ f"`{func.__name__}` does not support multiprocessing"
436
+ )
437
+ wandb.termwarn(message, repeat=False)
438
+ return cls.Dummy()
439
+
440
+ return func(self, *args, **kwargs)
441
+
442
+ return wrapper
443
+
444
+
445
+ @dataclass
446
+ class RunStatus:
447
+ sync_items_total: int = field(default=0)
448
+ sync_items_pending: int = field(default=0)
449
+ sync_time: datetime | None = field(default=None)
450
+
451
+
452
+ class Run:
453
+ """A unit of computation logged by wandb. Typically, this is an ML experiment.
454
+
455
+ Create a run with `wandb.init()`:
456
+ <!--yeadoc-test:run-object-basic-->
457
+ ```python
458
+ import wandb
459
+
460
+ run = wandb.init()
461
+ ```
462
+
463
+ There is only ever at most one active `wandb.Run` in any process,
464
+ and it is accessible as `wandb.run`:
465
+ <!--yeadoc-test:global-run-object-->
466
+ ```python
467
+ import wandb
468
+
469
+ assert wandb.run is None
470
+
471
+ wandb.init()
472
+
473
+ assert wandb.run is not None
474
+ ```
475
+ anything you log with `wandb.log` will be sent to that run.
476
+
477
+ If you want to start more runs in the same script or notebook, you'll need to
478
+ finish the run that is in-flight. Runs can be finished with `wandb.finish` or
479
+ by using them in a `with` block:
480
+ <!--yeadoc-test:run-context-manager-->
481
+ ```python
482
+ import wandb
483
+
484
+ wandb.init()
485
+ wandb.finish()
486
+
487
+ assert wandb.run is None
488
+
489
+ with wandb.init() as run:
490
+ pass # log data here
491
+
492
+ assert wandb.run is None
493
+ ```
494
+
495
+ See the documentation for `wandb.init` for more on creating runs, or check out
496
+ [our guide to `wandb.init`](https://docs.wandb.ai/guides/track/launch).
497
+
498
+ In distributed training, you can either create a single run in the rank 0 process
499
+ and then log information only from that process, or you can create a run in each process,
500
+ logging from each separately, and group the results together with the `group` argument
501
+ to `wandb.init`. For more details on distributed training with W&B, check out
502
+ [our guide](https://docs.wandb.ai/guides/track/log/distributed-training).
503
+
504
+ Currently, there is a parallel `Run` object in the `wandb.Api`. Eventually these
505
+ two objects will be merged.
506
+
507
+ Attributes:
508
+ summary: (Summary) Single values set for each `wandb.log()` key. By
509
+ default, summary is set to the last value logged. You can manually
510
+ set summary to the best value, like max accuracy, instead of the
511
+ final value.
512
+ """
513
+
514
+ _telemetry_obj: telemetry.TelemetryRecord
515
+ _telemetry_obj_active: bool
516
+ _telemetry_obj_dirty: bool
517
+ _telemetry_obj_flushed: bytes
518
+
519
+ _teardown_hooks: list[TeardownHook]
520
+
521
+ _backend: wandb.sdk.backend.backend.Backend | None
522
+ _internal_run_interface: wandb.sdk.interface.interface_queue.InterfaceQueue | None
523
+ _wl: _WandbSetup | None
524
+
525
+ _out_redir: redirect.RedirectBase | None
526
+ _err_redir: redirect.RedirectBase | None
527
+ _redirect_cb: Callable[[str, str], None] | None
528
+ _redirect_raw_cb: Callable[[str, str], None] | None
529
+ _output_writer: filesystem.CRDedupedFile | None
530
+
531
+ _atexit_cleanup_called: bool
532
+ _hooks: ExitHooks | None
533
+ _exit_code: int | None
534
+
535
+ _run_status_checker: RunStatusChecker | None
536
+
537
+ _sampled_history: SampledHistoryResponse | None
538
+ _final_summary: GetSummaryResponse | None
539
+ _poll_exit_handle: MailboxHandle | None
540
+ _poll_exit_response: PollExitResponse | None
541
+ _internal_messages_response: InternalMessagesResponse | None
542
+
543
+ _stdout_slave_fd: int | None
544
+ _stderr_slave_fd: int | None
545
+ _artifact_slots: list[str]
546
+
547
+ _init_pid: int
548
+ _attach_pid: int
549
+ _iface_pid: int | None
550
+ _iface_port: int | None
551
+
552
+ _attach_id: str | None
553
+ _is_attached: bool
554
+ _is_finished: bool
555
+ _settings: Settings
556
+
557
+ _forked: bool
558
+
559
+ _launch_artifacts: dict[str, Any] | None
560
+ _printer: printer.Printer
561
+
562
+ def __init__(
563
+ self,
564
+ settings: Settings,
565
+ config: dict[str, Any] | None = None,
566
+ sweep_config: dict[str, Any] | None = None,
567
+ launch_config: dict[str, Any] | None = None,
568
+ ) -> None:
569
+ # pid is set, so we know if this run object was initialized by this process
570
+ self._init_pid = os.getpid()
571
+
572
+ if settings._noop:
573
+ # TODO: properly handle setting for disabled mode
574
+ self._settings = settings
575
+ return
576
+
577
+ self._init(
578
+ settings=settings,
579
+ config=config,
580
+ sweep_config=sweep_config,
581
+ launch_config=launch_config,
582
+ )
583
+
584
+ def _init(
585
+ self,
586
+ settings: Settings,
587
+ config: dict[str, Any] | None = None,
588
+ sweep_config: dict[str, Any] | None = None,
589
+ launch_config: dict[str, Any] | None = None,
590
+ ) -> None:
591
+ self._settings = settings
592
+
593
+ self._config = wandb_config.Config()
594
+ self._config._set_callback(self._config_callback)
595
+ self._config._set_artifact_callback(self._config_artifact_callback)
596
+ self._config._set_settings(self._settings)
597
+
598
+ # todo: perhaps this should be a property that is a noop on a finished run
599
+ self.summary = wandb_summary.Summary(
600
+ self._summary_get_current_summary_callback,
601
+ )
602
+ self.summary._set_update_callback(self._summary_update_callback)
603
+
604
+ self._step = 0
605
+ self._starting_step = 0
606
+ # todo: eventually would be nice to make this configurable using self._settings._start_time
607
+ # need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
608
+ # does the total time get calculated right (not as 2 days and 15 minutes)?
609
+ self._start_time = time.time()
610
+
611
+ _datatypes_set_callback(self._datatypes_callback)
612
+
613
+ self._printer = printer.new_printer()
614
+
615
+ self._torch_history: wandb_torch.TorchHistory | None = None # type: ignore
616
+
617
+ self._backend = None
618
+ self._internal_run_interface = None
619
+ self._wl = None
620
+
621
+ self._hooks = None
622
+ self._teardown_hooks = []
623
+
624
+ self._output_writer = None
625
+ self._out_redir = None
626
+ self._err_redir = None
627
+ self._stdout_slave_fd = None
628
+ self._stderr_slave_fd = None
629
+
630
+ self._exit_code = None
631
+ self._exit_result = None
632
+
633
+ self._used_artifact_slots: dict[str, str] = {}
634
+
635
+ # Created when the run "starts".
636
+ self._run_status_checker = None
637
+
638
+ self._sampled_history = None
639
+ self._final_summary = None
640
+ self._poll_exit_response = None
641
+ self._internal_messages_response = None
642
+ self._poll_exit_handle = None
643
+
644
+ # Initialize telemetry object
645
+ self._telemetry_obj = telemetry.TelemetryRecord()
646
+ self._telemetry_obj_active = False
647
+ self._telemetry_obj_flushed = b""
648
+ self._telemetry_obj_dirty = False
649
+
650
+ self._atexit_cleanup_called = False
651
+
652
+ # Initial scope setup for sentry.
653
+ # This might get updated when the actual run comes back.
654
+ wandb._sentry.configure_scope(
655
+ tags=dict(self._settings),
656
+ process_context="user",
657
+ )
658
+
659
+ # Populate config
660
+ config = config or dict()
661
+ wandb_key = "_wandb"
662
+ config.setdefault(wandb_key, dict())
663
+ self._launch_artifact_mapping: dict[str, Any] = {}
664
+ self._unique_launch_artifact_sequence_names: dict[str, Any] = {}
665
+ if self._settings.save_code and self._settings.program_relpath:
666
+ config[wandb_key]["code_path"] = LogicalPath(
667
+ os.path.join("code", self._settings.program_relpath)
668
+ )
669
+
670
+ if self._settings.fork_from is not None:
671
+ config[wandb_key]["branch_point"] = {
672
+ "run_id": self._settings.fork_from.run,
673
+ "step": self._settings.fork_from.value,
674
+ }
675
+
676
+ if self._settings.resume_from is not None:
677
+ config[wandb_key]["branch_point"] = {
678
+ "run_id": self._settings.resume_from.run,
679
+ "step": self._settings.resume_from.value,
680
+ }
681
+
682
+ self._config._update(config, ignore_locked=True)
683
+
684
+ if sweep_config:
685
+ self._config.merge_locked(
686
+ sweep_config, user="sweep", _allow_val_change=True
687
+ )
688
+
689
+ if launch_config:
690
+ self._config.merge_locked(
691
+ launch_config, user="launch", _allow_val_change=True
692
+ )
693
+
694
+ # if run is from a launch queue, add queue id to _wandb config
695
+ launch_queue_name = wandb.env.get_launch_queue_name()
696
+ if launch_queue_name:
697
+ self._config[wandb_key]["launch_queue_name"] = launch_queue_name
698
+
699
+ launch_queue_entity = wandb.env.get_launch_queue_entity()
700
+ if launch_queue_entity:
701
+ self._config[wandb_key]["launch_queue_entity"] = launch_queue_entity
702
+
703
+ launch_trace_id = wandb.env.get_launch_trace_id()
704
+ if launch_trace_id:
705
+ self._config[wandb_key]["launch_trace_id"] = launch_trace_id
706
+
707
+ # interface pid and port configured when backend is configured (See _hack_set_run)
708
+ # TODO: using pid isn't the best for windows as pid reuse can happen more often than unix
709
+ self._iface_pid = None
710
+ self._iface_port = None
711
+ self._attach_id = None
712
+ self._is_attached = False
713
+ self._is_finished = False
714
+
715
+ self._attach_pid = os.getpid()
716
+ self._forked = False
717
+ # for now, use runid as attach id, this could/should be versioned in the future
718
+ if not self._settings.x_disable_service:
719
+ self._attach_id = self._settings.run_id
720
+
721
+ def _set_iface_pid(self, iface_pid: int) -> None:
722
+ self._iface_pid = iface_pid
723
+
724
+ def _set_iface_port(self, iface_port: int) -> None:
725
+ self._iface_port = iface_port
726
+
727
+ def _handle_launch_artifact_overrides(self) -> None:
728
+ if self._settings.launch and (os.environ.get("WANDB_ARTIFACTS") is not None):
729
+ try:
730
+ artifacts: dict[str, Any] = json.loads(
731
+ os.environ.get("WANDB_ARTIFACTS", "{}")
732
+ )
733
+ except (ValueError, SyntaxError):
734
+ wandb.termwarn("Malformed WANDB_ARTIFACTS, using original artifacts")
735
+ else:
736
+ self._initialize_launch_artifact_maps(artifacts)
737
+
738
+ elif (
739
+ self._settings.launch
740
+ and self._settings.launch_config_path
741
+ and os.path.exists(self._settings.launch_config_path)
742
+ ):
743
+ self.save(self._settings.launch_config_path)
744
+ with open(self._settings.launch_config_path) as fp:
745
+ launch_config = json.loads(fp.read())
746
+ if launch_config.get("overrides", {}).get("artifacts") is not None:
747
+ artifacts = launch_config.get("overrides").get("artifacts")
748
+ self._initialize_launch_artifact_maps(artifacts)
749
+
750
+ def _initialize_launch_artifact_maps(self, artifacts: dict[str, Any]) -> None:
751
+ for key, item in artifacts.items():
752
+ self._launch_artifact_mapping[key] = item
753
+ artifact_sequence_tuple_or_slot = key.split(":")
754
+
755
+ if len(artifact_sequence_tuple_or_slot) == 2:
756
+ sequence_name = artifact_sequence_tuple_or_slot[0].split("/")[-1]
757
+ if self._unique_launch_artifact_sequence_names.get(sequence_name):
758
+ self._unique_launch_artifact_sequence_names.pop(sequence_name)
759
+ else:
760
+ self._unique_launch_artifact_sequence_names[sequence_name] = item
761
+
762
+ def _telemetry_callback(self, telem_obj: telemetry.TelemetryRecord) -> None:
763
+ if not hasattr(self, "_telemetry_obj"):
764
+ return
765
+ self._telemetry_obj.MergeFrom(telem_obj)
766
+ self._telemetry_obj_dirty = True
767
+ self._telemetry_flush()
768
+
769
+ def _telemetry_flush(self) -> None:
770
+ if not hasattr(self, "_telemetry_obj"):
771
+ return
772
+ if not self._telemetry_obj_active:
773
+ return
774
+ if not self._telemetry_obj_dirty:
775
+ return
776
+ if self._backend and self._backend.interface:
777
+ serialized = self._telemetry_obj.SerializeToString()
778
+ if serialized == self._telemetry_obj_flushed:
779
+ return
780
+ self._backend.interface._publish_telemetry(self._telemetry_obj)
781
+ self._telemetry_obj_flushed = serialized
782
+ self._telemetry_obj_dirty = False
783
+
784
+ def _freeze(self) -> None:
785
+ self._frozen = True
786
+
787
+ def __setattr__(self, attr: str, value: object) -> None:
788
+ if getattr(self, "_frozen", None) and not hasattr(self, attr):
789
+ raise Exception(f"Attribute {attr} is not supported on Run object.")
790
+ super().__setattr__(attr, value)
791
+
792
+ def __deepcopy__(self, memo: dict[int, Any]) -> Run:
793
+ return self
794
+
795
+ def __getstate__(self) -> Any:
796
+ """Return run state as a custom pickle."""
797
+ # We only pickle in service mode
798
+ if not self._settings or self._settings.x_disable_service:
799
+ return
800
+
801
+ _attach_id = self._attach_id
802
+ if not _attach_id:
803
+ return
804
+
805
+ return dict(
806
+ _attach_id=_attach_id,
807
+ _init_pid=self._init_pid,
808
+ _is_finished=self._is_finished,
809
+ )
810
+
811
+ def __setstate__(self, state: Any) -> None:
812
+ """Set run state from a custom pickle."""
813
+ if not state:
814
+ return
815
+
816
+ _attach_id = state.get("_attach_id")
817
+ if not _attach_id:
818
+ return
819
+
820
+ if state["_init_pid"] == os.getpid():
821
+ raise RuntimeError("attach in the same process is not supported currently")
822
+
823
+ self.__dict__.update(state)
824
+
825
+ @property
826
+ def _torch(self) -> wandb_torch.TorchHistory: # type: ignore
827
+ if self._torch_history is None:
828
+ self._torch_history = wandb_torch.TorchHistory() # type: ignore
829
+ return self._torch_history
830
+
831
+ @property
832
+ @_run_decorator._attach
833
+ def settings(self) -> Settings:
834
+ """A frozen copy of run's Settings object."""
835
+ return self._settings.model_copy(deep=True)
836
+
837
+ @property
838
+ @_run_decorator._attach
839
+ def dir(self) -> str:
840
+ """The directory where files associated with the run are saved."""
841
+ return self._settings.files_dir
842
+
843
+ @property
844
+ @_run_decorator._attach
845
+ def config(self) -> wandb_config.Config:
846
+ """Config object associated with this run."""
847
+ return self._config
848
+
849
+ @property
850
+ @_run_decorator._attach
851
+ def config_static(self) -> wandb_config.ConfigStatic:
852
+ return wandb_config.ConfigStatic(self._config)
853
+
854
+ @property
855
+ @_run_decorator._attach
856
+ def name(self) -> str | None:
857
+ """Display name of the run.
858
+
859
+ Display names are not guaranteed to be unique and may be descriptive.
860
+ By default, they are randomly generated.
861
+ """
862
+ if self._settings.run_name:
863
+ return self._settings.run_name
864
+ return None
865
+
866
+ @name.setter
867
+ @_run_decorator._noop_on_finish()
868
+ def name(self, name: str) -> None:
869
+ with telemetry.context(run=self) as tel:
870
+ tel.feature.set_run_name = True
871
+ self._settings.run_name = name
872
+ if self._backend and self._backend.interface:
873
+ self._backend.interface.publish_run(self)
874
+
875
+ @property
876
+ @_run_decorator._attach
877
+ def notes(self) -> str | None:
878
+ """Notes associated with the run, if there are any.
879
+
880
+ Notes can be a multiline string and can also use markdown and latex equations
881
+ inside `$$`, like `$x + 3$`.
882
+ """
883
+ return self._settings.run_notes
884
+
885
+ @notes.setter
886
+ @_run_decorator._noop_on_finish()
887
+ def notes(self, notes: str) -> None:
888
+ self._settings.run_notes = notes
889
+ if self._backend and self._backend.interface:
890
+ self._backend.interface.publish_run(self)
891
+
892
+ @property
893
+ @_run_decorator._attach
894
+ def tags(self) -> tuple | None:
895
+ """Tags associated with the run, if there are any."""
896
+ return self._settings.run_tags or ()
897
+
898
+ @tags.setter
899
+ @_run_decorator._noop_on_finish()
900
+ def tags(self, tags: Sequence) -> None:
901
+ with telemetry.context(run=self) as tel:
902
+ tel.feature.set_run_tags = True
903
+ self._settings.run_tags = tuple(tags)
904
+ if self._backend and self._backend.interface:
905
+ self._backend.interface.publish_run(self)
906
+
907
+ @property
908
+ @_run_decorator._attach
909
+ def id(self) -> str:
910
+ """Identifier for this run."""
911
+ if TYPE_CHECKING:
912
+ assert self._settings.run_id is not None
913
+ return self._settings.run_id
914
+
915
+ @property
916
+ @_run_decorator._attach
917
+ def sweep_id(self) -> str | None:
918
+ """ID of the sweep associated with the run, if there is one."""
919
+ return self._settings.sweep_id
920
+
921
+ def _get_path(self) -> str:
922
+ return "/".join(
923
+ e
924
+ for e in [
925
+ self._settings.entity,
926
+ self._settings.project,
927
+ self._settings.run_id,
928
+ ]
929
+ if e is not None
930
+ )
931
+
932
+ @property
933
+ @_run_decorator._attach
934
+ def path(self) -> str:
935
+ """Path to the run.
936
+
937
+ Run paths include entity, project, and run ID, in the format
938
+ `entity/project/run_id`.
939
+ """
940
+ return self._get_path()
941
+
942
+ @property
943
+ @_run_decorator._attach
944
+ def start_time(self) -> float:
945
+ """Unix timestamp (in seconds) of when the run started."""
946
+ return self._start_time
947
+
948
+ @property
949
+ @_run_decorator._attach
950
+ def starting_step(self) -> int:
951
+ """The first step of the run."""
952
+ return self._starting_step
953
+
954
+ @property
955
+ @_run_decorator._attach
956
+ def resumed(self) -> bool:
957
+ """True if the run was resumed, False otherwise."""
958
+ return self._settings.resumed
959
+
960
+ @property
961
+ @_run_decorator._attach
962
+ def step(self) -> int:
963
+ """Current value of the step.
964
+
965
+ This counter is incremented by `wandb.log`.
966
+ """
967
+ return self._step
968
+
969
+ @property
970
+ @_run_decorator._attach
971
+ def mode(self) -> str:
972
+ """For compatibility with `0.9.x` and earlier, deprecate eventually."""
973
+ if hasattr(self, "_telemetry_obj"):
974
+ deprecate.deprecate(
975
+ field_name=deprecate.Deprecated.run__mode,
976
+ warning_message=(
977
+ "The mode property of wandb.run is deprecated "
978
+ "and will be removed in a future release."
979
+ ),
980
+ )
981
+ return "dryrun" if self._settings._offline else "run"
982
+
983
+ @property
984
+ @_run_decorator._attach
985
+ def offline(self) -> bool:
986
+ return self._settings._offline
987
+
988
+ @property
989
+ @_run_decorator._attach
990
+ def disabled(self) -> bool:
991
+ return self._settings._noop
992
+
993
+ @property
994
+ @_run_decorator._attach
995
+ def group(self) -> str:
996
+ """Name of the group associated with the run.
997
+
998
+ Setting a group helps the W&B UI organize runs in a sensible way.
999
+
1000
+ If you are doing a distributed training you should give all of the
1001
+ runs in the training the same group.
1002
+ If you are doing cross-validation you should give all the cross-validation
1003
+ folds the same group.
1004
+ """
1005
+ return self._settings.run_group or ""
1006
+
1007
+ @property
1008
+ @_run_decorator._attach
1009
+ def job_type(self) -> str:
1010
+ return self._settings.run_job_type or ""
1011
+
1012
+ def project_name(self) -> str:
1013
+ # TODO: deprecate this in favor of project
1014
+ return self._settings.project or ""
1015
+
1016
+ @property
1017
+ @_run_decorator._attach
1018
+ def project(self) -> str:
1019
+ """Name of the W&B project associated with the run."""
1020
+ return self.project_name()
1021
+
1022
+ @_run_decorator._noop_on_finish()
1023
+ @_run_decorator._attach
1024
+ def log_code(
1025
+ self,
1026
+ root: str | None = ".",
1027
+ name: str | None = None,
1028
+ include_fn: Callable[[str, str], bool]
1029
+ | Callable[[str], bool] = _is_py_requirements_or_dockerfile,
1030
+ exclude_fn: Callable[[str, str], bool]
1031
+ | Callable[[str], bool] = filenames.exclude_wandb_fn,
1032
+ ) -> Artifact | None:
1033
+ """Save the current state of your code to a W&B Artifact.
1034
+
1035
+ By default, it walks the current directory and logs all files that end with `.py`.
1036
+
1037
+ Args:
1038
+ root: The relative (to `os.getcwd()`) or absolute path to recursively find code from.
1039
+ name: (str, optional) The name of our code artifact. By default, we'll name
1040
+ the artifact `source-$PROJECT_ID-$ENTRYPOINT_RELPATH`. There may be scenarios where you want
1041
+ many runs to share the same artifact. Specifying name allows you to achieve that.
1042
+ include_fn: A callable that accepts a file path and (optionally) root path and
1043
+ returns True when it should be included and False otherwise. This
1044
+ defaults to: `lambda path, root: path.endswith(".py")`
1045
+ exclude_fn: A callable that accepts a file path and (optionally) root path and
1046
+ returns `True` when it should be excluded and `False` otherwise. This
1047
+ defaults to a function that excludes all files within `<root>/.wandb/`
1048
+ and `<root>/wandb/` directories.
1049
+
1050
+ Examples:
1051
+ Basic usage
1052
+ ```python
1053
+ run.log_code()
1054
+ ```
1055
+
1056
+ Advanced usage
1057
+ ```python
1058
+ run.log_code(
1059
+ "../",
1060
+ include_fn=lambda path: path.endswith(".py") or path.endswith(".ipynb"),
1061
+ exclude_fn=lambda path, root: os.path.relpath(path, root).startswith(
1062
+ "cache/"
1063
+ ),
1064
+ )
1065
+ ```
1066
+
1067
+ Returns:
1068
+ An `Artifact` object if code was logged
1069
+ """
1070
+ if name is None:
1071
+ if self.settings._jupyter:
1072
+ notebook_name = None
1073
+ if self.settings.notebook_name:
1074
+ notebook_name = self.settings.notebook_name
1075
+ elif self.settings.x_jupyter_path:
1076
+ if self.settings.x_jupyter_path.startswith("fileId="):
1077
+ notebook_name = self.settings.x_jupyter_name
1078
+ else:
1079
+ notebook_name = self.settings.x_jupyter_path
1080
+ name_string = f"{self._settings.project}-{notebook_name}"
1081
+ else:
1082
+ name_string = (
1083
+ f"{self._settings.project}-{self._settings.program_relpath}"
1084
+ )
1085
+ name = wandb.util.make_artifact_name_safe(f"source-{name_string}")
1086
+ art = wandb.Artifact(name, "code")
1087
+ files_added = False
1088
+ if root is not None:
1089
+ root = os.path.abspath(root)
1090
+ for file_path in filenames.filtered_dir(root, include_fn, exclude_fn):
1091
+ files_added = True
1092
+ save_name = os.path.relpath(file_path, root)
1093
+ art.add_file(file_path, name=save_name)
1094
+ # Add any manually staged files such as ipynb notebooks
1095
+ for dirpath, _, files in os.walk(self._settings._tmp_code_dir):
1096
+ for fname in files:
1097
+ file_path = os.path.join(dirpath, fname)
1098
+ save_name = os.path.relpath(file_path, self._settings._tmp_code_dir)
1099
+ files_added = True
1100
+ art.add_file(file_path, name=save_name)
1101
+ if not files_added:
1102
+ wandb.termwarn(
1103
+ "No relevant files were detected in the specified directory. No code will be logged to your run."
1104
+ )
1105
+ return None
1106
+
1107
+ return self._log_artifact(art)
1108
+
1109
+ def get_project_url(self) -> str | None:
1110
+ """Return the url for the W&B project associated with the run, if there is one.
1111
+
1112
+ Offline runs will not have a project url.
1113
+ """
1114
+ if self._settings._offline:
1115
+ wandb.termwarn("URL not available in offline run")
1116
+ return None
1117
+ return self._settings.project_url
1118
+
1119
+ def get_sweep_url(self) -> str | None:
1120
+ """Return the url for the sweep associated with the run, if there is one."""
1121
+ if self._settings._offline:
1122
+ wandb.termwarn("URL not available in offline run")
1123
+ return None
1124
+ return self._settings.sweep_url
1125
+
1126
+ def get_url(self) -> str | None:
1127
+ """Return the url for the W&B run, if there is one.
1128
+
1129
+ Offline runs will not have a url.
1130
+ """
1131
+ if self._settings._offline:
1132
+ wandb.termwarn("URL not available in offline run")
1133
+ return None
1134
+ return self._settings.run_url
1135
+
1136
+ @property
1137
+ @_run_decorator._attach
1138
+ def url(self) -> str | None:
1139
+ """The W&B url associated with the run."""
1140
+ return self.get_url()
1141
+
1142
+ @property
1143
+ @_run_decorator._attach
1144
+ def entity(self) -> str:
1145
+ """The name of the W&B entity associated with the run.
1146
+
1147
+ Entity can be a username or the name of a team or organization.
1148
+ """
1149
+ return self._settings.entity or ""
1150
+
1151
+ def _label_internal(
1152
+ self,
1153
+ code: str | None = None,
1154
+ repo: str | None = None,
1155
+ code_version: str | None = None,
1156
+ ) -> None:
1157
+ with telemetry.context(run=self) as tel:
1158
+ if code and RE_LABEL.match(code):
1159
+ tel.label.code_string = code
1160
+ if repo and RE_LABEL.match(repo):
1161
+ tel.label.repo_string = repo
1162
+ if code_version and RE_LABEL.match(code_version):
1163
+ tel.label.code_version = code_version
1164
+
1165
+ def _label(
1166
+ self,
1167
+ code: str | None = None,
1168
+ repo: str | None = None,
1169
+ code_version: str | None = None,
1170
+ **kwargs: str,
1171
+ ) -> None:
1172
+ if self._settings.label_disable:
1173
+ return
1174
+ for k, v in (("code", code), ("repo", repo), ("code_version", code_version)):
1175
+ if v and not RE_LABEL.match(v):
1176
+ wandb.termwarn(
1177
+ f"Label added for '{k}' with invalid identifier '{v}' (ignored).",
1178
+ repeat=False,
1179
+ )
1180
+ for v in kwargs:
1181
+ wandb.termwarn(
1182
+ f"Label added for unsupported key {v!r} (ignored).",
1183
+ repeat=False,
1184
+ )
1185
+
1186
+ self._label_internal(code=code, repo=repo, code_version=code_version)
1187
+
1188
+ # update telemetry in the backend immediately for _label() callers
1189
+ self._telemetry_flush()
1190
+
1191
+ def _label_probe_lines(self, lines: list[str]) -> None:
1192
+ if not lines:
1193
+ return
1194
+ parsed = telemetry._parse_label_lines(lines)
1195
+ if not parsed:
1196
+ return
1197
+ label_dict = {}
1198
+ code = parsed.get("code") or parsed.get("c")
1199
+ if code:
1200
+ label_dict["code"] = code
1201
+ repo = parsed.get("repo") or parsed.get("r")
1202
+ if repo:
1203
+ label_dict["repo"] = repo
1204
+ code_ver = parsed.get("version") or parsed.get("v")
1205
+ if code_ver:
1206
+ label_dict["code_version"] = code_ver
1207
+ self._label_internal(**label_dict)
1208
+
1209
+ def _label_probe_main(self) -> None:
1210
+ m = sys.modules.get("__main__")
1211
+ if not m:
1212
+ return
1213
+ doc = getattr(m, "__doc__", None)
1214
+ if not doc:
1215
+ return
1216
+
1217
+ doclines = doc.splitlines()
1218
+ self._label_probe_lines(doclines)
1219
+
1220
+ # TODO: annotate jupyter Notebook class
1221
+ def _label_probe_notebook(self, notebook: Any) -> None:
1222
+ logger.info("probe notebook")
1223
+ lines = None
1224
+ try:
1225
+ data = notebook.probe_ipynb()
1226
+ cell0 = data.get("cells", [])[0]
1227
+ lines = cell0.get("source")
1228
+ # kaggle returns a string instead of a list
1229
+ if isinstance(lines, str):
1230
+ lines = lines.split()
1231
+ except Exception as e:
1232
+ logger.info(f"Unable to probe notebook: {e}")
1233
+ return
1234
+ if lines:
1235
+ self._label_probe_lines(lines)
1236
+
1237
+ @_run_decorator._attach
1238
+ def display(self, height: int = 420, hidden: bool = False) -> bool:
1239
+ """Display this run in jupyter."""
1240
+ if self._settings.silent:
1241
+ return False
1242
+
1243
+ if not ipython.in_jupyter():
1244
+ return False
1245
+
1246
+ try:
1247
+ from IPython import display
1248
+
1249
+ display.display(display.HTML(self.to_html(height, hidden)))
1250
+ return True
1251
+
1252
+ except ImportError:
1253
+ wandb.termwarn(".display() only works in jupyter environments")
1254
+ return False
1255
+
1256
+ @_run_decorator._attach
1257
+ def to_html(self, height: int = 420, hidden: bool = False) -> str:
1258
+ """Generate HTML containing an iframe displaying the current run."""
1259
+ url = self._settings.run_url + "?jupyter=true"
1260
+ style = f"border:none;width:100%;height:{height}px;"
1261
+ prefix = ""
1262
+ if hidden:
1263
+ style += "display:none;"
1264
+ prefix = ipython.toggle_button()
1265
+ return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
1266
+
1267
+ def _repr_mimebundle_(
1268
+ self, include: Any | None = None, exclude: Any | None = None
1269
+ ) -> dict[str, str]:
1270
+ return {"text/html": self.to_html(hidden=True)}
1271
+
1272
+ @_run_decorator._noop_on_finish()
1273
+ def _config_callback(
1274
+ self,
1275
+ key: tuple[str, ...] | str | None = None,
1276
+ val: Any | None = None,
1277
+ data: dict[str, object] | None = None,
1278
+ ) -> None:
1279
+ logger.info(f"config_cb {key} {val} {data}")
1280
+ if self._backend and self._backend.interface:
1281
+ self._backend.interface.publish_config(key=key, val=val, data=data)
1282
+
1283
+ def _config_artifact_callback(
1284
+ self, key: str, val: str | Artifact | dict
1285
+ ) -> Artifact:
1286
+ # artifacts can look like dicts as they are passed into the run config
1287
+ # since the run config stores them on the backend as a dict with fields shown
1288
+ # in wandb.util.artifact_to_json
1289
+ if _is_artifact_version_weave_dict(val):
1290
+ assert isinstance(val, dict)
1291
+ public_api = self._public_api()
1292
+ artifact = Artifact._from_id(val["id"], public_api.client)
1293
+ return self.use_artifact(artifact, use_as=key)
1294
+ elif _is_artifact_string(val):
1295
+ # this will never fail, but is required to make mypy happy
1296
+ assert isinstance(val, str)
1297
+ artifact_string, base_url, is_id = parse_artifact_string(val)
1298
+ overrides = {}
1299
+ if base_url is not None:
1300
+ overrides = {"base_url": base_url}
1301
+ public_api = public.Api(overrides)
1302
+ else:
1303
+ public_api = self._public_api()
1304
+ if is_id:
1305
+ artifact = Artifact._from_id(artifact_string, public_api._client)
1306
+ else:
1307
+ artifact = public_api._artifact(name=artifact_string)
1308
+ # in the future we'll need to support using artifacts from
1309
+ # different instances of wandb.
1310
+
1311
+ return self.use_artifact(artifact, use_as=key)
1312
+ elif _is_artifact_object(val):
1313
+ return self.use_artifact(val, use_as=key)
1314
+ else:
1315
+ raise ValueError(
1316
+ f"Cannot call _config_artifact_callback on type {type(val)}"
1317
+ )
1318
+
1319
+ def _set_config_wandb(self, key: str, val: Any) -> None:
1320
+ self._config_callback(key=("_wandb", key), val=val)
1321
+
1322
+ @_run_decorator._noop_on_finish()
1323
+ def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
1324
+ with telemetry.context(run=self) as tel:
1325
+ tel.feature.set_summary = True
1326
+ if self._backend and self._backend.interface:
1327
+ self._backend.interface.publish_summary(summary_record)
1328
+
1329
+ def _on_progress_get_summary(self, handle: MailboxProgress) -> None:
1330
+ pass
1331
+ # TODO(jhr): enable printing for get_summary in later mailbox dev phase
1332
+ # line = "Waiting for run.summary data..."
1333
+ # self._printer.display(line)
1334
+
1335
+ def _summary_get_current_summary_callback(self) -> dict[str, Any]:
1336
+ if self._is_finished:
1337
+ # TODO: WB-18420: fetch summary from backend and stage it before run is finished
1338
+ wandb.termwarn("Summary data not available in finished run")
1339
+ return {}
1340
+ if not self._backend or not self._backend.interface:
1341
+ return {}
1342
+ handle = self._backend.interface.deliver_get_summary()
1343
+ result = handle.wait(
1344
+ timeout=self._settings.summary_timeout,
1345
+ on_progress=self._on_progress_get_summary,
1346
+ )
1347
+ if not result:
1348
+ return {}
1349
+ get_summary_response = result.response.get_summary_response
1350
+ return proto_util.dict_from_proto_list(get_summary_response.item)
1351
+
1352
+ def _metric_callback(self, metric_record: MetricRecord) -> None:
1353
+ if self._backend and self._backend.interface:
1354
+ self._backend.interface._publish_metric(metric_record)
1355
+
1356
+ def _datatypes_callback(self, fname: str) -> None:
1357
+ if not self._backend or not self._backend.interface:
1358
+ return
1359
+ files: FilesDict = dict(files=[(GlobStr(glob.escape(fname)), "now")])
1360
+ self._backend.interface.publish_files(files)
1361
+
1362
+ def _serialize_custom_charts(self, data: dict[str, Any]) -> dict[str, Any]:
1363
+ if not data:
1364
+ return data
1365
+
1366
+ chart_keys = set()
1367
+ for k, v in data.items():
1368
+ if isinstance(v, Visualize):
1369
+ data[k] = v.table
1370
+ v.set_key(k)
1371
+ self._config_callback(
1372
+ val=v.spec.config_value,
1373
+ key=v.spec.config_key,
1374
+ )
1375
+ elif isinstance(v, CustomChart):
1376
+ chart_keys.add(k)
1377
+ v.set_key(k)
1378
+ self._config_callback(
1379
+ key=v.spec.config_key,
1380
+ val=v.spec.config_value,
1381
+ )
1382
+
1383
+ for k in chart_keys:
1384
+ # remove the chart key from the row
1385
+ v = data.pop(k)
1386
+ if isinstance(v, CustomChart):
1387
+ data[v.spec.table_key] = v.table
1388
+ return data
1389
+
1390
+ def _partial_history_callback(
1391
+ self,
1392
+ data: dict[str, Any],
1393
+ step: int | None = None,
1394
+ commit: bool | None = None,
1395
+ ) -> None:
1396
+ if not (self._backend and self._backend.interface):
1397
+ return
1398
+
1399
+ data = data.copy() # avoid modifying the original data
1400
+
1401
+ # Serialize custom charts before publishing
1402
+ data = self._serialize_custom_charts(data)
1403
+
1404
+ not_using_tensorboard = len(wandb.patched["tensorboard"]) == 0
1405
+ self._backend.interface.publish_partial_history(
1406
+ data,
1407
+ user_step=self._step,
1408
+ step=step,
1409
+ flush=commit,
1410
+ publish_step=not_using_tensorboard,
1411
+ )
1412
+
1413
+ def _console_callback(self, name: str, data: str) -> None:
1414
+ # logger.info("console callback: %s, %s", name, data)
1415
+ if self._backend and self._backend.interface:
1416
+ self._backend.interface.publish_output(name, data)
1417
+
1418
+ @_run_decorator._noop_on_finish(only_warn=True)
1419
+ def _console_raw_callback(self, name: str, data: str) -> None:
1420
+ # logger.info("console callback: %s, %s", name, data)
1421
+
1422
+ # NOTE: console output is only allowed on the process which installed the callback
1423
+ # this will prevent potential corruption in the socket to the service. Other methods
1424
+ # are protected by the _attach run decorator, but this callback was installed on the
1425
+ # write function of stdout and stderr streams.
1426
+ console_pid = getattr(self, "_attach_pid", 0)
1427
+ if console_pid != os.getpid():
1428
+ return
1429
+
1430
+ if self._backend and self._backend.interface:
1431
+ self._backend.interface.publish_output_raw(name, data)
1432
+
1433
+ def _tensorboard_callback(
1434
+ self, logdir: str, save: bool = True, root_logdir: str = ""
1435
+ ) -> None:
1436
+ logger.info("tensorboard callback: %s, %s", logdir, save)
1437
+ if self._backend and self._backend.interface:
1438
+ self._backend.interface.publish_tbdata(logdir, save, root_logdir)
1439
+
1440
+ def _set_library(self, library: _WandbSetup) -> None:
1441
+ self._wl = library
1442
+
1443
+ def _set_backend(self, backend: wandb.sdk.backend.backend.Backend) -> None:
1444
+ self._backend = backend
1445
+
1446
+ def _set_internal_run_interface(
1447
+ self,
1448
+ interface: wandb.sdk.interface.interface_queue.InterfaceQueue,
1449
+ ) -> None:
1450
+ self._internal_run_interface = interface
1451
+
1452
+ def _set_teardown_hooks(self, hooks: list[TeardownHook]) -> None:
1453
+ self._teardown_hooks = hooks
1454
+
1455
+ def _set_run_obj(self, run_obj: RunRecord) -> None: # noqa: C901
1456
+ if run_obj.starting_step:
1457
+ self._starting_step = run_obj.starting_step
1458
+ self._step = run_obj.starting_step
1459
+
1460
+ if run_obj.start_time:
1461
+ self._start_time = run_obj.start_time.ToMicroseconds() / 1e6
1462
+
1463
+ # Grab the config from resuming
1464
+ if run_obj.config:
1465
+ c_dict = config_util.dict_no_value_from_proto_list(run_obj.config.update)
1466
+ # TODO: Windows throws a wild error when this is set...
1467
+ if "_wandb" in c_dict:
1468
+ del c_dict["_wandb"]
1469
+ # We update the config object here without triggering the callback
1470
+ self._config._update(c_dict, allow_val_change=True, ignore_locked=True)
1471
+ # Update the summary, this will trigger an un-needed graphql request :(
1472
+ if run_obj.summary:
1473
+ summary_dict = {}
1474
+ for orig in run_obj.summary.update:
1475
+ summary_dict[orig.key] = json.loads(orig.value_json)
1476
+ if summary_dict:
1477
+ self.summary.update(summary_dict)
1478
+
1479
+ # update settings from run_obj
1480
+ if run_obj.run_id:
1481
+ self._settings.run_id = run_obj.run_id
1482
+ if run_obj.entity:
1483
+ self._settings.entity = run_obj.entity
1484
+ if run_obj.project:
1485
+ self._settings.project = run_obj.project
1486
+ if run_obj.run_group:
1487
+ self._settings.run_group = run_obj.run_group
1488
+ if run_obj.job_type:
1489
+ self._settings.run_job_type = run_obj.job_type
1490
+ if run_obj.display_name:
1491
+ self._settings.run_name = run_obj.display_name
1492
+ if run_obj.notes:
1493
+ self._settings.run_notes = run_obj.notes
1494
+ if run_obj.tags:
1495
+ self._settings.run_tags = run_obj.tags
1496
+ if run_obj.sweep_id:
1497
+ self._settings.sweep_id = run_obj.sweep_id
1498
+ if run_obj.host:
1499
+ self._settings.host = run_obj.host
1500
+ if run_obj.resumed:
1501
+ self._settings.resumed = run_obj.resumed
1502
+ if run_obj.git:
1503
+ if run_obj.git.remote_url:
1504
+ self._settings.git_remote_url = run_obj.git.remote_url
1505
+ if run_obj.git.commit:
1506
+ self._settings.git_commit = run_obj.git.commit
1507
+
1508
+ if run_obj.forked:
1509
+ self._forked = run_obj.forked
1510
+
1511
+ wandb._sentry.configure_scope(
1512
+ process_context="user",
1513
+ tags=dict(self._settings),
1514
+ )
1515
+
1516
+ def _populate_git_info(self) -> None:
1517
+ from .lib.gitlib import GitRepo
1518
+
1519
+ # Use user provided git info if available otherwise resolve it from the environment
1520
+ try:
1521
+ repo = GitRepo(
1522
+ root=self._settings.git_root,
1523
+ remote=self._settings.git_remote,
1524
+ remote_url=self._settings.git_remote_url,
1525
+ commit=self._settings.git_commit,
1526
+ lazy=False,
1527
+ )
1528
+ self._settings.git_remote_url = repo.remote_url
1529
+ self._settings.git_commit = repo.last_commit
1530
+ except Exception:
1531
+ wandb.termwarn("Cannot find valid git repo associated with this directory.")
1532
+
1533
+ def _add_singleton(
1534
+ self, data_type: str, key: str, value: dict[int | str, str]
1535
+ ) -> None:
1536
+ """Store a singleton item to wandb config.
1537
+
1538
+ A singleton in this context is a piece of data that is continually
1539
+ logged with the same value in each history step, but represented
1540
+ as a single item in the config.
1541
+
1542
+ We do this to avoid filling up history with a lot of repeated unnecessary data
1543
+
1544
+ Add singleton can be called many times in one run, and it will only be
1545
+ updated when the value changes. The last value logged will be the one
1546
+ persisted to the server.
1547
+ """
1548
+ value_extra = {"type": data_type, "key": key, "value": value}
1549
+
1550
+ if data_type not in self._config["_wandb"]:
1551
+ self._config["_wandb"][data_type] = {}
1552
+
1553
+ if data_type in self._config["_wandb"][data_type]:
1554
+ old_value = self._config["_wandb"][data_type][key]
1555
+ else:
1556
+ old_value = None
1557
+
1558
+ if value_extra != old_value:
1559
+ self._config["_wandb"][data_type][key] = value_extra
1560
+ self._config.persist()
1561
+
1562
+ def _log(
1563
+ self,
1564
+ data: dict[str, Any],
1565
+ step: int | None = None,
1566
+ commit: bool | None = None,
1567
+ ) -> None:
1568
+ if not isinstance(data, Mapping):
1569
+ raise ValueError("wandb.log must be passed a dictionary")
1570
+
1571
+ if any(not isinstance(key, str) for key in data.keys()):
1572
+ raise ValueError("Key values passed to `wandb.log` must be strings.")
1573
+
1574
+ self._partial_history_callback(data, step, commit)
1575
+
1576
+ if step is not None:
1577
+ if os.getpid() != self._init_pid or self._is_attached:
1578
+ wandb.termwarn(
1579
+ "Note that setting step in multiprocessing can result in data loss. "
1580
+ "Please use `run.define_metric(...)` to define a custom metric "
1581
+ "to log your step values.",
1582
+ repeat=False,
1583
+ )
1584
+ # if step is passed in when tensorboard_sync is used we honor the step passed
1585
+ # to make decisions about how to close out the history record, but will strip
1586
+ # this history later on in publish_history()
1587
+ if len(wandb.patched["tensorboard"]) > 0:
1588
+ wandb.termwarn(
1589
+ "Step cannot be set when using tensorboard syncing. "
1590
+ "Please use `run.define_metric(...)` to define a custom metric "
1591
+ "to log your step values.",
1592
+ repeat=False,
1593
+ )
1594
+ if step > self._step:
1595
+ self._step = step
1596
+
1597
+ if (step is None and commit is None) or commit:
1598
+ self._step += 1
1599
+
1600
+ @_run_decorator._noop
1601
+ @_run_decorator._noop_on_finish()
1602
+ @_run_decorator._attach
1603
+ def log(
1604
+ self,
1605
+ data: dict[str, Any],
1606
+ step: int | None = None,
1607
+ commit: bool | None = None,
1608
+ sync: bool | None = None,
1609
+ ) -> None:
1610
+ """Upload run data.
1611
+
1612
+ Use `log` to log data from runs, such as scalars, images, video,
1613
+ histograms, plots, and tables.
1614
+
1615
+ See our [guides to logging](https://docs.wandb.ai/guides/track/log) for
1616
+ live examples, code snippets, best practices, and more.
1617
+
1618
+ The most basic usage is `run.log({"train-loss": 0.5, "accuracy": 0.9})`.
1619
+ This will save the loss and accuracy to the run's history and update
1620
+ the summary values for these metrics.
1621
+
1622
+ Visualize logged data in the workspace at [wandb.ai](https://wandb.ai),
1623
+ or locally on a [self-hosted instance](https://docs.wandb.ai/guides/hosting)
1624
+ of the W&B app, or export data to visualize and explore locally, e.g. in
1625
+ Jupyter notebooks, with [our API](https://docs.wandb.ai/guides/track/public-api-guide).
1626
+
1627
+ Logged values don't have to be scalars. Logging any wandb object is supported.
1628
+ For example `run.log({"example": wandb.Image("myimage.jpg")})` will log an
1629
+ example image which will be displayed nicely in the W&B UI.
1630
+ See the [reference documentation](https://docs.wandb.com/ref/python/data-types)
1631
+ for all of the different supported types or check out our
1632
+ [guides to logging](https://docs.wandb.ai/guides/track/log) for examples,
1633
+ from 3D molecular structures and segmentation masks to PR curves and histograms.
1634
+ You can use `wandb.Table` to log structured data. See our
1635
+ [guide to logging tables](https://docs.wandb.ai/guides/tables/tables-walkthrough)
1636
+ for details.
1637
+
1638
+ The W&B UI organizes metrics with a forward slash (`/`) in their name
1639
+ into sections named using the text before the final slash. For example,
1640
+ the following results in two sections named "train" and "validate":
1641
+
1642
+ ```
1643
+ run.log(
1644
+ {
1645
+ "train/accuracy": 0.9,
1646
+ "train/loss": 30,
1647
+ "validate/accuracy": 0.8,
1648
+ "validate/loss": 20,
1649
+ }
1650
+ )
1651
+ ```
1652
+
1653
+ Only one level of nesting is supported; `run.log({"a/b/c": 1})`
1654
+ produces a section named "a/b".
1655
+
1656
+ `run.log` is not intended to be called more than a few times per second.
1657
+ For optimal performance, limit your logging to once every N iterations,
1658
+ or collect data over multiple iterations and log it in a single step.
1659
+
1660
+ ### The W&B step
1661
+
1662
+ With basic usage, each call to `log` creates a new "step".
1663
+ The step must always increase, and it is not possible to log
1664
+ to a previous step.
1665
+
1666
+ Note that you can use any metric as the X axis in charts.
1667
+ In many cases, it is better to treat the W&B step like
1668
+ you'd treat a timestamp rather than a training step.
1669
+
1670
+ ```
1671
+ # Example: log an "epoch" metric for use as an X axis.
1672
+ run.log({"epoch": 40, "train-loss": 0.5})
1673
+ ```
1674
+
1675
+ See also [define_metric](https://docs.wandb.ai/ref/python/run#define_metric).
1676
+
1677
+ It is possible to use multiple `log` invocations to log to
1678
+ the same step with the `step` and `commit` parameters.
1679
+ The following are all equivalent:
1680
+
1681
+ ```
1682
+ # Normal usage:
1683
+ run.log({"train-loss": 0.5, "accuracy": 0.8})
1684
+ run.log({"train-loss": 0.4, "accuracy": 0.9})
1685
+
1686
+ # Implicit step without auto-incrementing:
1687
+ run.log({"train-loss": 0.5}, commit=False)
1688
+ run.log({"accuracy": 0.8})
1689
+ run.log({"train-loss": 0.4}, commit=False)
1690
+ run.log({"accuracy": 0.9})
1691
+
1692
+ # Explicit step:
1693
+ run.log({"train-loss": 0.5}, step=current_step)
1694
+ run.log({"accuracy": 0.8}, step=current_step)
1695
+ current_step += 1
1696
+ run.log({"train-loss": 0.4}, step=current_step)
1697
+ run.log({"accuracy": 0.9}, step=current_step)
1698
+ ```
1699
+
1700
+ Args:
1701
+ data: A `dict` with `str` keys and values that are serializable
1702
+ Python objects including: `int`, `float` and `string`;
1703
+ any of the `wandb.data_types`; lists, tuples and NumPy arrays
1704
+ of serializable Python objects; other `dict`s of this
1705
+ structure.
1706
+ step: The step number to log. If `None`, then an implicit
1707
+ auto-incrementing step is used. See the notes in
1708
+ the description.
1709
+ commit: If true, finalize and upload the step. If false, then
1710
+ accumulate data for the step. See the notes in the description.
1711
+ If `step` is `None`, then the default is `commit=True`;
1712
+ otherwise, the default is `commit=False`.
1713
+ sync: This argument is deprecated and does nothing.
1714
+
1715
+ Examples:
1716
+ For more and more detailed examples, see
1717
+ [our guides to logging](https://docs.wandb.com/guides/track/log).
1718
+
1719
+ ### Basic usage
1720
+ <!--yeadoc-test:init-and-log-basic-->
1721
+ ```python
1722
+ import wandb
1723
+
1724
+ run = wandb.init()
1725
+ run.log({"accuracy": 0.9, "epoch": 5})
1726
+ ```
1727
+
1728
+ ### Incremental logging
1729
+ <!--yeadoc-test:init-and-log-incremental-->
1730
+ ```python
1731
+ import wandb
1732
+
1733
+ run = wandb.init()
1734
+ run.log({"loss": 0.2}, commit=False)
1735
+ # Somewhere else when I'm ready to report this step:
1736
+ run.log({"accuracy": 0.8})
1737
+ ```
1738
+
1739
+ ### Histogram
1740
+ <!--yeadoc-test:init-and-log-histogram-->
1741
+ ```python
1742
+ import numpy as np
1743
+ import wandb
1744
+
1745
+ # sample gradients at random from normal distribution
1746
+ gradients = np.random.randn(100, 100)
1747
+ run = wandb.init()
1748
+ run.log({"gradients": wandb.Histogram(gradients)})
1749
+ ```
1750
+
1751
+ ### Image from numpy
1752
+ <!--yeadoc-test:init-and-log-image-numpy-->
1753
+ ```python
1754
+ import numpy as np
1755
+ import wandb
1756
+
1757
+ run = wandb.init()
1758
+ examples = []
1759
+ for i in range(3):
1760
+ pixels = np.random.randint(low=0, high=256, size=(100, 100, 3))
1761
+ image = wandb.Image(pixels, caption=f"random field {i}")
1762
+ examples.append(image)
1763
+ run.log({"examples": examples})
1764
+ ```
1765
+
1766
+ ### Image from PIL
1767
+ <!--yeadoc-test:init-and-log-image-pillow-->
1768
+ ```python
1769
+ import numpy as np
1770
+ from PIL import Image as PILImage
1771
+ import wandb
1772
+
1773
+ run = wandb.init()
1774
+ examples = []
1775
+ for i in range(3):
1776
+ pixels = np.random.randint(
1777
+ low=0, high=256, size=(100, 100, 3), dtype=np.uint8
1778
+ )
1779
+ pil_image = PILImage.fromarray(pixels, mode="RGB")
1780
+ image = wandb.Image(pil_image, caption=f"random field {i}")
1781
+ examples.append(image)
1782
+ run.log({"examples": examples})
1783
+ ```
1784
+
1785
+ ### Video from numpy
1786
+ <!--yeadoc-test:init-and-log-video-numpy-->
1787
+ ```python
1788
+ import numpy as np
1789
+ import wandb
1790
+
1791
+ run = wandb.init()
1792
+ # axes are (time, channel, height, width)
1793
+ frames = np.random.randint(
1794
+ low=0, high=256, size=(10, 3, 100, 100), dtype=np.uint8
1795
+ )
1796
+ run.log({"video": wandb.Video(frames, fps=4)})
1797
+ ```
1798
+
1799
+ ### Matplotlib Plot
1800
+ <!--yeadoc-test:init-and-log-matplotlib-->
1801
+ ```python
1802
+ from matplotlib import pyplot as plt
1803
+ import numpy as np
1804
+ import wandb
1805
+
1806
+ run = wandb.init()
1807
+ fig, ax = plt.subplots()
1808
+ x = np.linspace(0, 10)
1809
+ y = x * x
1810
+ ax.plot(x, y) # plot y = x^2
1811
+ run.log({"chart": fig})
1812
+ ```
1813
+
1814
+ ### PR Curve
1815
+ ```python
1816
+ import wandb
1817
+
1818
+ run = wandb.init()
1819
+ run.log({"pr": wandb.plot.pr_curve(y_test, y_probas, labels)})
1820
+ ```
1821
+
1822
+ ### 3D Object
1823
+ ```python
1824
+ import wandb
1825
+
1826
+ run = wandb.init()
1827
+ run.log(
1828
+ {
1829
+ "generated_samples": [
1830
+ wandb.Object3D(open("sample.obj")),
1831
+ wandb.Object3D(open("sample.gltf")),
1832
+ wandb.Object3D(open("sample.glb")),
1833
+ ]
1834
+ }
1835
+ )
1836
+ ```
1837
+
1838
+ Raises:
1839
+ wandb.Error: if called before `wandb.init`
1840
+ ValueError: if invalid data is passed
1841
+
1842
+ """
1843
+ if step is not None:
1844
+ with telemetry.context(run=self) as tel:
1845
+ tel.feature.set_step_log = True
1846
+
1847
+ if sync is not None:
1848
+ deprecate.deprecate(
1849
+ field_name=deprecate.Deprecated.run__log_sync,
1850
+ warning_message=(
1851
+ "`sync` argument is deprecated and does not affect the behaviour of `wandb.log`"
1852
+ ),
1853
+ )
1854
+ if self._settings._shared and step is not None:
1855
+ wandb.termwarn(
1856
+ "In shared mode, the use of `wandb.log` with the step argument is not supported "
1857
+ f"and will be ignored. Please refer to {url_registry.url('define-metric')} "
1858
+ "on how to customize your x-axis.",
1859
+ repeat=False,
1860
+ )
1861
+ self._log(data=data, step=step, commit=commit)
1862
+
1863
+ @_run_decorator._noop_on_finish()
1864
+ @_run_decorator._attach
1865
+ def save(
1866
+ self,
1867
+ glob_str: str | os.PathLike | None = None,
1868
+ base_path: str | os.PathLike | None = None,
1869
+ policy: PolicyName = "live",
1870
+ ) -> bool | list[str]:
1871
+ """Sync one or more files to W&B.
1872
+
1873
+ Relative paths are relative to the current working directory.
1874
+
1875
+ A Unix glob, such as "myfiles/*", is expanded at the time `save` is
1876
+ called regardless of the `policy`. In particular, new files are not
1877
+ picked up automatically.
1878
+
1879
+ A `base_path` may be provided to control the directory structure of
1880
+ uploaded files. It should be a prefix of `glob_str`, and the directory
1881
+ structure beneath it is preserved. It's best understood through
1882
+ examples:
1883
+
1884
+ ```
1885
+ wandb.save("these/are/myfiles/*")
1886
+ # => Saves files in a "these/are/myfiles/" folder in the run.
1887
+
1888
+ wandb.save("these/are/myfiles/*", base_path="these")
1889
+ # => Saves files in an "are/myfiles/" folder in the run.
1890
+
1891
+ wandb.save("/User/username/Documents/run123/*.txt")
1892
+ # => Saves files in a "run123/" folder in the run. See note below.
1893
+
1894
+ wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
1895
+ # => Saves files in a "username/Documents/run123/" folder in the run.
1896
+
1897
+ wandb.save("files/*/saveme.txt")
1898
+ # => Saves each "saveme.txt" file in an appropriate subdirectory
1899
+ # of "files/".
1900
+ ```
1901
+
1902
+ Note: when given an absolute path or glob and no `base_path`, one
1903
+ directory level is preserved as in the example above.
1904
+
1905
+ Args:
1906
+ glob_str: A relative or absolute path or Unix glob.
1907
+ base_path: A path to use to infer a directory structure; see examples.
1908
+ policy: One of `live`, `now`, or `end`.
1909
+ * live: upload the file as it changes, overwriting the previous version
1910
+ * now: upload the file once now
1911
+ * end: upload file when the run ends
1912
+
1913
+ Returns:
1914
+ Paths to the symlinks created for the matched files.
1915
+
1916
+ For historical reasons, this may return a boolean in legacy code.
1917
+ """
1918
+ if glob_str is None:
1919
+ # noop for historical reasons, run.save() may be called in legacy code
1920
+ deprecate.deprecate(
1921
+ field_name=deprecate.Deprecated.run__save_no_args,
1922
+ warning_message=(
1923
+ "Calling wandb.run.save without any arguments is deprecated."
1924
+ "Changes to attributes are automatically persisted."
1925
+ ),
1926
+ )
1927
+ return True
1928
+
1929
+ if isinstance(glob_str, bytes):
1930
+ # Preserved for backward compatibility: allow bytes inputs.
1931
+ glob_str = glob_str.decode("utf-8")
1932
+ if isinstance(glob_str, str) and (glob_str.startswith(("gs://", "s3://"))):
1933
+ # Provide a better error message for a common misuse.
1934
+ wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
1935
+ return []
1936
+ # NOTE: We use PurePath instead of Path because WindowsPath doesn't
1937
+ # like asterisks and errors out in resolve(). It also makes logical
1938
+ # sense: globs aren't real paths, they're just path-like strings.
1939
+ glob_path = pathlib.PurePath(glob_str)
1940
+ resolved_glob_path = pathlib.PurePath(os.path.abspath(glob_path))
1941
+
1942
+ if base_path is not None:
1943
+ base_path = pathlib.Path(base_path)
1944
+ elif not glob_path.is_absolute():
1945
+ base_path = pathlib.Path(".")
1946
+ else:
1947
+ # Absolute glob paths with no base path get special handling.
1948
+ wandb.termwarn(
1949
+ "Saving files without folders. If you want to preserve "
1950
+ "subdirectories pass base_path to wandb.save, i.e. "
1951
+ 'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
1952
+ repeat=False,
1953
+ )
1954
+ base_path = resolved_glob_path.parent.parent
1955
+
1956
+ if policy not in ("live", "end", "now"):
1957
+ raise ValueError(
1958
+ 'Only "live", "end" and "now" policies are currently supported.'
1959
+ )
1960
+
1961
+ resolved_base_path = pathlib.PurePath(os.path.abspath(base_path))
1962
+
1963
+ return self._save(
1964
+ resolved_glob_path,
1965
+ resolved_base_path,
1966
+ policy,
1967
+ )
1968
+
1969
+ def _save(
1970
+ self,
1971
+ glob_path: pathlib.PurePath,
1972
+ base_path: pathlib.PurePath,
1973
+ policy: PolicyName,
1974
+ ) -> list[str]:
1975
+ # Can't use is_relative_to() because that's added in Python 3.9,
1976
+ # but we support down to Python 3.8.
1977
+ if not str(glob_path).startswith(str(base_path)):
1978
+ raise ValueError("Glob may not walk above the base path")
1979
+
1980
+ if glob_path == base_path:
1981
+ raise ValueError("Glob cannot be the same as the base path")
1982
+
1983
+ relative_glob = glob_path.relative_to(base_path)
1984
+ if relative_glob.parts[0] == "*":
1985
+ raise ValueError("Glob may not start with '*' relative to the base path")
1986
+ relative_glob_str = GlobStr(str(relative_glob))
1987
+
1988
+ with telemetry.context(run=self) as tel:
1989
+ tel.feature.save = True
1990
+
1991
+ # Files in the files directory matched by the glob, including old and
1992
+ # new ones.
1993
+ globbed_files = set(
1994
+ pathlib.Path(
1995
+ self._settings.files_dir,
1996
+ ).glob(relative_glob_str)
1997
+ )
1998
+
1999
+ had_symlinked_files = len(globbed_files) > 0
2000
+ is_star_glob = "*" in relative_glob_str
2001
+
2002
+ # The base_path may itself be a glob, so we can't do
2003
+ # base_path.glob(relative_glob_str)
2004
+ for path_str in glob.glob(str(base_path / relative_glob_str)):
2005
+ source_path = pathlib.Path(path_str).absolute()
2006
+
2007
+ # We can't use relative_to() because base_path may be a glob.
2008
+ relative_path = pathlib.Path(*source_path.parts[len(base_path.parts) :])
2009
+
2010
+ target_path = pathlib.Path(self._settings.files_dir, relative_path)
2011
+ globbed_files.add(target_path)
2012
+
2013
+ # If the file is already where it needs to be, don't create a symlink.
2014
+ if source_path.resolve() == target_path.resolve():
2015
+ continue
2016
+
2017
+ target_path.parent.mkdir(parents=True, exist_ok=True)
2018
+
2019
+ # Delete the symlink if it exists.
2020
+ target_path.unlink(missing_ok=True)
2021
+
2022
+ target_path.symlink_to(source_path)
2023
+
2024
+ # Inform users that new files aren't detected automatically.
2025
+ if not had_symlinked_files and is_star_glob:
2026
+ file_str = f"{len(globbed_files)} file"
2027
+ if len(globbed_files) > 1:
2028
+ file_str += "s"
2029
+ wandb.termwarn(
2030
+ f"Symlinked {file_str} into the W&B run directory, "
2031
+ "call wandb.save again to sync new files."
2032
+ )
2033
+
2034
+ files_dict: FilesDict = {
2035
+ "files": [
2036
+ (
2037
+ GlobStr(str(f.relative_to(self._settings.files_dir))),
2038
+ policy,
2039
+ )
2040
+ for f in globbed_files
2041
+ ]
2042
+ }
2043
+ if self._backend and self._backend.interface:
2044
+ self._backend.interface.publish_files(files_dict)
2045
+
2046
+ return [str(f) for f in globbed_files]
2047
+
2048
+ @_run_decorator._attach
2049
+ def restore(
2050
+ self,
2051
+ name: str,
2052
+ run_path: str | None = None,
2053
+ replace: bool = False,
2054
+ root: str | None = None,
2055
+ ) -> None | TextIO:
2056
+ return restore(
2057
+ name,
2058
+ run_path or self._get_path(),
2059
+ replace,
2060
+ root or self._settings.files_dir,
2061
+ )
2062
+
2063
+ @_run_decorator._noop
2064
+ @_run_decorator._attach
2065
+ def finish(
2066
+ self,
2067
+ exit_code: int | None = None,
2068
+ quiet: bool | None = None,
2069
+ ) -> None:
2070
+ """Finish a run and upload any remaining data.
2071
+
2072
+ Marks the completion of a W&B run and ensures all data is synced to the server.
2073
+ The run's final state is determined by its exit conditions and sync status.
2074
+
2075
+ Run States:
2076
+ - Running: Active run that is logging data and/or sending heartbeats.
2077
+ - Crashed: Run that stopped sending heartbeats unexpectedly.
2078
+ - Finished: Run completed successfully (`exit_code=0`) with all data synced.
2079
+ - Failed: Run completed with errors (`exit_code!=0`).
2080
+
2081
+ Args:
2082
+ exit_code: Integer indicating the run's exit status. Use 0 for success,
2083
+ any other value marks the run as failed.
2084
+ quiet: Deprecated. Configure logging verbosity using `wandb.Settings(quiet=...)`.
2085
+ """
2086
+ if quiet is not None:
2087
+ deprecate.deprecate(
2088
+ field_name=deprecate.Deprecated.run__finish_quiet,
2089
+ warning_message=(
2090
+ "The `quiet` argument to `wandb.run.finish()` is deprecated, "
2091
+ "use `wandb.Settings(quiet=...)` to set this instead."
2092
+ ),
2093
+ )
2094
+ return self._finish(exit_code)
2095
+
2096
+ def _finish(
2097
+ self,
2098
+ exit_code: int | None = None,
2099
+ ) -> None:
2100
+ logger.info(f"finishing run {self._get_path()}")
2101
+ with telemetry.context(run=self) as tel:
2102
+ tel.feature.finish = True
2103
+
2104
+ # Pop this run (hopefully) from the run stack, to support the "reinit"
2105
+ # functionality of wandb.init().
2106
+ #
2107
+ # TODO: It's not clear how _global_run_stack could have length other
2108
+ # than 1 at this point in the code. If you're reading this, consider
2109
+ # refactoring this thing.
2110
+ if self._wl and len(self._wl._global_run_stack) > 0:
2111
+ self._wl._global_run_stack.pop()
2112
+
2113
+ # Run hooks that need to happen before the last messages to the
2114
+ # internal service, like Jupyter hooks.
2115
+ for hook in self._teardown_hooks:
2116
+ if hook.stage == TeardownStage.EARLY:
2117
+ hook.call()
2118
+
2119
+ # Early-stage hooks may use methods that require _is_finished
2120
+ # to be False, so we set this after running those hooks.
2121
+ self._is_finished = True
2122
+
2123
+ try:
2124
+ self._atexit_cleanup(exit_code=exit_code)
2125
+
2126
+ # Run hooks that should happen after the last messages to the
2127
+ # internal service, like detaching the logger.
2128
+ for hook in self._teardown_hooks:
2129
+ if hook.stage == TeardownStage.LATE:
2130
+ hook.call()
2131
+ self._teardown_hooks = []
2132
+
2133
+ # Inform the service that we're done sending messages for this run.
2134
+ #
2135
+ # TODO: Why not do this in _atexit_cleanup()?
2136
+ service = self._wl and self._wl.service
2137
+ if service and self._settings.run_id:
2138
+ service.inform_finish(run_id=self._settings.run_id)
2139
+
2140
+ finally:
2141
+ module.unset_globals()
2142
+ wandb._sentry.end_session()
2143
+
2144
+ @_run_decorator._noop
2145
+ @_run_decorator._attach
2146
+ def join(self, exit_code: int | None = None) -> None:
2147
+ """Deprecated alias for `finish()` - use finish instead."""
2148
+ if hasattr(self, "_telemetry_obj"):
2149
+ deprecate.deprecate(
2150
+ field_name=deprecate.Deprecated.run__join,
2151
+ warning_message=(
2152
+ "wandb.run.join() is deprecated, please use wandb.run.finish()."
2153
+ ),
2154
+ )
2155
+ self._finish(exit_code=exit_code)
2156
+
2157
+ @_run_decorator._noop_on_finish()
2158
+ @_run_decorator._attach
2159
+ def status(
2160
+ self,
2161
+ ) -> RunStatus:
2162
+ """Get sync info from the internal backend, about the current run's sync status."""
2163
+ if not self._backend or not self._backend.interface:
2164
+ return RunStatus()
2165
+
2166
+ handle_run_status = self._backend.interface.deliver_request_run_status()
2167
+ result = handle_run_status.wait(timeout=-1)
2168
+ assert result
2169
+ sync_data = result.response.run_status_response
2170
+
2171
+ sync_time = None
2172
+ if sync_data.sync_time.seconds:
2173
+ sync_time = datetime.fromtimestamp(
2174
+ sync_data.sync_time.seconds + sync_data.sync_time.nanos / 1e9
2175
+ )
2176
+ return RunStatus(
2177
+ sync_items_total=sync_data.sync_items_total,
2178
+ sync_items_pending=sync_data.sync_items_pending,
2179
+ sync_time=sync_time,
2180
+ )
2181
+
2182
+ def _add_panel(
2183
+ self, visualize_key: str, panel_type: str, panel_config: dict
2184
+ ) -> None:
2185
+ config = {
2186
+ "panel_type": panel_type,
2187
+ "panel_config": panel_config,
2188
+ }
2189
+ self._config_callback(val=config, key=("_wandb", "visualize", visualize_key))
2190
+
2191
+ def _set_globals(self) -> None:
2192
+ module.set_global(
2193
+ run=self,
2194
+ config=self.config,
2195
+ log=self.log,
2196
+ summary=self.summary,
2197
+ save=self.save,
2198
+ use_artifact=self.use_artifact,
2199
+ log_artifact=self.log_artifact,
2200
+ define_metric=self.define_metric,
2201
+ alert=self.alert,
2202
+ watch=self.watch,
2203
+ unwatch=self.unwatch,
2204
+ mark_preempting=self.mark_preempting,
2205
+ log_model=self.log_model,
2206
+ use_model=self.use_model,
2207
+ link_model=self.link_model,
2208
+ )
2209
+
2210
+ def _redirect(
2211
+ self,
2212
+ stdout_slave_fd: int | None,
2213
+ stderr_slave_fd: int | None,
2214
+ console: str | None = None,
2215
+ ) -> None:
2216
+ if console is None:
2217
+ console = self._settings.console
2218
+ # only use raw for service to minimize potential changes
2219
+ if console == "wrap":
2220
+ if not self._settings.x_disable_service:
2221
+ console = "wrap_raw"
2222
+ else:
2223
+ console = "wrap_emu"
2224
+ logger.info("redirect: %s", console)
2225
+
2226
+ out_redir: redirect.RedirectBase
2227
+ err_redir: redirect.RedirectBase
2228
+
2229
+ # raw output handles the output_log writing in the internal process
2230
+ if console in {"redirect", "wrap_emu"}:
2231
+ output_log_path = os.path.join(
2232
+ self._settings.files_dir, filenames.OUTPUT_FNAME
2233
+ )
2234
+ # output writer might have been set up, see wrap_fallback case
2235
+ if not self._output_writer:
2236
+ self._output_writer = filesystem.CRDedupedFile(
2237
+ open(output_log_path, "wb")
2238
+ )
2239
+
2240
+ if console == "redirect":
2241
+ logger.info("Redirecting console.")
2242
+ out_redir = redirect.Redirect(
2243
+ src="stdout",
2244
+ cbs=[
2245
+ lambda data: self._console_callback("stdout", data),
2246
+ self._output_writer.write, # type: ignore
2247
+ ],
2248
+ )
2249
+ err_redir = redirect.Redirect(
2250
+ src="stderr",
2251
+ cbs=[
2252
+ lambda data: self._console_callback("stderr", data),
2253
+ self._output_writer.write, # type: ignore
2254
+ ],
2255
+ )
2256
+ if os.name == "nt":
2257
+
2258
+ def wrap_fallback() -> None:
2259
+ if self._out_redir:
2260
+ self._out_redir.uninstall()
2261
+ if self._err_redir:
2262
+ self._err_redir.uninstall()
2263
+ msg = (
2264
+ "Tensorflow detected. Stream redirection is not supported "
2265
+ "on Windows when tensorflow is imported. Falling back to "
2266
+ "wrapping stdout/err."
2267
+ )
2268
+ wandb.termlog(msg)
2269
+ self._redirect(None, None, console="wrap")
2270
+
2271
+ add_import_hook("tensorflow", wrap_fallback)
2272
+ elif console == "wrap_emu":
2273
+ logger.info("Wrapping output streams.")
2274
+ out_redir = redirect.StreamWrapper(
2275
+ src="stdout",
2276
+ cbs=[
2277
+ lambda data: self._console_callback("stdout", data),
2278
+ self._output_writer.write, # type: ignore
2279
+ ],
2280
+ )
2281
+ err_redir = redirect.StreamWrapper(
2282
+ src="stderr",
2283
+ cbs=[
2284
+ lambda data: self._console_callback("stderr", data),
2285
+ self._output_writer.write, # type: ignore
2286
+ ],
2287
+ )
2288
+ elif console == "wrap_raw":
2289
+ logger.info("Wrapping output streams.")
2290
+ out_redir = redirect.StreamRawWrapper(
2291
+ src="stdout",
2292
+ cbs=[
2293
+ lambda data: self._console_raw_callback("stdout", data),
2294
+ ],
2295
+ )
2296
+ err_redir = redirect.StreamRawWrapper(
2297
+ src="stderr",
2298
+ cbs=[
2299
+ lambda data: self._console_raw_callback("stderr", data),
2300
+ ],
2301
+ )
2302
+ elif console == "off":
2303
+ return
2304
+ else:
2305
+ raise ValueError("unhandled console")
2306
+ try:
2307
+ # save stdout and stderr before installing new write functions
2308
+ out_redir.save()
2309
+ err_redir.save()
2310
+ out_redir.install()
2311
+ err_redir.install()
2312
+ self._out_redir = out_redir
2313
+ self._err_redir = err_redir
2314
+ logger.info("Redirects installed.")
2315
+ except Exception as e:
2316
+ print(e)
2317
+ logger.error("Failed to redirect.", exc_info=e)
2318
+ return
2319
+
2320
+ def _restore(self) -> None:
2321
+ logger.info("restore")
2322
+ # TODO(jhr): drain and shutdown all threads
2323
+ if self._out_redir:
2324
+ self._out_redir.uninstall()
2325
+ if self._err_redir:
2326
+ self._err_redir.uninstall()
2327
+ logger.info("restore done")
2328
+
2329
+ def _atexit_cleanup(self, exit_code: int | None = None) -> None:
2330
+ if self._backend is None:
2331
+ logger.warning("process exited without backend configured")
2332
+ return
2333
+ if self._atexit_cleanup_called:
2334
+ return
2335
+ self._atexit_cleanup_called = True
2336
+
2337
+ exit_code = exit_code or (self._hooks and self._hooks.exit_code) or 0
2338
+ self._exit_code = exit_code
2339
+ logger.info(f"got exitcode: {exit_code}")
2340
+
2341
+ # Delete this run's "resume" file if the run finished successfully.
2342
+ #
2343
+ # This is used by the "auto" resume mode, which resumes from the last
2344
+ # failed (or unfinished/crashed) run. If we reach this line, then this
2345
+ # run shouldn't be a candidate for "auto" resume.
2346
+ if exit_code == 0:
2347
+ if os.path.exists(self._settings.resume_fname):
2348
+ os.remove(self._settings.resume_fname)
2349
+
2350
+ try:
2351
+ self._on_finish()
2352
+
2353
+ except KeyboardInterrupt:
2354
+ if not wandb.wandb_agent._is_running(): # type: ignore
2355
+ wandb.termerror("Control-C detected -- Run data was not synced")
2356
+ raise
2357
+
2358
+ except Exception as e:
2359
+ self._console_stop()
2360
+ logger.error("Problem finishing run", exc_info=e)
2361
+ wandb.termerror("Problem finishing run")
2362
+ raise
2363
+
2364
+ Run._footer(
2365
+ sampled_history=self._sampled_history,
2366
+ final_summary=self._final_summary,
2367
+ poll_exit_response=self._poll_exit_response,
2368
+ internal_messages_response=self._internal_messages_response,
2369
+ settings=self._settings,
2370
+ printer=self._printer,
2371
+ )
2372
+
2373
+ def _console_start(self) -> None:
2374
+ logger.info("atexit reg")
2375
+ self._hooks = ExitHooks()
2376
+
2377
+ service = self._wl and self._wl.service
2378
+ if not service:
2379
+ self._hooks.hook()
2380
+ # NB: manager will perform atexit hook like behavior for outstanding runs
2381
+ atexit.register(lambda: self._atexit_cleanup())
2382
+
2383
+ self._redirect(self._stdout_slave_fd, self._stderr_slave_fd)
2384
+
2385
+ def _console_stop(self) -> None:
2386
+ self._restore()
2387
+ if self._output_writer:
2388
+ self._output_writer.close()
2389
+ self._output_writer = None
2390
+
2391
+ def _on_init(self) -> None:
2392
+ if self._settings._offline:
2393
+ return
2394
+
2395
+ def _on_start(self) -> None:
2396
+ # would like to move _set_global to _on_ready to unify _on_start and _on_attach
2397
+ # (we want to do the set globals after attach)
2398
+ # TODO(console) However _console_start calls Redirect that uses `wandb.run` hence breaks
2399
+ # TODO(jupyter) However _header calls _header_run_info that uses wandb.jupyter that uses
2400
+ # `wandb.run` and hence breaks
2401
+ self._set_globals()
2402
+ self._header(settings=self._settings, printer=self._printer)
2403
+
2404
+ if self._settings.save_code and self._settings.code_dir is not None:
2405
+ self.log_code(self._settings.code_dir)
2406
+
2407
+ if self._settings.x_save_requirements:
2408
+ if self._backend and self._backend.interface:
2409
+ from wandb.util import working_set
2410
+
2411
+ logger.debug(
2412
+ "Saving list of pip packages installed into the current environment"
2413
+ )
2414
+ self._backend.interface.publish_python_packages(working_set())
2415
+
2416
+ if self._backend and self._backend.interface and not self._settings._offline:
2417
+ self._run_status_checker = RunStatusChecker(
2418
+ interface=self._backend.interface,
2419
+ )
2420
+ self._run_status_checker.start()
2421
+
2422
+ self._console_start()
2423
+ self._on_ready()
2424
+
2425
+ def _on_attach(self) -> None:
2426
+ """Event triggered when run is attached to another run."""
2427
+ with telemetry.context(run=self) as tel:
2428
+ tel.feature.attach = True
2429
+
2430
+ self._set_globals()
2431
+ self._is_attached = True
2432
+ self._on_ready()
2433
+
2434
+ def _register_telemetry_import_hooks(
2435
+ self,
2436
+ ) -> None:
2437
+ def _telemetry_import_hook(
2438
+ run: Run,
2439
+ module: Any,
2440
+ ) -> None:
2441
+ with telemetry.context(run=run) as tel:
2442
+ try:
2443
+ name = getattr(module, "__name__", None)
2444
+ if name is not None:
2445
+ setattr(tel.imports_finish, name, True)
2446
+ except AttributeError:
2447
+ return
2448
+
2449
+ import_telemetry_set = telemetry.list_telemetry_imports()
2450
+ import_hook_fn = functools.partial(_telemetry_import_hook, self)
2451
+ if not self._settings.run_id:
2452
+ return
2453
+ for module_name in import_telemetry_set:
2454
+ register_post_import_hook(
2455
+ import_hook_fn,
2456
+ self._settings.run_id,
2457
+ module_name,
2458
+ )
2459
+
2460
+ def _on_ready(self) -> None:
2461
+ """Event triggered when run is ready for the user."""
2462
+ self._register_telemetry_import_hooks()
2463
+
2464
+ # start reporting any telemetry changes
2465
+ self._telemetry_obj_active = True
2466
+ self._telemetry_flush()
2467
+
2468
+ try:
2469
+ self._detect_and_apply_job_inputs()
2470
+ except Exception as e:
2471
+ logger.error("Problem applying launch job inputs", exc_info=e)
2472
+
2473
+ # object is about to be returned to the user, don't let them modify it
2474
+ self._freeze()
2475
+
2476
+ if not self._settings.resume:
2477
+ if os.path.exists(self._settings.resume_fname):
2478
+ os.remove(self._settings.resume_fname)
2479
+
2480
+ def _detect_and_apply_job_inputs(self) -> None:
2481
+ """If the user has staged launch inputs, apply them to the run."""
2482
+ from wandb.sdk.launch.inputs.internal import StagedLaunchInputs
2483
+
2484
+ StagedLaunchInputs().apply(self)
2485
+
2486
+ def _make_job_source_reqs(self) -> tuple[list[str], dict[str, Any], dict[str, Any]]:
2487
+ from wandb.util import working_set
2488
+
2489
+ installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
2490
+ input_types = TypeRegistry.type_of(self.config.as_dict()).to_json()
2491
+ output_types = TypeRegistry.type_of(self.summary._as_dict()).to_json()
2492
+
2493
+ return installed_packages_list, input_types, output_types
2494
+
2495
+ def _construct_job_artifact(
2496
+ self,
2497
+ name: str,
2498
+ source_dict: JobSourceDict,
2499
+ installed_packages_list: list[str],
2500
+ patch_path: os.PathLike | None = None,
2501
+ ) -> Artifact:
2502
+ job_artifact = job_builder.JobArtifact(name)
2503
+ if patch_path and os.path.exists(patch_path):
2504
+ job_artifact.add_file(FilePathStr(str(patch_path)), "diff.patch")
2505
+ with job_artifact.new_file("requirements.frozen.txt") as f:
2506
+ f.write("\n".join(installed_packages_list))
2507
+ with job_artifact.new_file("wandb-job.json") as f:
2508
+ f.write(json.dumps(source_dict))
2509
+
2510
+ return job_artifact
2511
+
2512
+ def _create_image_job(
2513
+ self,
2514
+ input_types: dict[str, Any],
2515
+ output_types: dict[str, Any],
2516
+ installed_packages_list: list[str],
2517
+ docker_image_name: str | None = None,
2518
+ args: list[str] | None = None,
2519
+ ) -> Artifact | None:
2520
+ docker_image_name = docker_image_name or os.getenv("WANDB_DOCKER")
2521
+
2522
+ if not docker_image_name:
2523
+ return None
2524
+
2525
+ name = wandb.util.make_artifact_name_safe(f"job-{docker_image_name}")
2526
+ s_args: Sequence[str] = args if args is not None else self._settings._args
2527
+ source_info: JobSourceDict = {
2528
+ "_version": "v0",
2529
+ "source_type": "image",
2530
+ "source": {"image": docker_image_name, "args": s_args},
2531
+ "input_types": input_types,
2532
+ "output_types": output_types,
2533
+ "runtime": self._settings._python,
2534
+ }
2535
+ job_artifact = self._construct_job_artifact(
2536
+ name, source_info, installed_packages_list
2537
+ )
2538
+
2539
+ return job_artifact
2540
+
2541
+ def _log_job_artifact_with_image(
2542
+ self, docker_image_name: str, args: list[str] | None = None
2543
+ ) -> Artifact:
2544
+ packages, in_types, out_types = self._make_job_source_reqs()
2545
+ job_artifact = self._create_image_job(
2546
+ in_types,
2547
+ out_types,
2548
+ packages,
2549
+ args=args,
2550
+ docker_image_name=docker_image_name,
2551
+ )
2552
+
2553
+ artifact = self.log_artifact(job_artifact)
2554
+
2555
+ if not artifact:
2556
+ raise wandb.Error(f"Job Artifact log unsuccessful: {artifact}")
2557
+ else:
2558
+ return artifact
2559
+
2560
+ def _on_probe_exit(self, probe_handle: MailboxProbe) -> None:
2561
+ handle = probe_handle.get_mailbox_handle()
2562
+ if handle:
2563
+ result = handle.wait(timeout=0, release=False)
2564
+ if not result:
2565
+ return
2566
+ probe_handle.set_probe_result(result)
2567
+ assert self._backend and self._backend.interface
2568
+ handle = self._backend.interface.deliver_poll_exit()
2569
+ probe_handle.set_mailbox_handle(handle)
2570
+
2571
+ def _on_progress_exit(
2572
+ self,
2573
+ progress_printer: progress.ProgressPrinter,
2574
+ progress_handle: MailboxProgress,
2575
+ ) -> None:
2576
+ probe_handles = progress_handle.get_probe_handles()
2577
+ if not probe_handles or len(probe_handles) != 1:
2578
+ return
2579
+
2580
+ result = probe_handles[0].get_probe_result()
2581
+ if not result:
2582
+ return
2583
+
2584
+ progress_printer.update([result.response.poll_exit_response])
2585
+
2586
+ def _on_finish(self) -> None:
2587
+ trigger.call("on_finished")
2588
+
2589
+ if self._run_status_checker is not None:
2590
+ self._run_status_checker.stop()
2591
+
2592
+ self._console_stop() # TODO: there's a race here with jupyter console logging
2593
+
2594
+ assert self._backend and self._backend.interface
2595
+
2596
+ if self._settings.x_update_finish_state:
2597
+ exit_handle = self._backend.interface.deliver_exit(self._exit_code)
2598
+ else:
2599
+ exit_handle = self._backend.interface.deliver_finish_without_exit()
2600
+
2601
+ exit_handle.add_probe(on_probe=self._on_probe_exit)
2602
+
2603
+ with progress.progress_printer(
2604
+ self._printer,
2605
+ self._settings,
2606
+ ) as progress_printer:
2607
+ # Wait for the run to complete.
2608
+ _ = exit_handle.wait(
2609
+ timeout=-1,
2610
+ on_progress=functools.partial(
2611
+ self._on_progress_exit,
2612
+ progress_printer,
2613
+ ),
2614
+ )
2615
+
2616
+ # Print some final statistics.
2617
+ poll_exit_handle = self._backend.interface.deliver_poll_exit()
2618
+ result = poll_exit_handle.wait(timeout=-1)
2619
+ assert result
2620
+ progress.print_sync_dedupe_stats(
2621
+ self._printer,
2622
+ result.response.poll_exit_response,
2623
+ )
2624
+
2625
+ self._poll_exit_response = result.response.poll_exit_response
2626
+ internal_messages_handle = self._backend.interface.deliver_internal_messages()
2627
+ result = internal_messages_handle.wait(timeout=-1)
2628
+ assert result
2629
+ self._internal_messages_response = result.response.internal_messages_response
2630
+
2631
+ # dispatch all our final requests
2632
+
2633
+ final_summary_handle = self._backend.interface.deliver_get_summary()
2634
+ sampled_history_handle = (
2635
+ self._backend.interface.deliver_request_sampled_history()
2636
+ )
2637
+
2638
+ result = sampled_history_handle.wait(timeout=-1)
2639
+ assert result
2640
+ self._sampled_history = result.response.sampled_history_response
2641
+
2642
+ result = final_summary_handle.wait(timeout=-1)
2643
+ assert result
2644
+ self._final_summary = result.response.get_summary_response
2645
+
2646
+ if self._backend:
2647
+ self._backend.cleanup()
2648
+
2649
+ if self._run_status_checker:
2650
+ self._run_status_checker.join()
2651
+
2652
+ if self._settings.run_id:
2653
+ self._unregister_telemetry_import_hooks(self._settings.run_id)
2654
+
2655
+ @staticmethod
2656
+ def _unregister_telemetry_import_hooks(run_id: str) -> None:
2657
+ import_telemetry_set = telemetry.list_telemetry_imports()
2658
+ for module_name in import_telemetry_set:
2659
+ unregister_post_import_hook(module_name, run_id)
2660
+
2661
+ @_run_decorator._noop_on_finish()
2662
+ @_run_decorator._attach
2663
+ def define_metric(
2664
+ self,
2665
+ name: str,
2666
+ step_metric: str | wandb_metric.Metric | None = None,
2667
+ step_sync: bool | None = None,
2668
+ hidden: bool | None = None,
2669
+ summary: str | None = None,
2670
+ goal: str | None = None,
2671
+ overwrite: bool | None = None,
2672
+ ) -> wandb_metric.Metric:
2673
+ """Customize metrics logged with `wandb.log()`.
2674
+
2675
+ Args:
2676
+ name: The name of the metric to customize.
2677
+ step_metric: The name of another metric to serve as the X-axis
2678
+ for this metric in automatically generated charts.
2679
+ step_sync: Automatically insert the last value of step_metric into
2680
+ `run.log()` if it is not provided explicitly. Defaults to True
2681
+ if step_metric is specified.
2682
+ hidden: Hide this metric from automatic plots.
2683
+ summary: Specify aggregate metrics added to summary.
2684
+ Supported aggregations include "min", "max", "mean", "last",
2685
+ "best", "copy" and "none". "best" is used together with the
2686
+ goal parameter. "none" prevents a summary from being generated.
2687
+ "copy" is deprecated and should not be used.
2688
+ goal: Specify how to interpret the "best" summary type.
2689
+ Supported options are "minimize" and "maximize".
2690
+ overwrite: If false, then this call is merged with previous
2691
+ `define_metric` calls for the same metric by using their
2692
+ values for any unspecified parameters. If true, then
2693
+ unspecified parameters overwrite values specified by
2694
+ previous calls.
2695
+
2696
+ Returns:
2697
+ An object that represents this call but can otherwise be discarded.
2698
+ """
2699
+ if summary and "copy" in summary:
2700
+ deprecate.deprecate(
2701
+ deprecate.Deprecated.run__define_metric_copy,
2702
+ "define_metric(summary='copy') is deprecated and will be removed.",
2703
+ self,
2704
+ )
2705
+
2706
+ if (summary and "best" in summary) or goal is not None:
2707
+ deprecate.deprecate(
2708
+ deprecate.Deprecated.run__define_metric_best_goal,
2709
+ "define_metric(summary='best', goal=...) is deprecated and will be removed. "
2710
+ "Use define_metric(summary='min') or define_metric(summary='max') instead.",
2711
+ self,
2712
+ )
2713
+
2714
+ return self._define_metric(
2715
+ name,
2716
+ step_metric,
2717
+ step_sync,
2718
+ hidden,
2719
+ summary,
2720
+ goal,
2721
+ overwrite,
2722
+ )
2723
+
2724
+ def _define_metric(
2725
+ self,
2726
+ name: str,
2727
+ step_metric: str | wandb_metric.Metric | None = None,
2728
+ step_sync: bool | None = None,
2729
+ hidden: bool | None = None,
2730
+ summary: str | None = None,
2731
+ goal: str | None = None,
2732
+ overwrite: bool | None = None,
2733
+ ) -> wandb_metric.Metric:
2734
+ if not name:
2735
+ raise wandb.Error("define_metric() requires non-empty name argument")
2736
+ if isinstance(step_metric, wandb_metric.Metric):
2737
+ step_metric = step_metric.name
2738
+ for arg_name, arg_val, exp_type in (
2739
+ ("name", name, str),
2740
+ ("step_metric", step_metric, str),
2741
+ ("step_sync", step_sync, bool),
2742
+ ("hidden", hidden, bool),
2743
+ ("summary", summary, str),
2744
+ ("goal", goal, str),
2745
+ ("overwrite", overwrite, bool),
2746
+ ):
2747
+ # NOTE: type checking is broken for isinstance and str
2748
+ if arg_val is not None and not isinstance(arg_val, exp_type):
2749
+ arg_type = type(arg_val).__name__
2750
+ raise wandb.Error(
2751
+ f"Unhandled define_metric() arg: {arg_name} type: {arg_type}"
2752
+ )
2753
+ stripped = name[:-1] if name.endswith("*") else name
2754
+ if "*" in stripped:
2755
+ raise wandb.Error(
2756
+ f"Unhandled define_metric() arg: name (glob suffixes only): {name}"
2757
+ )
2758
+ summary_ops: Sequence[str] | None = None
2759
+ if summary:
2760
+ summary_items = [s.lower() for s in summary.split(",")]
2761
+ summary_ops = []
2762
+ valid = {"min", "max", "mean", "best", "last", "copy", "none"}
2763
+ # TODO: deprecate copy and best
2764
+ for i in summary_items:
2765
+ if i not in valid:
2766
+ raise wandb.Error(f"Unhandled define_metric() arg: summary op: {i}")
2767
+ summary_ops.append(i)
2768
+ with telemetry.context(run=self) as tel:
2769
+ tel.feature.metric_summary = True
2770
+ # TODO: deprecate goal
2771
+ goal_cleaned: str | None = None
2772
+ if goal is not None:
2773
+ goal_cleaned = goal[:3].lower()
2774
+ valid_goal = {"min", "max"}
2775
+ if goal_cleaned not in valid_goal:
2776
+ raise wandb.Error(f"Unhandled define_metric() arg: goal: {goal}")
2777
+ with telemetry.context(run=self) as tel:
2778
+ tel.feature.metric_goal = True
2779
+ if hidden:
2780
+ with telemetry.context(run=self) as tel:
2781
+ tel.feature.metric_hidden = True
2782
+ if step_sync:
2783
+ with telemetry.context(run=self) as tel:
2784
+ tel.feature.metric_step_sync = True
2785
+
2786
+ with telemetry.context(run=self) as tel:
2787
+ tel.feature.metric = True
2788
+
2789
+ m = wandb_metric.Metric(
2790
+ name=name,
2791
+ step_metric=step_metric,
2792
+ step_sync=step_sync,
2793
+ summary=summary_ops,
2794
+ hidden=hidden,
2795
+ goal=goal_cleaned,
2796
+ overwrite=overwrite,
2797
+ )
2798
+ m._set_callback(self._metric_callback)
2799
+ m._commit()
2800
+ return m
2801
+
2802
+ @_run_decorator._attach
2803
+ def watch(
2804
+ self,
2805
+ models: torch.nn.Module | Sequence[torch.nn.Module],
2806
+ criterion: torch.F | None = None, # type: ignore
2807
+ log: Literal["gradients", "parameters", "all"] | None = "gradients",
2808
+ log_freq: int = 1000,
2809
+ idx: int | None = None,
2810
+ log_graph: bool = False,
2811
+ ) -> None:
2812
+ """Hooks into the given PyTorch model(s) to monitor gradients and the model's computational graph.
2813
+
2814
+ This function can track parameters, gradients, or both during training. It should be
2815
+ extended to support arbitrary machine learning models in the future.
2816
+
2817
+ Args:
2818
+ models (Union[torch.nn.Module, Sequence[torch.nn.Module]]):
2819
+ A single model or a sequence of models to be monitored.
2820
+ criterion (Optional[torch.F]):
2821
+ The loss function being optimized (optional).
2822
+ log (Optional[Literal["gradients", "parameters", "all"]]):
2823
+ Specifies whether to log "gradients", "parameters", or "all".
2824
+ Set to None to disable logging. (default="gradients")
2825
+ log_freq (int):
2826
+ Frequency (in batches) to log gradients and parameters. (default=1000)
2827
+ idx (Optional[int]):
2828
+ Index used when tracking multiple models with `wandb.watch`. (default=None)
2829
+ log_graph (bool):
2830
+ Whether to log the model's computational graph. (default=False)
2831
+
2832
+ Raises:
2833
+ ValueError:
2834
+ If `wandb.init` has not been called or if any of the models are not instances
2835
+ of `torch.nn.Module`.
2836
+ """
2837
+ wandb.sdk._watch(self, models, criterion, log, log_freq, idx, log_graph)
2838
+
2839
+ @_run_decorator._attach
2840
+ def unwatch(
2841
+ self, models: torch.nn.Module | Sequence[torch.nn.Module] | None = None
2842
+ ) -> None:
2843
+ """Remove pytorch model topology, gradient and parameter hooks.
2844
+
2845
+ Args:
2846
+ models (torch.nn.Module | Sequence[torch.nn.Module]):
2847
+ Optional list of pytorch models that have had watch called on them
2848
+ """
2849
+ wandb.sdk._unwatch(self, models=models)
2850
+
2851
+ # TODO(kdg): remove all artifact swapping logic
2852
+ def _swap_artifact_name(self, artifact_name: str, use_as: str | None) -> str:
2853
+ artifact_key_string = use_as or artifact_name
2854
+ replacement_artifact_info = self._launch_artifact_mapping.get(
2855
+ artifact_key_string
2856
+ )
2857
+ if replacement_artifact_info is not None:
2858
+ new_name = replacement_artifact_info.get("name")
2859
+ entity = replacement_artifact_info.get("entity")
2860
+ project = replacement_artifact_info.get("project")
2861
+ if new_name is None or entity is None or project is None:
2862
+ raise ValueError(
2863
+ "Misconfigured artifact in launch config. Must include name, project and entity keys."
2864
+ )
2865
+ return f"{entity}/{project}/{new_name}"
2866
+ elif replacement_artifact_info is None and use_as is None:
2867
+ sequence_name = artifact_name.split(":")[0].split("/")[-1]
2868
+ unique_artifact_replacement_info = (
2869
+ self._unique_launch_artifact_sequence_names.get(sequence_name)
2870
+ )
2871
+ if unique_artifact_replacement_info is not None:
2872
+ new_name = unique_artifact_replacement_info.get("name")
2873
+ entity = unique_artifact_replacement_info.get("entity")
2874
+ project = unique_artifact_replacement_info.get("project")
2875
+ if new_name is None or entity is None or project is None:
2876
+ raise ValueError(
2877
+ "Misconfigured artifact in launch config. Must include name, project and entity keys."
2878
+ )
2879
+ return f"{entity}/{project}/{new_name}"
2880
+
2881
+ else:
2882
+ return artifact_name
2883
+
2884
+ return artifact_name
2885
+
2886
+ def _detach(self) -> None:
2887
+ pass
2888
+
2889
+ @_run_decorator._noop_on_finish()
2890
+ @_run_decorator._attach
2891
+ def link_artifact(
2892
+ self,
2893
+ artifact: Artifact,
2894
+ target_path: str,
2895
+ aliases: list[str] | None = None,
2896
+ ) -> None:
2897
+ """Link the given artifact to a portfolio (a promoted collection of artifacts).
2898
+
2899
+ The linked artifact will be visible in the UI for the specified portfolio.
2900
+
2901
+ Args:
2902
+ artifact: the (public or local) artifact which will be linked
2903
+ target_path: `str` - takes the following forms: `{portfolio}`, `{project}/{portfolio}`,
2904
+ or `{entity}/{project}/{portfolio}`
2905
+ aliases: `List[str]` - optional alias(es) that will only be applied on this linked artifact
2906
+ inside the portfolio.
2907
+ The alias "latest" will always be applied to the latest version of an artifact that is linked.
2908
+
2909
+ Returns:
2910
+ None
2911
+
2912
+ """
2913
+ portfolio, project, entity = wandb.util._parse_entity_project_item(target_path)
2914
+ if aliases is None:
2915
+ aliases = []
2916
+
2917
+ if not self._backend or not self._backend.interface:
2918
+ return
2919
+
2920
+ if artifact.is_draft() and not artifact._is_draft_save_started():
2921
+ artifact = self._log_artifact(artifact)
2922
+
2923
+ if self._settings._offline:
2924
+ # TODO: implement offline mode + sync
2925
+ raise NotImplementedError
2926
+
2927
+ # Wait until the artifact is committed before trying to link it.
2928
+ artifact.wait()
2929
+
2930
+ organization = ""
2931
+ if is_artifact_registry_project(project):
2932
+ organization = entity
2933
+ # In a Registry linking, the entity is used to fetch the organization of the artifact
2934
+ # therefore the source artifact's entity is passed to the backend
2935
+ entity = artifact._source_entity
2936
+ handle = self._backend.interface.deliver_link_artifact(
2937
+ self,
2938
+ artifact,
2939
+ portfolio,
2940
+ aliases,
2941
+ entity,
2942
+ project,
2943
+ organization,
2944
+ )
2945
+ if artifact._ttl_duration_seconds is not None:
2946
+ wandb.termwarn(
2947
+ "Artifact TTL will be disabled for source artifacts that are linked to portfolios."
2948
+ )
2949
+ result = handle.wait(timeout=-1)
2950
+ if result is None:
2951
+ handle.abandon()
2952
+ else:
2953
+ response = result.response.link_artifact_response
2954
+ if response.error_message:
2955
+ wandb.termerror(response.error_message)
2956
+
2957
+ @_run_decorator._noop_on_finish()
2958
+ @_run_decorator._attach
2959
+ def use_artifact(
2960
+ self,
2961
+ artifact_or_name: str | Artifact,
2962
+ type: str | None = None,
2963
+ aliases: list[str] | None = None,
2964
+ use_as: str | None = None,
2965
+ ) -> Artifact:
2966
+ """Declare an artifact as an input to a run.
2967
+
2968
+ Call `download` or `file` on the returned object to get the contents locally.
2969
+
2970
+ Args:
2971
+ artifact_or_name: (str or Artifact) An artifact name.
2972
+ May be prefixed with project/ or entity/project/.
2973
+ If no entity is specified in the name, the Run or API setting's entity is used.
2974
+ Valid names can be in the following forms:
2975
+ - name:version
2976
+ - name:alias
2977
+ You can also pass an Artifact object created by calling `wandb.Artifact`
2978
+ type: (str, optional) The type of artifact to use.
2979
+ aliases: (list, optional) Aliases to apply to this artifact
2980
+ use_as: (string, optional) Optional string indicating what purpose the artifact was used with.
2981
+ Will be shown in UI.
2982
+
2983
+ Returns:
2984
+ An `Artifact` object.
2985
+ """
2986
+ if self._settings._offline:
2987
+ raise TypeError("Cannot use artifact when in offline mode.")
2988
+
2989
+ api = internal.Api(
2990
+ default_settings={
2991
+ "entity": self._settings.entity,
2992
+ "project": self._settings.project,
2993
+ }
2994
+ )
2995
+ api.set_current_run_id(self._settings.run_id)
2996
+
2997
+ if isinstance(artifact_or_name, str):
2998
+ if self._launch_artifact_mapping:
2999
+ name = self._swap_artifact_name(artifact_or_name, use_as)
3000
+ else:
3001
+ name = artifact_or_name
3002
+ public_api = self._public_api()
3003
+ artifact = public_api._artifact(type=type, name=name)
3004
+ if type is not None and type != artifact.type:
3005
+ raise ValueError(
3006
+ "Supplied type {} does not match type {} of artifact {}".format(
3007
+ type, artifact.type, artifact.name
3008
+ )
3009
+ )
3010
+ artifact._use_as = use_as or artifact_or_name
3011
+ if use_as:
3012
+ if (
3013
+ use_as in self._used_artifact_slots.keys()
3014
+ and self._used_artifact_slots[use_as] != artifact.id
3015
+ ):
3016
+ raise ValueError(
3017
+ "Cannot call use_artifact with the same use_as argument more than once"
3018
+ )
3019
+ elif ":" in use_as or "/" in use_as:
3020
+ raise ValueError(
3021
+ "use_as cannot contain special characters ':' or '/'"
3022
+ )
3023
+ self._used_artifact_slots[use_as] = artifact.id
3024
+ api.use_artifact(
3025
+ artifact.id,
3026
+ entity_name=self._settings.entity,
3027
+ project_name=self._settings.project,
3028
+ use_as=use_as or artifact_or_name,
3029
+ )
3030
+ else:
3031
+ artifact = artifact_or_name
3032
+ if aliases is None:
3033
+ aliases = []
3034
+ elif isinstance(aliases, str):
3035
+ aliases = [aliases]
3036
+ if isinstance(artifact_or_name, Artifact) and artifact.is_draft():
3037
+ if use_as is not None:
3038
+ wandb.termwarn(
3039
+ "Indicating use_as is not supported when using a draft artifact"
3040
+ )
3041
+ self._log_artifact(
3042
+ artifact,
3043
+ aliases=aliases,
3044
+ is_user_created=True,
3045
+ use_after_commit=True,
3046
+ )
3047
+ artifact.wait()
3048
+ artifact._use_as = use_as or artifact.name
3049
+ elif isinstance(artifact, Artifact) and not artifact.is_draft():
3050
+ if (
3051
+ self._launch_artifact_mapping
3052
+ and artifact.name in self._launch_artifact_mapping.keys()
3053
+ ):
3054
+ wandb.termwarn(
3055
+ "Swapping artifacts is not supported when using a non-draft artifact. "
3056
+ f"Using {artifact.name}."
3057
+ )
3058
+ artifact._use_as = use_as or artifact.name
3059
+ api.use_artifact(
3060
+ artifact.id, use_as=use_as or artifact._use_as or artifact.name
3061
+ )
3062
+ else:
3063
+ raise ValueError(
3064
+ 'You must pass an artifact name (e.g. "pedestrian-dataset:v1"), '
3065
+ "an instance of `wandb.Artifact`, or `wandb.Api().artifact()` to `use_artifact`"
3066
+ )
3067
+ if self._backend and self._backend.interface:
3068
+ self._backend.interface.publish_use_artifact(artifact)
3069
+ return artifact
3070
+
3071
+ @_run_decorator._noop_on_finish()
3072
+ @_run_decorator._attach
3073
+ def log_artifact(
3074
+ self,
3075
+ artifact_or_path: Artifact | StrPath,
3076
+ name: str | None = None,
3077
+ type: str | None = None,
3078
+ aliases: list[str] | None = None,
3079
+ tags: list[str] | None = None,
3080
+ ) -> Artifact:
3081
+ """Declare an artifact as an output of a run.
3082
+
3083
+ Args:
3084
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
3085
+ can be in the following forms:
3086
+ - `/local/directory`
3087
+ - `/local/directory/file.txt`
3088
+ - `s3://bucket/path`
3089
+ You can also pass an Artifact object created by calling
3090
+ `wandb.Artifact`.
3091
+ name: (str, optional) An artifact name. Valid names can be in the following forms:
3092
+ - name:version
3093
+ - name:alias
3094
+ - digest
3095
+ This will default to the basename of the path prepended with the current
3096
+ run id if not specified.
3097
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
3098
+ aliases: (list, optional) Aliases to apply to this artifact,
3099
+ defaults to `["latest"]`
3100
+ tags: (list, optional) Tags to apply to this artifact, if any.
3101
+
3102
+ Returns:
3103
+ An `Artifact` object.
3104
+ """
3105
+ return self._log_artifact(
3106
+ artifact_or_path,
3107
+ name=name,
3108
+ type=type,
3109
+ aliases=aliases,
3110
+ tags=tags,
3111
+ )
3112
+
3113
+ @_run_decorator._noop_on_finish()
3114
+ @_run_decorator._attach
3115
+ def upsert_artifact(
3116
+ self,
3117
+ artifact_or_path: Artifact | str,
3118
+ name: str | None = None,
3119
+ type: str | None = None,
3120
+ aliases: list[str] | None = None,
3121
+ distributed_id: str | None = None,
3122
+ ) -> Artifact:
3123
+ """Declare (or append to) a non-finalized artifact as output of a run.
3124
+
3125
+ Note that you must call run.finish_artifact() to finalize the artifact.
3126
+ This is useful when distributed jobs need to all contribute to the same artifact.
3127
+
3128
+ Args:
3129
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
3130
+ can be in the following forms:
3131
+ - `/local/directory`
3132
+ - `/local/directory/file.txt`
3133
+ - `s3://bucket/path`
3134
+ You can also pass an Artifact object created by calling
3135
+ `wandb.Artifact`.
3136
+ name: (str, optional) An artifact name. May be prefixed with entity/project.
3137
+ Valid names can be in the following forms:
3138
+ - name:version
3139
+ - name:alias
3140
+ - digest
3141
+ This will default to the basename of the path prepended with the current
3142
+ run id if not specified.
3143
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
3144
+ aliases: (list, optional) Aliases to apply to this artifact,
3145
+ defaults to `["latest"]`
3146
+ distributed_id: (string, optional) Unique string that all distributed jobs share. If None,
3147
+ defaults to the run's group name.
3148
+
3149
+ Returns:
3150
+ An `Artifact` object.
3151
+ """
3152
+ if self._settings.run_group is None and distributed_id is None:
3153
+ raise TypeError(
3154
+ "Cannot upsert artifact unless run is in a group or distributed_id is provided"
3155
+ )
3156
+ if distributed_id is None:
3157
+ distributed_id = self._settings.run_group or ""
3158
+ return self._log_artifact(
3159
+ artifact_or_path,
3160
+ name=name,
3161
+ type=type,
3162
+ aliases=aliases,
3163
+ distributed_id=distributed_id,
3164
+ finalize=False,
3165
+ )
3166
+
3167
+ @_run_decorator._noop_on_finish()
3168
+ @_run_decorator._attach
3169
+ def finish_artifact(
3170
+ self,
3171
+ artifact_or_path: Artifact | str,
3172
+ name: str | None = None,
3173
+ type: str | None = None,
3174
+ aliases: list[str] | None = None,
3175
+ distributed_id: str | None = None,
3176
+ ) -> Artifact:
3177
+ """Finishes a non-finalized artifact as output of a run.
3178
+
3179
+ Subsequent "upserts" with the same distributed ID will result in a new version.
3180
+
3181
+ Args:
3182
+ artifact_or_path: (str or Artifact) A path to the contents of this artifact,
3183
+ can be in the following forms:
3184
+ - `/local/directory`
3185
+ - `/local/directory/file.txt`
3186
+ - `s3://bucket/path`
3187
+ You can also pass an Artifact object created by calling
3188
+ `wandb.Artifact`.
3189
+ name: (str, optional) An artifact name. May be prefixed with entity/project.
3190
+ Valid names can be in the following forms:
3191
+ - name:version
3192
+ - name:alias
3193
+ - digest
3194
+ This will default to the basename of the path prepended with the current
3195
+ run id if not specified.
3196
+ type: (str) The type of artifact to log, examples include `dataset`, `model`
3197
+ aliases: (list, optional) Aliases to apply to this artifact,
3198
+ defaults to `["latest"]`
3199
+ distributed_id: (string, optional) Unique string that all distributed jobs share. If None,
3200
+ defaults to the run's group name.
3201
+
3202
+ Returns:
3203
+ An `Artifact` object.
3204
+ """
3205
+ if self._settings.run_group is None and distributed_id is None:
3206
+ raise TypeError(
3207
+ "Cannot finish artifact unless run is in a group or distributed_id is provided"
3208
+ )
3209
+ if distributed_id is None:
3210
+ distributed_id = self._settings.run_group or ""
3211
+
3212
+ return self._log_artifact(
3213
+ artifact_or_path,
3214
+ name,
3215
+ type,
3216
+ aliases,
3217
+ distributed_id=distributed_id,
3218
+ finalize=True,
3219
+ )
3220
+
3221
+ def _log_artifact(
3222
+ self,
3223
+ artifact_or_path: Artifact | StrPath,
3224
+ name: str | None = None,
3225
+ type: str | None = None,
3226
+ aliases: list[str] | None = None,
3227
+ tags: list[str] | None = None,
3228
+ distributed_id: str | None = None,
3229
+ finalize: bool = True,
3230
+ is_user_created: bool = False,
3231
+ use_after_commit: bool = False,
3232
+ ) -> Artifact:
3233
+ api = internal.Api()
3234
+ if api.settings().get("anonymous") == "true":
3235
+ wandb.termwarn(
3236
+ "Artifacts logged anonymously cannot be claimed and expire after 7 days."
3237
+ )
3238
+
3239
+ if not finalize and distributed_id is None:
3240
+ raise TypeError("Must provide distributed_id if artifact is not finalize")
3241
+
3242
+ if aliases is not None:
3243
+ aliases = validate_aliases(aliases)
3244
+
3245
+ # Check if artifact tags are supported
3246
+ if tags is not None:
3247
+ tags = validate_tags(tags)
3248
+
3249
+ artifact, aliases = self._prepare_artifact(
3250
+ artifact_or_path, name, type, aliases
3251
+ )
3252
+ artifact.distributed_id = distributed_id
3253
+ self._assert_can_log_artifact(artifact)
3254
+ if self._backend and self._backend.interface:
3255
+ if not self._settings._offline:
3256
+ future = self._backend.interface.communicate_artifact(
3257
+ self,
3258
+ artifact,
3259
+ aliases,
3260
+ tags,
3261
+ self.step,
3262
+ finalize=finalize,
3263
+ is_user_created=is_user_created,
3264
+ use_after_commit=use_after_commit,
3265
+ )
3266
+ artifact._set_save_future(future, self._public_api().client)
3267
+ else:
3268
+ self._backend.interface.publish_artifact(
3269
+ self,
3270
+ artifact,
3271
+ aliases,
3272
+ tags,
3273
+ finalize=finalize,
3274
+ is_user_created=is_user_created,
3275
+ use_after_commit=use_after_commit,
3276
+ )
3277
+ elif self._internal_run_interface:
3278
+ self._internal_run_interface.publish_artifact(
3279
+ self,
3280
+ artifact,
3281
+ aliases,
3282
+ tags,
3283
+ finalize=finalize,
3284
+ is_user_created=is_user_created,
3285
+ use_after_commit=use_after_commit,
3286
+ )
3287
+ return artifact
3288
+
3289
+ def _public_api(self, overrides: dict[str, str] | None = None) -> PublicApi:
3290
+ overrides = {"run": self._settings.run_id} # type: ignore
3291
+ if not self._settings._offline:
3292
+ overrides["entity"] = self._settings.entity or ""
3293
+ overrides["project"] = self._settings.project or ""
3294
+ return public.Api(overrides)
3295
+
3296
+ # TODO(jhr): annotate this
3297
+ def _assert_can_log_artifact(self, artifact) -> None: # type: ignore
3298
+ if self._settings._offline:
3299
+ return
3300
+ try:
3301
+ public_api = self._public_api()
3302
+ entity = public_api.settings["entity"]
3303
+ project = public_api.settings["project"]
3304
+ expected_type = Artifact._expected_type(
3305
+ entity, project, artifact.name, public_api.client
3306
+ )
3307
+ except requests.exceptions.RequestException:
3308
+ # Just return early if there is a network error. This is
3309
+ # ok, as this function is intended to help catch an invalid
3310
+ # type early, but not a hard requirement for valid operation.
3311
+ return
3312
+ if expected_type is not None and artifact.type != expected_type:
3313
+ raise ValueError(
3314
+ f"Artifact {artifact.name} already exists with type '{expected_type}'; "
3315
+ f"cannot create another with type '{artifact.type}'"
3316
+ )
3317
+ if entity and artifact._source_entity and entity != artifact._source_entity:
3318
+ raise ValueError(
3319
+ f"Artifact {artifact.name} is owned by entity "
3320
+ f"'{artifact._source_entity}'; it can't be moved to '{entity}'"
3321
+ )
3322
+ if project and artifact._source_project and project != artifact._source_project:
3323
+ raise ValueError(
3324
+ f"Artifact {artifact.name} exists in project "
3325
+ f"'{artifact._source_project}'; it can't be moved to '{project}'"
3326
+ )
3327
+
3328
+ def _prepare_artifact(
3329
+ self,
3330
+ artifact_or_path: Artifact | StrPath,
3331
+ name: str | None = None,
3332
+ type: str | None = None,
3333
+ aliases: list[str] | None = None,
3334
+ ) -> tuple[Artifact, list[str]]:
3335
+ if isinstance(artifact_or_path, (str, os.PathLike)):
3336
+ name = (
3337
+ name
3338
+ or f"run-{self._settings.run_id}-{os.path.basename(artifact_or_path)}"
3339
+ )
3340
+ artifact = wandb.Artifact(name, type or "unspecified")
3341
+ if os.path.isfile(artifact_or_path):
3342
+ artifact.add_file(str(artifact_or_path))
3343
+ elif os.path.isdir(artifact_or_path):
3344
+ artifact.add_dir(str(artifact_or_path))
3345
+ elif "://" in str(artifact_or_path):
3346
+ artifact.add_reference(str(artifact_or_path))
3347
+ else:
3348
+ raise ValueError(
3349
+ "path must be a file, directory or external"
3350
+ "reference like s3://bucket/path"
3351
+ )
3352
+ else:
3353
+ artifact = artifact_or_path
3354
+ if not isinstance(artifact, wandb.Artifact):
3355
+ raise ValueError(
3356
+ "You must pass an instance of wandb.Artifact or a "
3357
+ "valid file path to log_artifact"
3358
+ )
3359
+
3360
+ artifact.finalize()
3361
+ return artifact, _resolve_aliases(aliases)
3362
+
3363
+ @_run_decorator._noop_on_finish()
3364
+ @_run_decorator._attach
3365
+ def log_model(
3366
+ self,
3367
+ path: StrPath,
3368
+ name: str | None = None,
3369
+ aliases: list[str] | None = None,
3370
+ ) -> None:
3371
+ """Logs a model artifact containing the contents inside the 'path' to a run and marks it as an output to this run.
3372
+
3373
+ Args:
3374
+ path: (str) A path to the contents of this model,
3375
+ can be in the following forms:
3376
+ - `/local/directory`
3377
+ - `/local/directory/file.txt`
3378
+ - `s3://bucket/path`
3379
+ name: (str, optional) A name to assign to the model artifact that the file contents will be added to.
3380
+ The string must contain only the following alphanumeric characters: dashes, underscores, and dots.
3381
+ This will default to the basename of the path prepended with the current
3382
+ run id if not specified.
3383
+ aliases: (list, optional) Aliases to apply to the created model artifact,
3384
+ defaults to `["latest"]`
3385
+
3386
+ Examples:
3387
+ ```python
3388
+ run.log_model(
3389
+ path="/local/directory",
3390
+ name="my_model_artifact",
3391
+ aliases=["production"],
3392
+ )
3393
+ ```
3394
+
3395
+ Invalid usage
3396
+ ```python
3397
+ run.log_model(
3398
+ path="/local/directory",
3399
+ name="my_entity/my_project/my_model_artifact",
3400
+ aliases=["production"],
3401
+ )
3402
+ ```
3403
+
3404
+ Raises:
3405
+ ValueError: if name has invalid special characters
3406
+
3407
+ Returns:
3408
+ None
3409
+ """
3410
+ self._log_artifact(
3411
+ artifact_or_path=path, name=name, type="model", aliases=aliases
3412
+ )
3413
+
3414
+ @_run_decorator._noop_on_finish()
3415
+ @_run_decorator._attach
3416
+ def use_model(self, name: str) -> FilePathStr:
3417
+ """Download the files logged in a model artifact 'name'.
3418
+
3419
+ Args:
3420
+ name: (str) A model artifact name. 'name' must match the name of an existing logged
3421
+ model artifact.
3422
+ May be prefixed with entity/project/. Valid names
3423
+ can be in the following forms:
3424
+ - model_artifact_name:version
3425
+ - model_artifact_name:alias
3426
+
3427
+ Examples:
3428
+ ```python
3429
+ run.use_model(
3430
+ name="my_model_artifact:latest",
3431
+ )
3432
+
3433
+ run.use_model(
3434
+ name="my_project/my_model_artifact:v0",
3435
+ )
3436
+
3437
+ run.use_model(
3438
+ name="my_entity/my_project/my_model_artifact:<digest>",
3439
+ )
3440
+ ```
3441
+
3442
+ Invalid usage
3443
+ ```python
3444
+ run.use_model(
3445
+ name="my_entity/my_project/my_model_artifact",
3446
+ )
3447
+ ```
3448
+
3449
+ Raises:
3450
+ AssertionError: if model artifact 'name' is of a type that does not contain the substring 'model'.
3451
+
3452
+ Returns:
3453
+ path: (str) path to downloaded model artifact file(s).
3454
+ """
3455
+ artifact = self.use_artifact(artifact_or_name=name)
3456
+ assert (
3457
+ "model" in str(artifact.type.lower())
3458
+ ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
3459
+ path = artifact.download()
3460
+
3461
+ # If returned directory contains only one file, return path to that file
3462
+ dir_list = os.listdir(path)
3463
+ if len(dir_list) == 1:
3464
+ return FilePathStr(os.path.join(path, dir_list[0]))
3465
+ return path
3466
+
3467
+ @_run_decorator._noop_on_finish()
3468
+ @_run_decorator._attach
3469
+ def link_model(
3470
+ self,
3471
+ path: StrPath,
3472
+ registered_model_name: str,
3473
+ name: str | None = None,
3474
+ aliases: list[str] | None = None,
3475
+ ) -> None:
3476
+ """Log a model artifact version and link it to a registered model in the model registry.
3477
+
3478
+ The linked model version will be visible in the UI for the specified registered model.
3479
+
3480
+ Steps:
3481
+ - Check if 'name' model artifact has been logged. If so, use the artifact version that matches the files
3482
+ located at 'path' or log a new version. Otherwise log files under 'path' as a new model artifact, 'name'
3483
+ of type 'model'.
3484
+ - Check if registered model with name 'registered_model_name' exists in the 'model-registry' project.
3485
+ If not, create a new registered model with name 'registered_model_name'.
3486
+ - Link version of model artifact 'name' to registered model, 'registered_model_name'.
3487
+ - Attach aliases from 'aliases' list to the newly linked model artifact version.
3488
+
3489
+ Args:
3490
+ path: (str) A path to the contents of this model,
3491
+ can be in the following forms:
3492
+ - `/local/directory`
3493
+ - `/local/directory/file.txt`
3494
+ - `s3://bucket/path`
3495
+ registered_model_name: (str) - the name of the registered model that the model is to be linked to.
3496
+ A registered model is a collection of model versions linked to the model registry, typically representing a
3497
+ team's specific ML Task. The entity that this registered model belongs to will be derived from the run
3498
+ name: (str, optional) - the name of the model artifact that files in 'path' will be logged to. This will
3499
+ default to the basename of the path prepended with the current run id if not specified.
3500
+ aliases: (List[str], optional) - alias(es) that will only be applied on this linked artifact
3501
+ inside the registered model.
3502
+ The alias "latest" will always be applied to the latest version of an artifact that is linked.
3503
+
3504
+ Examples:
3505
+ ```python
3506
+ run.link_model(
3507
+ path="/local/directory",
3508
+ registered_model_name="my_reg_model",
3509
+ name="my_model_artifact",
3510
+ aliases=["production"],
3511
+ )
3512
+ ```
3513
+
3514
+ Invalid usage
3515
+ ```python
3516
+ run.link_model(
3517
+ path="/local/directory",
3518
+ registered_model_name="my_entity/my_project/my_reg_model",
3519
+ name="my_model_artifact",
3520
+ aliases=["production"],
3521
+ )
3522
+
3523
+ run.link_model(
3524
+ path="/local/directory",
3525
+ registered_model_name="my_reg_model",
3526
+ name="my_entity/my_project/my_model_artifact",
3527
+ aliases=["production"],
3528
+ )
3529
+ ```
3530
+
3531
+ Raises:
3532
+ AssertionError: if registered_model_name is a path or
3533
+ if model artifact 'name' is of a type that does not contain the substring 'model'
3534
+ ValueError: if name has invalid special characters
3535
+
3536
+ Returns:
3537
+ None
3538
+ """
3539
+ name_parts = registered_model_name.split("/")
3540
+ assert (
3541
+ len(name_parts) == 1
3542
+ ), "Please provide only the name of the registered model. Do not append the entity or project name."
3543
+ project = "model-registry"
3544
+ target_path = self.entity + "/" + project + "/" + registered_model_name
3545
+
3546
+ public_api = self._public_api()
3547
+ try:
3548
+ artifact = public_api._artifact(name=f"{name}:latest")
3549
+ assert (
3550
+ "model" in str(artifact.type.lower())
3551
+ ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
3552
+ artifact = self._log_artifact(
3553
+ artifact_or_path=path, name=name, type=artifact.type
3554
+ )
3555
+ except (ValueError, CommError):
3556
+ artifact = self._log_artifact(
3557
+ artifact_or_path=path, name=name, type="model"
3558
+ )
3559
+ self.link_artifact(artifact=artifact, target_path=target_path, aliases=aliases)
3560
+
3561
+ @_run_decorator._noop_on_finish()
3562
+ @_run_decorator._attach
3563
+ def alert(
3564
+ self,
3565
+ title: str,
3566
+ text: str,
3567
+ level: str | AlertLevel | None = None,
3568
+ wait_duration: int | float | timedelta | None = None,
3569
+ ) -> None:
3570
+ """Launch an alert with the given title and text.
3571
+
3572
+ Args:
3573
+ title: (str) The title of the alert, must be less than 64 characters long.
3574
+ text: (str) The text body of the alert.
3575
+ level: (str or AlertLevel, optional) The alert level to use, either: `INFO`, `WARN`, or `ERROR`.
3576
+ wait_duration: (int, float, or timedelta, optional) The time to wait (in seconds) before sending another
3577
+ alert with this title.
3578
+ """
3579
+ level = level or AlertLevel.INFO
3580
+ level_str: str = level.value if isinstance(level, AlertLevel) else level
3581
+ if level_str not in {lev.value for lev in AlertLevel}:
3582
+ raise ValueError("level must be one of 'INFO', 'WARN', or 'ERROR'")
3583
+
3584
+ wait_duration = wait_duration or timedelta(minutes=1)
3585
+ if isinstance(wait_duration, int) or isinstance(wait_duration, float):
3586
+ wait_duration = timedelta(seconds=wait_duration)
3587
+ elif not callable(getattr(wait_duration, "total_seconds", None)):
3588
+ raise ValueError(
3589
+ "wait_duration must be an int, float, or datetime.timedelta"
3590
+ )
3591
+ wait_duration = int(wait_duration.total_seconds() * 1000)
3592
+
3593
+ if self._backend and self._backend.interface:
3594
+ self._backend.interface.publish_alert(title, text, level_str, wait_duration)
3595
+
3596
+ def __enter__(self) -> Run:
3597
+ return self
3598
+
3599
+ def __exit__(
3600
+ self,
3601
+ exc_type: type[BaseException],
3602
+ exc_val: BaseException,
3603
+ exc_tb: TracebackType,
3604
+ ) -> bool:
3605
+ exception_raised = exc_type is not None
3606
+ if exception_raised:
3607
+ traceback.print_exception(exc_type, exc_val, exc_tb)
3608
+ exit_code = 1 if exception_raised else 0
3609
+ self._finish(exit_code=exit_code)
3610
+ return not exception_raised
3611
+
3612
+ @_run_decorator._noop_on_finish()
3613
+ @_run_decorator._attach
3614
+ def mark_preempting(self) -> None:
3615
+ """Mark this run as preempting.
3616
+
3617
+ Also tells the internal process to immediately report this to server.
3618
+ """
3619
+ if self._backend and self._backend.interface:
3620
+ self._backend.interface.publish_preempting()
3621
+
3622
+ @property
3623
+ @_run_decorator._noop_on_finish()
3624
+ @_run_decorator._attach
3625
+ def _system_metrics(self) -> dict[str, list[tuple[datetime, float]]]:
3626
+ """Returns a dictionary of system metrics.
3627
+
3628
+ Returns:
3629
+ A dictionary of system metrics.
3630
+ """
3631
+ from wandb.proto import wandb_internal_pb2
3632
+
3633
+ def pb_to_dict(
3634
+ system_metrics_pb: wandb_internal_pb2.GetSystemMetricsResponse,
3635
+ ) -> dict[str, list[tuple[datetime, float]]]:
3636
+ res = {}
3637
+
3638
+ for metric, records in system_metrics_pb.system_metrics.items():
3639
+ measurements = []
3640
+ for record in records.record:
3641
+ # Convert timestamp to datetime
3642
+ dt = datetime.fromtimestamp(
3643
+ record.timestamp.seconds, tz=timezone.utc
3644
+ )
3645
+ dt = dt.replace(microsecond=record.timestamp.nanos // 1000)
3646
+
3647
+ measurements.append((dt, record.value))
3648
+
3649
+ res[metric] = measurements
3650
+
3651
+ return res
3652
+
3653
+ if not self._backend or not self._backend.interface:
3654
+ return {}
3655
+
3656
+ handle = self._backend.interface.deliver_get_system_metrics()
3657
+ result = handle.wait(timeout=1)
3658
+
3659
+ if result:
3660
+ try:
3661
+ response = result.response.get_system_metrics_response
3662
+ if response:
3663
+ return pb_to_dict(response)
3664
+ except Exception as e:
3665
+ logger.error("Error getting system metrics: %s", e)
3666
+ return {}
3667
+
3668
+ # ------------------------------------------------------------------------------
3669
+ # HEADER
3670
+ # ------------------------------------------------------------------------------
3671
+ # Note: All the header methods are static methods since we want to share the printing logic
3672
+ # with the service execution path that doesn't have access to the run instance
3673
+ @staticmethod
3674
+ def _header(
3675
+ *,
3676
+ settings: Settings,
3677
+ printer: printer.Printer,
3678
+ ) -> None:
3679
+ Run._header_wandb_version_info(settings=settings, printer=printer)
3680
+ Run._header_sync_info(settings=settings, printer=printer)
3681
+ Run._header_run_info(settings=settings, printer=printer)
3682
+
3683
+ @staticmethod
3684
+ def _header_wandb_version_info(
3685
+ *,
3686
+ settings: Settings,
3687
+ printer: printer.Printer,
3688
+ ) -> None:
3689
+ if settings.quiet or settings.silent:
3690
+ return
3691
+
3692
+ # TODO: add this to a higher verbosity level
3693
+ printer.display(f"Tracking run with wandb version {wandb.__version__}")
3694
+
3695
+ @staticmethod
3696
+ def _header_sync_info(
3697
+ *,
3698
+ settings: Settings,
3699
+ printer: printer.Printer,
3700
+ ) -> None:
3701
+ if settings._offline:
3702
+ printer.display(
3703
+ [
3704
+ f"W&B syncing is set to {printer.code('`offline`')} in this directory. ",
3705
+ f"Run {printer.code('`wandb online`')} or set {printer.code('WANDB_MODE=online')} "
3706
+ "to enable cloud syncing.",
3707
+ ]
3708
+ )
3709
+ else:
3710
+ info = [f"Run data is saved locally in {printer.files(settings.sync_dir)}"]
3711
+ if not printer.supports_html:
3712
+ info.append(
3713
+ f"Run {printer.code('`wandb offline`')} to turn off syncing."
3714
+ )
3715
+ if not settings.quiet and not settings.silent:
3716
+ printer.display(info)
3717
+
3718
+ @staticmethod
3719
+ def _header_run_info(
3720
+ *,
3721
+ settings: Settings,
3722
+ printer: printer.Printer,
3723
+ ) -> None:
3724
+ if settings._offline or settings.silent:
3725
+ return
3726
+
3727
+ run_url = settings.run_url
3728
+ project_url = settings.project_url
3729
+ sweep_url = settings.sweep_url
3730
+
3731
+ run_state_str = (
3732
+ "Resuming run"
3733
+ if settings.resumed or settings.resume_from
3734
+ else "Syncing run"
3735
+ )
3736
+ run_name = settings.run_name
3737
+ if not run_name:
3738
+ return
3739
+
3740
+ if printer.supports_html:
3741
+ if not wandb.jupyter.maybe_display(): # type: ignore
3742
+ run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
3743
+ project_line, sweep_line = "", ""
3744
+
3745
+ # TODO(settings): make settings the source of truth
3746
+ if not wandb.jupyter.quiet(): # type: ignore
3747
+ doc_html = printer.link(url_registry.url("developer-guide"), "docs")
3748
+
3749
+ project_html = printer.link(project_url, "Weights & Biases")
3750
+ project_line = f"to {project_html} ({doc_html})"
3751
+
3752
+ if sweep_url:
3753
+ sweep_line = f"Sweep page: {printer.link(sweep_url, sweep_url)}"
3754
+
3755
+ printer.display(
3756
+ [f"{run_state_str} {run_line} {project_line}", sweep_line],
3757
+ )
3758
+
3759
+ elif run_name:
3760
+ printer.display(f"{run_state_str} {printer.name(run_name)}")
3761
+
3762
+ if not settings.quiet:
3763
+ # TODO: add verbosity levels and add this to higher levels
3764
+ printer.display(
3765
+ f'{printer.emoji("star")} View project at {printer.link(project_url)}'
3766
+ )
3767
+ if sweep_url:
3768
+ printer.display(
3769
+ f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
3770
+ )
3771
+ printer.display(
3772
+ f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
3773
+ )
3774
+
3775
+ # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
3776
+ if run_name and Api().api.settings().get("anonymous") == "true":
3777
+ printer.display(
3778
+ (
3779
+ "Do NOT share these links with anyone."
3780
+ " They can be used to claim your runs."
3781
+ ),
3782
+ level="warn",
3783
+ )
3784
+
3785
+ # ------------------------------------------------------------------------------
3786
+ # FOOTER
3787
+ # ------------------------------------------------------------------------------
3788
+ # Note: All the footer methods are static methods since we want to share the printing logic
3789
+ # with the service execution path that doesn't have access to the run instance
3790
+ @staticmethod
3791
+ def _footer(
3792
+ sampled_history: SampledHistoryResponse | None = None,
3793
+ final_summary: GetSummaryResponse | None = None,
3794
+ poll_exit_response: PollExitResponse | None = None,
3795
+ internal_messages_response: InternalMessagesResponse | None = None,
3796
+ *,
3797
+ settings: Settings,
3798
+ printer: printer.Printer,
3799
+ ) -> None:
3800
+ Run._footer_history_summary_info(
3801
+ history=sampled_history,
3802
+ summary=final_summary,
3803
+ settings=settings,
3804
+ printer=printer,
3805
+ )
3806
+
3807
+ Run._footer_sync_info(
3808
+ poll_exit_response=poll_exit_response,
3809
+ settings=settings,
3810
+ printer=printer,
3811
+ )
3812
+ Run._footer_log_dir_info(settings=settings, printer=printer)
3813
+ Run._footer_notify_wandb_core(
3814
+ settings=settings,
3815
+ printer=printer,
3816
+ )
3817
+ Run._footer_internal_messages(
3818
+ internal_messages_response=internal_messages_response,
3819
+ settings=settings,
3820
+ printer=printer,
3821
+ )
3822
+
3823
+ @staticmethod
3824
+ def _footer_sync_info(
3825
+ poll_exit_response: PollExitResponse | None = None,
3826
+ *,
3827
+ settings: Settings,
3828
+ printer: printer.Printer,
3829
+ ) -> None:
3830
+ if settings.silent:
3831
+ return
3832
+
3833
+ if settings._offline:
3834
+ if not settings.quiet:
3835
+ printer.display(
3836
+ [
3837
+ "You can sync this run to the cloud by running:",
3838
+ printer.code(f"wandb sync {settings.sync_dir}"),
3839
+ ],
3840
+ )
3841
+ return
3842
+
3843
+ info = []
3844
+ if settings.run_name and settings.run_url:
3845
+ info.append(
3846
+ f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
3847
+ )
3848
+ if settings.project_url:
3849
+ info.append(
3850
+ f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
3851
+ )
3852
+ if poll_exit_response and poll_exit_response.file_counts:
3853
+ logger.info("logging synced files")
3854
+ file_counts = poll_exit_response.file_counts
3855
+ info.append(
3856
+ f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
3857
+ f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
3858
+ )
3859
+ printer.display(info)
3860
+
3861
+ @staticmethod
3862
+ def _footer_log_dir_info(
3863
+ *,
3864
+ settings: Settings,
3865
+ printer: printer.Printer,
3866
+ ) -> None:
3867
+ if settings.quiet or settings.silent:
3868
+ return
3869
+
3870
+ log_dir = settings.log_user or settings.log_internal
3871
+ if log_dir:
3872
+ log_dir = os.path.dirname(log_dir.replace(os.getcwd(), "."))
3873
+ printer.display(
3874
+ f"Find logs at: {printer.files(log_dir)}",
3875
+ )
3876
+
3877
+ @staticmethod
3878
+ def _footer_history_summary_info(
3879
+ history: SampledHistoryResponse | None = None,
3880
+ summary: GetSummaryResponse | None = None,
3881
+ *,
3882
+ settings: Settings,
3883
+ printer: printer.Printer,
3884
+ ) -> None:
3885
+ if settings.quiet or settings.silent:
3886
+ return
3887
+
3888
+ panel = []
3889
+
3890
+ # Render history if available
3891
+ if history:
3892
+ logger.info("rendering history")
3893
+
3894
+ sampled_history = {
3895
+ item.key: wandb.util.downsample(
3896
+ item.values_float or item.values_int, 40
3897
+ )
3898
+ for item in history.item
3899
+ if not item.key.startswith("_")
3900
+ }
3901
+
3902
+ history_rows = []
3903
+ for key, values in sorted(sampled_history.items()):
3904
+ if any(not isinstance(value, numbers.Number) for value in values):
3905
+ continue
3906
+ sparkline = printer.sparklines(values)
3907
+ if sparkline:
3908
+ history_rows.append([key, sparkline])
3909
+ if history_rows:
3910
+ history_grid = printer.grid(
3911
+ history_rows,
3912
+ "Run history:",
3913
+ )
3914
+ panel.append(history_grid)
3915
+
3916
+ # Render summary if available
3917
+ if summary:
3918
+ final_summary = {}
3919
+ for item in summary.item:
3920
+ if item.key.startswith("_") or len(item.nested_key) > 0:
3921
+ continue
3922
+ final_summary[item.key] = json.loads(item.value_json)
3923
+
3924
+ logger.info("rendering summary")
3925
+ summary_rows = []
3926
+ for key, value in sorted(final_summary.items()):
3927
+ # arrays etc. might be too large. for now, we just don't print them
3928
+ if isinstance(value, str):
3929
+ value = value[:20] + "..." * (len(value) >= 20)
3930
+ summary_rows.append([key, value])
3931
+ elif isinstance(value, numbers.Number):
3932
+ value = round(value, 5) if isinstance(value, float) else value
3933
+ summary_rows.append([key, str(value)])
3934
+ else:
3935
+ continue
3936
+
3937
+ if summary_rows:
3938
+ summary_grid = printer.grid(
3939
+ summary_rows,
3940
+ "Run summary:",
3941
+ )
3942
+ panel.append(summary_grid)
3943
+
3944
+ if panel:
3945
+ printer.display(printer.panel(panel))
3946
+
3947
+ @staticmethod
3948
+ def _footer_internal_messages(
3949
+ internal_messages_response: InternalMessagesResponse | None = None,
3950
+ *,
3951
+ settings: Settings,
3952
+ printer: printer.Printer,
3953
+ ) -> None:
3954
+ if settings.quiet or settings.silent:
3955
+ return
3956
+
3957
+ if not internal_messages_response:
3958
+ return
3959
+
3960
+ for message in internal_messages_response.messages.warning:
3961
+ printer.display(message, level="warn")
3962
+
3963
+ @staticmethod
3964
+ def _footer_notify_wandb_core(
3965
+ *,
3966
+ settings: Settings,
3967
+ printer: printer.Printer,
3968
+ ) -> None:
3969
+ """Prints a message advertising the upcoming core release."""
3970
+ if settings.quiet or not settings.x_require_legacy_service:
3971
+ return
3972
+
3973
+ printer.display(
3974
+ "The legacy backend is deprecated. In future versions, `wandb-core` will become "
3975
+ "the sole backend service, and the `wandb.require('legacy-service')` flag will be removed. "
3976
+ f"For more information, visit {url_registry.url('wandb-core')}",
3977
+ level="warn",
3978
+ )
3979
+
3980
+
3981
+ # We define this outside of the run context to support restoring before init
3982
+ def restore(
3983
+ name: str,
3984
+ run_path: str | None = None,
3985
+ replace: bool = False,
3986
+ root: str | None = None,
3987
+ ) -> None | TextIO:
3988
+ """Download the specified file from cloud storage.
3989
+
3990
+ File is placed into the current directory or run directory.
3991
+ By default, will only download the file if it doesn't already exist.
3992
+
3993
+ Args:
3994
+ name: the name of the file
3995
+ run_path: optional path to a run to pull files from, i.e. `username/project_name/run_id`
3996
+ if wandb.init has not been called, this is required.
3997
+ replace: whether to download the file even if it already exists locally
3998
+ root: the directory to download the file to. Defaults to the current
3999
+ directory or the run directory if wandb.init was called.
4000
+
4001
+ Returns:
4002
+ None if it can't find the file, otherwise a file object open for reading
4003
+
4004
+ Raises:
4005
+ wandb.CommError: if we can't connect to the wandb backend
4006
+ ValueError: if the file is not found or can't find run_path
4007
+ """
4008
+ is_disabled = wandb.run is not None and wandb.run.disabled
4009
+ run = None if is_disabled else wandb.run
4010
+ if run_path is None:
4011
+ if run is not None:
4012
+ run_path = run.path
4013
+ else:
4014
+ raise ValueError(
4015
+ "run_path required when calling wandb.restore before wandb.init"
4016
+ )
4017
+ if root is None:
4018
+ if run is not None:
4019
+ root = run.dir
4020
+ api = public.Api()
4021
+ api_run = api.run(run_path)
4022
+ if root is None:
4023
+ root = os.getcwd()
4024
+ path = os.path.join(root, name)
4025
+ if os.path.exists(path) and replace is False:
4026
+ return open(path)
4027
+ if is_disabled:
4028
+ return None
4029
+ files = api_run.files([name])
4030
+ if len(files) == 0:
4031
+ return None
4032
+ # if the file does not exist, the file has an md5 of 0
4033
+ if files[0].md5 == "0":
4034
+ raise ValueError(f"File {name} not found in {run_path or root}.")
4035
+ return files[0].download(root=root, replace=True)
4036
+
4037
+
4038
+ # propagate our doc string to the runs restore method
4039
+ try:
4040
+ Run.restore.__doc__ = restore.__doc__
4041
+ except AttributeError:
4042
+ pass
4043
+
4044
+
4045
+ def finish(
4046
+ exit_code: int | None = None,
4047
+ quiet: bool | None = None,
4048
+ ) -> None:
4049
+ """Finish a run and upload any remaining data.
4050
+
4051
+ Marks the completion of a W&B run and ensures all data is synced to the server.
4052
+ The run's final state is determined by its exit conditions and sync status.
4053
+
4054
+ Run States:
4055
+ - Running: Active run that is logging data and/or sending heartbeats.
4056
+ - Crashed: Run that stopped sending heartbeats unexpectedly.
4057
+ - Finished: Run completed successfully (`exit_code=0`) with all data synced.
4058
+ - Failed: Run completed with errors (`exit_code!=0`).
4059
+
4060
+ Args:
4061
+ exit_code: Integer indicating the run's exit status. Use 0 for success,
4062
+ any other value marks the run as failed.
4063
+ quiet: Deprecated. Configure logging verbosity using `wandb.Settings(quiet=...)`.
4064
+ """
4065
+ if wandb.run:
4066
+ wandb.run.finish(exit_code=exit_code, quiet=quiet)