wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. wandb/__init__.py +1 -2
  2. wandb/__init__.pyi +3 -6
  3. wandb/_iterutils.py +26 -7
  4. wandb/_pydantic/__init__.py +2 -1
  5. wandb/_pydantic/utils.py +7 -0
  6. wandb/agents/pyagent.py +9 -15
  7. wandb/analytics/sentry.py +1 -2
  8. wandb/apis/attrs.py +3 -4
  9. wandb/apis/importers/internals/util.py +1 -1
  10. wandb/apis/importers/validation.py +2 -2
  11. wandb/apis/importers/wandb.py +30 -25
  12. wandb/apis/normalize.py +2 -2
  13. wandb/apis/public/__init__.py +1 -0
  14. wandb/apis/public/api.py +37 -33
  15. wandb/apis/public/artifacts.py +103 -72
  16. wandb/apis/public/jobs.py +3 -2
  17. wandb/apis/public/registries/registries_search.py +4 -2
  18. wandb/apis/public/registries/registry.py +1 -1
  19. wandb/apis/public/registries/utils.py +9 -9
  20. wandb/apis/public/runs.py +18 -6
  21. wandb/automations/_filters/expressions.py +1 -1
  22. wandb/automations/_filters/operators.py +1 -1
  23. wandb/automations/_filters/run_metrics.py +1 -1
  24. wandb/beta/workflows.py +6 -5
  25. wandb/bin/gpu_stats.exe +0 -0
  26. wandb/bin/wandb-core +0 -0
  27. wandb/cli/cli.py +54 -73
  28. wandb/docker/__init__.py +21 -74
  29. wandb/docker/names.py +40 -0
  30. wandb/env.py +0 -1
  31. wandb/errors/util.py +1 -1
  32. wandb/filesync/step_checksum.py +1 -1
  33. wandb/filesync/step_upload.py +1 -1
  34. wandb/integration/diffusers/resolvers/multimodal.py +1 -2
  35. wandb/integration/gym/__init__.py +5 -6
  36. wandb/integration/keras/callbacks/model_checkpoint.py +2 -2
  37. wandb/integration/keras/keras.py +13 -19
  38. wandb/integration/kfp/kfp_patch.py +2 -3
  39. wandb/integration/langchain/wandb_tracer.py +1 -1
  40. wandb/integration/metaflow/metaflow.py +13 -13
  41. wandb/integration/openai/fine_tuning.py +3 -2
  42. wandb/integration/sagemaker/auth.py +2 -1
  43. wandb/integration/sklearn/utils.py +2 -1
  44. wandb/integration/tensorboard/__init__.py +1 -1
  45. wandb/integration/tensorboard/log.py +2 -5
  46. wandb/integration/tensorflow/__init__.py +2 -2
  47. wandb/jupyter.py +20 -17
  48. wandb/plot/confusion_matrix.py +1 -1
  49. wandb/plot/utils.py +8 -7
  50. wandb/proto/v3/wandb_internal_pb2.py +355 -335
  51. wandb/proto/v3/wandb_settings_pb2.py +2 -2
  52. wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
  53. wandb/proto/v4/wandb_internal_pb2.py +339 -335
  54. wandb/proto/v4/wandb_settings_pb2.py +2 -2
  55. wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
  56. wandb/proto/v5/wandb_internal_pb2.py +339 -335
  57. wandb/proto/v5/wandb_settings_pb2.py +2 -2
  58. wandb/proto/v5/wandb_telemetry_pb2.py +12 -12
  59. wandb/proto/v6/wandb_internal_pb2.py +339 -335
  60. wandb/proto/v6/wandb_settings_pb2.py +2 -2
  61. wandb/proto/v6/wandb_telemetry_pb2.py +12 -12
  62. wandb/proto/wandb_deprecated.py +6 -8
  63. wandb/sdk/artifacts/_internal_artifact.py +43 -0
  64. wandb/sdk/artifacts/_validators.py +55 -35
  65. wandb/sdk/artifacts/artifact.py +117 -115
  66. wandb/sdk/artifacts/artifact_download_logger.py +2 -0
  67. wandb/sdk/artifacts/artifact_saver.py +1 -3
  68. wandb/sdk/artifacts/artifact_state.py +2 -0
  69. wandb/sdk/artifacts/artifact_ttl.py +2 -0
  70. wandb/sdk/artifacts/exceptions.py +14 -0
  71. wandb/sdk/artifacts/staging.py +2 -0
  72. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -6
  73. wandb/sdk/artifacts/storage_handlers/multi_handler.py +1 -1
  74. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -6
  75. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +1 -5
  76. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +1 -1
  77. wandb/sdk/artifacts/storage_layout.py +2 -0
  78. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +3 -3
  79. wandb/sdk/backend/backend.py +11 -182
  80. wandb/sdk/data_types/_dtypes.py +2 -6
  81. wandb/sdk/data_types/audio.py +20 -3
  82. wandb/sdk/data_types/base_types/media.py +12 -7
  83. wandb/sdk/data_types/base_types/wb_value.py +8 -18
  84. wandb/sdk/data_types/bokeh.py +19 -2
  85. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +17 -1
  86. wandb/sdk/data_types/helper_types/image_mask.py +7 -1
  87. wandb/sdk/data_types/html.py +4 -4
  88. wandb/sdk/data_types/image.py +178 -103
  89. wandb/sdk/data_types/molecule.py +6 -6
  90. wandb/sdk/data_types/object_3d.py +10 -5
  91. wandb/sdk/data_types/saved_model.py +11 -6
  92. wandb/sdk/data_types/table.py +313 -83
  93. wandb/sdk/data_types/table_decorators.py +108 -0
  94. wandb/sdk/data_types/utils.py +43 -7
  95. wandb/sdk/data_types/video.py +21 -3
  96. wandb/sdk/interface/interface.py +10 -0
  97. wandb/sdk/internal/datastore.py +2 -6
  98. wandb/sdk/internal/file_pusher.py +1 -5
  99. wandb/sdk/internal/file_stream.py +8 -17
  100. wandb/sdk/internal/handler.py +2 -2
  101. wandb/sdk/internal/incremental_table_util.py +53 -0
  102. wandb/sdk/internal/internal.py +3 -5
  103. wandb/sdk/internal/internal_api.py +66 -89
  104. wandb/sdk/internal/job_builder.py +2 -7
  105. wandb/sdk/internal/profiler.py +2 -2
  106. wandb/sdk/internal/progress.py +1 -3
  107. wandb/sdk/internal/run.py +1 -6
  108. wandb/sdk/internal/sender.py +24 -36
  109. wandb/sdk/internal/system/assets/aggregators.py +1 -7
  110. wandb/sdk/internal/system/assets/disk.py +3 -3
  111. wandb/sdk/internal/system/assets/gpu.py +4 -4
  112. wandb/sdk/internal/system/assets/gpu_amd.py +4 -4
  113. wandb/sdk/internal/system/assets/interfaces.py +6 -6
  114. wandb/sdk/internal/system/assets/tpu.py +1 -1
  115. wandb/sdk/internal/system/assets/trainium.py +6 -6
  116. wandb/sdk/internal/system/system_info.py +5 -7
  117. wandb/sdk/internal/system/system_monitor.py +4 -4
  118. wandb/sdk/internal/tb_watcher.py +5 -7
  119. wandb/sdk/launch/_launch.py +1 -1
  120. wandb/sdk/launch/_project_spec.py +19 -20
  121. wandb/sdk/launch/agent/agent.py +3 -3
  122. wandb/sdk/launch/agent/config.py +1 -1
  123. wandb/sdk/launch/agent/job_status_tracker.py +2 -2
  124. wandb/sdk/launch/builder/build.py +2 -3
  125. wandb/sdk/launch/builder/kaniko_builder.py +5 -4
  126. wandb/sdk/launch/environment/gcp_environment.py +1 -2
  127. wandb/sdk/launch/registry/azure_container_registry.py +2 -2
  128. wandb/sdk/launch/registry/elastic_container_registry.py +2 -2
  129. wandb/sdk/launch/registry/google_artifact_registry.py +3 -3
  130. wandb/sdk/launch/runner/abstract.py +5 -5
  131. wandb/sdk/launch/runner/kubernetes_monitor.py +2 -2
  132. wandb/sdk/launch/runner/kubernetes_runner.py +1 -1
  133. wandb/sdk/launch/runner/sagemaker_runner.py +2 -4
  134. wandb/sdk/launch/runner/vertex_runner.py +2 -7
  135. wandb/sdk/launch/sweeps/__init__.py +1 -1
  136. wandb/sdk/launch/sweeps/scheduler.py +2 -2
  137. wandb/sdk/launch/sweeps/utils.py +3 -3
  138. wandb/sdk/launch/utils.py +3 -4
  139. wandb/sdk/lib/apikey.py +5 -8
  140. wandb/sdk/lib/config_util.py +3 -3
  141. wandb/sdk/lib/fsm.py +3 -18
  142. wandb/sdk/lib/gitlib.py +6 -5
  143. wandb/sdk/lib/ipython.py +2 -2
  144. wandb/sdk/lib/json_util.py +9 -14
  145. wandb/sdk/lib/printer.py +3 -8
  146. wandb/sdk/lib/redirect.py +1 -1
  147. wandb/sdk/lib/retry.py +3 -7
  148. wandb/sdk/lib/run_moment.py +2 -2
  149. wandb/sdk/lib/service_connection.py +3 -1
  150. wandb/sdk/lib/service_token.py +1 -2
  151. wandb/sdk/mailbox/mailbox_handle.py +3 -7
  152. wandb/sdk/mailbox/response_handle.py +2 -6
  153. wandb/sdk/service/streams.py +3 -7
  154. wandb/sdk/verify/verify.py +5 -6
  155. wandb/sdk/wandb_config.py +1 -1
  156. wandb/sdk/wandb_init.py +38 -106
  157. wandb/sdk/wandb_login.py +7 -6
  158. wandb/sdk/wandb_run.py +52 -240
  159. wandb/sdk/wandb_settings.py +71 -60
  160. wandb/sdk/wandb_setup.py +40 -14
  161. wandb/sdk/wandb_watch.py +5 -7
  162. wandb/sync/__init__.py +1 -1
  163. wandb/sync/sync.py +13 -13
  164. wandb/util.py +17 -35
  165. wandb/wandb_agent.py +8 -11
  166. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/METADATA +5 -5
  167. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/RECORD +170 -168
  168. wandb/docker/auth.py +0 -435
  169. wandb/docker/www_authenticate.py +0 -94
  170. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/WHEEL +0 -0
  171. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/entry_points.txt +0 -0
  172. {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/licenses/LICENSE +0 -0
wandb/cli/cli.py CHANGED
@@ -20,9 +20,6 @@ import click
20
20
  import yaml
21
21
  from click.exceptions import ClickException
22
22
 
23
- # pycreds has a find_executable that works in windows
24
- from dockerpycreds.utils import find_executable
25
-
26
23
  import wandb
27
24
  import wandb.env
28
25
  import wandb.errors
@@ -31,6 +28,7 @@ from wandb import Config, Error, env, util, wandb_agent, wandb_sdk
31
28
  from wandb.apis import InternalApi, PublicApi
32
29
  from wandb.apis.public import RunQueue
33
30
  from wandb.errors.links import url_registry
31
+ from wandb.sdk import wandb_setup
34
32
  from wandb.sdk.artifacts._validators import is_artifact_registry_project
35
33
  from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
36
34
  from wandb.sdk.internal.internal_api import Api as SDKInternalApi
@@ -67,6 +65,9 @@ logging.basicConfig(
67
65
  logging.basicConfig(stream=sys.stdout, level=logging.INFO)
68
66
  logger = logging.getLogger("wandb")
69
67
 
68
+ _HAS_DOCKER = bool(shutil.which("docker"))
69
+ _HAS_NVIDIA_DOCKER = bool(shutil.which("nvidia-docker"))
70
+
70
71
  # Click Contexts
71
72
  CONTEXT = {"default_map": {}}
72
73
  RUN_CONTEXT = {
@@ -104,7 +105,7 @@ def display_error(func):
104
105
  except wandb.Error as e:
105
106
  exc_type, exc_value, exc_traceback = sys.exc_info()
106
107
  lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
107
- logger.error("".join(lines))
108
+ logger.exception("".join(lines))
108
109
  wandb.termerror(f"Find detailed error logs at: {_wandb_log_path}")
109
110
  click_exc = ClickWandbException(e)
110
111
  click_exc.orig_type = exc_type
@@ -118,9 +119,6 @@ _api = None # caching api instance allows patching from unit tests
118
119
 
119
120
  def _get_cling_api(reset=None):
120
121
  """Get a reference to the internal api with cling settings."""
121
- # TODO: move CLI to wandb-core backend
122
- wandb.require("legacy-service")
123
-
124
122
  global _api
125
123
  if reset:
126
124
  _api = None
@@ -128,7 +126,7 @@ def _get_cling_api(reset=None):
128
126
  if _api is None:
129
127
  # TODO(jhr): make a settings object that is better for non runs.
130
128
  # only override the necessary setting
131
- wandb.setup(settings=wandb.Settings(x_cli_only_mode=True))
129
+ wandb_setup.singleton().settings.x_cli_only_mode = True
132
130
  _api = InternalApi()
133
131
  return _api
134
132
 
@@ -195,9 +193,9 @@ def projects(entity, display=True):
195
193
  api = _get_cling_api()
196
194
  projects = api.list_projects(entity=entity)
197
195
  if len(projects) == 0:
198
- message = "No projects found for {}".format(entity)
196
+ message = f"No projects found for {entity}"
199
197
  else:
200
- message = 'Latest projects for "{}"'.format(entity)
198
+ message = f'Latest projects for "{entity}"'
201
199
  if display:
202
200
  click.echo(click.style(message, bold=True))
203
201
  for project in projects:
@@ -231,9 +229,6 @@ def projects(entity, display=True):
231
229
  )
232
230
  @display_error
233
231
  def login(key, host, cloud, relogin, anonymously, verify, no_offline=False):
234
- # TODO: move CLI to wandb-core backend
235
- wandb.require("legacy-service")
236
-
237
232
  # TODO: handle no_offline
238
233
  anon_mode = "must" if anonymously else "never"
239
234
 
@@ -242,12 +237,9 @@ def login(key, host, cloud, relogin, anonymously, verify, no_offline=False):
242
237
  key = key[0] if key is not None and len(key) > 0 else None
243
238
  relogin = True if key or relogin else False
244
239
 
245
- wandb.setup(
246
- settings=wandb.Settings(
247
- x_cli_only_mode=True,
248
- x_disable_viewer=relogin and not verify,
249
- )
250
- )
240
+ global_settings = wandb_setup.singleton().settings
241
+ global_settings.x_cli_only_mode = True
242
+ global_settings.x_disable_viewer = relogin and not verify
251
243
 
252
244
  wandb.login(
253
245
  anonymous=anon_mode,
@@ -421,7 +413,7 @@ def init(ctx, project, entity, reset, mode):
421
413
  """
422
414
  ).format(
423
415
  code1=click.style("import wandb", bold=True),
424
- code2=click.style('wandb.init(project="{}")'.format(project), bold=True),
416
+ code2=click.style(f'wandb.init(project="{project}")', bold=True),
425
417
  run=click.style("python <train.py>", bold=True),
426
418
  )
427
419
  )
@@ -777,7 +769,7 @@ def sweep(
777
769
  "resume": "Resuming",
778
770
  }
779
771
  wandb.termlog(f"{ings[state]} sweep {entity}/{project}/{sweep_id}")
780
- getattr(api, "{}_sweep".format(state))(sweep_id, entity=entity, project=project)
772
+ getattr(api, f"{state}_sweep")(sweep_id, entity=entity, project=project)
781
773
  wandb.termlog("Done.")
782
774
  return
783
775
  else:
@@ -1518,11 +1510,11 @@ def launch(
1518
1510
  wandb.termerror("Launched run exited with non-zero status")
1519
1511
  sys.exit(1)
1520
1512
  except LaunchError as e:
1521
- logger.error("=== %s ===", e)
1513
+ logger.exception("An error occurred.")
1522
1514
  wandb._sentry.exception(e)
1523
1515
  sys.exit(e)
1524
1516
  except ExecutionError as e:
1525
- logger.error("=== %s ===", e)
1517
+ logger.exception("An error occurred.")
1526
1518
  wandb._sentry.exception(e)
1527
1519
  sys.exit(e)
1528
1520
  except asyncio.CancelledError:
@@ -1552,7 +1544,7 @@ def launch(
1552
1544
 
1553
1545
  except Exception as e:
1554
1546
  wandb._sentry.exception(e)
1555
- raise e
1547
+ raise
1556
1548
 
1557
1549
 
1558
1550
  @cli.command(
@@ -1643,7 +1635,7 @@ def launch_agent(
1643
1635
  _launch.create_and_run_agent(api, agent_config)
1644
1636
  except Exception as e:
1645
1637
  wandb._sentry.exception(e)
1646
- raise e
1638
+ raise
1647
1639
 
1648
1640
 
1649
1641
  @cli.command(context_settings=CONTEXT, help="Run the W&B agent")
@@ -1721,7 +1713,7 @@ def scheduler(
1721
1713
  _scheduler.start()
1722
1714
  except Exception as e:
1723
1715
  wandb._sentry.exception(e)
1724
- raise e
1716
+ raise
1725
1717
 
1726
1718
 
1727
1719
  @cli.group(help="Commands for managing and viewing W&B jobs")
@@ -1974,13 +1966,13 @@ def docker_run(ctx, docker_run_args):
1974
1966
 
1975
1967
  See `docker run --help` for more details.
1976
1968
  """
1969
+ import wandb.docker
1970
+
1977
1971
  api = InternalApi()
1978
1972
  args = list(docker_run_args)
1979
1973
  if len(args) > 0 and args[0] == "run":
1980
1974
  args.pop(0)
1981
- if len([a for a in args if a.startswith("--runtime")]) == 0 and find_executable(
1982
- "nvidia-docker"
1983
- ):
1975
+ if len([a for a in args if a.startswith("--runtime")]) == 0 and _HAS_NVIDIA_DOCKER:
1984
1976
  args = ["--runtime", "nvidia"] + args
1985
1977
  # TODO: image_from_docker_args uses heuristics to find the docker image arg, there are likely cases
1986
1978
  # where this won't work
@@ -1989,13 +1981,13 @@ def docker_run(ctx, docker_run_args):
1989
1981
  if image:
1990
1982
  resolved_image = wandb.docker.image_id(image)
1991
1983
  if resolved_image:
1992
- args = ["-e", "WANDB_DOCKER={}".format(resolved_image)] + args
1984
+ args = ["-e", f"WANDB_DOCKER={resolved_image}"] + args
1993
1985
  else:
1994
1986
  wandb.termlog(
1995
1987
  "Couldn't detect image argument, running command without the WANDB_DOCKER env variable"
1996
1988
  )
1997
1989
  if api.api_key:
1998
- args = ["-e", "WANDB_API_KEY={}".format(api.api_key)] + args
1990
+ args = ["-e", f"WANDB_API_KEY={api.api_key}"] + args
1999
1991
  else:
2000
1992
  wandb.termlog(
2001
1993
  "Not logged in, run `wandb login` from the host machine to enable result logging"
@@ -2009,7 +2001,7 @@ def docker_run(ctx, docker_run_args):
2009
2001
  @click.argument("docker_image", required=False)
2010
2002
  @click.option(
2011
2003
  "--nvidia/--no-nvidia",
2012
- default=find_executable("nvidia-docker") is not None,
2004
+ default=_HAS_NVIDIA_DOCKER,
2013
2005
  help="Use the nvidia runtime, defaults to nvidia if nvidia-docker is present",
2014
2006
  )
2015
2007
  @click.option(
@@ -2066,8 +2058,11 @@ def docker(
2066
2058
  variable to an existing docker run command, see the wandb docker-run command.
2067
2059
  """
2068
2060
  api = InternalApi()
2069
- if not find_executable("docker"):
2061
+ if not _HAS_DOCKER:
2070
2062
  raise ClickException("Docker not installed, install it from https://docker.com")
2063
+
2064
+ import wandb.docker
2065
+
2071
2066
  args = list(docker_run_args)
2072
2067
  image = docker_image or ""
2073
2068
  # remove run for users used to nvidia-docker
@@ -2086,17 +2081,13 @@ def docker(
2086
2081
  resolved_image = wandb.docker.image_id(image)
2087
2082
  if resolved_image is None:
2088
2083
  raise ClickException(
2089
- "Couldn't find image locally or in a registry, try running `docker pull {}`".format(
2090
- image
2091
- )
2084
+ f"Couldn't find image locally or in a registry, try running `docker pull {image}`"
2092
2085
  )
2093
2086
  if digest:
2094
2087
  sys.stdout.write(resolved_image)
2095
2088
  exit(0)
2096
2089
 
2097
- existing = wandb.docker.shell(
2098
- ["ps", "-f", "ancestor={}".format(resolved_image), "-q"]
2099
- )
2090
+ existing = wandb.docker.shell(["ps", "-f", f"ancestor={resolved_image}", "-q"])
2100
2091
  if existing:
2101
2092
  if click.confirm(
2102
2093
  "Found running container with the same image, do you want to attach?"
@@ -2110,7 +2101,7 @@ def docker(
2110
2101
  "-e",
2111
2102
  "LANG=C.UTF-8",
2112
2103
  "-e",
2113
- "WANDB_DOCKER={}".format(resolved_image),
2104
+ f"WANDB_DOCKER={resolved_image}",
2114
2105
  "--ipc=host",
2115
2106
  "-v",
2116
2107
  wandb.docker.entrypoint + ":/wandb-entrypoint.sh",
@@ -2123,7 +2114,7 @@ def docker(
2123
2114
  # TODO: We should default to the working directory if defined
2124
2115
  command.extend(["-v", cwd + ":" + dir, "-w", dir])
2125
2116
  if api.api_key:
2126
- command.extend(["-e", "WANDB_API_KEY={}".format(api.api_key)])
2117
+ command.extend(["-e", f"WANDB_API_KEY={api.api_key}"])
2127
2118
  else:
2128
2119
  wandb.termlog(
2129
2120
  "Couldn't find WANDB_API_KEY, run `wandb login` to enable streaming metrics"
@@ -2131,15 +2122,13 @@ def docker(
2131
2122
  if jupyter:
2132
2123
  command.extend(["-e", "WANDB_ENSURE_JUPYTER=1", "-p", port + ":8888"])
2133
2124
  no_tty = True
2134
- cmd = "jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir {}".format(
2135
- dir
2136
- )
2125
+ cmd = f"jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir {dir}"
2137
2126
  command.extend(args)
2138
2127
  if no_tty:
2139
2128
  command.extend([image, shell, "-c", cmd])
2140
2129
  else:
2141
2130
  if cmd:
2142
- command.extend(["-e", "WANDB_COMMAND={}".format(cmd)])
2131
+ command.extend(["-e", f"WANDB_COMMAND={cmd}"])
2143
2132
  command.extend(["-it", image, shell])
2144
2133
  wandb.termlog("Launching docker container \U0001f6a2")
2145
2134
  subprocess.call(command)
@@ -2199,8 +2188,11 @@ def server():
2199
2188
  @display_error
2200
2189
  def start(ctx, port, env, daemon, upgrade, edge):
2201
2190
  api = InternalApi()
2202
- if not find_executable("docker"):
2191
+ if not _HAS_DOCKER:
2203
2192
  raise ClickException("Docker not installed, install it from https://docker.com")
2193
+
2194
+ import wandb.docker
2195
+
2204
2196
  local_image_sha = wandb.docker.image_id("wandb/local").split("wandb/local")[-1]
2205
2197
  registry_image_sha = wandb.docker.image_id_from_registry("wandb/local").split(
2206
2198
  "wandb/local"
@@ -2213,7 +2205,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
2213
2205
  "A new version of the W&B server is available, upgrade by calling `wandb server start --upgrade`"
2214
2206
  )
2215
2207
  running = subprocess.check_output(
2216
- ["docker", "ps", "--filter", "name=wandb-local", "--format", "{{.ID}}"]
2208
+ ["docker", "ps", "--filter", "name=^wandb-local$", "--format", "{{.ID}}"]
2217
2209
  )
2218
2210
  if running != b"":
2219
2211
  if upgrade:
@@ -2225,7 +2217,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
2225
2217
  exit(1)
2226
2218
  image = "docker.pkg.github.com/wandb/core/local" if edge else "wandb/local"
2227
2219
  username = getpass.getuser()
2228
- env_vars = ["-e", "LOCAL_USERNAME={}".format(username)]
2220
+ env_vars = ["-e", f"LOCAL_USERNAME={username}"]
2229
2221
  for e in env:
2230
2222
  env_vars.append("-e")
2231
2223
  env_vars.append(e)
@@ -2259,9 +2251,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
2259
2251
  )
2260
2252
  exit(1)
2261
2253
  else:
2262
- wandb.termlog(
2263
- "W&B server started at http://localhost:{} \U0001f680".format(port)
2264
- )
2254
+ wandb.termlog(f"W&B server started at http://localhost:{port} \U0001f680")
2265
2255
  wandb.termlog("You can stop the server by running `wandb server stop`")
2266
2256
  if not api.api_key:
2267
2257
  # Let the server start before potentially launching a browser
@@ -2271,7 +2261,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
2271
2261
 
2272
2262
  @server.command(context_settings=RUN_CONTEXT, help="Stop a local W&B server")
2273
2263
  def stop():
2274
- if not find_executable("docker"):
2264
+ if not _HAS_DOCKER:
2275
2265
  raise ClickException("Docker not installed, install it from https://docker.com")
2276
2266
  subprocess.call(["docker", "stop", "wandb-local"])
2277
2267
 
@@ -2407,7 +2397,7 @@ def get(path, root, type):
2407
2397
  )
2408
2398
  artifact = public_api.artifact(full_path, type=type)
2409
2399
  path = artifact.download(root=root)
2410
- wandb.termlog("Artifact downloaded to {}".format(path))
2400
+ wandb.termlog(f"Artifact downloaded to {path}")
2411
2401
  except ValueError:
2412
2402
  raise ClickException("Unable to download artifact")
2413
2403
 
@@ -2434,12 +2424,7 @@ def ls(path, type):
2434
2424
  )
2435
2425
  latest = next(versions)
2436
2426
  wandb.termlog(
2437
- "{:<15s}{:<15s}{:>15s} {:<20s}".format(
2438
- kind.type,
2439
- latest.updated_at,
2440
- util.to_human_size(latest.size),
2441
- latest.name,
2442
- )
2427
+ f"{kind.type:<15s}{latest.updated_at:<15s}{util.to_human_size(latest.size):>15s} {latest.name:<20s}"
2443
2428
  )
2444
2429
 
2445
2430
 
@@ -2485,17 +2470,17 @@ def pull(run, project, entity):
2485
2470
 
2486
2471
  for name in urls:
2487
2472
  if api.file_current(name, urls[name]["md5"]):
2488
- click.echo("File {} is up to date".format(name))
2473
+ click.echo(f"File {name} is up to date")
2489
2474
  else:
2490
2475
  length, response = api.download_file(urls[name]["url"])
2491
2476
  # TODO: I had to add this because some versions in CI broke click.progressbar
2492
- sys.stdout.write("File {}\r".format(name))
2477
+ sys.stdout.write(f"File {name}\r")
2493
2478
  dirname = os.path.dirname(name)
2494
2479
  if dirname != "":
2495
2480
  filesystem.mkdir_exists_ok(dirname)
2496
2481
  with click.progressbar(
2497
2482
  length=length,
2498
- label="File {}".format(name),
2483
+ label=f"File {name}",
2499
2484
  fill_char=click.style("&", fg="green"),
2500
2485
  ) as bar:
2501
2486
  with open(name, "wb") as f:
@@ -2541,8 +2526,8 @@ def restore(ctx, run, no_git, branch, project, entity):
2541
2526
  )
2542
2527
  repo = metadata.get("git", {}).get("repo")
2543
2528
  image = metadata.get("docker")
2544
- restore_message = """`wandb restore` needs to be run from the same git repository as the original run.
2545
- Run `git clone {}` and restore from there or pass the --no-git flag.""".format(repo)
2529
+ restore_message = f"""`wandb restore` needs to be run from the same git repository as the original run.
2530
+ Run `git clone {repo}` and restore from there or pass the --no-git flag."""
2546
2531
  if no_git:
2547
2532
  commit = None
2548
2533
  elif not api.git.enabled:
@@ -2587,21 +2572,17 @@ Run `git clone {}` and restore from there or pass the --no-git flag.""".format(r
2587
2572
  else:
2588
2573
  patch_path = None
2589
2574
 
2590
- branch_name = "wandb/{}".format(run)
2575
+ branch_name = f"wandb/{run}"
2591
2576
  if branch and branch_name not in api.git.repo.branches:
2592
2577
  api.git.repo.git.checkout(commit, b=branch_name)
2593
- wandb.termlog(
2594
- "Created branch {}".format(click.style(branch_name, bold=True))
2595
- )
2578
+ wandb.termlog(f"Created branch {click.style(branch_name, bold=True)}")
2596
2579
  elif branch:
2597
2580
  wandb.termlog(
2598
- "Using existing branch, run `git branch -D {}` from master for a clean checkout".format(
2599
- branch_name
2600
- )
2581
+ f"Using existing branch, run `git branch -D {branch_name}` from master for a clean checkout"
2601
2582
  )
2602
2583
  api.git.repo.git.checkout(branch_name)
2603
2584
  else:
2604
- wandb.termlog("Checking out {} in detached mode".format(commit))
2585
+ wandb.termlog(f"Checking out {commit} in detached mode")
2605
2586
  api.git.repo.git.checkout(commit)
2606
2587
 
2607
2588
  if patch_path:
@@ -2639,7 +2620,7 @@ Run `git clone {}` and restore from there or pass the --no-git flag.""".format(r
2639
2620
  with open(config_path, "w") as f:
2640
2621
  f.write(s)
2641
2622
 
2642
- wandb.termlog("Restored config variables to {}".format(config_path))
2623
+ wandb.termlog(f"Restored config variables to {config_path}")
2643
2624
  if image:
2644
2625
  if not metadata["program"].startswith("<") and metadata.get("args") is not None:
2645
2626
  # TODO: we may not want to default to python here.
wandb/docker/__init__.py CHANGED
@@ -1,13 +1,11 @@
1
1
  import json
2
2
  import logging
3
3
  import os
4
+ import shutil
4
5
  import subprocess
5
6
  from typing import Any, Dict, List, Optional, Tuple, Union
6
7
 
7
- import requests
8
- from dockerpycreds.utils import find_executable # type: ignore
9
-
10
- from wandb.docker import auth, www_authenticate
8
+ from wandb.docker import names
11
9
  from wandb.errors import Error
12
10
 
13
11
 
@@ -46,7 +44,6 @@ class DockerError(Error):
46
44
  entrypoint = os.path.join(
47
45
  os.path.dirname(os.path.abspath(__file__)), "wandb-entrypoint.sh"
48
46
  )
49
- auth_config = auth.load_config()
50
47
  log = logging.getLogger(__name__)
51
48
 
52
49
 
@@ -74,7 +71,7 @@ def is_buildx_installed() -> bool:
74
71
  global _buildx_installed
75
72
  if _buildx_installed is not None:
76
73
  return _buildx_installed # type: ignore
77
- if not find_executable("docker"):
74
+ if not shutil.which("docker"):
78
75
  _buildx_installed = False
79
76
  else:
80
77
  help_output = shell(["buildx", "--help"])
@@ -203,7 +200,7 @@ def default_image(gpu: bool = False) -> str:
203
200
  tag = "all"
204
201
  if not gpu:
205
202
  tag += "-cpu"
206
- return "wandb/deepo:{}".format(tag)
203
+ return f"wandb/deepo:{tag}"
207
204
 
208
205
 
209
206
  def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:
@@ -218,75 +215,25 @@ def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:
218
215
 
219
216
  def parse(image_name: str) -> Tuple[str, str, str]:
220
217
  repository, tag = parse_repository_tag(image_name)
221
- registry, repo_name = auth.resolve_repository_name(repository)
218
+ registry, repo_name = names.resolve_repository_name(repository)
222
219
  if registry == "docker.io":
223
220
  registry = "index.docker.io"
224
221
  return registry, repo_name, (tag or "latest")
225
222
 
226
223
 
227
- def auth_token(registry: str, repo: str) -> Dict[str, str]:
228
- """Make a request to the root of a v2 docker registry to get the auth url.
229
-
230
- Always returns a dictionary, if there's no token key we couldn't authenticate
231
- """
232
- # TODO: Cache tokens?
233
- auth_info = auth_config.resolve_authconfig(registry)
234
- if auth_info:
235
- normalized = {k.lower(): v for k, v in auth_info.items()}
236
- normalized_auth_info = (
237
- normalized.get("username"),
238
- normalized.get("password"),
239
- )
240
- else:
241
- normalized_auth_info = None
242
- response = requests.get(f"https://{registry}/v2/", timeout=3)
243
- if response.headers.get("www-authenticate"):
244
- try:
245
- info: Dict = www_authenticate.parse(response.headers["www-authenticate"])
246
- except ValueError:
247
- info = {}
248
- else:
249
- log.error(
250
- f"Received {response} when attempting to authenticate with {registry}"
251
- )
252
- info = {}
253
- if info.get("bearer"):
254
- res = requests.get(
255
- info["bearer"]["realm"]
256
- + "?service={}&scope=repository:{}:pull".format(
257
- info["bearer"]["service"], repo
258
- ),
259
- auth=normalized_auth_info, # type: ignore
260
- timeout=3,
261
- )
262
- res.raise_for_status()
263
- result_json: Dict[str, str] = res.json()
264
- return result_json
265
- return {}
224
+ def image_id_from_registry(image_name: str) -> Optional[str]:
225
+ """Query the image manifest to get its full ID including the digest.
266
226
 
227
+ Args:
228
+ image_name: The image name, such as "wandb/local".
267
229
 
268
- def image_id_from_registry(image_name: str) -> Optional[str]:
269
- """Get the docker id from a public or private registry."""
270
- registry, repository, tag = parse(image_name)
271
- res = None
272
- try:
273
- token = auth_token(registry, repository).get("token")
274
- # dockerhub is crazy
275
- if registry == "index.docker.io":
276
- registry = "registry-1.docker.io"
277
- res = requests.head(
278
- f"https://{registry}/v2/{repository}/manifests/{tag}",
279
- headers={
280
- "Authorization": f"Bearer {token}",
281
- "Accept": "application/vnd.docker.distribution.manifest.v2+json",
282
- },
283
- timeout=5,
284
- )
285
- res.raise_for_status()
286
- except requests.RequestException:
287
- log.error(f"Received {res} when attempting to get digest for {image_name}")
288
- return None
289
- return "@".join([registry + "/" + repository, res.headers["Docker-Content-Digest"]])
230
+ Returns:
231
+ The image name followed by its digest, like "wandb/local@sha256:...".
232
+ """
233
+ # https://docs.docker.com/reference/cli/docker/buildx/imagetools/inspect
234
+ inspect_cmd = ["buildx", "imagetools", "inspect", image_name]
235
+ format_args = ["--format", r"{{.Name}}@{{.Manifest.Digest}}"]
236
+ return shell([*inspect_cmd, *format_args])
290
237
 
291
238
 
292
239
  def image_id(image_name: str) -> Optional[str]:
@@ -295,11 +242,12 @@ def image_id(image_name: str) -> Optional[str]:
295
242
  return image_name
296
243
  else:
297
244
  digests = shell(["inspect", image_name, "--format", "{{json .RepoDigests}}"])
245
+
246
+ if digests is None:
247
+ return image_id_from_registry(image_name)
248
+
298
249
  try:
299
- if digests is None:
300
- raise ValueError
301
- im_id: str = json.loads(digests)[0]
302
- return im_id
250
+ return json.loads(digests)[0]
303
251
  except (ValueError, IndexError):
304
252
  return image_id_from_registry(image_name)
305
253
 
@@ -332,7 +280,6 @@ __all__ = [
332
280
  "image_id",
333
281
  "image_id_from_registry",
334
282
  "is_docker_installed",
335
- "auth_token",
336
283
  "parse",
337
284
  "parse_repository_tag",
338
285
  "default_image",
wandb/docker/names.py ADDED
@@ -0,0 +1,40 @@
1
+ from __future__ import annotations
2
+
3
+
4
+ class InvalidRepositoryError(Exception):
5
+ """The given string is not a valid repository name."""
6
+
7
+
8
+ def resolve_repository_name(repo_name: str) -> tuple[str, str]:
9
+ if "://" in repo_name:
10
+ raise InvalidRepositoryError(
11
+ f"Repository name cannot contain a scheme ({repo_name})"
12
+ )
13
+
14
+ index_name, remote_name = split_repo_name(repo_name)
15
+ if index_name[0] == "-" or index_name[-1] == "-":
16
+ raise InvalidRepositoryError(
17
+ f"Invalid index name ({index_name}). Cannot begin or end with a hyphen."
18
+ )
19
+ return resolve_index_name(index_name), remote_name
20
+
21
+
22
+ def resolve_index_name(index_name: str) -> str:
23
+ index_name = convert_to_hostname(index_name)
24
+ if index_name == "index.docker.io":
25
+ index_name = "docker.io"
26
+ return index_name
27
+
28
+
29
+ def split_repo_name(repo_name: str) -> tuple[str, str]:
30
+ parts = repo_name.split("/", 1)
31
+ if len(parts) == 1 or (
32
+ "." not in parts[0] and ":" not in parts[0] and parts[0] != "localhost"
33
+ ):
34
+ # This is a docker index repo (ex: username/foobar or ubuntu)
35
+ return "docker.io", repo_name
36
+ return parts[0], parts[1]
37
+
38
+
39
+ def convert_to_hostname(url: str) -> str:
40
+ return url.replace("http://", "").replace("https://", "").split("/", 1)[0]
wandb/env.py CHANGED
@@ -78,7 +78,6 @@ ARTIFACT_FETCH_FILE_URL_BATCH_SIZE = "WANDB_ARTIFACT_FETCH_FILE_URL_BATCH_SIZE"
78
78
  CACHE_DIR = "WANDB_CACHE_DIR"
79
79
  DISABLE_SSL = "WANDB_INSECURE_DISABLE_SSL"
80
80
  SERVICE = "WANDB_SERVICE"
81
- _DISABLE_SERVICE = "WANDB_DISABLE_SERVICE"
82
81
  SENTRY_DSN = "WANDB_SENTRY_DSN"
83
82
  INIT_TIMEOUT = "WANDB_INIT_TIMEOUT"
84
83
  GIT_COMMIT = "WANDB_GIT_COMMIT"
wandb/errors/util.py CHANGED
@@ -47,7 +47,7 @@ class ProtobufErrorHandler:
47
47
  The corresponding protobuf error message.
48
48
  """
49
49
  if not isinstance(exc, Error):
50
- raise ValueError("exc must be a subclass of wandb.errors.Error")
50
+ raise TypeError("exc must be a subclass of wandb.errors.Error")
51
51
 
52
52
  code = None
53
53
  for subclass in type(exc).__mro__:
@@ -128,7 +128,7 @@ class StepChecksum:
128
128
  elif isinstance(req, RequestFinish):
129
129
  break
130
130
  else:
131
- raise Exception("internal error")
131
+ raise TypeError
132
132
 
133
133
  self._output_queue.put(step_upload.RequestFinish(req.callback))
134
134
 
@@ -176,7 +176,7 @@ class StepUpload:
176
176
  self._artifacts[event.artifact_id]["pending_count"] += 1
177
177
  self._start_upload_job(event)
178
178
  else:
179
- raise Exception("Programming error: unhandled event: {}".format(str(event)))
179
+ raise TypeError(f"Event has unexpected type: {event!s}")
180
180
 
181
181
  def _start_upload_job(self, event: RequestUpload) -> None:
182
182
  # Operations on a single backend file must be serialized. if
@@ -669,8 +669,7 @@ class DiffusersMultiModalPipelineResolver:
669
669
  )
670
670
 
671
671
  # Return the WandB loggable dict
672
- loggable_dict = self.prepare_loggable_dict(pipeline, response, kwargs)
673
- return loggable_dict
672
+ return self.prepare_loggable_dict(pipeline, response, kwargs)
674
673
  except Exception as e:
675
674
  logger.warning(e)
676
675
  return None
@@ -9,8 +9,7 @@ _gym_version_lt_0_26: Optional[bool] = None
9
9
  _gymnasium_version_lt_1_0_0: Optional[bool] = None
10
10
 
11
11
  _required_error_msg = (
12
- "Couldn't import the gymnasium python package, "
13
- "install with `pip install gymnasium`"
12
+ "Couldn't import the gymnasium python package, install with `pip install gymnasium`"
14
13
  )
15
14
  GymLib = Literal["gym", "gymnasium"]
16
15
 
@@ -40,11 +39,11 @@ def monitor():
40
39
  else:
41
40
  import gymnasium as gym # type: ignore
42
41
 
43
- from wandb.util import parse_version
42
+ from packaging.version import parse
44
43
 
45
- gym_lib_version = parse_version(gym.__version__)
46
- _gym_version_lt_0_26 = gym_lib_version < parse_version("0.26.0")
47
- _gymnasium_version_lt_1_0_0 = gym_lib_version < parse_version("1.0.0a1")
44
+ gym_lib_version = parse(gym.__version__)
45
+ _gym_version_lt_0_26 = gym_lib_version < parse("0.26.0")
46
+ _gymnasium_version_lt_1_0_0 = gym_lib_version < parse("1.0.0a1")
48
47
 
49
48
  path = "path" # Default path
50
49
  if gym_lib == "gymnasium" and not _gymnasium_version_lt_1_0_0: