wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +1 -2
- wandb/__init__.pyi +3 -6
- wandb/_iterutils.py +26 -7
- wandb/_pydantic/__init__.py +2 -1
- wandb/_pydantic/utils.py +7 -0
- wandb/agents/pyagent.py +9 -15
- wandb/analytics/sentry.py +1 -2
- wandb/apis/attrs.py +3 -4
- wandb/apis/importers/internals/util.py +1 -1
- wandb/apis/importers/validation.py +2 -2
- wandb/apis/importers/wandb.py +30 -25
- wandb/apis/normalize.py +2 -2
- wandb/apis/public/__init__.py +1 -0
- wandb/apis/public/api.py +37 -33
- wandb/apis/public/artifacts.py +103 -72
- wandb/apis/public/jobs.py +3 -2
- wandb/apis/public/registries/registries_search.py +4 -2
- wandb/apis/public/registries/registry.py +1 -1
- wandb/apis/public/registries/utils.py +9 -9
- wandb/apis/public/runs.py +18 -6
- wandb/automations/_filters/expressions.py +1 -1
- wandb/automations/_filters/operators.py +1 -1
- wandb/automations/_filters/run_metrics.py +1 -1
- wandb/beta/workflows.py +6 -5
- wandb/bin/gpu_stats.exe +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +54 -73
- wandb/docker/__init__.py +21 -74
- wandb/docker/names.py +40 -0
- wandb/env.py +0 -1
- wandb/errors/util.py +1 -1
- wandb/filesync/step_checksum.py +1 -1
- wandb/filesync/step_upload.py +1 -1
- wandb/integration/diffusers/resolvers/multimodal.py +1 -2
- wandb/integration/gym/__init__.py +5 -6
- wandb/integration/keras/callbacks/model_checkpoint.py +2 -2
- wandb/integration/keras/keras.py +13 -19
- wandb/integration/kfp/kfp_patch.py +2 -3
- wandb/integration/langchain/wandb_tracer.py +1 -1
- wandb/integration/metaflow/metaflow.py +13 -13
- wandb/integration/openai/fine_tuning.py +3 -2
- wandb/integration/sagemaker/auth.py +2 -1
- wandb/integration/sklearn/utils.py +2 -1
- wandb/integration/tensorboard/__init__.py +1 -1
- wandb/integration/tensorboard/log.py +2 -5
- wandb/integration/tensorflow/__init__.py +2 -2
- wandb/jupyter.py +20 -17
- wandb/plot/confusion_matrix.py +1 -1
- wandb/plot/utils.py +8 -7
- wandb/proto/v3/wandb_internal_pb2.py +355 -335
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v4/wandb_internal_pb2.py +339 -335
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v5/wandb_internal_pb2.py +339 -335
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +12 -12
- wandb/proto/v6/wandb_internal_pb2.py +339 -335
- wandb/proto/v6/wandb_settings_pb2.py +2 -2
- wandb/proto/v6/wandb_telemetry_pb2.py +12 -12
- wandb/proto/wandb_deprecated.py +6 -8
- wandb/sdk/artifacts/_internal_artifact.py +43 -0
- wandb/sdk/artifacts/_validators.py +55 -35
- wandb/sdk/artifacts/artifact.py +117 -115
- wandb/sdk/artifacts/artifact_download_logger.py +2 -0
- wandb/sdk/artifacts/artifact_saver.py +1 -3
- wandb/sdk/artifacts/artifact_state.py +2 -0
- wandb/sdk/artifacts/artifact_ttl.py +2 -0
- wandb/sdk/artifacts/exceptions.py +14 -0
- wandb/sdk/artifacts/staging.py +2 -0
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -6
- wandb/sdk/artifacts/storage_handlers/multi_handler.py +1 -1
- wandb/sdk/artifacts/storage_handlers/tracking_handler.py +2 -6
- wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +1 -5
- wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +1 -1
- wandb/sdk/artifacts/storage_layout.py +2 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +3 -3
- wandb/sdk/backend/backend.py +11 -182
- wandb/sdk/data_types/_dtypes.py +2 -6
- wandb/sdk/data_types/audio.py +20 -3
- wandb/sdk/data_types/base_types/media.py +12 -7
- wandb/sdk/data_types/base_types/wb_value.py +8 -18
- wandb/sdk/data_types/bokeh.py +19 -2
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +17 -1
- wandb/sdk/data_types/helper_types/image_mask.py +7 -1
- wandb/sdk/data_types/html.py +4 -4
- wandb/sdk/data_types/image.py +178 -103
- wandb/sdk/data_types/molecule.py +6 -6
- wandb/sdk/data_types/object_3d.py +10 -5
- wandb/sdk/data_types/saved_model.py +11 -6
- wandb/sdk/data_types/table.py +313 -83
- wandb/sdk/data_types/table_decorators.py +108 -0
- wandb/sdk/data_types/utils.py +43 -7
- wandb/sdk/data_types/video.py +21 -3
- wandb/sdk/interface/interface.py +10 -0
- wandb/sdk/internal/datastore.py +2 -6
- wandb/sdk/internal/file_pusher.py +1 -5
- wandb/sdk/internal/file_stream.py +8 -17
- wandb/sdk/internal/handler.py +2 -2
- wandb/sdk/internal/incremental_table_util.py +53 -0
- wandb/sdk/internal/internal.py +3 -5
- wandb/sdk/internal/internal_api.py +66 -89
- wandb/sdk/internal/job_builder.py +2 -7
- wandb/sdk/internal/profiler.py +2 -2
- wandb/sdk/internal/progress.py +1 -3
- wandb/sdk/internal/run.py +1 -6
- wandb/sdk/internal/sender.py +24 -36
- wandb/sdk/internal/system/assets/aggregators.py +1 -7
- wandb/sdk/internal/system/assets/disk.py +3 -3
- wandb/sdk/internal/system/assets/gpu.py +4 -4
- wandb/sdk/internal/system/assets/gpu_amd.py +4 -4
- wandb/sdk/internal/system/assets/interfaces.py +6 -6
- wandb/sdk/internal/system/assets/tpu.py +1 -1
- wandb/sdk/internal/system/assets/trainium.py +6 -6
- wandb/sdk/internal/system/system_info.py +5 -7
- wandb/sdk/internal/system/system_monitor.py +4 -4
- wandb/sdk/internal/tb_watcher.py +5 -7
- wandb/sdk/launch/_launch.py +1 -1
- wandb/sdk/launch/_project_spec.py +19 -20
- wandb/sdk/launch/agent/agent.py +3 -3
- wandb/sdk/launch/agent/config.py +1 -1
- wandb/sdk/launch/agent/job_status_tracker.py +2 -2
- wandb/sdk/launch/builder/build.py +2 -3
- wandb/sdk/launch/builder/kaniko_builder.py +5 -4
- wandb/sdk/launch/environment/gcp_environment.py +1 -2
- wandb/sdk/launch/registry/azure_container_registry.py +2 -2
- wandb/sdk/launch/registry/elastic_container_registry.py +2 -2
- wandb/sdk/launch/registry/google_artifact_registry.py +3 -3
- wandb/sdk/launch/runner/abstract.py +5 -5
- wandb/sdk/launch/runner/kubernetes_monitor.py +2 -2
- wandb/sdk/launch/runner/kubernetes_runner.py +1 -1
- wandb/sdk/launch/runner/sagemaker_runner.py +2 -4
- wandb/sdk/launch/runner/vertex_runner.py +2 -7
- wandb/sdk/launch/sweeps/__init__.py +1 -1
- wandb/sdk/launch/sweeps/scheduler.py +2 -2
- wandb/sdk/launch/sweeps/utils.py +3 -3
- wandb/sdk/launch/utils.py +3 -4
- wandb/sdk/lib/apikey.py +5 -8
- wandb/sdk/lib/config_util.py +3 -3
- wandb/sdk/lib/fsm.py +3 -18
- wandb/sdk/lib/gitlib.py +6 -5
- wandb/sdk/lib/ipython.py +2 -2
- wandb/sdk/lib/json_util.py +9 -14
- wandb/sdk/lib/printer.py +3 -8
- wandb/sdk/lib/redirect.py +1 -1
- wandb/sdk/lib/retry.py +3 -7
- wandb/sdk/lib/run_moment.py +2 -2
- wandb/sdk/lib/service_connection.py +3 -1
- wandb/sdk/lib/service_token.py +1 -2
- wandb/sdk/mailbox/mailbox_handle.py +3 -7
- wandb/sdk/mailbox/response_handle.py +2 -6
- wandb/sdk/service/streams.py +3 -7
- wandb/sdk/verify/verify.py +5 -6
- wandb/sdk/wandb_config.py +1 -1
- wandb/sdk/wandb_init.py +38 -106
- wandb/sdk/wandb_login.py +7 -6
- wandb/sdk/wandb_run.py +52 -240
- wandb/sdk/wandb_settings.py +71 -60
- wandb/sdk/wandb_setup.py +40 -14
- wandb/sdk/wandb_watch.py +5 -7
- wandb/sync/__init__.py +1 -1
- wandb/sync/sync.py +13 -13
- wandb/util.py +17 -35
- wandb/wandb_agent.py +8 -11
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/METADATA +5 -5
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/RECORD +170 -168
- wandb/docker/auth.py +0 -435
- wandb/docker/www_authenticate.py +0 -94
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/WHEEL +0 -0
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.12rc1.dist-info → wandb-0.20.1.dist-info}/licenses/LICENSE +0 -0
wandb/cli/cli.py
CHANGED
@@ -20,9 +20,6 @@ import click
|
|
20
20
|
import yaml
|
21
21
|
from click.exceptions import ClickException
|
22
22
|
|
23
|
-
# pycreds has a find_executable that works in windows
|
24
|
-
from dockerpycreds.utils import find_executable
|
25
|
-
|
26
23
|
import wandb
|
27
24
|
import wandb.env
|
28
25
|
import wandb.errors
|
@@ -31,6 +28,7 @@ from wandb import Config, Error, env, util, wandb_agent, wandb_sdk
|
|
31
28
|
from wandb.apis import InternalApi, PublicApi
|
32
29
|
from wandb.apis.public import RunQueue
|
33
30
|
from wandb.errors.links import url_registry
|
31
|
+
from wandb.sdk import wandb_setup
|
34
32
|
from wandb.sdk.artifacts._validators import is_artifact_registry_project
|
35
33
|
from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
|
36
34
|
from wandb.sdk.internal.internal_api import Api as SDKInternalApi
|
@@ -67,6 +65,9 @@ logging.basicConfig(
|
|
67
65
|
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
|
68
66
|
logger = logging.getLogger("wandb")
|
69
67
|
|
68
|
+
_HAS_DOCKER = bool(shutil.which("docker"))
|
69
|
+
_HAS_NVIDIA_DOCKER = bool(shutil.which("nvidia-docker"))
|
70
|
+
|
70
71
|
# Click Contexts
|
71
72
|
CONTEXT = {"default_map": {}}
|
72
73
|
RUN_CONTEXT = {
|
@@ -104,7 +105,7 @@ def display_error(func):
|
|
104
105
|
except wandb.Error as e:
|
105
106
|
exc_type, exc_value, exc_traceback = sys.exc_info()
|
106
107
|
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
|
107
|
-
logger.
|
108
|
+
logger.exception("".join(lines))
|
108
109
|
wandb.termerror(f"Find detailed error logs at: {_wandb_log_path}")
|
109
110
|
click_exc = ClickWandbException(e)
|
110
111
|
click_exc.orig_type = exc_type
|
@@ -118,9 +119,6 @@ _api = None # caching api instance allows patching from unit tests
|
|
118
119
|
|
119
120
|
def _get_cling_api(reset=None):
|
120
121
|
"""Get a reference to the internal api with cling settings."""
|
121
|
-
# TODO: move CLI to wandb-core backend
|
122
|
-
wandb.require("legacy-service")
|
123
|
-
|
124
122
|
global _api
|
125
123
|
if reset:
|
126
124
|
_api = None
|
@@ -128,7 +126,7 @@ def _get_cling_api(reset=None):
|
|
128
126
|
if _api is None:
|
129
127
|
# TODO(jhr): make a settings object that is better for non runs.
|
130
128
|
# only override the necessary setting
|
131
|
-
|
129
|
+
wandb_setup.singleton().settings.x_cli_only_mode = True
|
132
130
|
_api = InternalApi()
|
133
131
|
return _api
|
134
132
|
|
@@ -195,9 +193,9 @@ def projects(entity, display=True):
|
|
195
193
|
api = _get_cling_api()
|
196
194
|
projects = api.list_projects(entity=entity)
|
197
195
|
if len(projects) == 0:
|
198
|
-
message = "No projects found for {}"
|
196
|
+
message = f"No projects found for {entity}"
|
199
197
|
else:
|
200
|
-
message = 'Latest projects for "{}"'
|
198
|
+
message = f'Latest projects for "{entity}"'
|
201
199
|
if display:
|
202
200
|
click.echo(click.style(message, bold=True))
|
203
201
|
for project in projects:
|
@@ -231,9 +229,6 @@ def projects(entity, display=True):
|
|
231
229
|
)
|
232
230
|
@display_error
|
233
231
|
def login(key, host, cloud, relogin, anonymously, verify, no_offline=False):
|
234
|
-
# TODO: move CLI to wandb-core backend
|
235
|
-
wandb.require("legacy-service")
|
236
|
-
|
237
232
|
# TODO: handle no_offline
|
238
233
|
anon_mode = "must" if anonymously else "never"
|
239
234
|
|
@@ -242,12 +237,9 @@ def login(key, host, cloud, relogin, anonymously, verify, no_offline=False):
|
|
242
237
|
key = key[0] if key is not None and len(key) > 0 else None
|
243
238
|
relogin = True if key or relogin else False
|
244
239
|
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
x_disable_viewer=relogin and not verify,
|
249
|
-
)
|
250
|
-
)
|
240
|
+
global_settings = wandb_setup.singleton().settings
|
241
|
+
global_settings.x_cli_only_mode = True
|
242
|
+
global_settings.x_disable_viewer = relogin and not verify
|
251
243
|
|
252
244
|
wandb.login(
|
253
245
|
anonymous=anon_mode,
|
@@ -421,7 +413,7 @@ def init(ctx, project, entity, reset, mode):
|
|
421
413
|
"""
|
422
414
|
).format(
|
423
415
|
code1=click.style("import wandb", bold=True),
|
424
|
-
code2=click.style('wandb.init(project="{}")'
|
416
|
+
code2=click.style(f'wandb.init(project="{project}")', bold=True),
|
425
417
|
run=click.style("python <train.py>", bold=True),
|
426
418
|
)
|
427
419
|
)
|
@@ -777,7 +769,7 @@ def sweep(
|
|
777
769
|
"resume": "Resuming",
|
778
770
|
}
|
779
771
|
wandb.termlog(f"{ings[state]} sweep {entity}/{project}/{sweep_id}")
|
780
|
-
getattr(api, "{}_sweep"
|
772
|
+
getattr(api, f"{state}_sweep")(sweep_id, entity=entity, project=project)
|
781
773
|
wandb.termlog("Done.")
|
782
774
|
return
|
783
775
|
else:
|
@@ -1518,11 +1510,11 @@ def launch(
|
|
1518
1510
|
wandb.termerror("Launched run exited with non-zero status")
|
1519
1511
|
sys.exit(1)
|
1520
1512
|
except LaunchError as e:
|
1521
|
-
logger.
|
1513
|
+
logger.exception("An error occurred.")
|
1522
1514
|
wandb._sentry.exception(e)
|
1523
1515
|
sys.exit(e)
|
1524
1516
|
except ExecutionError as e:
|
1525
|
-
logger.
|
1517
|
+
logger.exception("An error occurred.")
|
1526
1518
|
wandb._sentry.exception(e)
|
1527
1519
|
sys.exit(e)
|
1528
1520
|
except asyncio.CancelledError:
|
@@ -1552,7 +1544,7 @@ def launch(
|
|
1552
1544
|
|
1553
1545
|
except Exception as e:
|
1554
1546
|
wandb._sentry.exception(e)
|
1555
|
-
raise
|
1547
|
+
raise
|
1556
1548
|
|
1557
1549
|
|
1558
1550
|
@cli.command(
|
@@ -1643,7 +1635,7 @@ def launch_agent(
|
|
1643
1635
|
_launch.create_and_run_agent(api, agent_config)
|
1644
1636
|
except Exception as e:
|
1645
1637
|
wandb._sentry.exception(e)
|
1646
|
-
raise
|
1638
|
+
raise
|
1647
1639
|
|
1648
1640
|
|
1649
1641
|
@cli.command(context_settings=CONTEXT, help="Run the W&B agent")
|
@@ -1721,7 +1713,7 @@ def scheduler(
|
|
1721
1713
|
_scheduler.start()
|
1722
1714
|
except Exception as e:
|
1723
1715
|
wandb._sentry.exception(e)
|
1724
|
-
raise
|
1716
|
+
raise
|
1725
1717
|
|
1726
1718
|
|
1727
1719
|
@cli.group(help="Commands for managing and viewing W&B jobs")
|
@@ -1974,13 +1966,13 @@ def docker_run(ctx, docker_run_args):
|
|
1974
1966
|
|
1975
1967
|
See `docker run --help` for more details.
|
1976
1968
|
"""
|
1969
|
+
import wandb.docker
|
1970
|
+
|
1977
1971
|
api = InternalApi()
|
1978
1972
|
args = list(docker_run_args)
|
1979
1973
|
if len(args) > 0 and args[0] == "run":
|
1980
1974
|
args.pop(0)
|
1981
|
-
if len([a for a in args if a.startswith("--runtime")]) == 0 and
|
1982
|
-
"nvidia-docker"
|
1983
|
-
):
|
1975
|
+
if len([a for a in args if a.startswith("--runtime")]) == 0 and _HAS_NVIDIA_DOCKER:
|
1984
1976
|
args = ["--runtime", "nvidia"] + args
|
1985
1977
|
# TODO: image_from_docker_args uses heuristics to find the docker image arg, there are likely cases
|
1986
1978
|
# where this won't work
|
@@ -1989,13 +1981,13 @@ def docker_run(ctx, docker_run_args):
|
|
1989
1981
|
if image:
|
1990
1982
|
resolved_image = wandb.docker.image_id(image)
|
1991
1983
|
if resolved_image:
|
1992
|
-
args = ["-e", "WANDB_DOCKER={}"
|
1984
|
+
args = ["-e", f"WANDB_DOCKER={resolved_image}"] + args
|
1993
1985
|
else:
|
1994
1986
|
wandb.termlog(
|
1995
1987
|
"Couldn't detect image argument, running command without the WANDB_DOCKER env variable"
|
1996
1988
|
)
|
1997
1989
|
if api.api_key:
|
1998
|
-
args = ["-e", "WANDB_API_KEY={
|
1990
|
+
args = ["-e", f"WANDB_API_KEY={api.api_key}"] + args
|
1999
1991
|
else:
|
2000
1992
|
wandb.termlog(
|
2001
1993
|
"Not logged in, run `wandb login` from the host machine to enable result logging"
|
@@ -2009,7 +2001,7 @@ def docker_run(ctx, docker_run_args):
|
|
2009
2001
|
@click.argument("docker_image", required=False)
|
2010
2002
|
@click.option(
|
2011
2003
|
"--nvidia/--no-nvidia",
|
2012
|
-
default=
|
2004
|
+
default=_HAS_NVIDIA_DOCKER,
|
2013
2005
|
help="Use the nvidia runtime, defaults to nvidia if nvidia-docker is present",
|
2014
2006
|
)
|
2015
2007
|
@click.option(
|
@@ -2066,8 +2058,11 @@ def docker(
|
|
2066
2058
|
variable to an existing docker run command, see the wandb docker-run command.
|
2067
2059
|
"""
|
2068
2060
|
api = InternalApi()
|
2069
|
-
if not
|
2061
|
+
if not _HAS_DOCKER:
|
2070
2062
|
raise ClickException("Docker not installed, install it from https://docker.com")
|
2063
|
+
|
2064
|
+
import wandb.docker
|
2065
|
+
|
2071
2066
|
args = list(docker_run_args)
|
2072
2067
|
image = docker_image or ""
|
2073
2068
|
# remove run for users used to nvidia-docker
|
@@ -2086,17 +2081,13 @@ def docker(
|
|
2086
2081
|
resolved_image = wandb.docker.image_id(image)
|
2087
2082
|
if resolved_image is None:
|
2088
2083
|
raise ClickException(
|
2089
|
-
"Couldn't find image locally or in a registry, try running `docker pull {}`"
|
2090
|
-
image
|
2091
|
-
)
|
2084
|
+
f"Couldn't find image locally or in a registry, try running `docker pull {image}`"
|
2092
2085
|
)
|
2093
2086
|
if digest:
|
2094
2087
|
sys.stdout.write(resolved_image)
|
2095
2088
|
exit(0)
|
2096
2089
|
|
2097
|
-
existing = wandb.docker.shell(
|
2098
|
-
["ps", "-f", "ancestor={}".format(resolved_image), "-q"]
|
2099
|
-
)
|
2090
|
+
existing = wandb.docker.shell(["ps", "-f", f"ancestor={resolved_image}", "-q"])
|
2100
2091
|
if existing:
|
2101
2092
|
if click.confirm(
|
2102
2093
|
"Found running container with the same image, do you want to attach?"
|
@@ -2110,7 +2101,7 @@ def docker(
|
|
2110
2101
|
"-e",
|
2111
2102
|
"LANG=C.UTF-8",
|
2112
2103
|
"-e",
|
2113
|
-
"WANDB_DOCKER={}"
|
2104
|
+
f"WANDB_DOCKER={resolved_image}",
|
2114
2105
|
"--ipc=host",
|
2115
2106
|
"-v",
|
2116
2107
|
wandb.docker.entrypoint + ":/wandb-entrypoint.sh",
|
@@ -2123,7 +2114,7 @@ def docker(
|
|
2123
2114
|
# TODO: We should default to the working directory if defined
|
2124
2115
|
command.extend(["-v", cwd + ":" + dir, "-w", dir])
|
2125
2116
|
if api.api_key:
|
2126
|
-
command.extend(["-e", "WANDB_API_KEY={
|
2117
|
+
command.extend(["-e", f"WANDB_API_KEY={api.api_key}"])
|
2127
2118
|
else:
|
2128
2119
|
wandb.termlog(
|
2129
2120
|
"Couldn't find WANDB_API_KEY, run `wandb login` to enable streaming metrics"
|
@@ -2131,15 +2122,13 @@ def docker(
|
|
2131
2122
|
if jupyter:
|
2132
2123
|
command.extend(["-e", "WANDB_ENSURE_JUPYTER=1", "-p", port + ":8888"])
|
2133
2124
|
no_tty = True
|
2134
|
-
cmd = "jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir {}"
|
2135
|
-
dir
|
2136
|
-
)
|
2125
|
+
cmd = f"jupyter lab --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir {dir}"
|
2137
2126
|
command.extend(args)
|
2138
2127
|
if no_tty:
|
2139
2128
|
command.extend([image, shell, "-c", cmd])
|
2140
2129
|
else:
|
2141
2130
|
if cmd:
|
2142
|
-
command.extend(["-e", "WANDB_COMMAND={}"
|
2131
|
+
command.extend(["-e", f"WANDB_COMMAND={cmd}"])
|
2143
2132
|
command.extend(["-it", image, shell])
|
2144
2133
|
wandb.termlog("Launching docker container \U0001f6a2")
|
2145
2134
|
subprocess.call(command)
|
@@ -2199,8 +2188,11 @@ def server():
|
|
2199
2188
|
@display_error
|
2200
2189
|
def start(ctx, port, env, daemon, upgrade, edge):
|
2201
2190
|
api = InternalApi()
|
2202
|
-
if not
|
2191
|
+
if not _HAS_DOCKER:
|
2203
2192
|
raise ClickException("Docker not installed, install it from https://docker.com")
|
2193
|
+
|
2194
|
+
import wandb.docker
|
2195
|
+
|
2204
2196
|
local_image_sha = wandb.docker.image_id("wandb/local").split("wandb/local")[-1]
|
2205
2197
|
registry_image_sha = wandb.docker.image_id_from_registry("wandb/local").split(
|
2206
2198
|
"wandb/local"
|
@@ -2213,7 +2205,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
|
|
2213
2205
|
"A new version of the W&B server is available, upgrade by calling `wandb server start --upgrade`"
|
2214
2206
|
)
|
2215
2207
|
running = subprocess.check_output(
|
2216
|
-
["docker", "ps", "--filter", "name
|
2208
|
+
["docker", "ps", "--filter", "name=^wandb-local$", "--format", "{{.ID}}"]
|
2217
2209
|
)
|
2218
2210
|
if running != b"":
|
2219
2211
|
if upgrade:
|
@@ -2225,7 +2217,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
|
|
2225
2217
|
exit(1)
|
2226
2218
|
image = "docker.pkg.github.com/wandb/core/local" if edge else "wandb/local"
|
2227
2219
|
username = getpass.getuser()
|
2228
|
-
env_vars = ["-e", "LOCAL_USERNAME={}"
|
2220
|
+
env_vars = ["-e", f"LOCAL_USERNAME={username}"]
|
2229
2221
|
for e in env:
|
2230
2222
|
env_vars.append("-e")
|
2231
2223
|
env_vars.append(e)
|
@@ -2259,9 +2251,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
|
|
2259
2251
|
)
|
2260
2252
|
exit(1)
|
2261
2253
|
else:
|
2262
|
-
wandb.termlog(
|
2263
|
-
"W&B server started at http://localhost:{} \U0001f680".format(port)
|
2264
|
-
)
|
2254
|
+
wandb.termlog(f"W&B server started at http://localhost:{port} \U0001f680")
|
2265
2255
|
wandb.termlog("You can stop the server by running `wandb server stop`")
|
2266
2256
|
if not api.api_key:
|
2267
2257
|
# Let the server start before potentially launching a browser
|
@@ -2271,7 +2261,7 @@ def start(ctx, port, env, daemon, upgrade, edge):
|
|
2271
2261
|
|
2272
2262
|
@server.command(context_settings=RUN_CONTEXT, help="Stop a local W&B server")
|
2273
2263
|
def stop():
|
2274
|
-
if not
|
2264
|
+
if not _HAS_DOCKER:
|
2275
2265
|
raise ClickException("Docker not installed, install it from https://docker.com")
|
2276
2266
|
subprocess.call(["docker", "stop", "wandb-local"])
|
2277
2267
|
|
@@ -2407,7 +2397,7 @@ def get(path, root, type):
|
|
2407
2397
|
)
|
2408
2398
|
artifact = public_api.artifact(full_path, type=type)
|
2409
2399
|
path = artifact.download(root=root)
|
2410
|
-
wandb.termlog("Artifact downloaded to {}"
|
2400
|
+
wandb.termlog(f"Artifact downloaded to {path}")
|
2411
2401
|
except ValueError:
|
2412
2402
|
raise ClickException("Unable to download artifact")
|
2413
2403
|
|
@@ -2434,12 +2424,7 @@ def ls(path, type):
|
|
2434
2424
|
)
|
2435
2425
|
latest = next(versions)
|
2436
2426
|
wandb.termlog(
|
2437
|
-
"{:<15s}{:<15s}{:>15s} {:<20s}"
|
2438
|
-
kind.type,
|
2439
|
-
latest.updated_at,
|
2440
|
-
util.to_human_size(latest.size),
|
2441
|
-
latest.name,
|
2442
|
-
)
|
2427
|
+
f"{kind.type:<15s}{latest.updated_at:<15s}{util.to_human_size(latest.size):>15s} {latest.name:<20s}"
|
2443
2428
|
)
|
2444
2429
|
|
2445
2430
|
|
@@ -2485,17 +2470,17 @@ def pull(run, project, entity):
|
|
2485
2470
|
|
2486
2471
|
for name in urls:
|
2487
2472
|
if api.file_current(name, urls[name]["md5"]):
|
2488
|
-
click.echo("File {} is up to date"
|
2473
|
+
click.echo(f"File {name} is up to date")
|
2489
2474
|
else:
|
2490
2475
|
length, response = api.download_file(urls[name]["url"])
|
2491
2476
|
# TODO: I had to add this because some versions in CI broke click.progressbar
|
2492
|
-
sys.stdout.write("File {}\r"
|
2477
|
+
sys.stdout.write(f"File {name}\r")
|
2493
2478
|
dirname = os.path.dirname(name)
|
2494
2479
|
if dirname != "":
|
2495
2480
|
filesystem.mkdir_exists_ok(dirname)
|
2496
2481
|
with click.progressbar(
|
2497
2482
|
length=length,
|
2498
|
-
label="File {}"
|
2483
|
+
label=f"File {name}",
|
2499
2484
|
fill_char=click.style("&", fg="green"),
|
2500
2485
|
) as bar:
|
2501
2486
|
with open(name, "wb") as f:
|
@@ -2541,8 +2526,8 @@ def restore(ctx, run, no_git, branch, project, entity):
|
|
2541
2526
|
)
|
2542
2527
|
repo = metadata.get("git", {}).get("repo")
|
2543
2528
|
image = metadata.get("docker")
|
2544
|
-
restore_message = """`wandb restore` needs to be run from the same git repository as the original run.
|
2545
|
-
Run `git clone {}` and restore from there or pass the --no-git flag."""
|
2529
|
+
restore_message = f"""`wandb restore` needs to be run from the same git repository as the original run.
|
2530
|
+
Run `git clone {repo}` and restore from there or pass the --no-git flag."""
|
2546
2531
|
if no_git:
|
2547
2532
|
commit = None
|
2548
2533
|
elif not api.git.enabled:
|
@@ -2587,21 +2572,17 @@ Run `git clone {}` and restore from there or pass the --no-git flag.""".format(r
|
|
2587
2572
|
else:
|
2588
2573
|
patch_path = None
|
2589
2574
|
|
2590
|
-
branch_name = "wandb/{}"
|
2575
|
+
branch_name = f"wandb/{run}"
|
2591
2576
|
if branch and branch_name not in api.git.repo.branches:
|
2592
2577
|
api.git.repo.git.checkout(commit, b=branch_name)
|
2593
|
-
wandb.termlog(
|
2594
|
-
"Created branch {}".format(click.style(branch_name, bold=True))
|
2595
|
-
)
|
2578
|
+
wandb.termlog(f"Created branch {click.style(branch_name, bold=True)}")
|
2596
2579
|
elif branch:
|
2597
2580
|
wandb.termlog(
|
2598
|
-
"Using existing branch, run `git branch -D {}` from master for a clean checkout"
|
2599
|
-
branch_name
|
2600
|
-
)
|
2581
|
+
f"Using existing branch, run `git branch -D {branch_name}` from master for a clean checkout"
|
2601
2582
|
)
|
2602
2583
|
api.git.repo.git.checkout(branch_name)
|
2603
2584
|
else:
|
2604
|
-
wandb.termlog("Checking out {} in detached mode"
|
2585
|
+
wandb.termlog(f"Checking out {commit} in detached mode")
|
2605
2586
|
api.git.repo.git.checkout(commit)
|
2606
2587
|
|
2607
2588
|
if patch_path:
|
@@ -2639,7 +2620,7 @@ Run `git clone {}` and restore from there or pass the --no-git flag.""".format(r
|
|
2639
2620
|
with open(config_path, "w") as f:
|
2640
2621
|
f.write(s)
|
2641
2622
|
|
2642
|
-
wandb.termlog("Restored config variables to {}"
|
2623
|
+
wandb.termlog(f"Restored config variables to {config_path}")
|
2643
2624
|
if image:
|
2644
2625
|
if not metadata["program"].startswith("<") and metadata.get("args") is not None:
|
2645
2626
|
# TODO: we may not want to default to python here.
|
wandb/docker/__init__.py
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
import json
|
2
2
|
import logging
|
3
3
|
import os
|
4
|
+
import shutil
|
4
5
|
import subprocess
|
5
6
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
6
7
|
|
7
|
-
import
|
8
|
-
from dockerpycreds.utils import find_executable # type: ignore
|
9
|
-
|
10
|
-
from wandb.docker import auth, www_authenticate
|
8
|
+
from wandb.docker import names
|
11
9
|
from wandb.errors import Error
|
12
10
|
|
13
11
|
|
@@ -46,7 +44,6 @@ class DockerError(Error):
|
|
46
44
|
entrypoint = os.path.join(
|
47
45
|
os.path.dirname(os.path.abspath(__file__)), "wandb-entrypoint.sh"
|
48
46
|
)
|
49
|
-
auth_config = auth.load_config()
|
50
47
|
log = logging.getLogger(__name__)
|
51
48
|
|
52
49
|
|
@@ -74,7 +71,7 @@ def is_buildx_installed() -> bool:
|
|
74
71
|
global _buildx_installed
|
75
72
|
if _buildx_installed is not None:
|
76
73
|
return _buildx_installed # type: ignore
|
77
|
-
if not
|
74
|
+
if not shutil.which("docker"):
|
78
75
|
_buildx_installed = False
|
79
76
|
else:
|
80
77
|
help_output = shell(["buildx", "--help"])
|
@@ -203,7 +200,7 @@ def default_image(gpu: bool = False) -> str:
|
|
203
200
|
tag = "all"
|
204
201
|
if not gpu:
|
205
202
|
tag += "-cpu"
|
206
|
-
return "wandb/deepo:{}"
|
203
|
+
return f"wandb/deepo:{tag}"
|
207
204
|
|
208
205
|
|
209
206
|
def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:
|
@@ -218,75 +215,25 @@ def parse_repository_tag(repo_name: str) -> Tuple[str, Optional[str]]:
|
|
218
215
|
|
219
216
|
def parse(image_name: str) -> Tuple[str, str, str]:
|
220
217
|
repository, tag = parse_repository_tag(image_name)
|
221
|
-
registry, repo_name =
|
218
|
+
registry, repo_name = names.resolve_repository_name(repository)
|
222
219
|
if registry == "docker.io":
|
223
220
|
registry = "index.docker.io"
|
224
221
|
return registry, repo_name, (tag or "latest")
|
225
222
|
|
226
223
|
|
227
|
-
def
|
228
|
-
"""
|
229
|
-
|
230
|
-
Always returns a dictionary, if there's no token key we couldn't authenticate
|
231
|
-
"""
|
232
|
-
# TODO: Cache tokens?
|
233
|
-
auth_info = auth_config.resolve_authconfig(registry)
|
234
|
-
if auth_info:
|
235
|
-
normalized = {k.lower(): v for k, v in auth_info.items()}
|
236
|
-
normalized_auth_info = (
|
237
|
-
normalized.get("username"),
|
238
|
-
normalized.get("password"),
|
239
|
-
)
|
240
|
-
else:
|
241
|
-
normalized_auth_info = None
|
242
|
-
response = requests.get(f"https://{registry}/v2/", timeout=3)
|
243
|
-
if response.headers.get("www-authenticate"):
|
244
|
-
try:
|
245
|
-
info: Dict = www_authenticate.parse(response.headers["www-authenticate"])
|
246
|
-
except ValueError:
|
247
|
-
info = {}
|
248
|
-
else:
|
249
|
-
log.error(
|
250
|
-
f"Received {response} when attempting to authenticate with {registry}"
|
251
|
-
)
|
252
|
-
info = {}
|
253
|
-
if info.get("bearer"):
|
254
|
-
res = requests.get(
|
255
|
-
info["bearer"]["realm"]
|
256
|
-
+ "?service={}&scope=repository:{}:pull".format(
|
257
|
-
info["bearer"]["service"], repo
|
258
|
-
),
|
259
|
-
auth=normalized_auth_info, # type: ignore
|
260
|
-
timeout=3,
|
261
|
-
)
|
262
|
-
res.raise_for_status()
|
263
|
-
result_json: Dict[str, str] = res.json()
|
264
|
-
return result_json
|
265
|
-
return {}
|
224
|
+
def image_id_from_registry(image_name: str) -> Optional[str]:
|
225
|
+
"""Query the image manifest to get its full ID including the digest.
|
266
226
|
|
227
|
+
Args:
|
228
|
+
image_name: The image name, such as "wandb/local".
|
267
229
|
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
if registry == "index.docker.io":
|
276
|
-
registry = "registry-1.docker.io"
|
277
|
-
res = requests.head(
|
278
|
-
f"https://{registry}/v2/{repository}/manifests/{tag}",
|
279
|
-
headers={
|
280
|
-
"Authorization": f"Bearer {token}",
|
281
|
-
"Accept": "application/vnd.docker.distribution.manifest.v2+json",
|
282
|
-
},
|
283
|
-
timeout=5,
|
284
|
-
)
|
285
|
-
res.raise_for_status()
|
286
|
-
except requests.RequestException:
|
287
|
-
log.error(f"Received {res} when attempting to get digest for {image_name}")
|
288
|
-
return None
|
289
|
-
return "@".join([registry + "/" + repository, res.headers["Docker-Content-Digest"]])
|
230
|
+
Returns:
|
231
|
+
The image name followed by its digest, like "wandb/local@sha256:...".
|
232
|
+
"""
|
233
|
+
# https://docs.docker.com/reference/cli/docker/buildx/imagetools/inspect
|
234
|
+
inspect_cmd = ["buildx", "imagetools", "inspect", image_name]
|
235
|
+
format_args = ["--format", r"{{.Name}}@{{.Manifest.Digest}}"]
|
236
|
+
return shell([*inspect_cmd, *format_args])
|
290
237
|
|
291
238
|
|
292
239
|
def image_id(image_name: str) -> Optional[str]:
|
@@ -295,11 +242,12 @@ def image_id(image_name: str) -> Optional[str]:
|
|
295
242
|
return image_name
|
296
243
|
else:
|
297
244
|
digests = shell(["inspect", image_name, "--format", "{{json .RepoDigests}}"])
|
245
|
+
|
246
|
+
if digests is None:
|
247
|
+
return image_id_from_registry(image_name)
|
248
|
+
|
298
249
|
try:
|
299
|
-
|
300
|
-
raise ValueError
|
301
|
-
im_id: str = json.loads(digests)[0]
|
302
|
-
return im_id
|
250
|
+
return json.loads(digests)[0]
|
303
251
|
except (ValueError, IndexError):
|
304
252
|
return image_id_from_registry(image_name)
|
305
253
|
|
@@ -332,7 +280,6 @@ __all__ = [
|
|
332
280
|
"image_id",
|
333
281
|
"image_id_from_registry",
|
334
282
|
"is_docker_installed",
|
335
|
-
"auth_token",
|
336
283
|
"parse",
|
337
284
|
"parse_repository_tag",
|
338
285
|
"default_image",
|
wandb/docker/names.py
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
|
4
|
+
class InvalidRepositoryError(Exception):
|
5
|
+
"""The given string is not a valid repository name."""
|
6
|
+
|
7
|
+
|
8
|
+
def resolve_repository_name(repo_name: str) -> tuple[str, str]:
|
9
|
+
if "://" in repo_name:
|
10
|
+
raise InvalidRepositoryError(
|
11
|
+
f"Repository name cannot contain a scheme ({repo_name})"
|
12
|
+
)
|
13
|
+
|
14
|
+
index_name, remote_name = split_repo_name(repo_name)
|
15
|
+
if index_name[0] == "-" or index_name[-1] == "-":
|
16
|
+
raise InvalidRepositoryError(
|
17
|
+
f"Invalid index name ({index_name}). Cannot begin or end with a hyphen."
|
18
|
+
)
|
19
|
+
return resolve_index_name(index_name), remote_name
|
20
|
+
|
21
|
+
|
22
|
+
def resolve_index_name(index_name: str) -> str:
|
23
|
+
index_name = convert_to_hostname(index_name)
|
24
|
+
if index_name == "index.docker.io":
|
25
|
+
index_name = "docker.io"
|
26
|
+
return index_name
|
27
|
+
|
28
|
+
|
29
|
+
def split_repo_name(repo_name: str) -> tuple[str, str]:
|
30
|
+
parts = repo_name.split("/", 1)
|
31
|
+
if len(parts) == 1 or (
|
32
|
+
"." not in parts[0] and ":" not in parts[0] and parts[0] != "localhost"
|
33
|
+
):
|
34
|
+
# This is a docker index repo (ex: username/foobar or ubuntu)
|
35
|
+
return "docker.io", repo_name
|
36
|
+
return parts[0], parts[1]
|
37
|
+
|
38
|
+
|
39
|
+
def convert_to_hostname(url: str) -> str:
|
40
|
+
return url.replace("http://", "").replace("https://", "").split("/", 1)[0]
|
wandb/env.py
CHANGED
@@ -78,7 +78,6 @@ ARTIFACT_FETCH_FILE_URL_BATCH_SIZE = "WANDB_ARTIFACT_FETCH_FILE_URL_BATCH_SIZE"
|
|
78
78
|
CACHE_DIR = "WANDB_CACHE_DIR"
|
79
79
|
DISABLE_SSL = "WANDB_INSECURE_DISABLE_SSL"
|
80
80
|
SERVICE = "WANDB_SERVICE"
|
81
|
-
_DISABLE_SERVICE = "WANDB_DISABLE_SERVICE"
|
82
81
|
SENTRY_DSN = "WANDB_SENTRY_DSN"
|
83
82
|
INIT_TIMEOUT = "WANDB_INIT_TIMEOUT"
|
84
83
|
GIT_COMMIT = "WANDB_GIT_COMMIT"
|
wandb/errors/util.py
CHANGED
@@ -47,7 +47,7 @@ class ProtobufErrorHandler:
|
|
47
47
|
The corresponding protobuf error message.
|
48
48
|
"""
|
49
49
|
if not isinstance(exc, Error):
|
50
|
-
raise
|
50
|
+
raise TypeError("exc must be a subclass of wandb.errors.Error")
|
51
51
|
|
52
52
|
code = None
|
53
53
|
for subclass in type(exc).__mro__:
|
wandb/filesync/step_checksum.py
CHANGED
wandb/filesync/step_upload.py
CHANGED
@@ -176,7 +176,7 @@ class StepUpload:
|
|
176
176
|
self._artifacts[event.artifact_id]["pending_count"] += 1
|
177
177
|
self._start_upload_job(event)
|
178
178
|
else:
|
179
|
-
raise
|
179
|
+
raise TypeError(f"Event has unexpected type: {event!s}")
|
180
180
|
|
181
181
|
def _start_upload_job(self, event: RequestUpload) -> None:
|
182
182
|
# Operations on a single backend file must be serialized. if
|
@@ -669,8 +669,7 @@ class DiffusersMultiModalPipelineResolver:
|
|
669
669
|
)
|
670
670
|
|
671
671
|
# Return the WandB loggable dict
|
672
|
-
|
673
|
-
return loggable_dict
|
672
|
+
return self.prepare_loggable_dict(pipeline, response, kwargs)
|
674
673
|
except Exception as e:
|
675
674
|
logger.warning(e)
|
676
675
|
return None
|
@@ -9,8 +9,7 @@ _gym_version_lt_0_26: Optional[bool] = None
|
|
9
9
|
_gymnasium_version_lt_1_0_0: Optional[bool] = None
|
10
10
|
|
11
11
|
_required_error_msg = (
|
12
|
-
"Couldn't import the gymnasium python package, "
|
13
|
-
"install with `pip install gymnasium`"
|
12
|
+
"Couldn't import the gymnasium python package, install with `pip install gymnasium`"
|
14
13
|
)
|
15
14
|
GymLib = Literal["gym", "gymnasium"]
|
16
15
|
|
@@ -40,11 +39,11 @@ def monitor():
|
|
40
39
|
else:
|
41
40
|
import gymnasium as gym # type: ignore
|
42
41
|
|
43
|
-
from
|
42
|
+
from packaging.version import parse
|
44
43
|
|
45
|
-
gym_lib_version =
|
46
|
-
_gym_version_lt_0_26 = gym_lib_version <
|
47
|
-
_gymnasium_version_lt_1_0_0 = gym_lib_version <
|
44
|
+
gym_lib_version = parse(gym.__version__)
|
45
|
+
_gym_version_lt_0_26 = gym_lib_version < parse("0.26.0")
|
46
|
+
_gymnasium_version_lt_1_0_0 = gym_lib_version < parse("1.0.0a1")
|
48
47
|
|
49
48
|
path = "path" # Default path
|
50
49
|
if gym_lib == "gymnasium" and not _gymnasium_version_lt_1_0_0:
|