wandb 0.19.7__py3-none-musllinux_1_2_aarch64.whl → 0.19.9__py3-none-musllinux_1_2_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +5 -1
- wandb/__init__.pyi +43 -9
- wandb/_pydantic/__init__.py +23 -0
- wandb/_pydantic/base.py +113 -0
- wandb/_pydantic/v1_compat.py +262 -0
- wandb/apis/paginator.py +82 -38
- wandb/apis/public/api.py +10 -64
- wandb/apis/public/artifacts.py +73 -17
- wandb/apis/public/files.py +2 -2
- wandb/apis/public/projects.py +3 -2
- wandb/apis/public/reports.py +2 -2
- wandb/apis/public/runs.py +19 -11
- wandb/bin/gpu_stats +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/data_types.py +1 -1
- wandb/filesync/dir_watcher.py +2 -1
- wandb/integration/metaflow/metaflow.py +19 -17
- wandb/integration/sacred/__init__.py +1 -1
- wandb/jupyter.py +18 -15
- wandb/proto/v3/wandb_internal_pb2.py +7 -3
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +3 -3
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v5/wandb_internal_pb2.py +3 -3
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +10 -10
- wandb/proto/wandb_deprecated.py +2 -0
- wandb/sdk/artifacts/_graphql_fragments.py +18 -20
- wandb/sdk/artifacts/_validators.py +1 -0
- wandb/sdk/artifacts/artifact.py +81 -46
- wandb/sdk/artifacts/artifact_saver.py +16 -2
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +23 -2
- wandb/sdk/backend/backend.py +16 -5
- wandb/sdk/data_types/audio.py +1 -3
- wandb/sdk/data_types/base_types/media.py +11 -4
- wandb/sdk/data_types/image.py +44 -25
- wandb/sdk/data_types/molecule.py +1 -5
- wandb/sdk/data_types/object_3d.py +2 -1
- wandb/sdk/data_types/saved_model.py +7 -9
- wandb/sdk/data_types/video.py +1 -4
- wandb/sdk/interface/interface.py +65 -43
- wandb/sdk/interface/interface_queue.py +0 -7
- wandb/sdk/interface/interface_relay.py +6 -16
- wandb/sdk/interface/interface_shared.py +47 -40
- wandb/sdk/interface/interface_sock.py +1 -8
- wandb/sdk/interface/router.py +22 -54
- wandb/sdk/interface/router_queue.py +11 -10
- wandb/sdk/interface/router_relay.py +24 -12
- wandb/sdk/interface/router_sock.py +6 -11
- wandb/{apis/public → sdk/internal}/_generated/__init__.py +0 -6
- wandb/sdk/internal/_generated/base.py +226 -0
- wandb/{apis/public → sdk/internal}/_generated/server_features_query.py +3 -3
- wandb/{apis/public → sdk/internal}/_generated/typing_compat.py +1 -1
- wandb/sdk/internal/internal_api.py +138 -47
- wandb/sdk/internal/sender.py +5 -1
- wandb/sdk/internal/sender_config.py +8 -11
- wandb/sdk/internal/settings_static.py +24 -2
- wandb/sdk/lib/apikey.py +15 -16
- wandb/sdk/lib/console_capture.py +172 -0
- wandb/sdk/lib/redirect.py +102 -76
- wandb/sdk/lib/run_moment.py +4 -6
- wandb/sdk/lib/service_connection.py +37 -17
- wandb/sdk/lib/sock_client.py +2 -52
- wandb/sdk/lib/wb_logging.py +161 -0
- wandb/sdk/mailbox/__init__.py +3 -3
- wandb/sdk/mailbox/mailbox.py +31 -17
- wandb/sdk/mailbox/mailbox_handle.py +127 -0
- wandb/sdk/mailbox/{handles.py → response_handle.py} +34 -66
- wandb/sdk/mailbox/wait_with_progress.py +16 -15
- wandb/sdk/service/server_sock.py +4 -2
- wandb/sdk/service/streams.py +10 -5
- wandb/sdk/wandb_config.py +44 -43
- wandb/sdk/wandb_init.py +151 -92
- wandb/sdk/wandb_metadata.py +107 -91
- wandb/sdk/wandb_run.py +160 -54
- wandb/sdk/wandb_settings.py +410 -202
- wandb/sdk/wandb_setup.py +3 -1
- wandb/sdk/wandb_sync.py +1 -7
- {wandb-0.19.7.dist-info → wandb-0.19.9.dist-info}/METADATA +3 -3
- {wandb-0.19.7.dist-info → wandb-0.19.9.dist-info}/RECORD +644 -640
- wandb/apis/public/_generated/base.py +0 -128
- wandb/sdk/interface/message_future.py +0 -27
- wandb/sdk/interface/message_future_poll.py +0 -50
- /wandb/{apis/public → sdk/internal}/_generated/enums.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/input_types.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/operations.py +0 -0
- {wandb-0.19.7.dist-info → wandb-0.19.9.dist-info}/WHEEL +0 -0
- {wandb-0.19.7.dist-info → wandb-0.19.9.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.7.dist-info → wandb-0.19.9.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_settings.py
CHANGED
@@ -11,30 +11,28 @@ import re
|
|
11
11
|
import shutil
|
12
12
|
import socket
|
13
13
|
import sys
|
14
|
-
import tempfile
|
15
14
|
from datetime import datetime
|
16
|
-
from typing import Any, Literal, Sequence
|
17
|
-
from urllib.parse import quote, unquote, urlencode
|
18
15
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
# Optional and Union are used for type hinting instead of | because
|
17
|
+
# the latter is not supported in pydantic<2.6 and Python<3.10.
|
18
|
+
# Dict, List, and Tuple are used for backwards compatibility
|
19
|
+
# with pydantic v1 and Python<3.9.
|
20
|
+
from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union
|
21
|
+
from urllib.parse import quote, unquote, urlencode
|
23
22
|
|
24
23
|
from google.protobuf.wrappers_pb2 import BoolValue, DoubleValue, Int32Value, StringValue
|
25
|
-
from pydantic import
|
24
|
+
from pydantic import BaseModel, ConfigDict, Field
|
25
|
+
from typing_extensions import Self
|
26
|
+
|
27
|
+
import wandb
|
28
|
+
from wandb import env, termwarn, util
|
29
|
+
from wandb._pydantic import (
|
30
|
+
IS_PYDANTIC_V2,
|
26
31
|
AliasChoices,
|
27
|
-
BaseModel,
|
28
|
-
ConfigDict,
|
29
|
-
Field,
|
30
32
|
computed_field,
|
31
33
|
field_validator,
|
32
34
|
model_validator,
|
33
35
|
)
|
34
|
-
from pydantic_core import SchemaValidator, core_schema
|
35
|
-
|
36
|
-
import wandb
|
37
|
-
from wandb import env, termwarn, util
|
38
36
|
from wandb.errors import UsageError
|
39
37
|
from wandb.proto import wandb_settings_pb2
|
40
38
|
|
@@ -42,6 +40,117 @@ from .lib import apikey, credentials, ipython
|
|
42
40
|
from .lib.gitlib import GitRepo
|
43
41
|
from .lib.run_moment import RunMoment
|
44
42
|
|
43
|
+
validate_url: Callable[[str], None]
|
44
|
+
|
45
|
+
if IS_PYDANTIC_V2:
|
46
|
+
from pydantic_core import SchemaValidator, core_schema
|
47
|
+
|
48
|
+
def validate_url(url: str) -> None:
|
49
|
+
"""Validate a URL string."""
|
50
|
+
url_validator = SchemaValidator(
|
51
|
+
core_schema.url_schema(
|
52
|
+
allowed_schemes=["http", "https"],
|
53
|
+
strict=True,
|
54
|
+
)
|
55
|
+
)
|
56
|
+
url_validator.validate_python(url)
|
57
|
+
else:
|
58
|
+
from pydantic import root_validator
|
59
|
+
|
60
|
+
def validate_url(url: str) -> None:
|
61
|
+
"""Validate the base url of the wandb server.
|
62
|
+
|
63
|
+
param value: URL to validate
|
64
|
+
|
65
|
+
Based on the Django URLValidator, but with a few additional checks.
|
66
|
+
|
67
|
+
Copyright (c) Django Software Foundation and individual contributors.
|
68
|
+
All rights reserved.
|
69
|
+
|
70
|
+
Redistribution and use in source and binary forms, with or without modification,
|
71
|
+
are permitted provided that the following conditions are met:
|
72
|
+
|
73
|
+
1. Redistributions of source code must retain the above copyright notice,
|
74
|
+
this list of conditions and the following disclaimer.
|
75
|
+
|
76
|
+
2. Redistributions in binary form must reproduce the above copyright
|
77
|
+
notice, this list of conditions and the following disclaimer in the
|
78
|
+
documentation and/or other materials provided with the distribution.
|
79
|
+
|
80
|
+
3. Neither the name of Django nor the names of its contributors may be used
|
81
|
+
to endorse or promote products derived from this software without
|
82
|
+
specific prior written permission.
|
83
|
+
|
84
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
85
|
+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
86
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
87
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
88
|
+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
89
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
90
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
91
|
+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
92
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
93
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
94
|
+
"""
|
95
|
+
from urllib.parse import urlparse, urlsplit
|
96
|
+
|
97
|
+
if url is None:
|
98
|
+
return
|
99
|
+
|
100
|
+
ul = "\u00a1-\uffff" # Unicode letters range (must not be a raw string).
|
101
|
+
|
102
|
+
# IP patterns
|
103
|
+
ipv4_re = (
|
104
|
+
r"(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)"
|
105
|
+
r"(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}"
|
106
|
+
)
|
107
|
+
ipv6_re = r"\[[0-9a-f:.]+\]" # (simple regex, validated later)
|
108
|
+
|
109
|
+
# Host patterns
|
110
|
+
hostname_re = (
|
111
|
+
r"[a-z" + ul + r"0-9](?:[a-z" + ul + r"0-9-]{0,61}[a-z" + ul + r"0-9])?"
|
112
|
+
)
|
113
|
+
# Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
|
114
|
+
domain_re = r"(?:\.(?!-)[a-z" + ul + r"0-9-]{1,63}(?<!-))*"
|
115
|
+
tld_re = (
|
116
|
+
r"\." # dot
|
117
|
+
r"(?!-)" # can't start with a dash
|
118
|
+
r"(?:[a-z" + ul + "-]{2,63}" # domain label
|
119
|
+
r"|xn--[a-z0-9]{1,59})" # or punycode label
|
120
|
+
r"(?<!-)" # can't end with a dash
|
121
|
+
r"\.?" # may have a trailing dot
|
122
|
+
)
|
123
|
+
# host_re = "(" + hostname_re + domain_re + tld_re + "|localhost)"
|
124
|
+
# todo?: allow hostname to be just a hostname (no tld)?
|
125
|
+
host_re = "(" + hostname_re + domain_re + f"({tld_re})?" + "|localhost)"
|
126
|
+
|
127
|
+
regex = re.compile(
|
128
|
+
r"^(?:[a-z0-9.+-]*)://" # scheme is validated separately
|
129
|
+
r"(?:[^\s:@/]+(?::[^\s:@/]*)?@)?" # user:pass authentication
|
130
|
+
r"(?:" + ipv4_re + "|" + ipv6_re + "|" + host_re + ")"
|
131
|
+
r"(?::[0-9]{1,5})?" # port
|
132
|
+
r"(?:[/?#][^\s]*)?" # resource path
|
133
|
+
r"\Z",
|
134
|
+
re.IGNORECASE,
|
135
|
+
)
|
136
|
+
schemes = {"http", "https"}
|
137
|
+
unsafe_chars = frozenset("\t\r\n")
|
138
|
+
|
139
|
+
scheme = url.split("://")[0].lower()
|
140
|
+
split_url = urlsplit(url)
|
141
|
+
parsed_url = urlparse(url)
|
142
|
+
|
143
|
+
if parsed_url.netloc == "":
|
144
|
+
raise ValueError(f"Invalid URL: {url}")
|
145
|
+
elif unsafe_chars.intersection(url):
|
146
|
+
raise ValueError("URL cannot contain unsafe characters")
|
147
|
+
elif scheme not in schemes:
|
148
|
+
raise ValueError("URL must start with `http(s)://`")
|
149
|
+
elif not regex.search(url):
|
150
|
+
raise ValueError(f"{url} is not a valid server address")
|
151
|
+
elif split_url.hostname is None or len(split_url.hostname) > 253:
|
152
|
+
raise ValueError("hostname is invalid")
|
153
|
+
|
45
154
|
|
46
155
|
def _path_convert(*args: str) -> str:
|
47
156
|
"""Join path and apply os.path.expanduser to it."""
|
@@ -86,7 +195,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
86
195
|
allow_val_change: bool = False
|
87
196
|
"""Flag to allow modification of `Config` values after they've been set."""
|
88
197
|
|
89
|
-
anonymous: Literal["allow", "must", "never"]
|
198
|
+
anonymous: Optional[Literal["allow", "must", "never"]] = None
|
90
199
|
"""Controls anonymous data logging.
|
91
200
|
|
92
201
|
Possible values are:
|
@@ -100,19 +209,19 @@ class Settings(BaseModel, validate_assignment=True):
|
|
100
209
|
signed-up user account.
|
101
210
|
"""
|
102
211
|
|
103
|
-
api_key: str
|
212
|
+
api_key: Optional[str] = None
|
104
213
|
"""The W&B API key."""
|
105
214
|
|
106
|
-
azure_account_url_to_access_key:
|
215
|
+
azure_account_url_to_access_key: Optional[Dict[str, str]] = None
|
107
216
|
"""Mapping of Azure account URLs to their corresponding access keys for Azure integration."""
|
108
217
|
|
109
218
|
base_url: str = "https://api.wandb.ai"
|
110
219
|
"""The URL of the W&B backend for data synchronization."""
|
111
220
|
|
112
|
-
code_dir: str
|
221
|
+
code_dir: Optional[str] = None
|
113
222
|
"""Directory containing the code to be tracked by W&B."""
|
114
223
|
|
115
|
-
config_paths: Sequence[str]
|
224
|
+
config_paths: Optional[Sequence[str]] = None
|
116
225
|
"""Paths to files to load configuration from into the `Config` object."""
|
117
226
|
|
118
227
|
console: Literal["auto", "off", "wrap", "redirect", "wrap_raw", "wrap_emu"] = Field(
|
@@ -156,57 +265,57 @@ class Settings(BaseModel, validate_assignment=True):
|
|
156
265
|
disable_job_creation: bool = True
|
157
266
|
"""Whether to disable the creation of a job artifact for W&B Launch."""
|
158
267
|
|
159
|
-
docker: str
|
268
|
+
docker: Optional[str] = None
|
160
269
|
"""The Docker image used to execute the script."""
|
161
270
|
|
162
|
-
email: str
|
271
|
+
email: Optional[str] = None
|
163
272
|
"""The email address of the user."""
|
164
273
|
|
165
|
-
entity: str
|
274
|
+
entity: Optional[str] = None
|
166
275
|
"""The W&B entity, such as a user or a team."""
|
167
276
|
|
168
|
-
organization: str
|
277
|
+
organization: Optional[str] = None
|
169
278
|
"""The W&B organization."""
|
170
279
|
|
171
280
|
force: bool = False
|
172
281
|
"""Whether to pass the `force` flag to `wandb.login()`."""
|
173
282
|
|
174
|
-
fork_from: RunMoment
|
283
|
+
fork_from: Optional[RunMoment] = None
|
175
284
|
"""Specifies a point in a previous execution of a run to fork from.
|
176
285
|
|
177
286
|
The point is defined by the run ID, a metric, and its value.
|
178
287
|
Currently, only the metric '_step' is supported.
|
179
288
|
"""
|
180
289
|
|
181
|
-
git_commit: str
|
290
|
+
git_commit: Optional[str] = None
|
182
291
|
"""The git commit hash to associate with the run."""
|
183
292
|
|
184
293
|
git_remote: str = "origin"
|
185
294
|
"""The git remote to associate with the run."""
|
186
295
|
|
187
|
-
git_remote_url: str
|
296
|
+
git_remote_url: Optional[str] = None
|
188
297
|
"""The URL of the git remote repository."""
|
189
298
|
|
190
|
-
git_root: str
|
299
|
+
git_root: Optional[str] = None
|
191
300
|
"""Root directory of the git repository."""
|
192
301
|
|
193
302
|
heartbeat_seconds: int = 30
|
194
303
|
"""Interval in seconds between heartbeat signals sent to the W&B servers."""
|
195
304
|
|
196
|
-
host: str
|
305
|
+
host: Optional[str] = None
|
197
306
|
"""Hostname of the machine running the script."""
|
198
307
|
|
199
|
-
http_proxy: str
|
308
|
+
http_proxy: Optional[str] = None
|
200
309
|
"""Custom proxy servers for http requests to W&B."""
|
201
310
|
|
202
|
-
https_proxy: str
|
311
|
+
https_proxy: Optional[str] = None
|
203
312
|
"""Custom proxy servers for https requests to W&B."""
|
204
313
|
|
205
314
|
# Path to file containing an identity token (JWT) for authentication.
|
206
|
-
identity_token_file: str
|
315
|
+
identity_token_file: Optional[str] = None
|
207
316
|
"""Path to file containing an identity token (JWT) for authentication."""
|
208
317
|
|
209
|
-
ignore_globs:
|
318
|
+
ignore_globs: Sequence[str] = ()
|
210
319
|
"""Unix glob patterns relative to `files_dir` specifying files to exclude from upload."""
|
211
320
|
|
212
321
|
init_timeout: float = 90.0
|
@@ -215,10 +324,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
215
324
|
insecure_disable_ssl: bool = False
|
216
325
|
"""Whether to insecurely disable SSL verification."""
|
217
326
|
|
218
|
-
job_name: str
|
327
|
+
job_name: Optional[str] = None
|
219
328
|
"""Name of the Launch job running the script."""
|
220
329
|
|
221
|
-
job_source: Literal["repo", "artifact", "image"]
|
330
|
+
job_source: Optional[Literal["repo", "artifact", "image"]] = None
|
222
331
|
"""Source type for Launch."""
|
223
332
|
|
224
333
|
label_disable: bool = False
|
@@ -227,10 +336,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
227
336
|
launch: bool = False
|
228
337
|
"""Flag to indicate if the run is being launched through W&B Launch."""
|
229
338
|
|
230
|
-
launch_config_path: str
|
339
|
+
launch_config_path: Optional[str] = None
|
231
340
|
"""Path to the launch configuration file."""
|
232
341
|
|
233
|
-
login_timeout: float
|
342
|
+
login_timeout: Optional[float] = None
|
234
343
|
"""Time in seconds to wait for login operations before timing out."""
|
235
344
|
|
236
345
|
mode: Literal["online", "offline", "dryrun", "disabled", "run", "shared"] = Field(
|
@@ -239,13 +348,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
239
348
|
)
|
240
349
|
"""The operating mode for W&B logging and synchronization."""
|
241
350
|
|
242
|
-
notebook_name: str
|
351
|
+
notebook_name: Optional[str] = None
|
243
352
|
"""Name of the notebook if running in a Jupyter-like environment."""
|
244
353
|
|
245
|
-
program: str
|
354
|
+
program: Optional[str] = None
|
246
355
|
"""Path to the script that created the run, if available."""
|
247
356
|
|
248
|
-
program_abspath: str
|
357
|
+
program_abspath: Optional[str] = None
|
249
358
|
"""The absolute path from the root repository directory to the script that
|
250
359
|
created the run.
|
251
360
|
|
@@ -253,26 +362,39 @@ class Settings(BaseModel, validate_assignment=True):
|
|
253
362
|
.git directory, if it exists. Otherwise, it's the current working directory.
|
254
363
|
"""
|
255
364
|
|
256
|
-
program_relpath: str
|
365
|
+
program_relpath: Optional[str] = None
|
257
366
|
"""The relative path to the script that created the run."""
|
258
367
|
|
259
|
-
project: str
|
368
|
+
project: Optional[str] = None
|
260
369
|
"""The W&B project ID."""
|
261
370
|
|
262
371
|
quiet: bool = False
|
263
372
|
"""Flag to suppress non-essential output."""
|
264
373
|
|
265
|
-
reinit:
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
374
|
+
reinit: Union[
|
375
|
+
Literal[
|
376
|
+
"default",
|
377
|
+
"return_previous",
|
378
|
+
"finish_previous",
|
379
|
+
],
|
380
|
+
bool,
|
381
|
+
] = "default"
|
382
|
+
"""What to do when `wandb.init()` is called while a run is active.
|
383
|
+
|
384
|
+
Options:
|
385
|
+
- "default": Use "finish_previous" in notebooks and "return_previous"
|
386
|
+
otherwise.
|
387
|
+
- "return_previous": Return the active run.
|
388
|
+
- "finish_previous": Finish the active run, then return a new one.
|
389
|
+
|
390
|
+
Can also be a boolean, but this is deprecated. False is the same as
|
391
|
+
"return_previous", and True is the same as "finish_previous".
|
270
392
|
"""
|
271
393
|
|
272
394
|
relogin: bool = False
|
273
395
|
"""Flag to force a new login attempt."""
|
274
396
|
|
275
|
-
resume: Literal["allow", "must", "never", "auto"]
|
397
|
+
resume: Optional[Literal["allow", "must", "never", "auto"]] = None
|
276
398
|
"""Specifies the resume behavior for the run.
|
277
399
|
|
278
400
|
The available options are:
|
@@ -290,7 +412,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
290
412
|
machine.
|
291
413
|
"""
|
292
414
|
|
293
|
-
resume_from: RunMoment
|
415
|
+
resume_from: Optional[RunMoment] = None
|
294
416
|
"""Specifies a point in a previous execution of a run to resume from.
|
295
417
|
|
296
418
|
The point is defined by the run ID, a metric, and its value.
|
@@ -309,31 +431,31 @@ class Settings(BaseModel, validate_assignment=True):
|
|
309
431
|
In particular, this is used to derive the wandb directory and the run directory.
|
310
432
|
"""
|
311
433
|
|
312
|
-
run_group: str
|
434
|
+
run_group: Optional[str] = None
|
313
435
|
"""Group identifier for related runs.
|
314
436
|
|
315
437
|
Used for grouping runs in the UI.
|
316
438
|
"""
|
317
439
|
|
318
|
-
run_id: str
|
440
|
+
run_id: Optional[str] = None
|
319
441
|
"""The ID of the run."""
|
320
442
|
|
321
|
-
run_job_type: str
|
443
|
+
run_job_type: Optional[str] = None
|
322
444
|
"""Type of job being run (e.g., training, evaluation)."""
|
323
445
|
|
324
|
-
run_name: str
|
446
|
+
run_name: Optional[str] = None
|
325
447
|
"""Human-readable name for the run."""
|
326
448
|
|
327
|
-
run_notes: str
|
449
|
+
run_notes: Optional[str] = None
|
328
450
|
"""Additional notes or description for the run."""
|
329
451
|
|
330
|
-
run_tags:
|
452
|
+
run_tags: Optional[Tuple[str, ...]] = None
|
331
453
|
"""Tags to associate with the run for organization and filtering."""
|
332
454
|
|
333
455
|
sagemaker_disable: bool = False
|
334
456
|
"""Flag to disable SageMaker-specific functionality."""
|
335
457
|
|
336
|
-
save_code: bool
|
458
|
+
save_code: Optional[bool] = None
|
337
459
|
"""Whether to save the code associated with the run."""
|
338
460
|
|
339
461
|
settings_system: str = Field(
|
@@ -343,10 +465,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
343
465
|
)
|
344
466
|
"""Path to the system-wide settings file."""
|
345
467
|
|
346
|
-
show_colors: bool
|
468
|
+
show_colors: Optional[bool] = None
|
347
469
|
"""Whether to use colored output in the console."""
|
348
470
|
|
349
|
-
show_emoji: bool
|
471
|
+
show_emoji: Optional[bool] = None
|
350
472
|
"""Whether to show emoji in the console output."""
|
351
473
|
|
352
474
|
show_errors: bool = True
|
@@ -361,10 +483,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
361
483
|
silent: bool = False
|
362
484
|
"""Flag to suppress all output."""
|
363
485
|
|
364
|
-
start_method: str
|
486
|
+
start_method: Optional[str] = None
|
365
487
|
"""Method to use for starting subprocesses."""
|
366
488
|
|
367
|
-
strict: bool
|
489
|
+
strict: Optional[bool] = None
|
368
490
|
"""Whether to enable strict mode for validation and error checking."""
|
369
491
|
|
370
492
|
summary_timeout: int = 60
|
@@ -373,10 +495,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
373
495
|
summary_warnings: int = 5 # TODO: kill this with fire
|
374
496
|
"""Maximum number of summary warnings to display."""
|
375
497
|
|
376
|
-
sweep_id: str
|
498
|
+
sweep_id: Optional[str] = None
|
377
499
|
"""Identifier of the sweep this run belongs to."""
|
378
500
|
|
379
|
-
sweep_param_path: str
|
501
|
+
sweep_param_path: Optional[str] = None
|
380
502
|
"""Path to the sweep parameters configuration."""
|
381
503
|
|
382
504
|
symlink: bool = Field(
|
@@ -384,13 +506,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
384
506
|
)
|
385
507
|
"""Whether to use symlinks (True by default except on Windows)."""
|
386
508
|
|
387
|
-
sync_tensorboard: bool
|
509
|
+
sync_tensorboard: Optional[bool] = None
|
388
510
|
"""Whether to synchronize TensorBoard logs with W&B."""
|
389
511
|
|
390
512
|
table_raise_on_max_row_limit_exceeded: bool = False
|
391
513
|
"""Whether to raise an exception when table row limits are exceeded."""
|
392
514
|
|
393
|
-
username: str
|
515
|
+
username: Optional[str] = None
|
394
516
|
"""Username."""
|
395
517
|
|
396
518
|
# Internal settings.
|
@@ -424,13 +546,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
424
546
|
x_disable_machine_info: bool = False
|
425
547
|
"""Flag to disable automatic machine info collection."""
|
426
548
|
|
427
|
-
x_executable: str
|
549
|
+
x_executable: Optional[str] = None
|
428
550
|
"""Path to the Python executable."""
|
429
551
|
|
430
|
-
x_extra_http_headers:
|
552
|
+
x_extra_http_headers: Optional[Dict[str, str]] = None
|
431
553
|
"""Additional headers to add to all outgoing HTTP requests."""
|
432
554
|
|
433
|
-
x_file_stream_max_bytes: int
|
555
|
+
x_file_stream_max_bytes: Optional[int] = None
|
434
556
|
"""An approximate maximum request size for the filestream API.
|
435
557
|
|
436
558
|
Its purpose is to prevent HTTP requests from failing due to
|
@@ -438,50 +560,50 @@ class Settings(BaseModel, validate_assignment=True):
|
|
438
560
|
requests will be slightly larger.
|
439
561
|
"""
|
440
562
|
|
441
|
-
x_file_stream_max_line_bytes: int
|
563
|
+
x_file_stream_max_line_bytes: Optional[int] = None
|
442
564
|
"""Maximum line length for filestream JSONL files."""
|
443
565
|
|
444
|
-
x_file_stream_transmit_interval: float
|
566
|
+
x_file_stream_transmit_interval: Optional[float] = None
|
445
567
|
"""Interval in seconds between filestream transmissions."""
|
446
568
|
|
447
569
|
# Filestream retry client configuration.
|
448
570
|
|
449
|
-
x_file_stream_retry_max: int
|
571
|
+
x_file_stream_retry_max: Optional[int] = None
|
450
572
|
"""Max number of retries for filestream operations."""
|
451
573
|
|
452
|
-
x_file_stream_retry_wait_min_seconds: float
|
574
|
+
x_file_stream_retry_wait_min_seconds: Optional[float] = None
|
453
575
|
"""Minimum wait time between retries for filestream operations."""
|
454
576
|
|
455
|
-
x_file_stream_retry_wait_max_seconds: float
|
577
|
+
x_file_stream_retry_wait_max_seconds: Optional[float] = None
|
456
578
|
"""Maximum wait time between retries for filestream operations."""
|
457
579
|
|
458
|
-
x_file_stream_timeout_seconds: float
|
580
|
+
x_file_stream_timeout_seconds: Optional[float] = None
|
459
581
|
"""Timeout in seconds for individual filestream HTTP requests."""
|
460
582
|
|
461
583
|
# file transfer retry client configuration
|
462
584
|
|
463
|
-
x_file_transfer_retry_max: int
|
585
|
+
x_file_transfer_retry_max: Optional[int] = None
|
464
586
|
"""Max number of retries for file transfer operations."""
|
465
587
|
|
466
|
-
x_file_transfer_retry_wait_min_seconds: float
|
588
|
+
x_file_transfer_retry_wait_min_seconds: Optional[float] = None
|
467
589
|
"""Minimum wait time between retries for file transfer operations."""
|
468
590
|
|
469
|
-
x_file_transfer_retry_wait_max_seconds: float
|
591
|
+
x_file_transfer_retry_wait_max_seconds: Optional[float] = None
|
470
592
|
"""Maximum wait time between retries for file transfer operations."""
|
471
593
|
|
472
|
-
x_file_transfer_timeout_seconds: float
|
594
|
+
x_file_transfer_timeout_seconds: Optional[float] = None
|
473
595
|
"""Timeout in seconds for individual file transfer HTTP requests."""
|
474
596
|
|
475
|
-
x_files_dir: str
|
597
|
+
x_files_dir: Optional[str] = None
|
476
598
|
"""Override setting for the computed files_dir.."""
|
477
599
|
|
478
|
-
x_flow_control_custom: bool
|
600
|
+
x_flow_control_custom: Optional[bool] = None
|
479
601
|
"""Flag indicating custom flow control for filestream.
|
480
602
|
|
481
603
|
TODO: Not implemented in wandb-core.
|
482
604
|
"""
|
483
605
|
|
484
|
-
x_flow_control_disabled: bool
|
606
|
+
x_flow_control_disabled: Optional[bool] = None
|
485
607
|
"""Flag indicating flow control is disabled for filestream.
|
486
608
|
|
487
609
|
TODO: Not implemented in wandb-core.
|
@@ -489,47 +611,47 @@ class Settings(BaseModel, validate_assignment=True):
|
|
489
611
|
|
490
612
|
# graphql retry client configuration
|
491
613
|
|
492
|
-
x_graphql_retry_max: int
|
614
|
+
x_graphql_retry_max: Optional[int] = None
|
493
615
|
"""Max number of retries for GraphQL operations."""
|
494
616
|
|
495
|
-
x_graphql_retry_wait_min_seconds: float
|
617
|
+
x_graphql_retry_wait_min_seconds: Optional[float] = None
|
496
618
|
"""Minimum wait time between retries for GraphQL operations."""
|
497
619
|
|
498
|
-
x_graphql_retry_wait_max_seconds: float
|
620
|
+
x_graphql_retry_wait_max_seconds: Optional[float] = None
|
499
621
|
"""Maximum wait time between retries for GraphQL operations."""
|
500
622
|
|
501
|
-
x_graphql_timeout_seconds: float
|
623
|
+
x_graphql_timeout_seconds: Optional[float] = None
|
502
624
|
"""Timeout in seconds for individual GraphQL requests."""
|
503
625
|
|
504
626
|
x_internal_check_process: float = 8.0
|
505
627
|
"""Interval for internal process health checks in seconds."""
|
506
628
|
|
507
|
-
x_jupyter_name: str
|
629
|
+
x_jupyter_name: Optional[str] = None
|
508
630
|
"""Name of the Jupyter notebook."""
|
509
631
|
|
510
|
-
x_jupyter_path: str
|
632
|
+
x_jupyter_path: Optional[str] = None
|
511
633
|
"""Path to the Jupyter notebook."""
|
512
634
|
|
513
|
-
x_jupyter_root: str
|
635
|
+
x_jupyter_root: Optional[str] = None
|
514
636
|
"""Root directory of the Jupyter notebook."""
|
515
637
|
|
516
|
-
x_label: str
|
638
|
+
x_label: Optional[str] = None
|
517
639
|
"""Label to assign to system metrics and console logs collected for the run.
|
518
640
|
|
519
641
|
This is used to group data by on the frontend and can be used to distinguish data
|
520
642
|
from different processes in a distributed training job.
|
521
643
|
"""
|
522
644
|
|
523
|
-
x_live_policy_rate_limit: int
|
645
|
+
x_live_policy_rate_limit: Optional[int] = None
|
524
646
|
"""Rate limit for live policy updates in seconds."""
|
525
647
|
|
526
|
-
x_live_policy_wait_time: int
|
648
|
+
x_live_policy_wait_time: Optional[int] = None
|
527
649
|
"""Wait time between live policy updates in seconds."""
|
528
650
|
|
529
651
|
x_log_level: int = logging.INFO
|
530
652
|
"""Logging level for internal operations."""
|
531
653
|
|
532
|
-
x_network_buffer: int
|
654
|
+
x_network_buffer: Optional[int] = None
|
533
655
|
"""Size of the network buffer used in flow control.
|
534
656
|
|
535
657
|
TODO: Not implemented in wandb-core.
|
@@ -545,14 +667,14 @@ class Settings(BaseModel, validate_assignment=True):
|
|
545
667
|
as the primary process handles the main logging.
|
546
668
|
"""
|
547
669
|
|
548
|
-
x_proxies:
|
670
|
+
x_proxies: Optional[Dict[str, str]] = None
|
549
671
|
"""Custom proxy servers for requests to W&B.
|
550
672
|
|
551
673
|
This is deprecated and will be removed in future versions.
|
552
674
|
Please use `http_proxy` and `https_proxy` instead.
|
553
675
|
"""
|
554
676
|
|
555
|
-
x_runqueue_item_id: str
|
677
|
+
x_runqueue_item_id: Optional[str] = None
|
556
678
|
"""ID of the Launch run queue item being processed."""
|
557
679
|
|
558
680
|
x_require_legacy_service: bool = False
|
@@ -561,28 +683,34 @@ class Settings(BaseModel, validate_assignment=True):
|
|
561
683
|
x_save_requirements: bool = True
|
562
684
|
"""Flag to save the requirements file."""
|
563
685
|
|
564
|
-
|
686
|
+
x_server_side_derived_summary: bool = False
|
687
|
+
"""Flag to delegate automatic computation of summary from history to the server.
|
688
|
+
|
689
|
+
This does not disable user-provided summary updates.
|
690
|
+
"""
|
691
|
+
|
692
|
+
x_service_transport: Optional[str] = None
|
565
693
|
"""Transport method for communication with the wandb service."""
|
566
694
|
|
567
695
|
x_service_wait: float = 30.0
|
568
696
|
"""Time in seconds to wait for the wandb-core internal service to start."""
|
569
697
|
|
570
|
-
x_start_time: float
|
698
|
+
x_start_time: Optional[float] = None
|
571
699
|
"""The start time of the run in seconds since the Unix epoch."""
|
572
700
|
|
573
701
|
x_stats_pid: int = os.getpid()
|
574
702
|
"""PID of the process that started the wandb-core process to collect system stats for."""
|
575
703
|
|
576
|
-
x_stats_sampling_interval: float = Field(default=
|
704
|
+
x_stats_sampling_interval: float = Field(default=15.0)
|
577
705
|
"""Sampling interval for the system monitor in seconds."""
|
578
706
|
|
579
|
-
x_stats_neuron_monitor_config_path: str
|
707
|
+
x_stats_neuron_monitor_config_path: Optional[str] = None
|
580
708
|
"""Path to the default config file for the neuron-monitor tool.
|
581
709
|
|
582
710
|
This is used to monitor AWS Trainium devices.
|
583
711
|
"""
|
584
712
|
|
585
|
-
x_stats_dcgm_exporter: str
|
713
|
+
x_stats_dcgm_exporter: Optional[str] = None
|
586
714
|
"""Endpoint to extract Nvidia DCGM metrics from.
|
587
715
|
|
588
716
|
Two options are supported:
|
@@ -596,12 +724,12 @@ class Settings(BaseModel, validate_assignment=True):
|
|
596
724
|
- TODO: `http://192.168.0.1:9400/metrics`.
|
597
725
|
"""
|
598
726
|
|
599
|
-
x_stats_open_metrics_endpoints:
|
727
|
+
x_stats_open_metrics_endpoints: Optional[Dict[str, str]] = None
|
600
728
|
"""OpenMetrics `/metrics` endpoints to monitor for system metrics."""
|
601
729
|
|
602
|
-
x_stats_open_metrics_filters:
|
603
|
-
None
|
604
|
-
|
730
|
+
x_stats_open_metrics_filters: Union[
|
731
|
+
Dict[str, Dict[str, str]], Sequence[str], None
|
732
|
+
] = None
|
605
733
|
"""Filter to apply to metrics collected from OpenMetrics `/metrics` endpoints.
|
606
734
|
|
607
735
|
Supports two formats:
|
@@ -609,17 +737,17 @@ class Settings(BaseModel, validate_assignment=True):
|
|
609
737
|
- ("metric regex pattern 1", "metric regex pattern 2", ...)
|
610
738
|
"""
|
611
739
|
|
612
|
-
x_stats_open_metrics_http_headers:
|
740
|
+
x_stats_open_metrics_http_headers: Optional[Dict[str, str]] = None
|
613
741
|
"""HTTP headers to add to OpenMetrics requests."""
|
614
742
|
|
615
|
-
x_stats_disk_paths: Sequence[str]
|
743
|
+
x_stats_disk_paths: Optional[Sequence[str]] = Field(
|
616
744
|
default_factory=lambda: ("/", "/System/Volumes/Data")
|
617
745
|
if platform.system() == "Darwin"
|
618
746
|
else ("/",)
|
619
747
|
)
|
620
748
|
"""System paths to monitor for disk usage."""
|
621
749
|
|
622
|
-
x_stats_gpu_device_ids: Sequence[int]
|
750
|
+
x_stats_gpu_device_ids: Optional[Sequence[int]] = None
|
623
751
|
"""GPU device indices to monitor.
|
624
752
|
|
625
753
|
If not set, captures metrics for all GPUs.
|
@@ -659,24 +787,39 @@ class Settings(BaseModel, validate_assignment=True):
|
|
659
787
|
new_values[key] = values[key]
|
660
788
|
return new_values
|
661
789
|
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
self.resume,
|
670
|
-
|
671
|
-
|
672
|
-
)
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
790
|
+
if IS_PYDANTIC_V2:
|
791
|
+
|
792
|
+
@model_validator(mode="after")
|
793
|
+
def validate_mutual_exclusion_of_branching_args(self) -> Self:
|
794
|
+
if (
|
795
|
+
sum(
|
796
|
+
o is not None
|
797
|
+
for o in [self.fork_from, self.resume, self.resume_from]
|
798
|
+
)
|
799
|
+
> 1
|
800
|
+
):
|
801
|
+
raise ValueError(
|
802
|
+
"`fork_from`, `resume`, or `resume_from` are mutually exclusive. "
|
803
|
+
"Please specify only one of them."
|
804
|
+
)
|
805
|
+
return self
|
806
|
+
else:
|
807
|
+
|
808
|
+
@root_validator(pre=False) # type: ignore [call-overload]
|
809
|
+
@classmethod
|
810
|
+
def validate_mutual_exclusion_of_branching_args(cls, values):
|
811
|
+
if (
|
812
|
+
sum(
|
813
|
+
values.get(o) is not None
|
814
|
+
for o in ["fork_from", "resume", "resume_from"]
|
815
|
+
)
|
816
|
+
> 1
|
817
|
+
):
|
818
|
+
raise ValueError(
|
819
|
+
"`fork_from`, `resume`, or `resume_from` are mutually exclusive. "
|
820
|
+
"Please specify only one of them."
|
821
|
+
)
|
822
|
+
return values
|
680
823
|
|
681
824
|
# Field validators.
|
682
825
|
|
@@ -701,7 +844,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
701
844
|
@field_validator("base_url", mode="after")
|
702
845
|
@classmethod
|
703
846
|
def validate_base_url(cls, value):
|
704
|
-
|
847
|
+
validate_url(value)
|
705
848
|
# wandb.ai-specific checks
|
706
849
|
if re.match(r".*wandb\.ai[^\.]*$", value) and "api." not in value:
|
707
850
|
# user might guess app.wandb.ai or wandb.ai is the default cloud server
|
@@ -722,13 +865,21 @@ class Settings(BaseModel, validate_assignment=True):
|
|
722
865
|
|
723
866
|
@field_validator("console", mode="after")
|
724
867
|
@classmethod
|
725
|
-
def validate_console(cls, value,
|
868
|
+
def validate_console(cls, value, values):
|
726
869
|
if value != "auto":
|
727
870
|
return value
|
871
|
+
|
872
|
+
if hasattr(values, "data"):
|
873
|
+
# pydantic v2
|
874
|
+
values = values.data
|
875
|
+
else:
|
876
|
+
# pydantic v1
|
877
|
+
values = values
|
878
|
+
|
728
879
|
if (
|
729
880
|
ipython.in_jupyter()
|
730
|
-
or (
|
731
|
-
or not
|
881
|
+
or (values.get("start_method") == "thread")
|
882
|
+
or not values.get("x_disable_service")
|
732
883
|
or platform.system() == "Windows"
|
733
884
|
):
|
734
885
|
value = "wrap"
|
@@ -761,12 +912,20 @@ class Settings(BaseModel, validate_assignment=True):
|
|
761
912
|
|
762
913
|
@field_validator("fork_from", mode="before")
|
763
914
|
@classmethod
|
764
|
-
def validate_fork_from(cls, value,
|
915
|
+
def validate_fork_from(cls, value, values) -> Optional[RunMoment]:
|
765
916
|
run_moment = cls._runmoment_preprocessor(value)
|
917
|
+
|
918
|
+
if hasattr(values, "data"):
|
919
|
+
# pydantic v2
|
920
|
+
values = values.data
|
921
|
+
else:
|
922
|
+
# pydantic v1
|
923
|
+
values = values
|
924
|
+
|
766
925
|
if (
|
767
926
|
run_moment
|
768
|
-
and
|
769
|
-
and
|
927
|
+
and values.get("run_id") is not None
|
928
|
+
and values.get("run_id") == run_moment.run
|
770
929
|
):
|
771
930
|
raise ValueError(
|
772
931
|
"Provided `run_id` is the same as the run to `fork_from`. "
|
@@ -780,7 +939,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
780
939
|
def validate_http_proxy(cls, value):
|
781
940
|
if value is None:
|
782
941
|
return None
|
783
|
-
|
942
|
+
validate_url(value)
|
784
943
|
return value.rstrip("/")
|
785
944
|
|
786
945
|
@field_validator("https_proxy", mode="after")
|
@@ -788,7 +947,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
788
947
|
def validate_https_proxy(cls, value):
|
789
948
|
if value is None:
|
790
949
|
return None
|
791
|
-
|
950
|
+
validate_url(value)
|
792
951
|
return value.rstrip("/")
|
793
952
|
|
794
953
|
@field_validator("ignore_globs", mode="after")
|
@@ -822,7 +981,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
822
981
|
|
823
982
|
@field_validator("project", mode="after")
|
824
983
|
@classmethod
|
825
|
-
def validate_project(cls, value,
|
984
|
+
def validate_project(cls, value, values):
|
826
985
|
if value is None:
|
827
986
|
return None
|
828
987
|
invalid_chars_list = list("/\\#?%:")
|
@@ -848,12 +1007,20 @@ class Settings(BaseModel, validate_assignment=True):
|
|
848
1007
|
|
849
1008
|
@field_validator("resume_from", mode="before")
|
850
1009
|
@classmethod
|
851
|
-
def validate_resume_from(cls, value,
|
1010
|
+
def validate_resume_from(cls, value, values) -> Optional[RunMoment]:
|
852
1011
|
run_moment = cls._runmoment_preprocessor(value)
|
1012
|
+
|
1013
|
+
if hasattr(values, "data"):
|
1014
|
+
# pydantic v2
|
1015
|
+
values = values.data
|
1016
|
+
else:
|
1017
|
+
# pydantic v1
|
1018
|
+
values = values
|
1019
|
+
|
853
1020
|
if (
|
854
1021
|
run_moment
|
855
|
-
and
|
856
|
-
and
|
1022
|
+
and values.get("run_id") is not None
|
1023
|
+
and values.get("run_id") != run_moment.run
|
857
1024
|
):
|
858
1025
|
raise ValueError(
|
859
1026
|
"Both `run_id` and `resume_from` have been specified with different ids."
|
@@ -870,7 +1037,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
870
1037
|
|
871
1038
|
@field_validator("run_id", mode="after")
|
872
1039
|
@classmethod
|
873
|
-
def validate_run_id(cls, value,
|
1040
|
+
def validate_run_id(cls, value, values):
|
874
1041
|
if value is None:
|
875
1042
|
return None
|
876
1043
|
|
@@ -896,7 +1063,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
896
1063
|
raise UsageError("Service wait time cannot be negative")
|
897
1064
|
return value
|
898
1065
|
|
899
|
-
@field_validator("start_method")
|
1066
|
+
@field_validator("start_method", mode="after")
|
900
1067
|
@classmethod
|
901
1068
|
def validate_start_method(cls, value):
|
902
1069
|
if value is None:
|
@@ -978,7 +1145,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
978
1145
|
|
979
1146
|
@computed_field # type: ignore[prop-decorator]
|
980
1147
|
@property
|
981
|
-
def _args(self) ->
|
1148
|
+
def _args(self) -> List[str]:
|
982
1149
|
if not self._jupyter:
|
983
1150
|
return sys.argv[1:]
|
984
1151
|
return []
|
@@ -1000,7 +1167,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1000
1167
|
|
1001
1168
|
@computed_field # type: ignore[prop-decorator]
|
1002
1169
|
@property
|
1003
|
-
def _code_path_local(self) -> str
|
1170
|
+
def _code_path_local(self) -> Optional[str]:
|
1004
1171
|
"""The relative path from the current working directory to the code path.
|
1005
1172
|
|
1006
1173
|
For example, if the code path is /home/user/project/example.py, and the
|
@@ -1083,12 +1250,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1083
1250
|
@computed_field # type: ignore[prop-decorator]
|
1084
1251
|
@property
|
1085
1252
|
def _tmp_code_dir(self) -> str:
|
1086
|
-
return _path_convert(
|
1087
|
-
self.wandb_dir,
|
1088
|
-
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1089
|
-
"tmp",
|
1090
|
-
"code",
|
1091
|
-
)
|
1253
|
+
return _path_convert(self.sync_dir, "tmp", "code")
|
1092
1254
|
|
1093
1255
|
@computed_field # type: ignore[prop-decorator]
|
1094
1256
|
@property
|
@@ -1097,7 +1259,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1097
1259
|
|
1098
1260
|
@computed_field # type: ignore[prop-decorator]
|
1099
1261
|
@property
|
1100
|
-
def colab_url(self) -> str
|
1262
|
+
def colab_url(self) -> Optional[str]:
|
1101
1263
|
"""The URL to the Colab notebook, if running in Colab."""
|
1102
1264
|
if not self._colab:
|
1103
1265
|
return None
|
@@ -1115,11 +1277,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1115
1277
|
@property
|
1116
1278
|
def files_dir(self) -> str:
|
1117
1279
|
"""Absolute path to the local directory where the run's files are stored."""
|
1118
|
-
return self.x_files_dir or _path_convert(
|
1119
|
-
self.wandb_dir,
|
1120
|
-
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1121
|
-
"files",
|
1122
|
-
)
|
1280
|
+
return self.x_files_dir or _path_convert(self.sync_dir, "files")
|
1123
1281
|
|
1124
1282
|
@computed_field # type: ignore[prop-decorator]
|
1125
1283
|
@property
|
@@ -1130,9 +1288,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1130
1288
|
@property
|
1131
1289
|
def log_dir(self) -> str:
|
1132
1290
|
"""The directory for storing log files."""
|
1133
|
-
return _path_convert(
|
1134
|
-
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", "logs"
|
1135
|
-
)
|
1291
|
+
return _path_convert(self.sync_dir, "logs")
|
1136
1292
|
|
1137
1293
|
@computed_field # type: ignore[prop-decorator]
|
1138
1294
|
@property
|
@@ -1213,7 +1369,8 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1213
1369
|
@property
|
1214
1370
|
def sync_dir(self) -> str:
|
1215
1371
|
return _path_convert(
|
1216
|
-
self.wandb_dir,
|
1372
|
+
self.wandb_dir,
|
1373
|
+
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1217
1374
|
)
|
1218
1375
|
|
1219
1376
|
@computed_field # type: ignore[prop-decorator]
|
@@ -1235,29 +1392,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1235
1392
|
@computed_field # type: ignore[prop-decorator]
|
1236
1393
|
@property
|
1237
1394
|
def wandb_dir(self) -> str:
|
1238
|
-
"""Full path to the wandb directory.
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1245
|
-
# We use the hidden version if it already exists, otherwise non-hidden.
|
1246
|
-
if os.path.exists(os.path.join(root_dir, ".wandb")):
|
1247
|
-
__stage_dir__ = ".wandb" + os.sep
|
1248
|
-
else:
|
1249
|
-
__stage_dir__ = "wandb" + os.sep
|
1250
|
-
|
1251
|
-
path = os.path.join(root_dir, __stage_dir__)
|
1252
|
-
if not os.access(root_dir or ".", os.W_OK):
|
1253
|
-
termwarn(
|
1254
|
-
f"Path {path} wasn't writable, using system temp directory.",
|
1255
|
-
repeat=False,
|
1256
|
-
)
|
1257
|
-
path = os.path.join(
|
1258
|
-
tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep)
|
1259
|
-
)
|
1260
|
-
|
1395
|
+
"""Full path to the wandb directory."""
|
1396
|
+
stage_dir = (
|
1397
|
+
".wandb" + os.sep
|
1398
|
+
if os.path.exists(os.path.join(self.root_dir, ".wandb"))
|
1399
|
+
else "wandb" + os.sep
|
1400
|
+
)
|
1401
|
+
path = os.path.join(self.root_dir, stage_dir)
|
1261
1402
|
return os.path.expanduser(path)
|
1262
1403
|
|
1263
1404
|
# Methods to collect and update settings from different sources.
|
@@ -1283,7 +1424,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1283
1424
|
if value is not None:
|
1284
1425
|
setattr(self, key, value)
|
1285
1426
|
|
1286
|
-
def update_from_env_vars(self, environ:
|
1427
|
+
def update_from_env_vars(self, environ: Dict[str, Any]):
|
1287
1428
|
"""Update settings from environment variables."""
|
1288
1429
|
env_prefix: str = "WANDB_"
|
1289
1430
|
private_env_prefix: str = env_prefix + "_"
|
@@ -1401,7 +1542,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1401
1542
|
|
1402
1543
|
self.program = program
|
1403
1544
|
|
1404
|
-
def update_from_dict(self, settings:
|
1545
|
+
def update_from_dict(self, settings: Dict[str, Any]) -> None:
|
1405
1546
|
"""Update settings from a dictionary."""
|
1406
1547
|
for key, value in dict(settings).items():
|
1407
1548
|
if value is not None:
|
@@ -1419,7 +1560,11 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1419
1560
|
"""Generate a protobuf representation of the settings."""
|
1420
1561
|
settings_proto = wandb_settings_pb2.Settings()
|
1421
1562
|
for k, v in self.model_dump(exclude_none=True).items():
|
1422
|
-
#
|
1563
|
+
# Client-only settings that don't exist on the protobuf.
|
1564
|
+
if k in ("reinit",):
|
1565
|
+
continue
|
1566
|
+
|
1567
|
+
# Special case for x_stats_open_metrics_filters.
|
1423
1568
|
if k == "x_stats_open_metrics_filters":
|
1424
1569
|
if isinstance(v, (list, set, tuple)):
|
1425
1570
|
setting = getattr(settings_proto, k)
|
@@ -1433,12 +1578,16 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1433
1578
|
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1434
1579
|
continue
|
1435
1580
|
|
1436
|
-
#
|
1581
|
+
# Special case for RunMoment fields.
|
1437
1582
|
if k in ("fork_from", "resume_from"):
|
1438
|
-
run_moment =
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1583
|
+
run_moment = (
|
1584
|
+
v
|
1585
|
+
if isinstance(v, RunMoment)
|
1586
|
+
else RunMoment(
|
1587
|
+
run=v.get("run"),
|
1588
|
+
value=v.get("value"),
|
1589
|
+
metric=v.get("metric"),
|
1590
|
+
)
|
1442
1591
|
)
|
1443
1592
|
getattr(settings_proto, k).CopyFrom(
|
1444
1593
|
wandb_settings_pb2.RunMoment(
|
@@ -1474,18 +1623,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1474
1623
|
|
1475
1624
|
return settings_proto
|
1476
1625
|
|
1477
|
-
|
1478
|
-
def validate_url(url: str) -> None:
|
1479
|
-
"""Validate a URL string."""
|
1480
|
-
url_validator = SchemaValidator(
|
1481
|
-
core_schema.url_schema(
|
1482
|
-
allowed_schemes=["http", "https"],
|
1483
|
-
strict=True,
|
1484
|
-
)
|
1485
|
-
)
|
1486
|
-
url_validator.validate_python(url)
|
1487
|
-
|
1488
|
-
def _get_program(self) -> str | None:
|
1626
|
+
def _get_program(self) -> Optional[str]:
|
1489
1627
|
"""Get the program that started the current process."""
|
1490
1628
|
if not self._jupyter:
|
1491
1629
|
# If not in a notebook, try to get the program from the environment
|
@@ -1515,7 +1653,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1515
1653
|
return self.x_jupyter_path
|
1516
1654
|
|
1517
1655
|
@staticmethod
|
1518
|
-
def _get_program_relpath(program: str, root: str
|
1656
|
+
def _get_program_relpath(program: str, root: Optional[str] = None) -> Optional[str]:
|
1519
1657
|
"""Get the relative path to the program from the root directory."""
|
1520
1658
|
if not program:
|
1521
1659
|
return None
|
@@ -1541,7 +1679,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1541
1679
|
parser = configparser.ConfigParser()
|
1542
1680
|
parser.add_section(section)
|
1543
1681
|
parser.read(file_name)
|
1544
|
-
config:
|
1682
|
+
config: Dict[str, Any] = dict()
|
1545
1683
|
for k in parser[section]:
|
1546
1684
|
config[k] = parser[section][k]
|
1547
1685
|
if k == "ignore_globs":
|
@@ -1567,9 +1705,79 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1567
1705
|
return f"?{urlencode({'apiKey': api_key})}"
|
1568
1706
|
|
1569
1707
|
@staticmethod
|
1570
|
-
def _runmoment_preprocessor(
|
1708
|
+
def _runmoment_preprocessor(
|
1709
|
+
val: Union[RunMoment, str, None],
|
1710
|
+
) -> Optional[RunMoment]:
|
1571
1711
|
"""Preprocess the setting for forking or resuming a run."""
|
1572
1712
|
if isinstance(val, RunMoment) or val is None:
|
1573
1713
|
return val
|
1574
1714
|
elif isinstance(val, str):
|
1575
1715
|
return RunMoment.from_uri(val)
|
1716
|
+
|
1717
|
+
if not IS_PYDANTIC_V2:
|
1718
|
+
|
1719
|
+
def model_copy(self, *args, **kwargs):
|
1720
|
+
return self.copy(*args, **kwargs)
|
1721
|
+
|
1722
|
+
def model_dump(self, **kwargs):
|
1723
|
+
"""Compatibility method for Pydantic v1 to mimic v2's model_dump.
|
1724
|
+
|
1725
|
+
In v1, this is equivalent to dict() but also includes computed properties.
|
1726
|
+
|
1727
|
+
Args:
|
1728
|
+
**kwargs: Options passed to the dict method
|
1729
|
+
- exclude_none: Whether to exclude fields with None values
|
1730
|
+
|
1731
|
+
Returns:
|
1732
|
+
A dictionary of the model's fields and computed properties
|
1733
|
+
"""
|
1734
|
+
# Handle exclude_none separately since it's named differently in v1
|
1735
|
+
exclude_none = kwargs.pop("exclude_none", False)
|
1736
|
+
|
1737
|
+
# Start with regular fields from dict()
|
1738
|
+
result = self.dict(**kwargs)
|
1739
|
+
|
1740
|
+
# Get all computed properties
|
1741
|
+
for name in dir(self.__class__):
|
1742
|
+
attr = getattr(self.__class__, name, None)
|
1743
|
+
if isinstance(attr, property):
|
1744
|
+
try:
|
1745
|
+
# Only include properties that don't raise errors
|
1746
|
+
value = getattr(self, name)
|
1747
|
+
result[name] = value
|
1748
|
+
except (AttributeError, NotImplementedError, TypeError, ValueError):
|
1749
|
+
# Skip properties that can't be accessed or raise errors
|
1750
|
+
pass
|
1751
|
+
elif isinstance(attr, RunMoment):
|
1752
|
+
value = getattr(self, name)
|
1753
|
+
result[name] = value
|
1754
|
+
|
1755
|
+
# Special Pydantic attributes that should always be excluded
|
1756
|
+
exclude_fields = {
|
1757
|
+
"model_config",
|
1758
|
+
"model_fields",
|
1759
|
+
"model_fields_set",
|
1760
|
+
"__fields__",
|
1761
|
+
"__model_fields_set",
|
1762
|
+
"__pydantic_self__",
|
1763
|
+
"__pydantic_initialised__",
|
1764
|
+
}
|
1765
|
+
|
1766
|
+
# Remove special Pydantic attributes
|
1767
|
+
for field in exclude_fields:
|
1768
|
+
if field in result:
|
1769
|
+
del result[field]
|
1770
|
+
|
1771
|
+
if exclude_none:
|
1772
|
+
# Remove None values from the result
|
1773
|
+
return {k: v for k, v in result.items() if v is not None}
|
1774
|
+
|
1775
|
+
return result
|
1776
|
+
|
1777
|
+
@property
|
1778
|
+
def model_fields_set(self) -> set:
|
1779
|
+
"""Return a set of fields that have been explicitly set.
|
1780
|
+
|
1781
|
+
This is a compatibility property for Pydantic v1 to mimic v2's model_fields_set.
|
1782
|
+
"""
|
1783
|
+
return getattr(self, "__fields_set__", set())
|