wandb 0.19.8__py3-none-any.whl → 0.19.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +5 -1
- wandb/__init__.pyi +15 -8
- wandb/_pydantic/__init__.py +30 -0
- wandb/_pydantic/base.py +148 -0
- wandb/_pydantic/utils.py +66 -0
- wandb/_pydantic/v1_compat.py +284 -0
- wandb/apis/paginator.py +82 -38
- wandb/apis/public/__init__.py +2 -2
- wandb/apis/public/api.py +111 -53
- wandb/apis/public/artifacts.py +387 -639
- wandb/apis/public/automations.py +69 -0
- wandb/apis/public/files.py +2 -2
- wandb/apis/public/integrations.py +168 -0
- wandb/apis/public/projects.py +32 -2
- wandb/apis/public/reports.py +2 -2
- wandb/apis/public/runs.py +19 -11
- wandb/apis/public/utils.py +107 -1
- wandb/automations/__init__.py +81 -0
- wandb/automations/_filters/__init__.py +40 -0
- wandb/automations/_filters/expressions.py +179 -0
- wandb/automations/_filters/operators.py +267 -0
- wandb/automations/_filters/run_metrics.py +183 -0
- wandb/automations/_generated/__init__.py +184 -0
- wandb/automations/_generated/create_filter_trigger.py +21 -0
- wandb/automations/_generated/create_generic_webhook_integration.py +43 -0
- wandb/automations/_generated/delete_trigger.py +19 -0
- wandb/automations/_generated/enums.py +33 -0
- wandb/automations/_generated/fragments.py +343 -0
- wandb/automations/_generated/generic_webhook_integrations_by_entity.py +22 -0
- wandb/automations/_generated/get_triggers.py +24 -0
- wandb/automations/_generated/get_triggers_by_entity.py +24 -0
- wandb/automations/_generated/input_types.py +104 -0
- wandb/automations/_generated/integrations_by_entity.py +22 -0
- wandb/automations/_generated/operations.py +710 -0
- wandb/automations/_generated/slack_integrations_by_entity.py +22 -0
- wandb/automations/_generated/update_filter_trigger.py +21 -0
- wandb/automations/_utils.py +123 -0
- wandb/automations/_validators.py +73 -0
- wandb/automations/actions.py +205 -0
- wandb/automations/automations.py +109 -0
- wandb/automations/events.py +235 -0
- wandb/automations/integrations.py +26 -0
- wandb/automations/scopes.py +76 -0
- wandb/beta/workflows.py +9 -10
- wandb/bin/gpu_stats +0 -0
- wandb/cli/cli.py +3 -3
- wandb/integration/keras/keras.py +2 -1
- wandb/integration/langchain/wandb_tracer.py +2 -1
- wandb/integration/metaflow/metaflow.py +19 -17
- wandb/integration/sacred/__init__.py +1 -1
- wandb/jupyter.py +155 -133
- wandb/old/summary.py +0 -2
- wandb/proto/v3/wandb_internal_pb2.py +297 -292
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +292 -292
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v5/wandb_internal_pb2.py +292 -292
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v6/wandb_base_pb2.py +41 -0
- wandb/proto/v6/wandb_internal_pb2.py +393 -0
- wandb/proto/v6/wandb_server_pb2.py +78 -0
- wandb/proto/v6/wandb_settings_pb2.py +58 -0
- wandb/proto/v6/wandb_telemetry_pb2.py +52 -0
- wandb/proto/wandb_base_pb2.py +2 -0
- wandb/proto/wandb_deprecated.py +10 -0
- wandb/proto/wandb_internal_pb2.py +3 -1
- wandb/proto/wandb_server_pb2.py +2 -0
- wandb/proto/wandb_settings_pb2.py +2 -0
- wandb/proto/wandb_telemetry_pb2.py +2 -0
- wandb/sdk/artifacts/_generated/__init__.py +248 -0
- wandb/sdk/artifacts/_generated/artifact_collection_membership_files.py +43 -0
- wandb/sdk/artifacts/_generated/artifact_version_files.py +36 -0
- wandb/sdk/artifacts/_generated/create_artifact_collection_tag_assignments.py +36 -0
- wandb/sdk/artifacts/_generated/delete_artifact_collection_tag_assignments.py +25 -0
- wandb/sdk/artifacts/_generated/delete_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/delete_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_generated/enums.py +17 -0
- wandb/sdk/artifacts/_generated/fragments.py +186 -0
- wandb/sdk/artifacts/_generated/input_types.py +16 -0
- wandb/sdk/artifacts/_generated/move_artifact_collection.py +35 -0
- wandb/sdk/artifacts/_generated/operations.py +510 -0
- wandb/sdk/artifacts/_generated/project_artifact_collection.py +101 -0
- wandb/sdk/artifacts/_generated/project_artifact_collections.py +33 -0
- wandb/sdk/artifacts/_generated/project_artifact_type.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifact_types.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifacts.py +42 -0
- wandb/sdk/artifacts/_generated/run_input_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/run_output_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/update_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/update_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_graphql_fragments.py +56 -81
- wandb/sdk/artifacts/_validators.py +1 -0
- wandb/sdk/artifacts/artifact.py +110 -49
- wandb/sdk/artifacts/artifact_manifest_entry.py +2 -1
- wandb/sdk/artifacts/artifact_saver.py +16 -2
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +1 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +23 -2
- wandb/sdk/data_types/audio.py +1 -3
- wandb/sdk/data_types/base_types/media.py +13 -7
- wandb/sdk/data_types/base_types/wb_value.py +34 -11
- wandb/sdk/data_types/html.py +36 -9
- wandb/sdk/data_types/image.py +56 -37
- wandb/sdk/data_types/molecule.py +1 -5
- wandb/sdk/data_types/object_3d.py +2 -1
- wandb/sdk/data_types/saved_model.py +7 -9
- wandb/sdk/data_types/table.py +5 -0
- wandb/sdk/data_types/trace_tree.py +2 -0
- wandb/sdk/data_types/utils.py +1 -1
- wandb/sdk/data_types/video.py +15 -30
- wandb/sdk/interface/interface.py +2 -0
- wandb/{apis/public → sdk/internal}/_generated/__init__.py +0 -6
- wandb/{apis/public → sdk/internal}/_generated/server_features_query.py +3 -3
- wandb/sdk/internal/internal_api.py +138 -47
- wandb/sdk/internal/profiler.py +6 -5
- wandb/sdk/internal/run.py +13 -6
- wandb/sdk/internal/sender.py +2 -0
- wandb/sdk/internal/sender_config.py +8 -11
- wandb/sdk/internal/settings_static.py +24 -2
- wandb/sdk/lib/apikey.py +40 -20
- wandb/sdk/lib/asyncio_compat.py +1 -1
- wandb/sdk/lib/deprecate.py +13 -22
- wandb/sdk/lib/disabled.py +2 -1
- wandb/sdk/lib/printer.py +37 -8
- wandb/sdk/lib/printer_asyncio.py +46 -0
- wandb/sdk/lib/redirect.py +10 -5
- wandb/sdk/lib/run_moment.py +4 -6
- wandb/sdk/lib/wb_logging.py +161 -0
- wandb/sdk/service/server_sock.py +19 -14
- wandb/sdk/service/service.py +9 -7
- wandb/sdk/service/streams.py +5 -0
- wandb/sdk/verify/verify.py +6 -3
- wandb/sdk/wandb_config.py +44 -43
- wandb/sdk/wandb_init.py +323 -141
- wandb/sdk/wandb_login.py +13 -4
- wandb/sdk/wandb_metadata.py +107 -91
- wandb/sdk/wandb_run.py +529 -325
- wandb/sdk/wandb_settings.py +422 -202
- wandb/sdk/wandb_setup.py +52 -1
- wandb/util.py +29 -29
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/METADATA +7 -7
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/RECORD +150 -93
- wandb/_globals.py +0 -19
- wandb/apis/public/_generated/base.py +0 -128
- wandb/apis/public/_generated/typing_compat.py +0 -14
- /wandb/{apis/public → sdk/internal}/_generated/enums.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/input_types.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/operations.py +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/WHEEL +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_settings.py
CHANGED
@@ -11,30 +11,28 @@ import re
|
|
11
11
|
import shutil
|
12
12
|
import socket
|
13
13
|
import sys
|
14
|
-
import tempfile
|
15
14
|
from datetime import datetime
|
16
|
-
from typing import Any, Literal, Sequence
|
17
|
-
from urllib.parse import quote, unquote, urlencode
|
18
15
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
# Optional and Union are used for type hinting instead of | because
|
17
|
+
# the latter is not supported in pydantic<2.6 and Python<3.10.
|
18
|
+
# Dict, List, and Tuple are used for backwards compatibility
|
19
|
+
# with pydantic v1 and Python<3.9.
|
20
|
+
from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union
|
21
|
+
from urllib.parse import quote, unquote, urlencode
|
23
22
|
|
24
23
|
from google.protobuf.wrappers_pb2 import BoolValue, DoubleValue, Int32Value, StringValue
|
25
|
-
from pydantic import
|
24
|
+
from pydantic import BaseModel, ConfigDict, Field
|
25
|
+
from typing_extensions import Self
|
26
|
+
|
27
|
+
import wandb
|
28
|
+
from wandb import env, termwarn, util
|
29
|
+
from wandb._pydantic import (
|
30
|
+
IS_PYDANTIC_V2,
|
26
31
|
AliasChoices,
|
27
|
-
BaseModel,
|
28
|
-
ConfigDict,
|
29
|
-
Field,
|
30
32
|
computed_field,
|
31
33
|
field_validator,
|
32
34
|
model_validator,
|
33
35
|
)
|
34
|
-
from pydantic_core import SchemaValidator, core_schema
|
35
|
-
|
36
|
-
import wandb
|
37
|
-
from wandb import env, termwarn, util
|
38
36
|
from wandb.errors import UsageError
|
39
37
|
from wandb.proto import wandb_settings_pb2
|
40
38
|
|
@@ -42,6 +40,117 @@ from .lib import apikey, credentials, ipython
|
|
42
40
|
from .lib.gitlib import GitRepo
|
43
41
|
from .lib.run_moment import RunMoment
|
44
42
|
|
43
|
+
validate_url: Callable[[str], None]
|
44
|
+
|
45
|
+
if IS_PYDANTIC_V2:
|
46
|
+
from pydantic_core import SchemaValidator, core_schema
|
47
|
+
|
48
|
+
def validate_url(url: str) -> None:
|
49
|
+
"""Validate a URL string."""
|
50
|
+
url_validator = SchemaValidator(
|
51
|
+
core_schema.url_schema(
|
52
|
+
allowed_schemes=["http", "https"],
|
53
|
+
strict=True,
|
54
|
+
)
|
55
|
+
)
|
56
|
+
url_validator.validate_python(url)
|
57
|
+
else:
|
58
|
+
from pydantic import root_validator
|
59
|
+
|
60
|
+
def validate_url(url: str) -> None:
|
61
|
+
"""Validate the base url of the wandb server.
|
62
|
+
|
63
|
+
param value: URL to validate
|
64
|
+
|
65
|
+
Based on the Django URLValidator, but with a few additional checks.
|
66
|
+
|
67
|
+
Copyright (c) Django Software Foundation and individual contributors.
|
68
|
+
All rights reserved.
|
69
|
+
|
70
|
+
Redistribution and use in source and binary forms, with or without modification,
|
71
|
+
are permitted provided that the following conditions are met:
|
72
|
+
|
73
|
+
1. Redistributions of source code must retain the above copyright notice,
|
74
|
+
this list of conditions and the following disclaimer.
|
75
|
+
|
76
|
+
2. Redistributions in binary form must reproduce the above copyright
|
77
|
+
notice, this list of conditions and the following disclaimer in the
|
78
|
+
documentation and/or other materials provided with the distribution.
|
79
|
+
|
80
|
+
3. Neither the name of Django nor the names of its contributors may be used
|
81
|
+
to endorse or promote products derived from this software without
|
82
|
+
specific prior written permission.
|
83
|
+
|
84
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
85
|
+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
86
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
87
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
88
|
+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
89
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
90
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
91
|
+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
92
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
93
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
94
|
+
"""
|
95
|
+
from urllib.parse import urlparse, urlsplit
|
96
|
+
|
97
|
+
if url is None:
|
98
|
+
return
|
99
|
+
|
100
|
+
ul = "\u00a1-\uffff" # Unicode letters range (must not be a raw string).
|
101
|
+
|
102
|
+
# IP patterns
|
103
|
+
ipv4_re = (
|
104
|
+
r"(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)"
|
105
|
+
r"(?:\.(?:0|25[0-5]|2[0-4][0-9]|1[0-9]?[0-9]?|[1-9][0-9]?)){3}"
|
106
|
+
)
|
107
|
+
ipv6_re = r"\[[0-9a-f:.]+\]" # (simple regex, validated later)
|
108
|
+
|
109
|
+
# Host patterns
|
110
|
+
hostname_re = (
|
111
|
+
r"[a-z" + ul + r"0-9](?:[a-z" + ul + r"0-9-]{0,61}[a-z" + ul + r"0-9])?"
|
112
|
+
)
|
113
|
+
# Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
|
114
|
+
domain_re = r"(?:\.(?!-)[a-z" + ul + r"0-9-]{1,63}(?<!-))*"
|
115
|
+
tld_re = (
|
116
|
+
r"\." # dot
|
117
|
+
r"(?!-)" # can't start with a dash
|
118
|
+
r"(?:[a-z" + ul + "-]{2,63}" # domain label
|
119
|
+
r"|xn--[a-z0-9]{1,59})" # or punycode label
|
120
|
+
r"(?<!-)" # can't end with a dash
|
121
|
+
r"\.?" # may have a trailing dot
|
122
|
+
)
|
123
|
+
# host_re = "(" + hostname_re + domain_re + tld_re + "|localhost)"
|
124
|
+
# todo?: allow hostname to be just a hostname (no tld)?
|
125
|
+
host_re = "(" + hostname_re + domain_re + f"({tld_re})?" + "|localhost)"
|
126
|
+
|
127
|
+
regex = re.compile(
|
128
|
+
r"^(?:[a-z0-9.+-]*)://" # scheme is validated separately
|
129
|
+
r"(?:[^\s:@/]+(?::[^\s:@/]*)?@)?" # user:pass authentication
|
130
|
+
r"(?:" + ipv4_re + "|" + ipv6_re + "|" + host_re + ")"
|
131
|
+
r"(?::[0-9]{1,5})?" # port
|
132
|
+
r"(?:[/?#][^\s]*)?" # resource path
|
133
|
+
r"\Z",
|
134
|
+
re.IGNORECASE,
|
135
|
+
)
|
136
|
+
schemes = {"http", "https"}
|
137
|
+
unsafe_chars = frozenset("\t\r\n")
|
138
|
+
|
139
|
+
scheme = url.split("://")[0].lower()
|
140
|
+
split_url = urlsplit(url)
|
141
|
+
parsed_url = urlparse(url)
|
142
|
+
|
143
|
+
if parsed_url.netloc == "":
|
144
|
+
raise ValueError(f"Invalid URL: {url}")
|
145
|
+
elif unsafe_chars.intersection(url):
|
146
|
+
raise ValueError("URL cannot contain unsafe characters")
|
147
|
+
elif scheme not in schemes:
|
148
|
+
raise ValueError("URL must start with `http(s)://`")
|
149
|
+
elif not regex.search(url):
|
150
|
+
raise ValueError(f"{url} is not a valid server address")
|
151
|
+
elif split_url.hostname is None or len(split_url.hostname) > 253:
|
152
|
+
raise ValueError("hostname is invalid")
|
153
|
+
|
45
154
|
|
46
155
|
def _path_convert(*args: str) -> str:
|
47
156
|
"""Join path and apply os.path.expanduser to it."""
|
@@ -86,7 +195,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
86
195
|
allow_val_change: bool = False
|
87
196
|
"""Flag to allow modification of `Config` values after they've been set."""
|
88
197
|
|
89
|
-
anonymous: Literal["allow", "must", "never"]
|
198
|
+
anonymous: Optional[Literal["allow", "must", "never"]] = None
|
90
199
|
"""Controls anonymous data logging.
|
91
200
|
|
92
201
|
Possible values are:
|
@@ -100,19 +209,19 @@ class Settings(BaseModel, validate_assignment=True):
|
|
100
209
|
signed-up user account.
|
101
210
|
"""
|
102
211
|
|
103
|
-
api_key: str
|
212
|
+
api_key: Optional[str] = None
|
104
213
|
"""The W&B API key."""
|
105
214
|
|
106
|
-
azure_account_url_to_access_key:
|
215
|
+
azure_account_url_to_access_key: Optional[Dict[str, str]] = None
|
107
216
|
"""Mapping of Azure account URLs to their corresponding access keys for Azure integration."""
|
108
217
|
|
109
218
|
base_url: str = "https://api.wandb.ai"
|
110
219
|
"""The URL of the W&B backend for data synchronization."""
|
111
220
|
|
112
|
-
code_dir: str
|
221
|
+
code_dir: Optional[str] = None
|
113
222
|
"""Directory containing the code to be tracked by W&B."""
|
114
223
|
|
115
|
-
config_paths: Sequence[str]
|
224
|
+
config_paths: Optional[Sequence[str]] = None
|
116
225
|
"""Paths to files to load configuration from into the `Config` object."""
|
117
226
|
|
118
227
|
console: Literal["auto", "off", "wrap", "redirect", "wrap_raw", "wrap_emu"] = Field(
|
@@ -137,7 +246,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
137
246
|
|
138
247
|
"wrap_emu" - Same as "wrap" but captures output through an emulator.
|
139
248
|
Derived from the `wrap` setting and should not be set manually.
|
140
|
-
|
249
|
+
"""
|
141
250
|
|
142
251
|
console_multipart: bool = False
|
143
252
|
"""Whether to produce multipart console log files."""
|
@@ -156,57 +265,57 @@ class Settings(BaseModel, validate_assignment=True):
|
|
156
265
|
disable_job_creation: bool = True
|
157
266
|
"""Whether to disable the creation of a job artifact for W&B Launch."""
|
158
267
|
|
159
|
-
docker: str
|
268
|
+
docker: Optional[str] = None
|
160
269
|
"""The Docker image used to execute the script."""
|
161
270
|
|
162
|
-
email: str
|
271
|
+
email: Optional[str] = None
|
163
272
|
"""The email address of the user."""
|
164
273
|
|
165
|
-
entity: str
|
274
|
+
entity: Optional[str] = None
|
166
275
|
"""The W&B entity, such as a user or a team."""
|
167
276
|
|
168
|
-
organization: str
|
277
|
+
organization: Optional[str] = None
|
169
278
|
"""The W&B organization."""
|
170
279
|
|
171
280
|
force: bool = False
|
172
281
|
"""Whether to pass the `force` flag to `wandb.login()`."""
|
173
282
|
|
174
|
-
fork_from: RunMoment
|
283
|
+
fork_from: Optional[RunMoment] = None
|
175
284
|
"""Specifies a point in a previous execution of a run to fork from.
|
176
285
|
|
177
286
|
The point is defined by the run ID, a metric, and its value.
|
178
287
|
Currently, only the metric '_step' is supported.
|
179
288
|
"""
|
180
289
|
|
181
|
-
git_commit: str
|
290
|
+
git_commit: Optional[str] = None
|
182
291
|
"""The git commit hash to associate with the run."""
|
183
292
|
|
184
293
|
git_remote: str = "origin"
|
185
294
|
"""The git remote to associate with the run."""
|
186
295
|
|
187
|
-
git_remote_url: str
|
296
|
+
git_remote_url: Optional[str] = None
|
188
297
|
"""The URL of the git remote repository."""
|
189
298
|
|
190
|
-
git_root: str
|
299
|
+
git_root: Optional[str] = None
|
191
300
|
"""Root directory of the git repository."""
|
192
301
|
|
193
302
|
heartbeat_seconds: int = 30
|
194
303
|
"""Interval in seconds between heartbeat signals sent to the W&B servers."""
|
195
304
|
|
196
|
-
host: str
|
305
|
+
host: Optional[str] = None
|
197
306
|
"""Hostname of the machine running the script."""
|
198
307
|
|
199
|
-
http_proxy: str
|
308
|
+
http_proxy: Optional[str] = None
|
200
309
|
"""Custom proxy servers for http requests to W&B."""
|
201
310
|
|
202
|
-
https_proxy: str
|
311
|
+
https_proxy: Optional[str] = None
|
203
312
|
"""Custom proxy servers for https requests to W&B."""
|
204
313
|
|
205
314
|
# Path to file containing an identity token (JWT) for authentication.
|
206
|
-
identity_token_file: str
|
315
|
+
identity_token_file: Optional[str] = None
|
207
316
|
"""Path to file containing an identity token (JWT) for authentication."""
|
208
317
|
|
209
|
-
ignore_globs:
|
318
|
+
ignore_globs: Sequence[str] = ()
|
210
319
|
"""Unix glob patterns relative to `files_dir` specifying files to exclude from upload."""
|
211
320
|
|
212
321
|
init_timeout: float = 90.0
|
@@ -215,10 +324,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
215
324
|
insecure_disable_ssl: bool = False
|
216
325
|
"""Whether to insecurely disable SSL verification."""
|
217
326
|
|
218
|
-
job_name: str
|
327
|
+
job_name: Optional[str] = None
|
219
328
|
"""Name of the Launch job running the script."""
|
220
329
|
|
221
|
-
job_source: Literal["repo", "artifact", "image"]
|
330
|
+
job_source: Optional[Literal["repo", "artifact", "image"]] = None
|
222
331
|
"""Source type for Launch."""
|
223
332
|
|
224
333
|
label_disable: bool = False
|
@@ -227,10 +336,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
227
336
|
launch: bool = False
|
228
337
|
"""Flag to indicate if the run is being launched through W&B Launch."""
|
229
338
|
|
230
|
-
launch_config_path: str
|
339
|
+
launch_config_path: Optional[str] = None
|
231
340
|
"""Path to the launch configuration file."""
|
232
341
|
|
233
|
-
login_timeout: float
|
342
|
+
login_timeout: Optional[float] = None
|
234
343
|
"""Time in seconds to wait for login operations before timing out."""
|
235
344
|
|
236
345
|
mode: Literal["online", "offline", "dryrun", "disabled", "run", "shared"] = Field(
|
@@ -239,13 +348,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
239
348
|
)
|
240
349
|
"""The operating mode for W&B logging and synchronization."""
|
241
350
|
|
242
|
-
notebook_name: str
|
351
|
+
notebook_name: Optional[str] = None
|
243
352
|
"""Name of the notebook if running in a Jupyter-like environment."""
|
244
353
|
|
245
|
-
program: str
|
354
|
+
program: Optional[str] = None
|
246
355
|
"""Path to the script that created the run, if available."""
|
247
356
|
|
248
|
-
program_abspath: str
|
357
|
+
program_abspath: Optional[str] = None
|
249
358
|
"""The absolute path from the root repository directory to the script that
|
250
359
|
created the run.
|
251
360
|
|
@@ -253,26 +362,46 @@ class Settings(BaseModel, validate_assignment=True):
|
|
253
362
|
.git directory, if it exists. Otherwise, it's the current working directory.
|
254
363
|
"""
|
255
364
|
|
256
|
-
program_relpath: str
|
365
|
+
program_relpath: Optional[str] = None
|
257
366
|
"""The relative path to the script that created the run."""
|
258
367
|
|
259
|
-
project: str
|
368
|
+
project: Optional[str] = None
|
260
369
|
"""The W&B project ID."""
|
261
370
|
|
262
371
|
quiet: bool = False
|
263
372
|
"""Flag to suppress non-essential output."""
|
264
373
|
|
265
|
-
reinit:
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
374
|
+
reinit: Union[
|
375
|
+
Literal[
|
376
|
+
"default",
|
377
|
+
"return_previous",
|
378
|
+
"finish_previous",
|
379
|
+
"create_new",
|
380
|
+
],
|
381
|
+
bool,
|
382
|
+
] = "default"
|
383
|
+
"""What to do when `wandb.init()` is called while a run is active.
|
384
|
+
|
385
|
+
Options:
|
386
|
+
- "default": Use "finish_previous" in notebooks and "return_previous"
|
387
|
+
otherwise.
|
388
|
+
- "return_previous": Return the most recently created run
|
389
|
+
that is not yet finished. This does not update `wandb.run`; see
|
390
|
+
the "create_new" option.
|
391
|
+
- "finish_previous": Finish all active runs, then return a new run.
|
392
|
+
- "create_new": Create a new run without modifying other active runs.
|
393
|
+
Does not update `wandb.run` and top-level functions like `wandb.log`.
|
394
|
+
Because of this, some older integrations that rely on the global run
|
395
|
+
will not work.
|
396
|
+
|
397
|
+
Can also be a boolean, but this is deprecated. False is the same as
|
398
|
+
"return_previous", and True is the same as "finish_previous".
|
270
399
|
"""
|
271
400
|
|
272
401
|
relogin: bool = False
|
273
402
|
"""Flag to force a new login attempt."""
|
274
403
|
|
275
|
-
resume: Literal["allow", "must", "never", "auto"]
|
404
|
+
resume: Optional[Literal["allow", "must", "never", "auto"]] = None
|
276
405
|
"""Specifies the resume behavior for the run.
|
277
406
|
|
278
407
|
The available options are:
|
@@ -290,7 +419,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
290
419
|
machine.
|
291
420
|
"""
|
292
421
|
|
293
|
-
resume_from: RunMoment
|
422
|
+
resume_from: Optional[RunMoment] = None
|
294
423
|
"""Specifies a point in a previous execution of a run to resume from.
|
295
424
|
|
296
425
|
The point is defined by the run ID, a metric, and its value.
|
@@ -309,31 +438,31 @@ class Settings(BaseModel, validate_assignment=True):
|
|
309
438
|
In particular, this is used to derive the wandb directory and the run directory.
|
310
439
|
"""
|
311
440
|
|
312
|
-
run_group: str
|
441
|
+
run_group: Optional[str] = None
|
313
442
|
"""Group identifier for related runs.
|
314
443
|
|
315
444
|
Used for grouping runs in the UI.
|
316
445
|
"""
|
317
446
|
|
318
|
-
run_id: str
|
447
|
+
run_id: Optional[str] = None
|
319
448
|
"""The ID of the run."""
|
320
449
|
|
321
|
-
run_job_type: str
|
450
|
+
run_job_type: Optional[str] = None
|
322
451
|
"""Type of job being run (e.g., training, evaluation)."""
|
323
452
|
|
324
|
-
run_name: str
|
453
|
+
run_name: Optional[str] = None
|
325
454
|
"""Human-readable name for the run."""
|
326
455
|
|
327
|
-
run_notes: str
|
456
|
+
run_notes: Optional[str] = None
|
328
457
|
"""Additional notes or description for the run."""
|
329
458
|
|
330
|
-
run_tags:
|
459
|
+
run_tags: Optional[Tuple[str, ...]] = None
|
331
460
|
"""Tags to associate with the run for organization and filtering."""
|
332
461
|
|
333
462
|
sagemaker_disable: bool = False
|
334
463
|
"""Flag to disable SageMaker-specific functionality."""
|
335
464
|
|
336
|
-
save_code: bool
|
465
|
+
save_code: Optional[bool] = None
|
337
466
|
"""Whether to save the code associated with the run."""
|
338
467
|
|
339
468
|
settings_system: str = Field(
|
@@ -343,10 +472,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
343
472
|
)
|
344
473
|
"""Path to the system-wide settings file."""
|
345
474
|
|
346
|
-
show_colors: bool
|
475
|
+
show_colors: Optional[bool] = None
|
347
476
|
"""Whether to use colored output in the console."""
|
348
477
|
|
349
|
-
show_emoji: bool
|
478
|
+
show_emoji: Optional[bool] = None
|
350
479
|
"""Whether to show emoji in the console output."""
|
351
480
|
|
352
481
|
show_errors: bool = True
|
@@ -361,10 +490,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
361
490
|
silent: bool = False
|
362
491
|
"""Flag to suppress all output."""
|
363
492
|
|
364
|
-
start_method: str
|
493
|
+
start_method: Optional[str] = None
|
365
494
|
"""Method to use for starting subprocesses."""
|
366
495
|
|
367
|
-
strict: bool
|
496
|
+
strict: Optional[bool] = None
|
368
497
|
"""Whether to enable strict mode for validation and error checking."""
|
369
498
|
|
370
499
|
summary_timeout: int = 60
|
@@ -373,10 +502,10 @@ class Settings(BaseModel, validate_assignment=True):
|
|
373
502
|
summary_warnings: int = 5 # TODO: kill this with fire
|
374
503
|
"""Maximum number of summary warnings to display."""
|
375
504
|
|
376
|
-
sweep_id: str
|
505
|
+
sweep_id: Optional[str] = None
|
377
506
|
"""Identifier of the sweep this run belongs to."""
|
378
507
|
|
379
|
-
sweep_param_path: str
|
508
|
+
sweep_param_path: Optional[str] = None
|
380
509
|
"""Path to the sweep parameters configuration."""
|
381
510
|
|
382
511
|
symlink: bool = Field(
|
@@ -384,13 +513,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
384
513
|
)
|
385
514
|
"""Whether to use symlinks (True by default except on Windows)."""
|
386
515
|
|
387
|
-
sync_tensorboard: bool
|
516
|
+
sync_tensorboard: Optional[bool] = None
|
388
517
|
"""Whether to synchronize TensorBoard logs with W&B."""
|
389
518
|
|
390
519
|
table_raise_on_max_row_limit_exceeded: bool = False
|
391
520
|
"""Whether to raise an exception when table row limits are exceeded."""
|
392
521
|
|
393
|
-
username: str
|
522
|
+
username: Optional[str] = None
|
394
523
|
"""Username."""
|
395
524
|
|
396
525
|
# Internal settings.
|
@@ -424,13 +553,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
424
553
|
x_disable_machine_info: bool = False
|
425
554
|
"""Flag to disable automatic machine info collection."""
|
426
555
|
|
427
|
-
x_executable: str
|
556
|
+
x_executable: Optional[str] = None
|
428
557
|
"""Path to the Python executable."""
|
429
558
|
|
430
|
-
x_extra_http_headers:
|
559
|
+
x_extra_http_headers: Optional[Dict[str, str]] = None
|
431
560
|
"""Additional headers to add to all outgoing HTTP requests."""
|
432
561
|
|
433
|
-
x_file_stream_max_bytes: int
|
562
|
+
x_file_stream_max_bytes: Optional[int] = None
|
434
563
|
"""An approximate maximum request size for the filestream API.
|
435
564
|
|
436
565
|
Its purpose is to prevent HTTP requests from failing due to
|
@@ -438,50 +567,50 @@ class Settings(BaseModel, validate_assignment=True):
|
|
438
567
|
requests will be slightly larger.
|
439
568
|
"""
|
440
569
|
|
441
|
-
x_file_stream_max_line_bytes: int
|
570
|
+
x_file_stream_max_line_bytes: Optional[int] = None
|
442
571
|
"""Maximum line length for filestream JSONL files."""
|
443
572
|
|
444
|
-
x_file_stream_transmit_interval: float
|
573
|
+
x_file_stream_transmit_interval: Optional[float] = None
|
445
574
|
"""Interval in seconds between filestream transmissions."""
|
446
575
|
|
447
576
|
# Filestream retry client configuration.
|
448
577
|
|
449
|
-
x_file_stream_retry_max: int
|
578
|
+
x_file_stream_retry_max: Optional[int] = None
|
450
579
|
"""Max number of retries for filestream operations."""
|
451
580
|
|
452
|
-
x_file_stream_retry_wait_min_seconds: float
|
581
|
+
x_file_stream_retry_wait_min_seconds: Optional[float] = None
|
453
582
|
"""Minimum wait time between retries for filestream operations."""
|
454
583
|
|
455
|
-
x_file_stream_retry_wait_max_seconds: float
|
584
|
+
x_file_stream_retry_wait_max_seconds: Optional[float] = None
|
456
585
|
"""Maximum wait time between retries for filestream operations."""
|
457
586
|
|
458
|
-
x_file_stream_timeout_seconds: float
|
587
|
+
x_file_stream_timeout_seconds: Optional[float] = None
|
459
588
|
"""Timeout in seconds for individual filestream HTTP requests."""
|
460
589
|
|
461
590
|
# file transfer retry client configuration
|
462
591
|
|
463
|
-
x_file_transfer_retry_max: int
|
592
|
+
x_file_transfer_retry_max: Optional[int] = None
|
464
593
|
"""Max number of retries for file transfer operations."""
|
465
594
|
|
466
|
-
x_file_transfer_retry_wait_min_seconds: float
|
595
|
+
x_file_transfer_retry_wait_min_seconds: Optional[float] = None
|
467
596
|
"""Minimum wait time between retries for file transfer operations."""
|
468
597
|
|
469
|
-
x_file_transfer_retry_wait_max_seconds: float
|
598
|
+
x_file_transfer_retry_wait_max_seconds: Optional[float] = None
|
470
599
|
"""Maximum wait time between retries for file transfer operations."""
|
471
600
|
|
472
|
-
x_file_transfer_timeout_seconds: float
|
601
|
+
x_file_transfer_timeout_seconds: Optional[float] = None
|
473
602
|
"""Timeout in seconds for individual file transfer HTTP requests."""
|
474
603
|
|
475
|
-
x_files_dir: str
|
604
|
+
x_files_dir: Optional[str] = None
|
476
605
|
"""Override setting for the computed files_dir.."""
|
477
606
|
|
478
|
-
x_flow_control_custom: bool
|
607
|
+
x_flow_control_custom: Optional[bool] = None
|
479
608
|
"""Flag indicating custom flow control for filestream.
|
480
609
|
|
481
610
|
TODO: Not implemented in wandb-core.
|
482
611
|
"""
|
483
612
|
|
484
|
-
x_flow_control_disabled: bool
|
613
|
+
x_flow_control_disabled: Optional[bool] = None
|
485
614
|
"""Flag indicating flow control is disabled for filestream.
|
486
615
|
|
487
616
|
TODO: Not implemented in wandb-core.
|
@@ -489,47 +618,47 @@ class Settings(BaseModel, validate_assignment=True):
|
|
489
618
|
|
490
619
|
# graphql retry client configuration
|
491
620
|
|
492
|
-
x_graphql_retry_max: int
|
621
|
+
x_graphql_retry_max: Optional[int] = None
|
493
622
|
"""Max number of retries for GraphQL operations."""
|
494
623
|
|
495
|
-
x_graphql_retry_wait_min_seconds: float
|
624
|
+
x_graphql_retry_wait_min_seconds: Optional[float] = None
|
496
625
|
"""Minimum wait time between retries for GraphQL operations."""
|
497
626
|
|
498
|
-
x_graphql_retry_wait_max_seconds: float
|
627
|
+
x_graphql_retry_wait_max_seconds: Optional[float] = None
|
499
628
|
"""Maximum wait time between retries for GraphQL operations."""
|
500
629
|
|
501
|
-
x_graphql_timeout_seconds: float
|
630
|
+
x_graphql_timeout_seconds: Optional[float] = None
|
502
631
|
"""Timeout in seconds for individual GraphQL requests."""
|
503
632
|
|
504
633
|
x_internal_check_process: float = 8.0
|
505
634
|
"""Interval for internal process health checks in seconds."""
|
506
635
|
|
507
|
-
x_jupyter_name: str
|
636
|
+
x_jupyter_name: Optional[str] = None
|
508
637
|
"""Name of the Jupyter notebook."""
|
509
638
|
|
510
|
-
x_jupyter_path: str
|
639
|
+
x_jupyter_path: Optional[str] = None
|
511
640
|
"""Path to the Jupyter notebook."""
|
512
641
|
|
513
|
-
x_jupyter_root: str
|
642
|
+
x_jupyter_root: Optional[str] = None
|
514
643
|
"""Root directory of the Jupyter notebook."""
|
515
644
|
|
516
|
-
x_label: str
|
645
|
+
x_label: Optional[str] = None
|
517
646
|
"""Label to assign to system metrics and console logs collected for the run.
|
518
647
|
|
519
648
|
This is used to group data by on the frontend and can be used to distinguish data
|
520
649
|
from different processes in a distributed training job.
|
521
650
|
"""
|
522
651
|
|
523
|
-
x_live_policy_rate_limit: int
|
652
|
+
x_live_policy_rate_limit: Optional[int] = None
|
524
653
|
"""Rate limit for live policy updates in seconds."""
|
525
654
|
|
526
|
-
x_live_policy_wait_time: int
|
655
|
+
x_live_policy_wait_time: Optional[int] = None
|
527
656
|
"""Wait time between live policy updates in seconds."""
|
528
657
|
|
529
658
|
x_log_level: int = logging.INFO
|
530
659
|
"""Logging level for internal operations."""
|
531
660
|
|
532
|
-
x_network_buffer: int
|
661
|
+
x_network_buffer: Optional[int] = None
|
533
662
|
"""Size of the network buffer used in flow control.
|
534
663
|
|
535
664
|
TODO: Not implemented in wandb-core.
|
@@ -545,14 +674,14 @@ class Settings(BaseModel, validate_assignment=True):
|
|
545
674
|
as the primary process handles the main logging.
|
546
675
|
"""
|
547
676
|
|
548
|
-
x_proxies:
|
677
|
+
x_proxies: Optional[Dict[str, str]] = None
|
549
678
|
"""Custom proxy servers for requests to W&B.
|
550
679
|
|
551
680
|
This is deprecated and will be removed in future versions.
|
552
681
|
Please use `http_proxy` and `https_proxy` instead.
|
553
682
|
"""
|
554
683
|
|
555
|
-
x_runqueue_item_id: str
|
684
|
+
x_runqueue_item_id: Optional[str] = None
|
556
685
|
"""ID of the Launch run queue item being processed."""
|
557
686
|
|
558
687
|
x_require_legacy_service: bool = False
|
@@ -567,13 +696,19 @@ class Settings(BaseModel, validate_assignment=True):
|
|
567
696
|
This does not disable user-provided summary updates.
|
568
697
|
"""
|
569
698
|
|
570
|
-
|
699
|
+
x_server_side_expand_glob_metrics: bool = False
|
700
|
+
"""Flag to delegate glob matching of metrics in define_metric to the server.
|
701
|
+
|
702
|
+
If the server does not support this, the client will perform the glob matching.
|
703
|
+
"""
|
704
|
+
|
705
|
+
x_service_transport: Optional[str] = None
|
571
706
|
"""Transport method for communication with the wandb service."""
|
572
707
|
|
573
708
|
x_service_wait: float = 30.0
|
574
709
|
"""Time in seconds to wait for the wandb-core internal service to start."""
|
575
710
|
|
576
|
-
x_start_time: float
|
711
|
+
x_start_time: Optional[float] = None
|
577
712
|
"""The start time of the run in seconds since the Unix epoch."""
|
578
713
|
|
579
714
|
x_stats_pid: int = os.getpid()
|
@@ -582,13 +717,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
582
717
|
x_stats_sampling_interval: float = Field(default=15.0)
|
583
718
|
"""Sampling interval for the system monitor in seconds."""
|
584
719
|
|
585
|
-
x_stats_neuron_monitor_config_path: str
|
720
|
+
x_stats_neuron_monitor_config_path: Optional[str] = None
|
586
721
|
"""Path to the default config file for the neuron-monitor tool.
|
587
722
|
|
588
723
|
This is used to monitor AWS Trainium devices.
|
589
724
|
"""
|
590
725
|
|
591
|
-
x_stats_dcgm_exporter: str
|
726
|
+
x_stats_dcgm_exporter: Optional[str] = None
|
592
727
|
"""Endpoint to extract Nvidia DCGM metrics from.
|
593
728
|
|
594
729
|
Two options are supported:
|
@@ -602,12 +737,12 @@ class Settings(BaseModel, validate_assignment=True):
|
|
602
737
|
- TODO: `http://192.168.0.1:9400/metrics`.
|
603
738
|
"""
|
604
739
|
|
605
|
-
x_stats_open_metrics_endpoints:
|
740
|
+
x_stats_open_metrics_endpoints: Optional[Dict[str, str]] = None
|
606
741
|
"""OpenMetrics `/metrics` endpoints to monitor for system metrics."""
|
607
742
|
|
608
|
-
x_stats_open_metrics_filters:
|
609
|
-
None
|
610
|
-
|
743
|
+
x_stats_open_metrics_filters: Union[
|
744
|
+
Dict[str, Dict[str, str]], Sequence[str], None
|
745
|
+
] = None
|
611
746
|
"""Filter to apply to metrics collected from OpenMetrics `/metrics` endpoints.
|
612
747
|
|
613
748
|
Supports two formats:
|
@@ -615,17 +750,17 @@ class Settings(BaseModel, validate_assignment=True):
|
|
615
750
|
- ("metric regex pattern 1", "metric regex pattern 2", ...)
|
616
751
|
"""
|
617
752
|
|
618
|
-
x_stats_open_metrics_http_headers:
|
753
|
+
x_stats_open_metrics_http_headers: Optional[Dict[str, str]] = None
|
619
754
|
"""HTTP headers to add to OpenMetrics requests."""
|
620
755
|
|
621
|
-
x_stats_disk_paths: Sequence[str]
|
756
|
+
x_stats_disk_paths: Optional[Sequence[str]] = Field(
|
622
757
|
default_factory=lambda: ("/", "/System/Volumes/Data")
|
623
758
|
if platform.system() == "Darwin"
|
624
759
|
else ("/",)
|
625
760
|
)
|
626
761
|
"""System paths to monitor for disk usage."""
|
627
762
|
|
628
|
-
x_stats_gpu_device_ids: Sequence[int]
|
763
|
+
x_stats_gpu_device_ids: Optional[Sequence[int]] = None
|
629
764
|
"""GPU device indices to monitor.
|
630
765
|
|
631
766
|
If not set, captures metrics for all GPUs.
|
@@ -665,24 +800,39 @@ class Settings(BaseModel, validate_assignment=True):
|
|
665
800
|
new_values[key] = values[key]
|
666
801
|
return new_values
|
667
802
|
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
self.resume,
|
676
|
-
|
677
|
-
|
678
|
-
)
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
803
|
+
if IS_PYDANTIC_V2:
|
804
|
+
|
805
|
+
@model_validator(mode="after")
|
806
|
+
def validate_mutual_exclusion_of_branching_args(self) -> Self:
|
807
|
+
if (
|
808
|
+
sum(
|
809
|
+
o is not None
|
810
|
+
for o in [self.fork_from, self.resume, self.resume_from]
|
811
|
+
)
|
812
|
+
> 1
|
813
|
+
):
|
814
|
+
raise ValueError(
|
815
|
+
"`fork_from`, `resume`, or `resume_from` are mutually exclusive. "
|
816
|
+
"Please specify only one of them."
|
817
|
+
)
|
818
|
+
return self
|
819
|
+
else:
|
820
|
+
|
821
|
+
@root_validator(pre=False) # type: ignore [call-overload]
|
822
|
+
@classmethod
|
823
|
+
def validate_mutual_exclusion_of_branching_args(cls, values):
|
824
|
+
if (
|
825
|
+
sum(
|
826
|
+
values.get(o) is not None
|
827
|
+
for o in ["fork_from", "resume", "resume_from"]
|
828
|
+
)
|
829
|
+
> 1
|
830
|
+
):
|
831
|
+
raise ValueError(
|
832
|
+
"`fork_from`, `resume`, or `resume_from` are mutually exclusive. "
|
833
|
+
"Please specify only one of them."
|
834
|
+
)
|
835
|
+
return values
|
686
836
|
|
687
837
|
# Field validators.
|
688
838
|
|
@@ -707,7 +857,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
707
857
|
@field_validator("base_url", mode="after")
|
708
858
|
@classmethod
|
709
859
|
def validate_base_url(cls, value):
|
710
|
-
|
860
|
+
validate_url(value)
|
711
861
|
# wandb.ai-specific checks
|
712
862
|
if re.match(r".*wandb\.ai[^\.]*$", value) and "api." not in value:
|
713
863
|
# user might guess app.wandb.ai or wandb.ai is the default cloud server
|
@@ -728,13 +878,21 @@ class Settings(BaseModel, validate_assignment=True):
|
|
728
878
|
|
729
879
|
@field_validator("console", mode="after")
|
730
880
|
@classmethod
|
731
|
-
def validate_console(cls, value,
|
881
|
+
def validate_console(cls, value, values):
|
732
882
|
if value != "auto":
|
733
883
|
return value
|
884
|
+
|
885
|
+
if hasattr(values, "data"):
|
886
|
+
# pydantic v2
|
887
|
+
values = values.data
|
888
|
+
else:
|
889
|
+
# pydantic v1
|
890
|
+
values = values
|
891
|
+
|
734
892
|
if (
|
735
893
|
ipython.in_jupyter()
|
736
|
-
or (
|
737
|
-
or not
|
894
|
+
or (values.get("start_method") == "thread")
|
895
|
+
or not values.get("x_disable_service")
|
738
896
|
or platform.system() == "Windows"
|
739
897
|
):
|
740
898
|
value = "wrap"
|
@@ -767,12 +925,20 @@ class Settings(BaseModel, validate_assignment=True):
|
|
767
925
|
|
768
926
|
@field_validator("fork_from", mode="before")
|
769
927
|
@classmethod
|
770
|
-
def validate_fork_from(cls, value,
|
928
|
+
def validate_fork_from(cls, value, values) -> Optional[RunMoment]:
|
771
929
|
run_moment = cls._runmoment_preprocessor(value)
|
930
|
+
|
931
|
+
if hasattr(values, "data"):
|
932
|
+
# pydantic v2
|
933
|
+
values = values.data
|
934
|
+
else:
|
935
|
+
# pydantic v1
|
936
|
+
values = values
|
937
|
+
|
772
938
|
if (
|
773
939
|
run_moment
|
774
|
-
and
|
775
|
-
and
|
940
|
+
and values.get("run_id") is not None
|
941
|
+
and values.get("run_id") == run_moment.run
|
776
942
|
):
|
777
943
|
raise ValueError(
|
778
944
|
"Provided `run_id` is the same as the run to `fork_from`. "
|
@@ -786,7 +952,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
786
952
|
def validate_http_proxy(cls, value):
|
787
953
|
if value is None:
|
788
954
|
return None
|
789
|
-
|
955
|
+
validate_url(value)
|
790
956
|
return value.rstrip("/")
|
791
957
|
|
792
958
|
@field_validator("https_proxy", mode="after")
|
@@ -794,7 +960,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
794
960
|
def validate_https_proxy(cls, value):
|
795
961
|
if value is None:
|
796
962
|
return None
|
797
|
-
|
963
|
+
validate_url(value)
|
798
964
|
return value.rstrip("/")
|
799
965
|
|
800
966
|
@field_validator("ignore_globs", mode="after")
|
@@ -828,7 +994,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
828
994
|
|
829
995
|
@field_validator("project", mode="after")
|
830
996
|
@classmethod
|
831
|
-
def validate_project(cls, value,
|
997
|
+
def validate_project(cls, value, values):
|
832
998
|
if value is None:
|
833
999
|
return None
|
834
1000
|
invalid_chars_list = list("/\\#?%:")
|
@@ -854,12 +1020,20 @@ class Settings(BaseModel, validate_assignment=True):
|
|
854
1020
|
|
855
1021
|
@field_validator("resume_from", mode="before")
|
856
1022
|
@classmethod
|
857
|
-
def validate_resume_from(cls, value,
|
1023
|
+
def validate_resume_from(cls, value, values) -> Optional[RunMoment]:
|
858
1024
|
run_moment = cls._runmoment_preprocessor(value)
|
1025
|
+
|
1026
|
+
if hasattr(values, "data"):
|
1027
|
+
# pydantic v2
|
1028
|
+
values = values.data
|
1029
|
+
else:
|
1030
|
+
# pydantic v1
|
1031
|
+
values = values
|
1032
|
+
|
859
1033
|
if (
|
860
1034
|
run_moment
|
861
|
-
and
|
862
|
-
and
|
1035
|
+
and values.get("run_id") is not None
|
1036
|
+
and values.get("run_id") != run_moment.run
|
863
1037
|
):
|
864
1038
|
raise ValueError(
|
865
1039
|
"Both `run_id` and `resume_from` have been specified with different ids."
|
@@ -876,7 +1050,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
876
1050
|
|
877
1051
|
@field_validator("run_id", mode="after")
|
878
1052
|
@classmethod
|
879
|
-
def validate_run_id(cls, value,
|
1053
|
+
def validate_run_id(cls, value, values):
|
880
1054
|
if value is None:
|
881
1055
|
return None
|
882
1056
|
|
@@ -902,7 +1076,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
902
1076
|
raise UsageError("Service wait time cannot be negative")
|
903
1077
|
return value
|
904
1078
|
|
905
|
-
@field_validator("start_method")
|
1079
|
+
@field_validator("start_method", mode="after")
|
906
1080
|
@classmethod
|
907
1081
|
def validate_start_method(cls, value):
|
908
1082
|
if value is None:
|
@@ -984,7 +1158,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
984
1158
|
|
985
1159
|
@computed_field # type: ignore[prop-decorator]
|
986
1160
|
@property
|
987
|
-
def _args(self) ->
|
1161
|
+
def _args(self) -> List[str]:
|
988
1162
|
if not self._jupyter:
|
989
1163
|
return sys.argv[1:]
|
990
1164
|
return []
|
@@ -1006,7 +1180,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1006
1180
|
|
1007
1181
|
@computed_field # type: ignore[prop-decorator]
|
1008
1182
|
@property
|
1009
|
-
def _code_path_local(self) -> str
|
1183
|
+
def _code_path_local(self) -> Optional[str]:
|
1010
1184
|
"""The relative path from the current working directory to the code path.
|
1011
1185
|
|
1012
1186
|
For example, if the code path is /home/user/project/example.py, and the
|
@@ -1089,12 +1263,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1089
1263
|
@computed_field # type: ignore[prop-decorator]
|
1090
1264
|
@property
|
1091
1265
|
def _tmp_code_dir(self) -> str:
|
1092
|
-
return _path_convert(
|
1093
|
-
self.wandb_dir,
|
1094
|
-
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1095
|
-
"tmp",
|
1096
|
-
"code",
|
1097
|
-
)
|
1266
|
+
return _path_convert(self.sync_dir, "tmp", "code")
|
1098
1267
|
|
1099
1268
|
@computed_field # type: ignore[prop-decorator]
|
1100
1269
|
@property
|
@@ -1103,7 +1272,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1103
1272
|
|
1104
1273
|
@computed_field # type: ignore[prop-decorator]
|
1105
1274
|
@property
|
1106
|
-
def colab_url(self) -> str
|
1275
|
+
def colab_url(self) -> Optional[str]:
|
1107
1276
|
"""The URL to the Colab notebook, if running in Colab."""
|
1108
1277
|
if not self._colab:
|
1109
1278
|
return None
|
@@ -1121,11 +1290,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1121
1290
|
@property
|
1122
1291
|
def files_dir(self) -> str:
|
1123
1292
|
"""Absolute path to the local directory where the run's files are stored."""
|
1124
|
-
return self.x_files_dir or _path_convert(
|
1125
|
-
self.wandb_dir,
|
1126
|
-
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1127
|
-
"files",
|
1128
|
-
)
|
1293
|
+
return self.x_files_dir or _path_convert(self.sync_dir, "files")
|
1129
1294
|
|
1130
1295
|
@computed_field # type: ignore[prop-decorator]
|
1131
1296
|
@property
|
@@ -1136,9 +1301,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1136
1301
|
@property
|
1137
1302
|
def log_dir(self) -> str:
|
1138
1303
|
"""The directory for storing log files."""
|
1139
|
-
return _path_convert(
|
1140
|
-
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", "logs"
|
1141
|
-
)
|
1304
|
+
return _path_convert(self.sync_dir, "logs")
|
1142
1305
|
|
1143
1306
|
@computed_field # type: ignore[prop-decorator]
|
1144
1307
|
@property
|
@@ -1219,7 +1382,8 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1219
1382
|
@property
|
1220
1383
|
def sync_dir(self) -> str:
|
1221
1384
|
return _path_convert(
|
1222
|
-
self.wandb_dir,
|
1385
|
+
self.wandb_dir,
|
1386
|
+
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1223
1387
|
)
|
1224
1388
|
|
1225
1389
|
@computed_field # type: ignore[prop-decorator]
|
@@ -1241,29 +1405,13 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1241
1405
|
@computed_field # type: ignore[prop-decorator]
|
1242
1406
|
@property
|
1243
1407
|
def wandb_dir(self) -> str:
|
1244
|
-
"""Full path to the wandb directory.
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
# We use the hidden version if it already exists, otherwise non-hidden.
|
1252
|
-
if os.path.exists(os.path.join(root_dir, ".wandb")):
|
1253
|
-
__stage_dir__ = ".wandb" + os.sep
|
1254
|
-
else:
|
1255
|
-
__stage_dir__ = "wandb" + os.sep
|
1256
|
-
|
1257
|
-
path = os.path.join(root_dir, __stage_dir__)
|
1258
|
-
if not os.access(root_dir or ".", os.W_OK):
|
1259
|
-
termwarn(
|
1260
|
-
f"Path {path} wasn't writable, using system temp directory.",
|
1261
|
-
repeat=False,
|
1262
|
-
)
|
1263
|
-
path = os.path.join(
|
1264
|
-
tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep)
|
1265
|
-
)
|
1266
|
-
|
1408
|
+
"""Full path to the wandb directory."""
|
1409
|
+
stage_dir = (
|
1410
|
+
".wandb" + os.sep
|
1411
|
+
if os.path.exists(os.path.join(self.root_dir, ".wandb"))
|
1412
|
+
else "wandb" + os.sep
|
1413
|
+
)
|
1414
|
+
path = os.path.join(self.root_dir, stage_dir)
|
1267
1415
|
return os.path.expanduser(path)
|
1268
1416
|
|
1269
1417
|
# Methods to collect and update settings from different sources.
|
@@ -1289,7 +1437,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1289
1437
|
if value is not None:
|
1290
1438
|
setattr(self, key, value)
|
1291
1439
|
|
1292
|
-
def update_from_env_vars(self, environ:
|
1440
|
+
def update_from_env_vars(self, environ: Dict[str, Any]):
|
1293
1441
|
"""Update settings from environment variables."""
|
1294
1442
|
env_prefix: str = "WANDB_"
|
1295
1443
|
private_env_prefix: str = env_prefix + "_"
|
@@ -1407,7 +1555,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1407
1555
|
|
1408
1556
|
self.program = program
|
1409
1557
|
|
1410
|
-
def update_from_dict(self, settings:
|
1558
|
+
def update_from_dict(self, settings: Dict[str, Any]) -> None:
|
1411
1559
|
"""Update settings from a dictionary."""
|
1412
1560
|
for key, value in dict(settings).items():
|
1413
1561
|
if value is not None:
|
@@ -1425,7 +1573,11 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1425
1573
|
"""Generate a protobuf representation of the settings."""
|
1426
1574
|
settings_proto = wandb_settings_pb2.Settings()
|
1427
1575
|
for k, v in self.model_dump(exclude_none=True).items():
|
1428
|
-
#
|
1576
|
+
# Client-only settings that don't exist on the protobuf.
|
1577
|
+
if k in ("reinit",):
|
1578
|
+
continue
|
1579
|
+
|
1580
|
+
# Special case for x_stats_open_metrics_filters.
|
1429
1581
|
if k == "x_stats_open_metrics_filters":
|
1430
1582
|
if isinstance(v, (list, set, tuple)):
|
1431
1583
|
setting = getattr(settings_proto, k)
|
@@ -1439,12 +1591,16 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1439
1591
|
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1440
1592
|
continue
|
1441
1593
|
|
1442
|
-
#
|
1594
|
+
# Special case for RunMoment fields.
|
1443
1595
|
if k in ("fork_from", "resume_from"):
|
1444
|
-
run_moment =
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1596
|
+
run_moment = (
|
1597
|
+
v
|
1598
|
+
if isinstance(v, RunMoment)
|
1599
|
+
else RunMoment(
|
1600
|
+
run=v.get("run"),
|
1601
|
+
value=v.get("value"),
|
1602
|
+
metric=v.get("metric"),
|
1603
|
+
)
|
1448
1604
|
)
|
1449
1605
|
getattr(settings_proto, k).CopyFrom(
|
1450
1606
|
wandb_settings_pb2.RunMoment(
|
@@ -1480,18 +1636,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1480
1636
|
|
1481
1637
|
return settings_proto
|
1482
1638
|
|
1483
|
-
|
1484
|
-
def validate_url(url: str) -> None:
|
1485
|
-
"""Validate a URL string."""
|
1486
|
-
url_validator = SchemaValidator(
|
1487
|
-
core_schema.url_schema(
|
1488
|
-
allowed_schemes=["http", "https"],
|
1489
|
-
strict=True,
|
1490
|
-
)
|
1491
|
-
)
|
1492
|
-
url_validator.validate_python(url)
|
1493
|
-
|
1494
|
-
def _get_program(self) -> str | None:
|
1639
|
+
def _get_program(self) -> Optional[str]:
|
1495
1640
|
"""Get the program that started the current process."""
|
1496
1641
|
if not self._jupyter:
|
1497
1642
|
# If not in a notebook, try to get the program from the environment
|
@@ -1521,7 +1666,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1521
1666
|
return self.x_jupyter_path
|
1522
1667
|
|
1523
1668
|
@staticmethod
|
1524
|
-
def _get_program_relpath(program: str, root: str
|
1669
|
+
def _get_program_relpath(program: str, root: Optional[str] = None) -> Optional[str]:
|
1525
1670
|
"""Get the relative path to the program from the root directory."""
|
1526
1671
|
if not program:
|
1527
1672
|
return None
|
@@ -1530,6 +1675,11 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1530
1675
|
if not root:
|
1531
1676
|
return None
|
1532
1677
|
|
1678
|
+
# For windows if the root and program are on different drives,
|
1679
|
+
# os.path.relpath will raise a ValueError.
|
1680
|
+
if not util.are_paths_on_same_drive(root, program):
|
1681
|
+
return None
|
1682
|
+
|
1533
1683
|
full_path_to_program = os.path.join(
|
1534
1684
|
root, os.path.relpath(os.getcwd(), root), program
|
1535
1685
|
)
|
@@ -1547,7 +1697,7 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1547
1697
|
parser = configparser.ConfigParser()
|
1548
1698
|
parser.add_section(section)
|
1549
1699
|
parser.read(file_name)
|
1550
|
-
config:
|
1700
|
+
config: Dict[str, Any] = dict()
|
1551
1701
|
for k in parser[section]:
|
1552
1702
|
config[k] = parser[section][k]
|
1553
1703
|
if k == "ignore_globs":
|
@@ -1573,9 +1723,79 @@ class Settings(BaseModel, validate_assignment=True):
|
|
1573
1723
|
return f"?{urlencode({'apiKey': api_key})}"
|
1574
1724
|
|
1575
1725
|
@staticmethod
|
1576
|
-
def _runmoment_preprocessor(
|
1726
|
+
def _runmoment_preprocessor(
|
1727
|
+
val: Union[RunMoment, str, None],
|
1728
|
+
) -> Optional[RunMoment]:
|
1577
1729
|
"""Preprocess the setting for forking or resuming a run."""
|
1578
1730
|
if isinstance(val, RunMoment) or val is None:
|
1579
1731
|
return val
|
1580
1732
|
elif isinstance(val, str):
|
1581
1733
|
return RunMoment.from_uri(val)
|
1734
|
+
|
1735
|
+
if not IS_PYDANTIC_V2:
|
1736
|
+
|
1737
|
+
def model_copy(self, *args, **kwargs):
|
1738
|
+
return self.copy(*args, **kwargs)
|
1739
|
+
|
1740
|
+
def model_dump(self, **kwargs):
|
1741
|
+
"""Compatibility method for Pydantic v1 to mimic v2's model_dump.
|
1742
|
+
|
1743
|
+
In v1, this is equivalent to dict() but also includes computed properties.
|
1744
|
+
|
1745
|
+
Args:
|
1746
|
+
**kwargs: Options passed to the dict method
|
1747
|
+
- exclude_none: Whether to exclude fields with None values
|
1748
|
+
|
1749
|
+
Returns:
|
1750
|
+
A dictionary of the model's fields and computed properties
|
1751
|
+
"""
|
1752
|
+
# Handle exclude_none separately since it's named differently in v1
|
1753
|
+
exclude_none = kwargs.pop("exclude_none", False)
|
1754
|
+
|
1755
|
+
# Start with regular fields from dict()
|
1756
|
+
result = self.dict(**kwargs)
|
1757
|
+
|
1758
|
+
# Get all computed properties
|
1759
|
+
for name in dir(self.__class__):
|
1760
|
+
attr = getattr(self.__class__, name, None)
|
1761
|
+
if isinstance(attr, property):
|
1762
|
+
try:
|
1763
|
+
# Only include properties that don't raise errors
|
1764
|
+
value = getattr(self, name)
|
1765
|
+
result[name] = value
|
1766
|
+
except (AttributeError, NotImplementedError, TypeError, ValueError):
|
1767
|
+
# Skip properties that can't be accessed or raise errors
|
1768
|
+
pass
|
1769
|
+
elif isinstance(attr, RunMoment):
|
1770
|
+
value = getattr(self, name)
|
1771
|
+
result[name] = value
|
1772
|
+
|
1773
|
+
# Special Pydantic attributes that should always be excluded
|
1774
|
+
exclude_fields = {
|
1775
|
+
"model_config",
|
1776
|
+
"model_fields",
|
1777
|
+
"model_fields_set",
|
1778
|
+
"__fields__",
|
1779
|
+
"__model_fields_set",
|
1780
|
+
"__pydantic_self__",
|
1781
|
+
"__pydantic_initialised__",
|
1782
|
+
}
|
1783
|
+
|
1784
|
+
# Remove special Pydantic attributes
|
1785
|
+
for field in exclude_fields:
|
1786
|
+
if field in result:
|
1787
|
+
del result[field]
|
1788
|
+
|
1789
|
+
if exclude_none:
|
1790
|
+
# Remove None values from the result
|
1791
|
+
return {k: v for k, v in result.items() if v is not None}
|
1792
|
+
|
1793
|
+
return result
|
1794
|
+
|
1795
|
+
@property
|
1796
|
+
def model_fields_set(self) -> set:
|
1797
|
+
"""Return a set of fields that have been explicitly set.
|
1798
|
+
|
1799
|
+
This is a compatibility property for Pydantic v1 to mimic v2's model_fields_set.
|
1800
|
+
"""
|
1801
|
+
return getattr(self, "__fields_set__", set())
|