wandb 0.18.7__py3-none-win_amd64.whl → 0.19.0rc1__py3-none-win_amd64.whl
Sign up to get free protection for your applications and to get access to all the features.
- package_readme.md +8 -0
- wandb/__init__.py +5 -7
- wandb/__init__.pyi +51 -30
- wandb/analytics/sentry.py +4 -10
- wandb/apis/importers/internals/internal.py +6 -6
- wandb/apis/importers/internals/protocols.py +11 -7
- wandb/apis/public/jobs.py +1 -7
- wandb/apis/public/reports.py +6 -17
- wandb/apis/public/runs.py +12 -10
- wandb/bin/gpu_stats.exe +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +9 -45
- wandb/env.py +3 -5
- wandb/errors/links.py +1 -1
- wandb/errors/term.py +1 -6
- wandb/filesync/dir_watcher.py +3 -3
- wandb/filesync/step_upload.py +2 -5
- wandb/integration/fastai/__init__.py +1 -6
- wandb/integration/gym/__init__.py +1 -7
- wandb/integration/keras/callbacks/metrics_logger.py +1 -8
- wandb/integration/keras/callbacks/model_checkpoint.py +1 -8
- wandb/integration/keras/keras.py +3 -5
- wandb/integration/lightgbm/__init__.py +1 -1
- wandb/integration/sb3/sb3.py +1 -7
- wandb/integration/sklearn/utils.py +1 -1
- wandb/integration/tensorboard/log.py +1 -2
- wandb/integration/torch/wandb_torch.py +1 -1
- wandb/integration/ultralytics/bbox_utils.py +9 -2
- wandb/jupyter.py +4 -4
- wandb/proto/v3/wandb_internal_pb2.py +31 -31
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_internal_pb2.py +31 -31
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_internal_pb2.py +31 -31
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/wandb_deprecated.py +1 -11
- wandb/proto/wandb_generate_deprecated.py +3 -7
- wandb/sdk/artifacts/artifact.py +2 -10
- wandb/sdk/artifacts/artifact_file_cache.py +2 -5
- wandb/sdk/artifacts/artifact_saver.py +2 -6
- wandb/sdk/artifacts/storage_handlers/gcs_handler.py +2 -4
- wandb/sdk/artifacts/storage_handlers/local_file_handler.py +2 -4
- wandb/sdk/artifacts/storage_handlers/s3_handler.py +2 -4
- wandb/sdk/backend/backend.py +1 -1
- wandb/sdk/data_types/histogram.py +1 -3
- wandb/sdk/data_types/object_3d.py +2 -6
- wandb/sdk/data_types/table.py +1 -1
- wandb/sdk/data_types/utils.py +1 -2
- wandb/sdk/data_types/video.py +15 -4
- wandb/sdk/integration_utils/auto_logging.py +1 -8
- wandb/sdk/interface/interface.py +12 -5
- wandb/sdk/interface/interface_shared.py +9 -0
- wandb/sdk/internal/file_stream.py +1 -4
- wandb/sdk/internal/flow_control.py +1 -1
- wandb/sdk/internal/handler.py +7 -2
- wandb/sdk/internal/internal.py +3 -3
- wandb/sdk/internal/internal_api.py +3 -10
- wandb/sdk/internal/job_builder.py +20 -12
- wandb/sdk/internal/progress.py +1 -5
- wandb/sdk/internal/sender.py +9 -13
- wandb/sdk/internal/settings_static.py +4 -10
- wandb/sdk/internal/system/assets/cpu.py +2 -2
- wandb/sdk/internal/system/assets/disk.py +3 -3
- wandb/sdk/internal/system/assets/gpu.py +7 -7
- wandb/sdk/internal/system/assets/gpu_amd.py +1 -7
- wandb/sdk/internal/system/assets/interfaces.py +11 -13
- wandb/sdk/internal/system/assets/ipu.py +1 -1
- wandb/sdk/internal/system/assets/memory.py +2 -2
- wandb/sdk/internal/system/assets/open_metrics.py +2 -8
- wandb/sdk/internal/system/assets/trainium.py +3 -9
- wandb/sdk/internal/system/system_info.py +14 -13
- wandb/sdk/internal/system/system_monitor.py +5 -12
- wandb/sdk/internal/tb_watcher.py +1 -1
- wandb/sdk/internal/writer.py +1 -1
- wandb/sdk/launch/agent/run_queue_item_file_saver.py +1 -7
- wandb/sdk/launch/create_job.py +2 -3
- wandb/sdk/launch/runner/abstract.py +1 -6
- wandb/sdk/launch/runner/kubernetes_monitor.py +2 -4
- wandb/sdk/lib/apikey.py +2 -6
- wandb/sdk/lib/fsm.py +12 -6
- wandb/sdk/lib/ipython.py +1 -6
- wandb/sdk/lib/module.py +0 -3
- wandb/sdk/lib/progress.py +2 -3
- wandb/sdk/lib/run_moment.py +1 -7
- wandb/sdk/lib/server.py +10 -24
- wandb/sdk/service/server.py +1 -1
- wandb/sdk/service/service.py +5 -5
- wandb/sdk/wandb_init.py +215 -166
- wandb/sdk/wandb_login.py +17 -27
- wandb/sdk/wandb_run.py +90 -116
- wandb/sdk/wandb_settings.py +978 -1778
- wandb/sdk/wandb_setup.py +86 -89
- wandb/sdk/wandb_watch.py +1 -1
- wandb/sync/sync.py +1 -2
- wandb/util.py +6 -39
- wandb/wandb_controller.py +10 -12
- {wandb-0.18.7.dist-info → wandb-0.19.0rc1.dist-info}/METADATA +14 -4
- {wandb-0.18.7.dist-info → wandb-0.19.0rc1.dist-info}/RECORD +101 -106
- {wandb-0.18.7.dist-info → wandb-0.19.0rc1.dist-info}/WHEEL +1 -1
- wandb/integration/magic.py +0 -556
- wandb/magic.py +0 -3
- wandb/sdk/lib/_settings_toposort_generate.py +0 -159
- wandb/sdk/lib/_settings_toposort_generated.py +0 -251
- wandb/sdk/lib/reporting.py +0 -99
- {wandb-0.18.7.dist-info → wandb-0.19.0rc1.dist-info}/entry_points.txt +0 -0
- {wandb-0.18.7.dist-info → wandb-0.19.0rc1.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_settings.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
import
|
1
|
+
from __future__ import annotations
|
2
|
+
|
2
3
|
import configparser
|
3
|
-
import enum
|
4
4
|
import getpass
|
5
5
|
import json
|
6
6
|
import logging
|
@@ -12,1045 +12,517 @@ import shutil
|
|
12
12
|
import socket
|
13
13
|
import sys
|
14
14
|
import tempfile
|
15
|
-
import time
|
16
|
-
from dataclasses import dataclass
|
17
15
|
from datetime import datetime
|
18
|
-
from
|
19
|
-
from
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
ItemsView,
|
26
|
-
Iterable,
|
27
|
-
Mapping,
|
28
|
-
Optional,
|
29
|
-
Sequence,
|
30
|
-
Set,
|
31
|
-
Tuple,
|
32
|
-
Union,
|
33
|
-
no_type_check,
|
34
|
-
)
|
35
|
-
from urllib.parse import quote, unquote, urlencode, urlparse, urlsplit
|
16
|
+
from typing import Any, Literal, Sequence
|
17
|
+
from urllib.parse import quote, unquote, urlencode
|
18
|
+
|
19
|
+
if sys.version_info >= (3, 11):
|
20
|
+
from typing import Self
|
21
|
+
else:
|
22
|
+
from typing_extensions import Self
|
36
23
|
|
37
24
|
from google.protobuf.wrappers_pb2 import BoolValue, DoubleValue, Int32Value, StringValue
|
25
|
+
from pydantic import (
|
26
|
+
BaseModel,
|
27
|
+
ConfigDict,
|
28
|
+
Field,
|
29
|
+
computed_field,
|
30
|
+
field_validator,
|
31
|
+
model_validator,
|
32
|
+
)
|
33
|
+
from pydantic_core import SchemaValidator, core_schema
|
38
34
|
|
39
35
|
import wandb
|
40
|
-
import
|
41
|
-
from wandb import util
|
36
|
+
from wandb import env, termwarn, util
|
42
37
|
from wandb.apis.internal import Api
|
43
38
|
from wandb.errors import UsageError
|
44
39
|
from wandb.proto import wandb_settings_pb2
|
45
|
-
from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
|
46
|
-
from wandb.sdk.lib import credentials, filesystem
|
47
|
-
from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
|
48
|
-
from wandb.sdk.lib.run_moment import RunMoment
|
49
|
-
from wandb.sdk.wandb_setup import _EarlyLogger
|
50
40
|
|
51
|
-
from .lib import apikey, ipython
|
41
|
+
from .lib import apikey, credentials, filesystem, ipython
|
52
42
|
from .lib.gitlib import GitRepo
|
43
|
+
from .lib.run_moment import RunMoment
|
53
44
|
from .lib.runid import generate_id
|
54
45
|
|
55
|
-
if sys.version_info >= (3, 8):
|
56
|
-
from typing import get_args, get_origin, get_type_hints
|
57
|
-
else:
|
58
|
-
from typing_extensions import get_args, get_origin, get_type_hints
|
59
|
-
|
60
|
-
|
61
|
-
class SettingsPreprocessingError(UsageError):
|
62
|
-
"""Raised when the value supplied to a wandb.Settings() setting does not pass preprocessing."""
|
63
|
-
|
64
|
-
|
65
|
-
class SettingsValidationError(UsageError):
|
66
|
-
"""Raised when the value supplied to a wandb.Settings() setting does not pass validation."""
|
67
|
-
|
68
|
-
|
69
|
-
class SettingsUnexpectedArgsError(UsageError):
|
70
|
-
"""Raised when unexpected arguments are passed to wandb.Settings()."""
|
71
|
-
|
72
|
-
|
73
|
-
def _get_wandb_dir(root_dir: str) -> str:
|
74
|
-
"""Get the full path to the wandb directory.
|
75
46
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
# We use the hidden version if it already exists, otherwise non-hidden.
|
80
|
-
if os.path.exists(os.path.join(root_dir, ".wandb")):
|
81
|
-
__stage_dir__ = ".wandb" + os.sep
|
82
|
-
else:
|
83
|
-
__stage_dir__ = "wandb" + os.sep
|
47
|
+
def _path_convert(*args: str) -> str:
|
48
|
+
"""Join path and apply os.path.expanduser to it."""
|
49
|
+
return os.path.expanduser(os.path.join(*args))
|
84
50
|
|
85
|
-
path = os.path.join(root_dir, __stage_dir__)
|
86
|
-
if not os.access(root_dir or ".", os.W_OK):
|
87
|
-
wandb.termwarn(
|
88
|
-
f"Path {path} wasn't writable, using system temp directory.",
|
89
|
-
repeat=False,
|
90
|
-
)
|
91
|
-
path = os.path.join(tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep))
|
92
|
-
|
93
|
-
return os.path.expanduser(path)
|
94
|
-
|
95
|
-
|
96
|
-
def _str_as_bool(val: Union[str, bool]) -> bool:
|
97
|
-
"""Parse a string as a bool."""
|
98
|
-
if isinstance(val, bool):
|
99
|
-
return val
|
100
|
-
try:
|
101
|
-
ret_val = bool(strtobool(str(val)))
|
102
|
-
return ret_val
|
103
|
-
except (AttributeError, ValueError):
|
104
|
-
pass
|
105
|
-
|
106
|
-
raise UsageError(f"Could not parse value {val} as a bool.")
|
107
|
-
|
108
|
-
|
109
|
-
def _str_as_json(val: Union[str, Dict[str, Any]]) -> Any:
|
110
|
-
"""Parse a string as a json object."""
|
111
|
-
if not isinstance(val, str):
|
112
|
-
return val
|
113
|
-
try:
|
114
|
-
return json.loads(val)
|
115
|
-
except (AttributeError, ValueError):
|
116
|
-
pass
|
117
|
-
|
118
|
-
raise UsageError(f"Could not parse value {val} as JSON.")
|
119
|
-
|
120
|
-
|
121
|
-
def _str_as_tuple(val: Union[str, Sequence[str]]) -> Tuple[str, ...]:
|
122
|
-
"""Parse a (potentially comma-separated) string as a tuple."""
|
123
|
-
if isinstance(val, str):
|
124
|
-
return tuple(val.split(","))
|
125
|
-
return tuple(val)
|
126
|
-
|
127
|
-
|
128
|
-
def _datetime_as_str(val: Union[datetime, str]) -> str:
|
129
|
-
"""Parse a datetime object as a string."""
|
130
|
-
if isinstance(val, datetime):
|
131
|
-
return datetime.strftime(val, "%Y%m%d_%H%M%S")
|
132
|
-
return val
|
133
|
-
|
134
|
-
|
135
|
-
def _redact_dict(
|
136
|
-
d: Dict[str, Any],
|
137
|
-
unsafe_keys: Union[Set[str], FrozenSet[str]] = frozenset({"api_key"}),
|
138
|
-
redact_str: str = "***REDACTED***",
|
139
|
-
) -> Dict[str, Any]:
|
140
|
-
"""Redact a dict of unsafe values specified by their key."""
|
141
|
-
if not d or unsafe_keys.isdisjoint(d):
|
142
|
-
return d
|
143
|
-
safe_dict = d.copy()
|
144
|
-
safe_dict.update({k: redact_str for k in unsafe_keys.intersection(d)})
|
145
|
-
return safe_dict
|
146
|
-
|
147
|
-
|
148
|
-
def _get_program() -> Optional[str]:
|
149
|
-
program = os.getenv(wandb.env.PROGRAM)
|
150
|
-
if program is not None:
|
151
|
-
return program
|
152
|
-
try:
|
153
|
-
import __main__
|
154
|
-
|
155
|
-
if __main__.__spec__ is None:
|
156
|
-
return __main__.__file__
|
157
|
-
# likely run as `python -m ...`
|
158
|
-
return f"-m {__main__.__spec__.name}"
|
159
|
-
except (ImportError, AttributeError):
|
160
|
-
return None
|
161
|
-
|
162
|
-
|
163
|
-
def _preprocess_file_stream_max_line_bytes(val: Any) -> Optional[int]:
|
164
|
-
"""Preprocess the file_stream_max_line_bytes setting.
|
165
|
-
|
166
|
-
For now treat negative values as 0, which means use the default.
|
167
|
-
"""
|
168
|
-
try:
|
169
|
-
value = int(val)
|
170
|
-
if value < 0:
|
171
|
-
return None
|
172
|
-
return value
|
173
|
-
except ValueError:
|
174
|
-
return None
|
175
|
-
|
176
|
-
|
177
|
-
def _runmoment_preprocessor(val: Any) -> Optional[RunMoment]:
|
178
|
-
if isinstance(val, RunMoment) or val is None:
|
179
|
-
return val
|
180
|
-
elif isinstance(val, str):
|
181
|
-
return RunMoment.from_uri(val)
|
182
|
-
raise UsageError(f"Could not parse value {val} as a RunMoment.")
|
183
|
-
|
184
|
-
|
185
|
-
def _get_program_relpath(
|
186
|
-
program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
|
187
|
-
) -> Optional[str]:
|
188
|
-
if not program:
|
189
|
-
if _logger is not None:
|
190
|
-
_logger.warning("Empty program passed to get_program_relpath")
|
191
|
-
return None
|
192
51
|
|
193
|
-
|
194
|
-
|
195
|
-
return None
|
52
|
+
class Settings(BaseModel, validate_assignment=True):
|
53
|
+
"""Settings for the W&B SDK."""
|
196
54
|
|
197
|
-
|
198
|
-
|
55
|
+
# Pydantic configuration.
|
56
|
+
model_config = ConfigDict(
|
57
|
+
extra="forbid", # throw an error if extra fields are provided
|
58
|
+
# validate_default=True, # validate default values
|
199
59
|
)
|
200
|
-
if os.path.exists(full_path_to_program):
|
201
|
-
relative_path = os.path.relpath(full_path_to_program, start=root)
|
202
|
-
if "../" in relative_path:
|
203
|
-
if _logger is not None:
|
204
|
-
_logger.warning(f"Could not save program above cwd: {program}")
|
205
|
-
return None
|
206
|
-
return relative_path
|
207
|
-
|
208
|
-
if _logger is not None:
|
209
|
-
_logger.warning(f"Could not find program at {program}")
|
210
|
-
return None
|
211
|
-
|
212
|
-
|
213
|
-
def is_instance_recursive(obj: Any, type_hint: Any) -> bool: # noqa: C901
|
214
|
-
if type_hint is Any:
|
215
|
-
return True
|
216
60
|
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
elif len(args) == 2 and args[-1] == ...:
|
251
|
-
item_type = args[0]
|
252
|
-
for item in obj:
|
253
|
-
if not is_instance_recursive(item, item_type):
|
254
|
-
return False
|
255
|
-
elif len(args) == len(obj):
|
256
|
-
for item, item_type in zip(obj, args):
|
257
|
-
if not is_instance_recursive(item, item_type):
|
258
|
-
return False
|
259
|
-
else:
|
260
|
-
return False
|
261
|
-
|
262
|
-
return True
|
263
|
-
|
264
|
-
if issubclass(origin, collections.abc.Set):
|
265
|
-
if not isinstance(obj, collections.abc.Set):
|
266
|
-
return False
|
267
|
-
|
268
|
-
(item_type,) = args
|
269
|
-
for item in obj:
|
270
|
-
if not is_instance_recursive(item, item_type):
|
271
|
-
return False
|
272
|
-
|
273
|
-
return True
|
274
|
-
|
275
|
-
return False
|
276
|
-
|
277
|
-
|
278
|
-
@enum.unique
|
279
|
-
class Source(enum.IntEnum):
|
280
|
-
OVERRIDE: int = 0
|
281
|
-
BASE: int = 1 # todo: audit this
|
282
|
-
ORG: int = 2
|
283
|
-
ENTITY: int = 3
|
284
|
-
PROJECT: int = 4
|
285
|
-
USER: int = 5
|
286
|
-
SYSTEM: int = 6
|
287
|
-
WORKSPACE: int = 7
|
288
|
-
ENV: int = 8
|
289
|
-
SETUP: int = 9
|
290
|
-
LOGIN: int = 10
|
291
|
-
INIT: int = 11
|
292
|
-
SETTINGS: int = 12
|
293
|
-
ARGS: int = 13
|
294
|
-
RUN: int = 14
|
295
|
-
|
296
|
-
|
297
|
-
ConsoleValue = {
|
298
|
-
"auto",
|
299
|
-
"off",
|
300
|
-
"wrap",
|
301
|
-
"redirect",
|
302
|
-
# internal console states
|
303
|
-
"wrap_raw",
|
304
|
-
"wrap_emu",
|
305
|
-
}
|
306
|
-
|
307
|
-
|
308
|
-
@dataclass()
|
309
|
-
class SettingsData:
|
310
|
-
"""Settings for the W&B SDK."""
|
311
|
-
|
312
|
-
_args: Sequence[str]
|
313
|
-
_aws_lambda: bool
|
314
|
-
_cli_only_mode: bool # Avoid running any code specific for runs
|
315
|
-
_code_path_local: str
|
316
|
-
_colab: bool
|
317
|
-
# _config_dict: Config
|
318
|
-
_cuda: str
|
319
|
-
_disable_meta: bool # Do not collect system metadata
|
320
|
-
_disable_service: (
|
321
|
-
bool # Disable wandb-service, spin up internal process the old way
|
61
|
+
# Public settings.
|
62
|
+
|
63
|
+
# Flag to allow table artifacts to be synced in offline mode.
|
64
|
+
#
|
65
|
+
# To revert to the old behavior, set this to False.
|
66
|
+
allow_offline_artifacts: bool = True
|
67
|
+
allow_val_change: bool = False
|
68
|
+
anonymous: Literal["allow", "must", "never"] | None = None
|
69
|
+
# The W&B API key.
|
70
|
+
api_key: str | None = None
|
71
|
+
azure_account_url_to_access_key: dict[str, str] | None = None
|
72
|
+
# The URL of the W&B backend, used for GraphQL and filestream operations.
|
73
|
+
base_url: str = "https://api.wandb.ai"
|
74
|
+
code_dir: str | None = None
|
75
|
+
config_paths: Sequence[str] | None = None
|
76
|
+
# The type of console capture to be applied. Possible values are:
|
77
|
+
# "auto" - Automatically selects the console capture method based on the
|
78
|
+
# system environment and settings.
|
79
|
+
#
|
80
|
+
# "off" - Disables console capture.
|
81
|
+
#
|
82
|
+
# "redirect" - Redirects low-level file descriptors for capturing output.
|
83
|
+
#
|
84
|
+
# "wrap" - Overrides the write methods of sys.stdout/sys.stderr. Will be
|
85
|
+
# mapped to either "wrap_raw" or "wrap_emu" based on the state of the system.
|
86
|
+
#
|
87
|
+
# "wrap_raw" - Same as "wrap" but captures raw output directly instead of
|
88
|
+
# through an emulator.
|
89
|
+
#
|
90
|
+
# "wrap_emu" - Same as "wrap" but captures output through an emulator.
|
91
|
+
console: Literal["auto", "off", "wrap", "redirect", "wrap_raw", "wrap_emu"] = Field(
|
92
|
+
default="auto",
|
93
|
+
validate_default=True,
|
322
94
|
)
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
_platform: str
|
368
|
-
_proxies: Mapping[
|
369
|
-
str, str
|
370
|
-
] # custom proxy servers for the requests to W&B [scheme -> url]
|
371
|
-
_python: str
|
372
|
-
_runqueue_item_id: str
|
373
|
-
_require_legacy_service: bool
|
374
|
-
_save_requirements: bool
|
375
|
-
_service_transport: str
|
376
|
-
_service_wait: float
|
377
|
-
_shared: bool
|
378
|
-
_start_datetime: str
|
379
|
-
_start_time: float
|
380
|
-
_stats_pid: int # (internal) base pid for system stats
|
381
|
-
_stats_sampling_interval: float # sampling interval for system stats
|
382
|
-
_stats_sample_rate_seconds: float # badly-named sampling interval, deprecated
|
383
|
-
_stats_samples_to_average: (
|
384
|
-
int # number of samples to average before reporting, deprecated
|
95
|
+
# Whether to produce multipart console log files.
|
96
|
+
console_multipart: bool = False
|
97
|
+
# Path to file for writing temporary access tokens.
|
98
|
+
credentials_file: str = Field(
|
99
|
+
default_factory=lambda: str(credentials.DEFAULT_WANDB_CREDENTIALS_FILE)
|
100
|
+
)
|
101
|
+
# Whether to disable code saving.
|
102
|
+
disable_code: bool = False
|
103
|
+
# Whether to disable capturing the git state.
|
104
|
+
disable_git: bool = False
|
105
|
+
# Whether to disable the creation of a job artifact for W&B Launch.
|
106
|
+
disable_job_creation: bool = True
|
107
|
+
# The Docker image used to execute the script.
|
108
|
+
docker: str | None = None
|
109
|
+
# The email address of the user.
|
110
|
+
email: str | None = None
|
111
|
+
# The W&B entity, like a user or a team.
|
112
|
+
entity: str | None = None
|
113
|
+
force: bool = False
|
114
|
+
fork_from: RunMoment | None = None
|
115
|
+
git_commit: str | None = None
|
116
|
+
git_remote: str = "origin"
|
117
|
+
git_remote_url: str | None = None
|
118
|
+
git_root: str | None = None
|
119
|
+
heartbeat_seconds: int = 30
|
120
|
+
host: str | None = None
|
121
|
+
# The custom proxy servers for http requests to W&B.
|
122
|
+
http_proxy: str | None = None
|
123
|
+
# The custom proxy servers for https requests to W&B.
|
124
|
+
https_proxy: str | None = None
|
125
|
+
# Path to file containing an identity token (JWT) for authentication.
|
126
|
+
identity_token_file: str | None = None
|
127
|
+
# Unix glob patterns relative to `files_dir` to not upload.
|
128
|
+
ignore_globs: tuple[str, ...] = ()
|
129
|
+
init_timeout: float = 90.0
|
130
|
+
job_name: str | None = None
|
131
|
+
job_source: Literal["repo", "artifact", "image"] | None = None
|
132
|
+
label_disable: bool = False
|
133
|
+
launch: bool = False
|
134
|
+
launch_config_path: str | None = None
|
135
|
+
login_timeout: float | None = None
|
136
|
+
mode: Literal["online", "offline", "dryrun", "disabled", "run", "shared"] = Field(
|
137
|
+
default="online",
|
138
|
+
validate_default=True,
|
385
139
|
)
|
386
|
-
|
387
|
-
|
140
|
+
notebook_name: str | None = None
|
141
|
+
# Path to the script that created the run, if available.
|
142
|
+
program: str | None = None
|
143
|
+
# The absolute path from the root repository directory to the script that
|
144
|
+
# created the run.
|
145
|
+
#
|
146
|
+
# Root repository directory is defined as the directory containing the
|
147
|
+
# .git directory, if it exists. Otherwise, it's the current working directory.
|
148
|
+
program_abspath: str | None = None
|
149
|
+
program_relpath: str | None = None
|
150
|
+
# The W&B project ID.
|
151
|
+
project: str | None = None
|
152
|
+
quiet: bool = False
|
153
|
+
reinit: bool = False
|
154
|
+
relogin: bool = False
|
155
|
+
# Specifies the resume behavior for the run. The available options are:
|
156
|
+
#
|
157
|
+
# "must": Resumes from an existing run with the same ID. If no such run exists,
|
158
|
+
# it will result in failure.
|
159
|
+
#
|
160
|
+
# "allow": Attempts to resume from an existing run with the same ID. If none is
|
161
|
+
# found, a new run will be created.
|
162
|
+
#
|
163
|
+
# "never": Always starts a new run. If a run with the same ID already exists,
|
164
|
+
# it will result in failure.
|
165
|
+
#
|
166
|
+
# "auto": Automatically resumes from the most recent failed run on the same
|
167
|
+
# machine.
|
168
|
+
resume: Literal["allow", "must", "never", "auto"] | None = None
|
169
|
+
resume_from: RunMoment | None = None
|
170
|
+
# Indication from the server about the state of the run.
|
171
|
+
#
|
172
|
+
# This is different from resume, a user provided flag.
|
173
|
+
resumed: bool = False
|
174
|
+
# The root directory that will be used to derive other paths,
|
175
|
+
# such as the wandb directory, and the run directory.
|
176
|
+
root_dir: str = Field(default_factory=lambda: os.path.abspath(os.getcwd()))
|
177
|
+
run_group: str | None = None
|
178
|
+
# The ID of the run.
|
179
|
+
run_id: str | None = None
|
180
|
+
run_job_type: str | None = None
|
181
|
+
run_name: str | None = None
|
182
|
+
run_notes: str | None = None
|
183
|
+
run_tags: tuple[str, ...] | None = None
|
184
|
+
sagemaker_disable: bool = False
|
185
|
+
save_code: bool | None = None
|
186
|
+
settings_system: str = Field(
|
187
|
+
default_factory=lambda: _path_convert(
|
188
|
+
os.path.join("~", ".config", "wandb", "settings")
|
189
|
+
)
|
388
190
|
)
|
389
|
-
|
390
|
-
|
191
|
+
show_colors: bool | None = None
|
192
|
+
show_emoji: bool | None = None
|
193
|
+
show_errors: bool = True
|
194
|
+
show_info: bool = True
|
195
|
+
show_warnings: bool = True
|
196
|
+
silent: bool = False
|
197
|
+
start_method: str | None = None
|
198
|
+
strict: bool | None = None
|
199
|
+
summary_timeout: int = 60
|
200
|
+
summary_warnings: int = 5 # TODO: kill this with fire
|
201
|
+
sweep_id: str | None = None
|
202
|
+
sweep_param_path: str | None = None
|
203
|
+
symlink: bool = Field(
|
204
|
+
default_factory=lambda: False if platform.system() == "Windows" else True
|
391
205
|
)
|
392
|
-
|
393
|
-
|
206
|
+
sync_tensorboard: bool | None = None
|
207
|
+
table_raise_on_max_row_limit_exceeded: bool = False
|
208
|
+
username: str | None = None
|
209
|
+
|
210
|
+
# Internal settings.
|
211
|
+
#
|
212
|
+
# These are typically not meant to be set by the user and should not be considered
|
213
|
+
# a part of the public API as they may change or be removed in future versions.
|
214
|
+
|
215
|
+
# CLI mode.
|
216
|
+
x_cli_only_mode: bool = False
|
217
|
+
# Disable the collection of system metadata.
|
218
|
+
x_disable_meta: bool = False
|
219
|
+
# Pre-wandb-core, this setting was used to disable the (now legacy) wandb service.
|
220
|
+
#
|
221
|
+
# TODO: this is deprecated and will be removed in future versions.
|
222
|
+
x_disable_service: bool = False
|
223
|
+
# Do not use setproctitle for internal process in legacy service.
|
224
|
+
x_disable_setproctitle: bool = False
|
225
|
+
# Disable system metrics collection.
|
226
|
+
x_disable_stats: bool = False
|
227
|
+
# Disable check for latest version of wandb, from PyPI.
|
228
|
+
x_disable_update_check: bool = False
|
229
|
+
# Prevent early viewer query.
|
230
|
+
x_disable_viewer: bool = False
|
231
|
+
# Disable automatic machine info collection.
|
232
|
+
x_disable_machine_info: bool = False
|
233
|
+
# Python executable
|
234
|
+
x_executable: str | None = None
|
235
|
+
# Additional headers to add to all outgoing HTTP requests.
|
236
|
+
x_extra_http_headers: dict[str, str] | None = None
|
237
|
+
# An approximate maximum request size for the filestream API.
|
238
|
+
#
|
239
|
+
# This applies when wandb-core is enabled. Its purpose is to prevent
|
240
|
+
# HTTP requests from failing due to containing too much data.
|
241
|
+
#
|
242
|
+
# This number is approximate: requests will be slightly larger.
|
243
|
+
x_file_stream_max_bytes: int | None = None
|
244
|
+
# Max line length for filestream jsonl files.
|
245
|
+
x_file_stream_max_line_bytes: int | None = None
|
246
|
+
# Interval in seconds between filestream transmissions.
|
247
|
+
x_file_stream_transmit_interval: float | None = None
|
248
|
+
# Filestream retry client configuration.
|
249
|
+
# max number of retries
|
250
|
+
x_file_stream_retry_max: int | None = None
|
251
|
+
# min wait time between retries
|
252
|
+
x_file_stream_retry_wait_min_seconds: float | None = None
|
253
|
+
# max wait time between retries
|
254
|
+
x_file_stream_retry_wait_max_seconds: float | None = None
|
255
|
+
# timeout for individual HTTP requests
|
256
|
+
x_file_stream_timeout_seconds: float | None = None
|
257
|
+
# file transfer retry client configuration
|
258
|
+
x_file_transfer_retry_max: int | None = None
|
259
|
+
x_file_transfer_retry_wait_min_seconds: float | None = None
|
260
|
+
x_file_transfer_retry_wait_max_seconds: float | None = None
|
261
|
+
x_file_transfer_timeout_seconds: float | None = None
|
262
|
+
# override setting for the computed files_dir
|
263
|
+
x_files_dir: str | None = None
|
264
|
+
# flow control configuration for file stream
|
265
|
+
x_flow_control_custom: bool | None = None
|
266
|
+
x_flow_control_disabled: bool | None = None
|
267
|
+
# graphql retry client configuration
|
268
|
+
x_graphql_retry_max: int | None = None
|
269
|
+
x_graphql_retry_wait_min_seconds: float | None = None
|
270
|
+
x_graphql_retry_wait_max_seconds: float | None = None
|
271
|
+
x_graphql_timeout_seconds: float | None = None
|
272
|
+
x_internal_check_process: float = 8.0
|
273
|
+
x_jupyter_name: str | None = None
|
274
|
+
x_jupyter_path: str | None = None
|
275
|
+
x_jupyter_root: str | None = None
|
276
|
+
# Label to assign to system metrics and console logs collected for the run
|
277
|
+
# to group by on the frontend. Can be used to distinguish data from different
|
278
|
+
# nodes in a distributed training job.
|
279
|
+
x_label: str | None = None
|
280
|
+
x_live_policy_rate_limit: int | None = None
|
281
|
+
x_live_policy_wait_time: int | None = None
|
282
|
+
x_log_level: int = logging.INFO
|
283
|
+
x_network_buffer: int | None = None
|
284
|
+
# Determines whether to save internal wandb files and metadata.
|
285
|
+
# In a distributed setting, this is useful for avoiding file overwrites on secondary nodes
|
286
|
+
# when only system metrics and logs are needed, as the primary node handles the main logging.
|
287
|
+
x_primary_node: bool = True
|
288
|
+
# [deprecated, use http(s)_proxy] custom proxy servers for the requests to W&B
|
289
|
+
# [scheme -> url].
|
290
|
+
x_proxies: dict[str, str] | None = None
|
291
|
+
x_runqueue_item_id: str | None = None
|
292
|
+
x_require_legacy_service: bool = False
|
293
|
+
x_save_requirements: bool = True
|
294
|
+
x_service_transport: str | None = None
|
295
|
+
x_service_wait: float = 30.0
|
296
|
+
x_show_operation_stats: bool = False
|
297
|
+
# The start time of the run in seconds since the Unix epoch.
|
298
|
+
x_start_time: float | None = None
|
299
|
+
# PID of the process that started the wandb-core process to collect system stats for.
|
300
|
+
x_stats_pid: int = os.getpid()
|
301
|
+
# Sampling interval for the system monitor in seconds.
|
302
|
+
x_stats_sampling_interval: float = Field(default=10.0)
|
303
|
+
# Path to store the default config file for the neuron-monitor tool
|
304
|
+
# used to monitor AWS Trainium devices.
|
305
|
+
x_stats_neuron_monitor_config_path: str | None = None
|
306
|
+
# Open metrics endpoint names and urls.
|
307
|
+
x_stats_open_metrics_endpoints: dict[str, str] | None = None
|
308
|
+
# Filter to apply to metrics collected from OpenMetrics endpoints.
|
309
|
+
# Supports two formats:
|
394
310
|
# - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
|
395
311
|
# - ("metric regex pattern 1", "metric regex pattern 2", ...)
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
host: str
|
434
|
-
http_proxy: str # proxy server for the http requests to W&B
|
435
|
-
https_proxy: str # proxy server for the https requests to W&B
|
436
|
-
identity_token_file: str # file path to supply a jwt for authentication
|
437
|
-
ignore_globs: Tuple[str]
|
438
|
-
init_timeout: float
|
439
|
-
is_local: bool
|
440
|
-
job_name: str
|
441
|
-
job_source: str
|
442
|
-
label_disable: bool
|
443
|
-
launch: bool
|
444
|
-
launch_config_path: str
|
445
|
-
log_dir: str
|
446
|
-
log_internal: str
|
447
|
-
log_symlink_internal: str
|
448
|
-
log_symlink_user: str
|
449
|
-
log_user: str
|
450
|
-
login_timeout: float
|
451
|
-
# magic: Union[str, bool, dict] # never used in code, deprecated
|
452
|
-
mode: str
|
453
|
-
notebook_name: str
|
454
|
-
program: str
|
455
|
-
program_abspath: str
|
456
|
-
program_relpath: str
|
457
|
-
project: str
|
458
|
-
project_url: str
|
459
|
-
quiet: bool
|
460
|
-
reinit: bool
|
461
|
-
relogin: bool
|
462
|
-
# todo: add a preprocessing step to convert this to string
|
463
|
-
resume: Union[str, bool]
|
464
|
-
resume_fname: str
|
465
|
-
resumed: bool # indication from the server about the state of the run (different from resume - user provided flag)
|
466
|
-
root_dir: str
|
467
|
-
run_group: str
|
468
|
-
run_id: str
|
469
|
-
run_job_type: str
|
470
|
-
run_mode: str
|
471
|
-
run_name: str
|
472
|
-
run_notes: str
|
473
|
-
run_tags: Tuple[str]
|
474
|
-
run_url: str
|
475
|
-
sagemaker_disable: bool
|
476
|
-
save_code: bool
|
477
|
-
settings_system: str
|
478
|
-
settings_workspace: str
|
479
|
-
show_colors: bool
|
480
|
-
show_emoji: bool
|
481
|
-
show_errors: bool
|
482
|
-
show_info: bool
|
483
|
-
show_warnings: bool
|
484
|
-
silent: bool
|
485
|
-
start_method: str
|
486
|
-
strict: bool
|
487
|
-
summary_errors: int
|
488
|
-
summary_timeout: int
|
489
|
-
summary_warnings: int
|
490
|
-
sweep_id: str
|
491
|
-
sweep_param_path: str
|
492
|
-
sweep_url: str
|
493
|
-
symlink: bool
|
494
|
-
sync_dir: str
|
495
|
-
sync_file: str
|
496
|
-
sync_symlink_latest: str
|
497
|
-
table_raise_on_max_row_limit_exceeded: bool
|
498
|
-
timespec: str
|
499
|
-
tmp_dir: str
|
500
|
-
username: str
|
501
|
-
wandb_dir: str
|
502
|
-
|
503
|
-
|
504
|
-
class Property:
|
505
|
-
"""A class to represent attributes (individual settings) of the Settings object.
|
506
|
-
|
507
|
-
- Encapsulates the logic of how to preprocess and validate values of settings
|
508
|
-
throughout the lifetime of a class instance.
|
509
|
-
- Allows for runtime modification of settings with hooks, e.g. in the case when
|
510
|
-
a setting depends on another setting.
|
511
|
-
- The update() method is used to update the value of a setting.
|
512
|
-
- The `is_policy` attribute determines the source priority when updating the property value.
|
513
|
-
E.g. if `is_policy` is True, the smallest `Source` value takes precedence.
|
514
|
-
"""
|
515
|
-
|
516
|
-
def __init__( # pylint: disable=unused-argument
|
517
|
-
self,
|
518
|
-
name: str,
|
519
|
-
value: Optional[Any] = None,
|
520
|
-
preprocessor: Union[Callable, Sequence[Callable], None] = None,
|
521
|
-
# validators allow programming by contract
|
522
|
-
validator: Union[Callable, Sequence[Callable], None] = None,
|
523
|
-
# runtime converter (hook): properties can be e.g. tied to other properties
|
524
|
-
hook: Union[Callable, Sequence[Callable], None] = None,
|
525
|
-
# always apply hook even if value is None. can be used to replace @property's
|
526
|
-
auto_hook: bool = False,
|
527
|
-
is_policy: bool = False,
|
528
|
-
frozen: bool = False,
|
529
|
-
source: int = Source.BASE,
|
530
|
-
**kwargs: Any,
|
531
|
-
):
|
532
|
-
self.name = name
|
533
|
-
self._preprocessor = preprocessor
|
534
|
-
self._validator = validator
|
535
|
-
self._hook = hook
|
536
|
-
self._auto_hook = auto_hook
|
537
|
-
self._is_policy = is_policy
|
538
|
-
self._source = source
|
539
|
-
|
540
|
-
# preprocess and validate value
|
541
|
-
self._value = self._validate(self._preprocess(value))
|
542
|
-
|
543
|
-
self.__frozen = frozen
|
312
|
+
x_stats_open_metrics_filters: dict[str, dict[str, str]] | Sequence[str] | None = (
|
313
|
+
None
|
314
|
+
)
|
315
|
+
# HTTP headers to add to OpenMetrics requests.
|
316
|
+
x_stats_open_metrics_http_headers: dict[str, str] | None = None
|
317
|
+
# System paths to monitor for disk usage.
|
318
|
+
x_stats_disk_paths: Sequence[str] | None = Field(
|
319
|
+
default_factory=lambda: ("/", "/System/Volumes/Data")
|
320
|
+
if platform.system() == "Darwin"
|
321
|
+
else ("/",)
|
322
|
+
)
|
323
|
+
# Number of system metric samples to buffer in memory in the wandb-core process.
|
324
|
+
# Can be accessed via run._system_metrics.
|
325
|
+
x_stats_buffer_size: int = 0
|
326
|
+
# Flag to indicate whether we are syncing a run from the transaction log.
|
327
|
+
x_sync: bool = False
|
328
|
+
# Controls whether this process can update the run's final state (finished/failed) on the server.
|
329
|
+
# Set to False in distributed training when only the main process should determine the final state.
|
330
|
+
x_update_finish_state: bool = True
|
331
|
+
|
332
|
+
# Model validator to catch legacy settings.
|
333
|
+
@model_validator(mode="before")
|
334
|
+
@classmethod
|
335
|
+
def catch_private_settings(cls, values):
|
336
|
+
"""Check if a private field is provided and assign to the corrsponding public one.
|
337
|
+
|
338
|
+
This is a compatibility layer to handle previous versions of the settings.
|
339
|
+
"""
|
340
|
+
new_values = {}
|
341
|
+
for key in values:
|
342
|
+
# Internal settings are prefixed with "x_" instead of "_"
|
343
|
+
# as Pydantic does not allow "_" in field names.
|
344
|
+
if key.startswith("_"):
|
345
|
+
new_values["x" + key] = values[key]
|
346
|
+
else:
|
347
|
+
new_values[key] = values[key]
|
348
|
+
return new_values
|
544
349
|
|
545
|
-
@
|
546
|
-
def
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
350
|
+
@model_validator(mode="after")
|
351
|
+
def validate_mutual_exclusion_of_branching_args(self) -> Self:
|
352
|
+
if (
|
353
|
+
sum(
|
354
|
+
o is not None
|
355
|
+
for o in [
|
356
|
+
self.fork_from,
|
357
|
+
self.resume,
|
358
|
+
self.resume_from,
|
359
|
+
]
|
360
|
+
)
|
361
|
+
> 1
|
362
|
+
):
|
363
|
+
raise ValueError(
|
364
|
+
"`fork_from`, `resume`, or `resume_from` are mutually exclusive. "
|
365
|
+
"Please specify only one of them."
|
366
|
+
)
|
367
|
+
return self
|
554
368
|
|
555
|
-
|
556
|
-
def is_policy(self) -> bool:
|
557
|
-
return self._is_policy
|
369
|
+
# Field validators.
|
558
370
|
|
559
|
-
@
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
if callable(self._preprocessor)
|
568
|
-
else self._preprocessor
|
371
|
+
@field_validator("x_disable_service", mode="after")
|
372
|
+
@classmethod
|
373
|
+
def validate_disable_service(cls, value):
|
374
|
+
if value:
|
375
|
+
termwarn(
|
376
|
+
"Disabling the wandb service is deprecated as of version 0.18.0 "
|
377
|
+
"and will be removed in future versions. ",
|
378
|
+
repeat=False,
|
569
379
|
)
|
570
|
-
for p in _preprocessor:
|
571
|
-
try:
|
572
|
-
value = p(value)
|
573
|
-
except Exception:
|
574
|
-
raise SettingsPreprocessingError(
|
575
|
-
f"Unable to preprocess value for property {self.name}: {value}."
|
576
|
-
)
|
577
380
|
return value
|
578
381
|
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
)
|
584
|
-
for v in _validator:
|
585
|
-
if not v(value):
|
586
|
-
# failed validation will likely cause a downstream error
|
587
|
-
# when trying to convert to protobuf, so we raise a hard error
|
588
|
-
raise SettingsValidationError(
|
589
|
-
f"Invalid value for property {self.name}: {value}."
|
590
|
-
)
|
382
|
+
@field_validator("api_key", mode="after")
|
383
|
+
@classmethod
|
384
|
+
def validate_api_key(cls, value):
|
385
|
+
if value is not None and (len(value) > len(value.strip())):
|
386
|
+
raise UsageError("API key cannot start or end with whitespace")
|
591
387
|
return value
|
592
388
|
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
# -
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
(source == Source.OVERRIDE)
|
603
|
-
or (
|
604
|
-
self._is_policy
|
605
|
-
and self._source != Source.OVERRIDE
|
606
|
-
and source <= self._source
|
607
|
-
)
|
608
|
-
or (
|
609
|
-
not self._is_policy
|
610
|
-
and self._source != Source.OVERRIDE
|
611
|
-
and source >= self._source
|
389
|
+
@field_validator("base_url", mode="after")
|
390
|
+
@classmethod
|
391
|
+
def validate_base_url(cls, value):
|
392
|
+
cls.validate_url(value)
|
393
|
+
# wandb.ai-specific checks
|
394
|
+
if re.match(r".*wandb\.ai[^\.]*$", value) and "api." not in value:
|
395
|
+
# user might guess app.wandb.ai or wandb.ai is the default cloud server
|
396
|
+
raise ValueError(
|
397
|
+
f"{value} is not a valid server address, did you mean https://api.wandb.ai?"
|
612
398
|
)
|
399
|
+
elif re.match(r".*wandb\.ai[^\.]*$", value) and not value.startswith("https"):
|
400
|
+
raise ValueError("http is not secure, please use https://api.wandb.ai")
|
401
|
+
return value.rstrip("/")
|
402
|
+
|
403
|
+
@field_validator("console", mode="after")
|
404
|
+
@classmethod
|
405
|
+
def validate_console(cls, value, info):
|
406
|
+
if value != "auto":
|
407
|
+
return value
|
408
|
+
if (
|
409
|
+
ipython.in_jupyter()
|
410
|
+
or (info.data.get("start_method") == "thread")
|
411
|
+
or not info.data.get("disable_service")
|
412
|
+
or platform.system() == "Windows"
|
613
413
|
):
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
def __setattr__(self, key: str, value: Any) -> None:
|
619
|
-
if "_Property__frozen" in self.__dict__ and self.__frozen:
|
620
|
-
raise TypeError(f"Property object {self.name} is frozen")
|
621
|
-
if key == "value":
|
622
|
-
raise AttributeError("Use update() to update property value")
|
623
|
-
self.__dict__[key] = value
|
624
|
-
|
625
|
-
def __str__(self) -> str:
|
626
|
-
return f"{self.value!r}" if isinstance(self.value, str) else f"{self.value}"
|
627
|
-
|
628
|
-
def __repr__(self) -> str:
|
629
|
-
return (
|
630
|
-
f"<Property {self.name}: value={self.value} "
|
631
|
-
f"_value={self._value} source={self._source} is_policy={self._is_policy}>"
|
632
|
-
)
|
633
|
-
# return f"<Property {self.name}: value={self.value}>"
|
634
|
-
# return self.__dict__.__repr__()
|
414
|
+
value = "wrap"
|
415
|
+
else:
|
416
|
+
value = "redirect"
|
417
|
+
return value
|
635
418
|
|
419
|
+
@field_validator("x_file_stream_max_line_bytes", mode="after")
|
420
|
+
@classmethod
|
421
|
+
def validate_file_stream_max_line_bytes(cls, value):
|
422
|
+
if value is not None and value < 1:
|
423
|
+
raise ValueError("File stream max line bytes must be greater than 0")
|
424
|
+
return value
|
636
425
|
|
637
|
-
|
638
|
-
|
426
|
+
@field_validator("fork_from", mode="before")
|
427
|
+
@classmethod
|
428
|
+
def validate_fork_from(cls, value, info) -> RunMoment | None:
|
429
|
+
run_moment = cls._runmoment_preprocessor(value)
|
430
|
+
if run_moment and info.data.get("run_id") == run_moment.run:
|
431
|
+
raise ValueError(
|
432
|
+
"Provided `run_id` is the same as the run to `fork_from`. "
|
433
|
+
"Please provide a different `run_id` or remove the `run_id` argument. "
|
434
|
+
"If you want to rewind the current run, please use `resume_from` instead."
|
435
|
+
)
|
436
|
+
return run_moment
|
639
437
|
|
640
|
-
|
641
|
-
|
438
|
+
@field_validator("http_proxy", mode="after")
|
439
|
+
@classmethod
|
440
|
+
def validate_http_proxy(cls, value):
|
441
|
+
if value is None:
|
442
|
+
return None
|
443
|
+
cls.validate_url(value)
|
444
|
+
return value.rstrip("/")
|
642
445
|
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
},
|
651
|
-
_code_path_local={
|
652
|
-
"hook": lambda _: _get_program_relpath(self.program),
|
653
|
-
"auto_hook": True,
|
654
|
-
},
|
655
|
-
_colab={
|
656
|
-
"hook": lambda _: "google.colab" in sys.modules,
|
657
|
-
"auto_hook": True,
|
658
|
-
},
|
659
|
-
_disable_machine_info={
|
660
|
-
"value": False,
|
661
|
-
"preprocessor": _str_as_bool,
|
662
|
-
},
|
663
|
-
_disable_meta={
|
664
|
-
"value": False,
|
665
|
-
"preprocessor": _str_as_bool,
|
666
|
-
"hook": lambda x: self._disable_machine_info or x,
|
667
|
-
},
|
668
|
-
_disable_service={
|
669
|
-
"value": False,
|
670
|
-
"preprocessor": self._process_disable_service,
|
671
|
-
"is_policy": True,
|
672
|
-
},
|
673
|
-
_disable_setproctitle={"value": False, "preprocessor": _str_as_bool},
|
674
|
-
_disable_stats={
|
675
|
-
"value": False,
|
676
|
-
"preprocessor": _str_as_bool,
|
677
|
-
"hook": lambda x: self._disable_machine_info or x,
|
678
|
-
},
|
679
|
-
_disable_update_check={"preprocessor": _str_as_bool},
|
680
|
-
_disable_viewer={"preprocessor": _str_as_bool},
|
681
|
-
_extra_http_headers={"preprocessor": _str_as_json},
|
682
|
-
_file_stream_max_bytes={"preprocessor": int},
|
683
|
-
_file_stream_transmit_interval={"preprocessor": float},
|
684
|
-
_file_stream_retry_max={"preprocessor": int},
|
685
|
-
_file_stream_retry_wait_min_seconds={"preprocessor": float},
|
686
|
-
_file_stream_retry_wait_max_seconds={"preprocessor": float},
|
687
|
-
_file_stream_timeout_seconds={"preprocessor": float},
|
688
|
-
_file_stream_max_line_bytes={
|
689
|
-
"preprocessor": _preprocess_file_stream_max_line_bytes,
|
690
|
-
},
|
691
|
-
_file_transfer_retry_max={"preprocessor": int},
|
692
|
-
_file_transfer_retry_wait_min_seconds={"preprocessor": float},
|
693
|
-
_file_transfer_retry_wait_max_seconds={"preprocessor": float},
|
694
|
-
_file_transfer_timeout_seconds={"preprocessor": float},
|
695
|
-
_flow_control_disabled={
|
696
|
-
"hook": lambda _: self._network_buffer == 0,
|
697
|
-
"auto_hook": True,
|
698
|
-
},
|
699
|
-
_flow_control_custom={
|
700
|
-
"hook": lambda _: bool(self._network_buffer),
|
701
|
-
"auto_hook": True,
|
702
|
-
},
|
703
|
-
_graphql_retry_max={"preprocessor": int},
|
704
|
-
_graphql_retry_wait_min_seconds={"preprocessor": float},
|
705
|
-
_graphql_retry_wait_max_seconds={"preprocessor": float},
|
706
|
-
_graphql_timeout_seconds={"preprocessor": float},
|
707
|
-
_internal_check_process={"value": 8, "preprocessor": float},
|
708
|
-
_internal_queue_timeout={"value": 2, "preprocessor": float},
|
709
|
-
_ipython={
|
710
|
-
"hook": lambda _: ipython.in_ipython(),
|
711
|
-
"auto_hook": True,
|
712
|
-
},
|
713
|
-
_jupyter={
|
714
|
-
"hook": lambda _: ipython.in_jupyter(),
|
715
|
-
"auto_hook": True,
|
716
|
-
},
|
717
|
-
_kaggle={"hook": lambda _: util._is_likely_kaggle(), "auto_hook": True},
|
718
|
-
_log_level={"value": logging.DEBUG},
|
719
|
-
_network_buffer={"preprocessor": int},
|
720
|
-
_noop={"hook": lambda _: self.mode == "disabled", "auto_hook": True},
|
721
|
-
_notebook={
|
722
|
-
"hook": lambda _: self._ipython
|
723
|
-
or self._jupyter
|
724
|
-
or self._colab
|
725
|
-
or self._kaggle,
|
726
|
-
"auto_hook": True,
|
727
|
-
},
|
728
|
-
_offline={
|
729
|
-
"hook": (
|
730
|
-
lambda _: True
|
731
|
-
if self.disabled or (self.mode in ("dryrun", "offline"))
|
732
|
-
else False
|
733
|
-
),
|
734
|
-
"auto_hook": True,
|
735
|
-
},
|
736
|
-
_platform={"value": util.get_platform_name()},
|
737
|
-
_proxies={
|
738
|
-
# TODO: deprecate and ask the user to use http_proxy and https_proxy instead
|
739
|
-
"preprocessor": _str_as_json,
|
740
|
-
},
|
741
|
-
_require_legacy_service={"value": False, "preprocessor": _str_as_bool},
|
742
|
-
_save_requirements={"value": True, "preprocessor": _str_as_bool},
|
743
|
-
_service_wait={
|
744
|
-
"value": 30,
|
745
|
-
"preprocessor": float,
|
746
|
-
"validator": self._validate__service_wait,
|
747
|
-
},
|
748
|
-
_shared={
|
749
|
-
"hook": lambda _: self.mode == "shared",
|
750
|
-
"auto_hook": True,
|
751
|
-
},
|
752
|
-
_start_datetime={"preprocessor": _datetime_as_str},
|
753
|
-
_stats_sampling_interval={
|
754
|
-
"value": 10.0,
|
755
|
-
"preprocessor": float,
|
756
|
-
"validator": self._validate__stats_sampling_interval,
|
757
|
-
},
|
758
|
-
_stats_sample_rate_seconds={
|
759
|
-
"value": 2.0,
|
760
|
-
"preprocessor": float,
|
761
|
-
"validator": self._validate__stats_sample_rate_seconds,
|
762
|
-
},
|
763
|
-
_stats_samples_to_average={
|
764
|
-
"value": 15,
|
765
|
-
"preprocessor": int,
|
766
|
-
"validator": self._validate__stats_samples_to_average,
|
767
|
-
},
|
768
|
-
_stats_join_assets={"value": True, "preprocessor": _str_as_bool},
|
769
|
-
_stats_neuron_monitor_config_path={
|
770
|
-
"hook": lambda x: self._path_convert(x),
|
771
|
-
},
|
772
|
-
_stats_open_metrics_endpoints={
|
773
|
-
"preprocessor": _str_as_json,
|
774
|
-
},
|
775
|
-
_stats_open_metrics_filters={
|
776
|
-
# capture all metrics on all endpoints by default
|
777
|
-
"value": (".*",),
|
778
|
-
"preprocessor": _str_as_json,
|
779
|
-
},
|
780
|
-
_stats_disk_paths={
|
781
|
-
"value": ("/",),
|
782
|
-
"preprocessor": _str_as_json,
|
783
|
-
},
|
784
|
-
_stats_buffer_size={
|
785
|
-
"value": 0,
|
786
|
-
"preprocessor": int,
|
787
|
-
},
|
788
|
-
_sync={"value": False},
|
789
|
-
_tmp_code_dir={
|
790
|
-
"value": "code",
|
791
|
-
"hook": lambda x: self._path_convert(self.tmp_dir, x),
|
792
|
-
},
|
793
|
-
_windows={
|
794
|
-
"hook": lambda _: platform.system() == "Windows",
|
795
|
-
"auto_hook": True,
|
796
|
-
},
|
797
|
-
_show_operation_stats={"preprocessor": _str_as_bool},
|
798
|
-
allow_offline_artifacts={"value": "True", "preprocessor": _str_as_bool},
|
799
|
-
anonymous={"validator": self._validate_anonymous},
|
800
|
-
api_key={"validator": self._validate_api_key},
|
801
|
-
base_url={
|
802
|
-
"value": "https://api.wandb.ai",
|
803
|
-
"preprocessor": lambda x: str(x).strip().rstrip("/"),
|
804
|
-
"validator": self._validate_base_url,
|
805
|
-
},
|
806
|
-
colab_url={
|
807
|
-
"hook": lambda _: self._get_colab_url(),
|
808
|
-
"auto_hook": True,
|
809
|
-
},
|
810
|
-
config_paths={"preprocessor": _str_as_tuple},
|
811
|
-
console={
|
812
|
-
"value": "auto",
|
813
|
-
"validator": self._validate_console,
|
814
|
-
"hook": lambda x: self._convert_console(x),
|
815
|
-
"auto_hook": True,
|
816
|
-
},
|
817
|
-
console_multipart={"value": False, "preprocessor": _str_as_bool},
|
818
|
-
credentials_file={
|
819
|
-
"value": str(credentials.DEFAULT_WANDB_CREDENTIALS_FILE),
|
820
|
-
"preprocessor": str,
|
821
|
-
},
|
822
|
-
deployment={
|
823
|
-
"hook": lambda _: "local" if self.is_local else "cloud",
|
824
|
-
"auto_hook": True,
|
825
|
-
},
|
826
|
-
disable_code={
|
827
|
-
"value": False,
|
828
|
-
"preprocessor": _str_as_bool,
|
829
|
-
"hook": lambda x: self._disable_machine_info or x,
|
830
|
-
},
|
831
|
-
disable_hints={"preprocessor": _str_as_bool},
|
832
|
-
disable_git={
|
833
|
-
"value": False,
|
834
|
-
"preprocessor": _str_as_bool,
|
835
|
-
"hook": lambda x: self._disable_machine_info or x,
|
836
|
-
},
|
837
|
-
disable_job_creation={
|
838
|
-
"value": False,
|
839
|
-
"preprocessor": _str_as_bool,
|
840
|
-
"hook": lambda x: self._disable_machine_info or x,
|
841
|
-
},
|
842
|
-
disabled={"value": False, "preprocessor": _str_as_bool},
|
843
|
-
files_dir={
|
844
|
-
"value": "files",
|
845
|
-
"hook": lambda x: self._path_convert(
|
846
|
-
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", x
|
847
|
-
),
|
848
|
-
},
|
849
|
-
force={"preprocessor": _str_as_bool},
|
850
|
-
fork_from={
|
851
|
-
"value": None,
|
852
|
-
"preprocessor": _runmoment_preprocessor,
|
853
|
-
},
|
854
|
-
resume_from={
|
855
|
-
"value": None,
|
856
|
-
"preprocessor": _runmoment_preprocessor,
|
857
|
-
},
|
858
|
-
git_remote={"value": "origin"},
|
859
|
-
heartbeat_seconds={"value": 30},
|
860
|
-
http_proxy={
|
861
|
-
"hook": lambda x: self._proxies and self._proxies.get("http") or x,
|
862
|
-
"auto_hook": True,
|
863
|
-
},
|
864
|
-
https_proxy={
|
865
|
-
"hook": lambda x: self._proxies and self._proxies.get("https") or x,
|
866
|
-
"auto_hook": True,
|
867
|
-
},
|
868
|
-
identity_token_file={"value": None, "preprocessor": str},
|
869
|
-
ignore_globs={
|
870
|
-
"value": tuple(),
|
871
|
-
"preprocessor": lambda x: tuple(x) if not isinstance(x, tuple) else x,
|
872
|
-
},
|
873
|
-
init_timeout={"value": 90, "preprocessor": lambda x: float(x)},
|
874
|
-
is_local={
|
875
|
-
"hook": (
|
876
|
-
lambda _: self.base_url != "https://api.wandb.ai"
|
877
|
-
if self.base_url is not None
|
878
|
-
else False
|
879
|
-
),
|
880
|
-
"auto_hook": True,
|
881
|
-
},
|
882
|
-
job_name={"preprocessor": str},
|
883
|
-
job_source={"validator": self._validate_job_source},
|
884
|
-
label_disable={"preprocessor": _str_as_bool},
|
885
|
-
launch={"preprocessor": _str_as_bool},
|
886
|
-
log_dir={
|
887
|
-
"value": "logs",
|
888
|
-
"hook": lambda x: self._path_convert(
|
889
|
-
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", x
|
890
|
-
),
|
891
|
-
},
|
892
|
-
log_internal={
|
893
|
-
"value": "debug-internal.log",
|
894
|
-
"hook": lambda x: self._path_convert(self.log_dir, x),
|
895
|
-
},
|
896
|
-
log_symlink_internal={
|
897
|
-
"value": "debug-internal.log",
|
898
|
-
"hook": lambda x: self._path_convert(self.wandb_dir, x),
|
899
|
-
},
|
900
|
-
log_symlink_user={
|
901
|
-
"value": "debug.log",
|
902
|
-
"hook": lambda x: self._path_convert(self.wandb_dir, x),
|
903
|
-
},
|
904
|
-
log_user={
|
905
|
-
"value": "debug.log",
|
906
|
-
"hook": lambda x: self._path_convert(self.log_dir, x),
|
907
|
-
},
|
908
|
-
login_timeout={"preprocessor": lambda x: float(x)},
|
909
|
-
mode={"value": "online", "validator": self._validate_mode},
|
910
|
-
program={
|
911
|
-
"hook": lambda x: self._get_program(x),
|
912
|
-
},
|
913
|
-
project={
|
914
|
-
"validator": self._validate_project,
|
915
|
-
},
|
916
|
-
project_url={"hook": lambda _: self._project_url(), "auto_hook": True},
|
917
|
-
quiet={"preprocessor": _str_as_bool},
|
918
|
-
reinit={"preprocessor": _str_as_bool},
|
919
|
-
relogin={"preprocessor": _str_as_bool},
|
920
|
-
# todo: hack to make to_proto() always happy
|
921
|
-
resume={"preprocessor": lambda x: None if x is False else x},
|
922
|
-
resume_fname={
|
923
|
-
"value": "wandb-resume.json",
|
924
|
-
"hook": lambda x: self._path_convert(self.wandb_dir, x),
|
925
|
-
},
|
926
|
-
resumed={"value": "False", "preprocessor": _str_as_bool},
|
927
|
-
root_dir={
|
928
|
-
"preprocessor": lambda x: str(x),
|
929
|
-
"value": os.path.abspath(os.getcwd()),
|
930
|
-
},
|
931
|
-
run_id={
|
932
|
-
"validator": self._validate_run_id,
|
933
|
-
},
|
934
|
-
run_mode={
|
935
|
-
"hook": lambda _: "offline-run" if self._offline else "run",
|
936
|
-
"auto_hook": True,
|
937
|
-
},
|
938
|
-
run_tags={
|
939
|
-
"preprocessor": lambda x: tuple(x) if not isinstance(x, tuple) else x,
|
940
|
-
},
|
941
|
-
run_url={"hook": lambda _: self._run_url(), "auto_hook": True},
|
942
|
-
sagemaker_disable={"preprocessor": _str_as_bool},
|
943
|
-
save_code={"preprocessor": _str_as_bool},
|
944
|
-
settings_system={
|
945
|
-
"value": os.path.join("~", ".config", "wandb", "settings"),
|
946
|
-
"hook": lambda x: self._path_convert(x),
|
947
|
-
},
|
948
|
-
settings_workspace={
|
949
|
-
"value": "settings",
|
950
|
-
"hook": lambda x: self._path_convert(self.wandb_dir, x),
|
951
|
-
},
|
952
|
-
show_colors={"preprocessor": _str_as_bool},
|
953
|
-
show_emoji={"preprocessor": _str_as_bool},
|
954
|
-
show_errors={"value": "True", "preprocessor": _str_as_bool},
|
955
|
-
show_info={"value": "True", "preprocessor": _str_as_bool},
|
956
|
-
show_warnings={"value": "True", "preprocessor": _str_as_bool},
|
957
|
-
silent={"value": "False", "preprocessor": _str_as_bool},
|
958
|
-
start_method={"validator": self._validate_start_method},
|
959
|
-
strict={"preprocessor": _str_as_bool},
|
960
|
-
summary_timeout={"value": 60, "preprocessor": lambda x: int(x)},
|
961
|
-
summary_warnings={
|
962
|
-
"value": 5,
|
963
|
-
"preprocessor": lambda x: int(x),
|
964
|
-
"is_policy": True,
|
965
|
-
},
|
966
|
-
sweep_url={"hook": lambda _: self._sweep_url(), "auto_hook": True},
|
967
|
-
symlink={"preprocessor": _str_as_bool},
|
968
|
-
sync_dir={
|
969
|
-
"hook": [
|
970
|
-
lambda _: self._path_convert(
|
971
|
-
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}"
|
972
|
-
)
|
973
|
-
],
|
974
|
-
"auto_hook": True,
|
975
|
-
},
|
976
|
-
sync_file={
|
977
|
-
"hook": lambda _: self._path_convert(
|
978
|
-
self.sync_dir, f"run-{self.run_id}.wandb"
|
979
|
-
),
|
980
|
-
"auto_hook": True,
|
981
|
-
},
|
982
|
-
sync_symlink_latest={
|
983
|
-
"value": "latest-run",
|
984
|
-
"hook": lambda x: self._path_convert(self.wandb_dir, x),
|
985
|
-
},
|
986
|
-
table_raise_on_max_row_limit_exceeded={
|
987
|
-
"value": False,
|
988
|
-
"preprocessor": _str_as_bool,
|
989
|
-
},
|
990
|
-
timespec={
|
991
|
-
"hook": lambda _: self._start_datetime,
|
992
|
-
"auto_hook": True,
|
993
|
-
},
|
994
|
-
tmp_dir={
|
995
|
-
"value": "tmp",
|
996
|
-
"hook": lambda x: (
|
997
|
-
self._path_convert(
|
998
|
-
self.wandb_dir,
|
999
|
-
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
1000
|
-
x,
|
1001
|
-
)
|
1002
|
-
or tempfile.gettempdir()
|
1003
|
-
),
|
1004
|
-
},
|
1005
|
-
wandb_dir={
|
1006
|
-
"hook": lambda _: _get_wandb_dir(self.root_dir or ""),
|
1007
|
-
"auto_hook": True,
|
1008
|
-
},
|
1009
|
-
)
|
1010
|
-
return props
|
446
|
+
@field_validator("https_proxy", mode="after")
|
447
|
+
@classmethod
|
448
|
+
def validate_https_proxy(cls, value):
|
449
|
+
if value is None:
|
450
|
+
return None
|
451
|
+
cls.validate_url(value)
|
452
|
+
return value.rstrip("/")
|
1011
453
|
|
1012
|
-
|
1013
|
-
@
|
1014
|
-
def
|
1015
|
-
|
454
|
+
@field_validator("ignore_globs", mode="after")
|
455
|
+
@classmethod
|
456
|
+
def validate_ignore_globs(cls, value):
|
457
|
+
return tuple(value) if not isinstance(value, tuple) else value
|
1016
458
|
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
|
459
|
+
@field_validator("project", mode="after")
|
460
|
+
@classmethod
|
461
|
+
def validate_project(cls, value, info):
|
462
|
+
if value is None:
|
463
|
+
return None
|
464
|
+
invalid_chars_list = list("/\\#?%:")
|
465
|
+
if len(value) > 128:
|
466
|
+
raise UsageError(f"Invalid project name {value!r}: exceeded 128 characters")
|
467
|
+
invalid_chars = {char for char in invalid_chars_list if char in value}
|
468
|
+
if invalid_chars:
|
469
|
+
raise UsageError(
|
470
|
+
f"Invalid project name {value!r}: "
|
471
|
+
f"cannot contain characters {','.join(invalid_chars_list)!r}, "
|
472
|
+
f"found {','.join(invalid_chars)!r}"
|
473
|
+
)
|
474
|
+
return value
|
1023
475
|
|
1024
|
-
|
476
|
+
@field_validator("resume", mode="before")
|
477
|
+
@classmethod
|
478
|
+
def validate_resume(cls, value):
|
479
|
+
if value is False:
|
480
|
+
return None
|
481
|
+
if value is True:
|
482
|
+
return "auto"
|
483
|
+
return value
|
1025
484
|
|
1026
|
-
|
485
|
+
@field_validator("resume_from", mode="before")
|
486
|
+
@classmethod
|
487
|
+
def validate_resume_from(cls, value, info) -> RunMoment | None:
|
488
|
+
run_moment = cls._runmoment_preprocessor(value)
|
489
|
+
if run_moment and info.data.get("run_id") != run_moment.run:
|
490
|
+
raise ValueError(
|
491
|
+
"Both `run_id` and `resume_from` have been specified with different ids."
|
492
|
+
)
|
493
|
+
return run_moment
|
1027
494
|
|
1028
|
-
@
|
1029
|
-
|
1030
|
-
|
1031
|
-
if value
|
1032
|
-
|
1033
|
-
return True
|
495
|
+
@field_validator("run_id", mode="after")
|
496
|
+
@classmethod
|
497
|
+
def validate_run_id(cls, value, info):
|
498
|
+
if value is None:
|
499
|
+
return None
|
1034
500
|
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1040
|
-
|
1041
|
-
|
1042
|
-
)
|
1043
|
-
invalid_chars = {char for char in invalid_chars_list if char in value}
|
1044
|
-
if invalid_chars:
|
1045
|
-
raise UsageError(
|
1046
|
-
f"Invalid project name {value!r}: "
|
1047
|
-
f"cannot contain characters {','.join(invalid_chars_list)!r}, "
|
1048
|
-
f"found {','.join(invalid_chars)!r}"
|
1049
|
-
)
|
1050
|
-
return True
|
501
|
+
if len(value) == 0:
|
502
|
+
raise UsageError("Run ID cannot be empty")
|
503
|
+
if len(value) > len(value.strip()):
|
504
|
+
raise UsageError("Run ID cannot start or end with whitespace")
|
505
|
+
if not bool(value.strip()):
|
506
|
+
raise UsageError("Run ID cannot contain only whitespace")
|
507
|
+
return value
|
1051
508
|
|
1052
|
-
@
|
1053
|
-
|
509
|
+
@field_validator("settings_system", mode="after")
|
510
|
+
@classmethod
|
511
|
+
def validate_settings_system(cls, value):
|
512
|
+
return _path_convert(value)
|
513
|
+
|
514
|
+
@field_validator("x_service_wait", mode="after")
|
515
|
+
@classmethod
|
516
|
+
def validate_service_wait(cls, value):
|
517
|
+
if value < 0:
|
518
|
+
raise UsageError("Service wait time cannot be negative")
|
519
|
+
return
|
520
|
+
|
521
|
+
@field_validator("start_method")
|
522
|
+
@classmethod
|
523
|
+
def validate_start_method(cls, value):
|
524
|
+
if value is None:
|
525
|
+
return value
|
1054
526
|
available_methods = ["thread"]
|
1055
527
|
if hasattr(multiprocessing, "get_all_start_methods"):
|
1056
528
|
available_methods += multiprocessing.get_all_start_methods()
|
@@ -1058,251 +530,237 @@ class Settings(SettingsData):
|
|
1058
530
|
raise UsageError(
|
1059
531
|
f"Settings field `start_method`: {value!r} not in {available_methods}"
|
1060
532
|
)
|
1061
|
-
return
|
1062
|
-
|
1063
|
-
@staticmethod
|
1064
|
-
def _validate_console(value: str) -> bool:
|
1065
|
-
choices = ConsoleValue
|
1066
|
-
if value not in choices:
|
1067
|
-
# do not advertise internal console states
|
1068
|
-
choices -= {"wrap_emu", "wrap_raw"}
|
1069
|
-
raise UsageError(f"Settings field `console`: {value!r} not in {choices}")
|
1070
|
-
return True
|
533
|
+
return value
|
1071
534
|
|
1072
|
-
@
|
1073
|
-
|
1074
|
-
|
1075
|
-
if value
|
1076
|
-
raise UsageError(
|
1077
|
-
return
|
535
|
+
@field_validator("x_stats_sampling_interval", mode="after")
|
536
|
+
@classmethod
|
537
|
+
def validate_stats_sampling_interval(cls, value):
|
538
|
+
if value < 0.1:
|
539
|
+
raise UsageError("Stats sampling interval cannot be less than 0.1 seconds")
|
540
|
+
return value
|
1078
541
|
|
1079
|
-
@
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
542
|
+
@field_validator("x_stats_open_metrics_endpoints", mode="before")
|
543
|
+
@classmethod
|
544
|
+
def validate_stats_open_metrics_endpoints(cls, value):
|
545
|
+
if isinstance(value, str):
|
546
|
+
return json.loads(value)
|
547
|
+
return value
|
1084
548
|
|
1085
|
-
@
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
549
|
+
@field_validator("x_stats_open_metrics_filters", mode="before")
|
550
|
+
@classmethod
|
551
|
+
def validate_stats_open_metrics_filters(cls, value):
|
552
|
+
if isinstance(value, str):
|
553
|
+
return json.loads(value)
|
554
|
+
return value
|
1089
555
|
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1093
|
-
|
1094
|
-
|
1095
|
-
|
556
|
+
@field_validator("x_stats_open_metrics_http_headers", mode="before")
|
557
|
+
@classmethod
|
558
|
+
def validate_stats_open_metrics_http_headers(cls, value):
|
559
|
+
if isinstance(value, str):
|
560
|
+
return json.loads(value)
|
561
|
+
return value
|
1096
562
|
|
1097
|
-
|
563
|
+
@field_validator("sweep_id", mode="after")
|
564
|
+
@classmethod
|
565
|
+
def validate_sweep_id(cls, value):
|
566
|
+
if value is None:
|
567
|
+
return None
|
568
|
+
if len(value) == 0:
|
569
|
+
raise UsageError("Sweep ID cannot be empty")
|
570
|
+
if len(value) > len(value.strip()):
|
571
|
+
raise UsageError("Sweep ID cannot start or end with whitespace")
|
572
|
+
if not bool(value.strip()):
|
573
|
+
raise UsageError("Sweep ID cannot contain only whitespace")
|
574
|
+
return value
|
1098
575
|
|
1099
|
-
|
1100
|
-
def _validate_base_url(value: Optional[str]) -> bool:
|
1101
|
-
"""Validate the base url of the wandb server.
|
576
|
+
# Computed fields.
|
1102
577
|
|
1103
|
-
|
578
|
+
@computed_field # type: ignore[prop-decorator]
|
579
|
+
@property
|
580
|
+
def _args(self) -> list[str]:
|
581
|
+
if not self._jupyter:
|
582
|
+
return sys.argv[1:]
|
583
|
+
return []
|
1104
584
|
|
1105
|
-
|
585
|
+
@computed_field # type: ignore[prop-decorator]
|
586
|
+
@property
|
587
|
+
def _aws_lambda(self) -> bool:
|
588
|
+
"""Check if we are running in a lambda environment."""
|
589
|
+
from sentry_sdk.integrations.aws_lambda import ( # type: ignore[import-not-found]
|
590
|
+
get_lambda_bootstrap,
|
591
|
+
)
|
1106
592
|
|
1107
|
-
|
1108
|
-
|
593
|
+
lambda_bootstrap = get_lambda_bootstrap()
|
594
|
+
if not lambda_bootstrap or not hasattr(
|
595
|
+
lambda_bootstrap, "handle_event_request"
|
596
|
+
):
|
597
|
+
return False
|
598
|
+
return True
|
1109
599
|
|
1110
|
-
|
1111
|
-
|
600
|
+
@computed_field # type: ignore[prop-decorator]
|
601
|
+
@property
|
602
|
+
def _code_path_local(self) -> str | None:
|
603
|
+
"""The relative path from the current working directory to the code path.
|
1112
604
|
|
1113
|
-
|
1114
|
-
|
605
|
+
For example, if the code path is /home/user/project/example.py, and the
|
606
|
+
current working directory is /home/user/project, then the code path local
|
607
|
+
is example.py.
|
1115
608
|
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
609
|
+
If couldn't find the relative path, this will be an empty string.
|
610
|
+
"""
|
611
|
+
return self._get_program_relpath(self.program) if self.program else None
|
1119
612
|
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
613
|
+
@computed_field # type: ignore[prop-decorator]
|
614
|
+
@property
|
615
|
+
def _colab(self) -> bool:
|
616
|
+
return "google.colab" in sys.modules
|
1123
617
|
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
1129
|
-
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
1130
|
-
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
1131
|
-
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
1132
|
-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
1133
|
-
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
1134
|
-
"""
|
1135
|
-
if value is None:
|
1136
|
-
return True
|
618
|
+
@computed_field # type: ignore[prop-decorator]
|
619
|
+
@property
|
620
|
+
def _ipython(self) -> bool:
|
621
|
+
return ipython.in_ipython()
|
1137
622
|
|
1138
|
-
|
623
|
+
@computed_field # type: ignore[prop-decorator]
|
624
|
+
@property
|
625
|
+
def _jupyter(self) -> bool:
|
626
|
+
return ipython.in_jupyter()
|
1139
627
|
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
)
|
1145
|
-
ipv6_re = r"\[[0-9a-f:.]+\]" # (simple regex, validated later)
|
628
|
+
@computed_field # type: ignore[prop-decorator]
|
629
|
+
@property
|
630
|
+
def _kaggle(self) -> bool:
|
631
|
+
return util._is_likely_kaggle()
|
1146
632
|
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
# Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
|
1152
|
-
domain_re = r"(?:\.(?!-)[a-z" + ul + r"0-9-]{1,63}(?<!-))*"
|
1153
|
-
tld_re = (
|
1154
|
-
r"\." # dot
|
1155
|
-
r"(?!-)" # can't start with a dash
|
1156
|
-
r"(?:[a-z" + ul + "-]{2,63}" # domain label
|
1157
|
-
r"|xn--[a-z0-9]{1,59})" # or punycode label
|
1158
|
-
r"(?<!-)" # can't end with a dash
|
1159
|
-
r"\.?" # may have a trailing dot
|
1160
|
-
)
|
1161
|
-
# host_re = "(" + hostname_re + domain_re + tld_re + "|localhost)"
|
1162
|
-
# todo?: allow hostname to be just a hostname (no tld)?
|
1163
|
-
host_re = "(" + hostname_re + domain_re + f"({tld_re})?" + "|localhost)"
|
1164
|
-
|
1165
|
-
regex = re.compile(
|
1166
|
-
r"^(?:[a-z0-9.+-]*)://" # scheme is validated separately
|
1167
|
-
r"(?:[^\s:@/]+(?::[^\s:@/]*)?@)?" # user:pass authentication
|
1168
|
-
r"(?:" + ipv4_re + "|" + ipv6_re + "|" + host_re + ")"
|
1169
|
-
r"(?::[0-9]{1,5})?" # port
|
1170
|
-
r"(?:[/?#][^\s]*)?" # resource path
|
1171
|
-
r"\Z",
|
1172
|
-
re.IGNORECASE,
|
1173
|
-
)
|
1174
|
-
schemes = {"http", "https"}
|
1175
|
-
unsafe_chars = frozenset("\t\r\n")
|
633
|
+
@computed_field # type: ignore[prop-decorator]
|
634
|
+
@property
|
635
|
+
def _noop(self) -> bool:
|
636
|
+
return self.mode == "disabled"
|
1176
637
|
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
638
|
+
@computed_field # type: ignore[prop-decorator]
|
639
|
+
@property
|
640
|
+
def _notebook(self) -> bool:
|
641
|
+
return self._ipython or self._jupyter or self._colab or self._kaggle
|
1180
642
|
|
1181
|
-
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
)
|
1186
|
-
elif re.match(r".*wandb\.ai[^\.]*$", value) and scheme != "https":
|
1187
|
-
raise UsageError("http is not secure, please use https://api.wandb.ai")
|
1188
|
-
elif parsed_url.netloc == "":
|
1189
|
-
raise UsageError(f"Invalid URL: {value}")
|
1190
|
-
elif unsafe_chars.intersection(value):
|
1191
|
-
raise UsageError("URL cannot contain unsafe characters")
|
1192
|
-
elif scheme not in schemes:
|
1193
|
-
raise UsageError("URL must start with `http(s)://`")
|
1194
|
-
elif not regex.search(value):
|
1195
|
-
raise UsageError(f"{value} is not a valid server address")
|
1196
|
-
elif split_url.hostname is None or len(split_url.hostname) > 253:
|
1197
|
-
raise UsageError("hostname is invalid")
|
643
|
+
@computed_field # type: ignore[prop-decorator]
|
644
|
+
@property
|
645
|
+
def _offline(self) -> bool:
|
646
|
+
return self.mode in ("offline", "dryrun")
|
1198
647
|
|
1199
|
-
|
648
|
+
@computed_field # type: ignore[prop-decorator]
|
649
|
+
@property
|
650
|
+
def _os(self) -> str:
|
651
|
+
"""The operating system of the machine running the script."""
|
652
|
+
return platform.platform(aliased=True)
|
1200
653
|
|
1201
|
-
@
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
wandb.termwarn(
|
1206
|
-
"Disabling the wandb service is deprecated as of version 0.18.0 and will be removed in future versions. ",
|
1207
|
-
repeat=False,
|
1208
|
-
)
|
1209
|
-
return value
|
654
|
+
@computed_field # type: ignore[prop-decorator]
|
655
|
+
@property
|
656
|
+
def _platform(self) -> str:
|
657
|
+
return f"{platform.system()}-{platform.machine()}".lower()
|
1210
658
|
|
1211
|
-
@
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
return True
|
659
|
+
@computed_field # type: ignore[prop-decorator]
|
660
|
+
@property
|
661
|
+
def _python(self) -> str:
|
662
|
+
return f"{platform.python_implementation()} {platform.python_version()}"
|
1216
663
|
|
1217
|
-
@
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1221
|
-
return True
|
664
|
+
@computed_field # type: ignore[prop-decorator]
|
665
|
+
@property
|
666
|
+
def _shared(self) -> bool:
|
667
|
+
"""Whether we are in shared mode.
|
1222
668
|
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
return True
|
669
|
+
In "shared" mode, multiple processes can write to the same run,
|
670
|
+
for example from different machines.
|
671
|
+
"""
|
672
|
+
return self.mode == "shared"
|
1228
673
|
|
1229
|
-
@
|
1230
|
-
|
1231
|
-
|
1232
|
-
|
1233
|
-
|
674
|
+
@computed_field # type: ignore[prop-decorator]
|
675
|
+
@property
|
676
|
+
def _start_datetime(self) -> str:
|
677
|
+
if self.x_start_time is None:
|
678
|
+
return ""
|
679
|
+
datetime_now = datetime.fromtimestamp(self.x_start_time)
|
680
|
+
return datetime_now.strftime("%Y%m%d_%H%M%S")
|
1234
681
|
|
1235
|
-
@
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
682
|
+
@computed_field # type: ignore[prop-decorator]
|
683
|
+
@property
|
684
|
+
def _tmp_code_dir(self) -> str:
|
685
|
+
return _path_convert(
|
686
|
+
self.wandb_dir,
|
687
|
+
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
688
|
+
"tmp",
|
689
|
+
"code",
|
690
|
+
)
|
1243
691
|
|
1244
|
-
#
|
1245
|
-
@
|
1246
|
-
def
|
1247
|
-
|
1248
|
-
return os.path.expanduser(os.path.join(*args))
|
1249
|
-
|
1250
|
-
def _convert_console(self, console: str) -> str:
|
1251
|
-
if console == "auto":
|
1252
|
-
if (
|
1253
|
-
self._jupyter
|
1254
|
-
or (self.start_method == "thread")
|
1255
|
-
or not self._disable_service
|
1256
|
-
or self._windows
|
1257
|
-
):
|
1258
|
-
console = "wrap"
|
1259
|
-
else:
|
1260
|
-
console = "redirect"
|
1261
|
-
return console
|
692
|
+
@computed_field # type: ignore[prop-decorator]
|
693
|
+
@property
|
694
|
+
def _windows(self) -> bool:
|
695
|
+
return platform.system() == "Windows"
|
1262
696
|
|
1263
|
-
|
697
|
+
@computed_field # type: ignore[prop-decorator]
|
698
|
+
@property
|
699
|
+
def colab_url(self) -> str | None:
|
700
|
+
"""The URL to the Colab notebook, if running in Colab."""
|
1264
701
|
if not self._colab:
|
1265
702
|
return None
|
1266
|
-
if self.
|
1267
|
-
unescaped = unquote(self.
|
703
|
+
if self.x_jupyter_path and self.x_jupyter_path.startswith("fileId="):
|
704
|
+
unescaped = unquote(self.x_jupyter_path)
|
1268
705
|
return "https://colab.research.google.com/notebook#" + unescaped
|
1269
706
|
return None
|
1270
707
|
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
if not self._jupyter:
|
1276
|
-
return program
|
1277
|
-
|
1278
|
-
if self.notebook_name:
|
1279
|
-
return self.notebook_name
|
708
|
+
@computed_field # type: ignore[prop-decorator]
|
709
|
+
@property
|
710
|
+
def deployment(self) -> Literal["local", "cloud"]:
|
711
|
+
return "local" if self.is_local else "cloud"
|
1280
712
|
|
1281
|
-
|
1282
|
-
|
713
|
+
@computed_field # type: ignore[prop-decorator]
|
714
|
+
@property
|
715
|
+
def files_dir(self) -> str:
|
716
|
+
"""Absolute path to the local directory where the run's files are stored."""
|
717
|
+
return self.x_files_dir or _path_convert(
|
718
|
+
self.wandb_dir,
|
719
|
+
f"{self.run_mode}-{self.timespec}-{self.run_id}",
|
720
|
+
"files",
|
721
|
+
)
|
1283
722
|
|
1284
|
-
|
1285
|
-
|
1286
|
-
|
1287
|
-
|
723
|
+
@computed_field # type: ignore[prop-decorator]
|
724
|
+
@property
|
725
|
+
def is_local(self) -> bool:
|
726
|
+
return str(self.base_url) != "https://api.wandb.ai"
|
1288
727
|
|
1289
|
-
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
728
|
+
@computed_field # type: ignore[prop-decorator]
|
729
|
+
@property
|
730
|
+
def log_dir(self) -> str:
|
731
|
+
"""The directory for storing log files."""
|
732
|
+
return _path_convert(
|
733
|
+
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}", "logs"
|
734
|
+
)
|
1293
735
|
|
1294
|
-
|
736
|
+
@computed_field # type: ignore[prop-decorator]
|
737
|
+
@property
|
738
|
+
def log_internal(self) -> str:
|
739
|
+
"""The path to the file to use for internal logs."""
|
740
|
+
return _path_convert(self.log_dir, "debug-internal.log")
|
1295
741
|
|
1296
|
-
|
742
|
+
@computed_field # type: ignore[prop-decorator]
|
743
|
+
@property
|
744
|
+
def log_symlink_internal(self) -> str:
|
745
|
+
"""The path to the symlink to the internal log file of the most recent run."""
|
746
|
+
return _path_convert(self.wandb_dir, "debug-internal.log")
|
1297
747
|
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
748
|
+
@computed_field # type: ignore[prop-decorator]
|
749
|
+
@property
|
750
|
+
def log_symlink_user(self) -> str:
|
751
|
+
"""The path to the symlink to the user-process log file of the most recent run."""
|
752
|
+
return _path_convert(self.wandb_dir, "debug.log")
|
1301
753
|
|
1302
|
-
|
1303
|
-
|
754
|
+
@computed_field # type: ignore[prop-decorator]
|
755
|
+
@property
|
756
|
+
def log_user(self) -> str:
|
757
|
+
"""The path to the file to use for user-process logs."""
|
758
|
+
return _path_convert(self.log_dir, "debug.log")
|
1304
759
|
|
1305
|
-
|
760
|
+
@computed_field # type: ignore[prop-decorator]
|
761
|
+
@property
|
762
|
+
def project_url(self) -> str:
|
763
|
+
"""The W&B URL where the project can be viewed."""
|
1306
764
|
project_url = self._project_url_base()
|
1307
765
|
if not project_url:
|
1308
766
|
return ""
|
@@ -1311,400 +769,133 @@ class Settings(SettingsData):
|
|
1311
769
|
|
1312
770
|
return f"{project_url}{query}"
|
1313
771
|
|
1314
|
-
|
1315
|
-
|
772
|
+
@computed_field # type: ignore[prop-decorator]
|
773
|
+
@property
|
774
|
+
def resume_fname(self) -> str:
|
775
|
+
"""The path to the resume file."""
|
776
|
+
return _path_convert(self.wandb_dir, "wandb-resume.json")
|
777
|
+
|
778
|
+
@computed_field # type: ignore[prop-decorator]
|
779
|
+
@property
|
780
|
+
def run_mode(self) -> Literal["run", "offline-run"]:
|
781
|
+
return "run" if not self._offline else "offline-run"
|
782
|
+
|
783
|
+
@computed_field # type: ignore[prop-decorator]
|
784
|
+
@property
|
785
|
+
def run_url(self) -> str:
|
786
|
+
"""The W&B URL where the run can be viewed."""
|
1316
787
|
project_url = self._project_url_base()
|
1317
788
|
if not all([project_url, self.run_id]):
|
1318
789
|
return ""
|
1319
790
|
|
1320
791
|
query = self._get_url_query_string()
|
1321
|
-
return f"{project_url}/runs/{quote(self.run_id)}{query}"
|
792
|
+
return f"{project_url}/runs/{quote(self.run_id or '')}{query}"
|
1322
793
|
|
1323
|
-
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1327
|
-
""
|
1328
|
-
time_stamp: float = time.time()
|
1329
|
-
datetime_now: datetime = datetime.fromtimestamp(time_stamp)
|
1330
|
-
datetime_now_str = _datetime_as_str(datetime_now)
|
1331
|
-
object.__setattr__(self, "_Settings_start_datetime", datetime_now_str)
|
1332
|
-
object.__setattr__(self, "_Settings_start_time", time_stamp)
|
1333
|
-
self.update(
|
1334
|
-
_start_datetime=datetime_now_str,
|
1335
|
-
_start_time=time_stamp,
|
1336
|
-
source=source,
|
1337
|
-
)
|
794
|
+
@computed_field # type: ignore[prop-decorator]
|
795
|
+
@property
|
796
|
+
def settings_workspace(self) -> str:
|
797
|
+
"""The path to the workspace settings file."""
|
798
|
+
return _path_convert(self.wandb_dir, "settings")
|
1338
799
|
|
1339
|
-
|
1340
|
-
|
800
|
+
@computed_field # type: ignore[prop-decorator]
|
801
|
+
@property
|
802
|
+
def sweep_url(self) -> str:
|
803
|
+
"""The W&B URL where the sweep can be viewed."""
|
1341
804
|
project_url = self._project_url_base()
|
1342
805
|
if not all([project_url, self.sweep_id]):
|
1343
806
|
return ""
|
1344
807
|
|
1345
808
|
query = self._get_url_query_string()
|
1346
|
-
return f"{project_url}/sweeps/{quote(self.sweep_id)}{query}"
|
1347
|
-
|
1348
|
-
def __init__(self, **kwargs: Any) -> None:
|
1349
|
-
self.__frozen: bool = False
|
1350
|
-
self.__initialized: bool = False
|
1351
|
-
|
1352
|
-
self.__modification_order = SETTINGS_TOPOLOGICALLY_SORTED
|
1353
|
-
|
1354
|
-
# Set default settings values
|
1355
|
-
# We start off with the class attributes and `default_props` dicts
|
1356
|
-
# and then create Property objects.
|
1357
|
-
# Once initialized, attributes are to only be updated using the `update` method
|
1358
|
-
default_props = self._default_props()
|
1359
|
-
|
1360
|
-
# Init instance attributes as Property objects.
|
1361
|
-
# Type hints of class attributes are used to generate a type validator function
|
1362
|
-
# for runtime checks for each attribute.
|
1363
|
-
# These are defaults, using Source.BASE for non-policy attributes and Source.RUN for policies.
|
1364
|
-
for prop, type_hint in get_type_hints(SettingsData).items():
|
1365
|
-
validators = [self._validator_factory(type_hint)]
|
1366
|
-
|
1367
|
-
if prop in default_props:
|
1368
|
-
validator = default_props[prop].pop("validator", [])
|
1369
|
-
# Property validator could be either Callable or Sequence[Callable]
|
1370
|
-
if callable(validator):
|
1371
|
-
validators.append(validator)
|
1372
|
-
elif isinstance(validator, Sequence):
|
1373
|
-
validators.extend(list(validator))
|
1374
|
-
object.__setattr__(
|
1375
|
-
self,
|
1376
|
-
prop,
|
1377
|
-
Property(
|
1378
|
-
name=prop,
|
1379
|
-
**default_props[prop],
|
1380
|
-
validator=validators,
|
1381
|
-
# todo: double-check this logic:
|
1382
|
-
source=Source.RUN
|
1383
|
-
if default_props[prop].get("is_policy", False)
|
1384
|
-
else Source.BASE,
|
1385
|
-
),
|
1386
|
-
)
|
1387
|
-
else:
|
1388
|
-
object.__setattr__(
|
1389
|
-
self,
|
1390
|
-
prop,
|
1391
|
-
Property(
|
1392
|
-
name=prop,
|
1393
|
-
validator=validators,
|
1394
|
-
source=Source.BASE,
|
1395
|
-
),
|
1396
|
-
)
|
809
|
+
return f"{project_url}/sweeps/{quote(self.sweep_id or '')}{query}"
|
1397
810
|
|
1398
|
-
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
|
1403
|
-
|
1404
|
-
)
|
811
|
+
@computed_field # type: ignore[prop-decorator]
|
812
|
+
@property
|
813
|
+
def sync_dir(self) -> str:
|
814
|
+
return _path_convert(
|
815
|
+
self.wandb_dir, f"{self.run_mode}-{self.timespec}-{self.run_id}"
|
816
|
+
)
|
1405
817
|
|
1406
|
-
|
1407
|
-
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
self.update({prop: kwargs[prop]}, source=source)
|
1412
|
-
kwargs.pop(prop)
|
1413
|
-
|
1414
|
-
for k, v in kwargs.items():
|
1415
|
-
# todo: double-check this logic:
|
1416
|
-
source = Source.RUN if self.__dict__[k].is_policy else Source.BASE
|
1417
|
-
self.update({k: v}, source=source)
|
1418
|
-
|
1419
|
-
# setup private attributes
|
1420
|
-
object.__setattr__(self, "_Settings_start_datetime", None)
|
1421
|
-
object.__setattr__(self, "_Settings_start_time", None)
|
1422
|
-
|
1423
|
-
# done with init, use self.update() to update attributes from now on
|
1424
|
-
self.__initialized = True
|
1425
|
-
|
1426
|
-
# todo? freeze settings to prevent accidental changes
|
1427
|
-
# self.freeze()
|
1428
|
-
|
1429
|
-
def __str__(self) -> str:
|
1430
|
-
# get attributes that are instances of the Property class:
|
1431
|
-
representation = {
|
1432
|
-
k: v.value for k, v in self.__dict__.items() if isinstance(v, Property)
|
1433
|
-
}
|
1434
|
-
return f"<Settings {_redact_dict(representation)}>"
|
1435
|
-
|
1436
|
-
def __repr__(self) -> str:
|
1437
|
-
# private attributes
|
1438
|
-
private = {k: v for k, v in self.__dict__.items() if k.startswith("_Settings")}
|
1439
|
-
# get attributes that are instances of the Property class:
|
1440
|
-
attributes = {
|
1441
|
-
k: f"<Property value={v.value} source={v.source}>"
|
1442
|
-
for k, v in self.__dict__.items()
|
1443
|
-
if isinstance(v, Property)
|
1444
|
-
}
|
1445
|
-
representation = {**private, **attributes}
|
1446
|
-
return f"<Settings {representation}>"
|
818
|
+
@computed_field # type: ignore[prop-decorator]
|
819
|
+
@property
|
820
|
+
def sync_file(self) -> str:
|
821
|
+
"""Path to the append-only binary transaction log file."""
|
822
|
+
return _path_convert(self.sync_dir, f"run-{self.run_id}.wandb")
|
1447
823
|
|
1448
|
-
|
1449
|
-
|
824
|
+
@computed_field # type: ignore[prop-decorator]
|
825
|
+
@property
|
826
|
+
def sync_symlink_latest(self) -> str:
|
827
|
+
return _path_convert(self.wandb_dir, "latest-run")
|
1450
828
|
|
1451
|
-
|
1452
|
-
|
1453
|
-
|
1454
|
-
|
1455
|
-
new = Settings()
|
1456
|
-
# update properties that have deps or are dependent on in the topologically-sorted order
|
1457
|
-
for prop in self.__modification_order:
|
1458
|
-
new.update({prop: attributes[prop]._value}, source=attributes[prop].source)
|
1459
|
-
attributes.pop(prop)
|
1460
|
-
|
1461
|
-
# update the remaining attributes
|
1462
|
-
for k, v in attributes.items():
|
1463
|
-
# make sure to use the raw property value (v._value),
|
1464
|
-
# not the potential result of runtime hooks applied to it (v.value)
|
1465
|
-
new.update({k: v._value}, source=v.source)
|
1466
|
-
new.unfreeze()
|
1467
|
-
|
1468
|
-
return new
|
1469
|
-
|
1470
|
-
def __deepcopy__(self, memo: dict) -> "Settings":
|
1471
|
-
return self.__copy__()
|
1472
|
-
|
1473
|
-
# attribute access methods
|
1474
|
-
@no_type_check # this is a hack to make mypy happy
|
1475
|
-
def __getattribute__(self, name: str) -> Any:
|
1476
|
-
"""Expose `attribute.value` if `attribute` is a Property."""
|
1477
|
-
item = object.__getattribute__(self, name)
|
1478
|
-
if isinstance(item, Property):
|
1479
|
-
return item.value
|
1480
|
-
return item
|
1481
|
-
|
1482
|
-
def __setattr__(self, key: str, value: Any) -> None:
|
1483
|
-
if "_Settings__initialized" in self.__dict__ and self.__initialized:
|
1484
|
-
raise TypeError(f"Please use update() to update attribute `{key}` value")
|
1485
|
-
object.__setattr__(self, key, value)
|
1486
|
-
|
1487
|
-
def __iter__(self) -> Iterable:
|
1488
|
-
return iter(self.to_dict())
|
1489
|
-
|
1490
|
-
def copy(self) -> "Settings":
|
1491
|
-
return self.__copy__()
|
1492
|
-
|
1493
|
-
# implement the Mapping interface
|
1494
|
-
def keys(self) -> Iterable[str]:
|
1495
|
-
return self.to_dict().keys()
|
1496
|
-
|
1497
|
-
@no_type_check # this is a hack to make mypy happy
|
1498
|
-
def __getitem__(self, name: str) -> Any:
|
1499
|
-
"""Expose attribute.value if attribute is a Property."""
|
1500
|
-
item = object.__getattribute__(self, name)
|
1501
|
-
if isinstance(item, Property):
|
1502
|
-
return item.value
|
1503
|
-
return item
|
1504
|
-
|
1505
|
-
def update(
|
1506
|
-
self,
|
1507
|
-
settings: Optional[Union[Dict[str, Any], "Settings"]] = None,
|
1508
|
-
source: int = Source.OVERRIDE,
|
1509
|
-
**kwargs: Any,
|
1510
|
-
) -> None:
|
1511
|
-
"""Update individual settings."""
|
1512
|
-
if "_Settings__frozen" in self.__dict__ and self.__frozen:
|
1513
|
-
raise TypeError("Settings object is frozen")
|
1514
|
-
|
1515
|
-
if isinstance(settings, Settings):
|
1516
|
-
# If a Settings object is passed, detect the settings that differ
|
1517
|
-
# from defaults, collect them into a dict, and apply them using `source`.
|
1518
|
-
# This comes up in `wandb.init(settings=wandb.Settings(...))` and
|
1519
|
-
# seems like the behavior that the user would expect when calling init that way.
|
1520
|
-
defaults = Settings()
|
1521
|
-
settings_dict = dict()
|
1522
|
-
for k, v in settings.__dict__.items():
|
1523
|
-
if isinstance(v, Property):
|
1524
|
-
if v._value != defaults.__dict__[k]._value:
|
1525
|
-
settings_dict[k] = v._value
|
1526
|
-
# replace with the generated dict
|
1527
|
-
settings = settings_dict
|
1528
|
-
|
1529
|
-
# add kwargs to settings
|
1530
|
-
settings = settings or dict()
|
1531
|
-
# explicit kwargs take precedence over settings
|
1532
|
-
settings = {**settings, **kwargs}
|
1533
|
-
unknown_properties = []
|
1534
|
-
for key in settings.keys():
|
1535
|
-
# only allow updating known Properties
|
1536
|
-
if key not in self.__dict__ or not isinstance(self.__dict__[key], Property):
|
1537
|
-
unknown_properties.append(key)
|
1538
|
-
if unknown_properties:
|
1539
|
-
raise KeyError(f"Unknown settings: {unknown_properties}")
|
1540
|
-
# only if all keys are valid, update them
|
1541
|
-
|
1542
|
-
# store settings to be updated in a dict to preserve stats on preprocessing and validation errors
|
1543
|
-
settings.copy()
|
1544
|
-
|
1545
|
-
# update properties that have deps or are dependent on in the topologically-sorted order
|
1546
|
-
for key in self.__modification_order:
|
1547
|
-
if key in settings:
|
1548
|
-
self.__dict__[key].update(settings.pop(key), source=source)
|
1549
|
-
|
1550
|
-
# update the remaining properties
|
1551
|
-
for key, value in settings.items():
|
1552
|
-
self.__dict__[key].update(value, source)
|
1553
|
-
|
1554
|
-
def items(self) -> ItemsView[str, Any]:
|
1555
|
-
return self.to_dict().items()
|
1556
|
-
|
1557
|
-
def get(self, key: str, default: Optional[Any] = None) -> Any:
|
1558
|
-
return self.to_dict().get(key, default)
|
1559
|
-
|
1560
|
-
def freeze(self) -> None:
|
1561
|
-
object.__setattr__(self, "_Settings__frozen", True)
|
1562
|
-
|
1563
|
-
def unfreeze(self) -> None:
|
1564
|
-
object.__setattr__(self, "_Settings__frozen", False)
|
1565
|
-
|
1566
|
-
def is_frozen(self) -> bool:
|
1567
|
-
return self.__frozen
|
1568
|
-
|
1569
|
-
def to_dict(self) -> Dict[str, Any]:
|
1570
|
-
"""Return a dict representation of the settings."""
|
1571
|
-
# get attributes that are instances of the Property class:
|
1572
|
-
attributes = {
|
1573
|
-
k: v.value for k, v in self.__dict__.items() if isinstance(v, Property)
|
1574
|
-
}
|
1575
|
-
return attributes
|
829
|
+
@computed_field # type: ignore[prop-decorator]
|
830
|
+
@property
|
831
|
+
def timespec(self) -> str:
|
832
|
+
return self._start_datetime
|
1576
833
|
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
settings = wandb_settings_pb2.Settings()
|
1582
|
-
for field in fields(SettingsData):
|
1583
|
-
k = field.name
|
1584
|
-
v = getattr(self, k)
|
1585
|
-
# special case for _stats_open_metrics_filters
|
1586
|
-
if k == "_stats_open_metrics_filters":
|
1587
|
-
if isinstance(v, (list, set, tuple)):
|
1588
|
-
setting = getattr(settings, k)
|
1589
|
-
setting.sequence.value.extend(v)
|
1590
|
-
elif isinstance(v, dict):
|
1591
|
-
setting = getattr(settings, k)
|
1592
|
-
for key, value in v.items():
|
1593
|
-
for kk, vv in value.items():
|
1594
|
-
setting.mapping.value[key].value[kk] = vv
|
1595
|
-
else:
|
1596
|
-
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1597
|
-
continue
|
834
|
+
@computed_field # type: ignore[prop-decorator]
|
835
|
+
@property
|
836
|
+
def wandb_dir(self) -> str:
|
837
|
+
"""Full path to the wandb directory.
|
1598
838
|
|
1599
|
-
|
1600
|
-
|
1601
|
-
|
1602
|
-
|
1603
|
-
elif isinstance(v, float):
|
1604
|
-
getattr(settings, k).CopyFrom(DoubleValue(value=v))
|
1605
|
-
elif isinstance(v, str):
|
1606
|
-
getattr(settings, k).CopyFrom(StringValue(value=v))
|
1607
|
-
elif isinstance(v, (list, set, tuple)):
|
1608
|
-
# we only support sequences of strings for now
|
1609
|
-
sequence = getattr(settings, k)
|
1610
|
-
sequence.value.extend(v)
|
1611
|
-
elif isinstance(v, dict):
|
1612
|
-
mapping = getattr(settings, k)
|
1613
|
-
for key, value in v.items():
|
1614
|
-
# we only support dicts with string values for now
|
1615
|
-
mapping.value[key] = value
|
1616
|
-
elif isinstance(v, RunMoment):
|
1617
|
-
getattr(settings, k).CopyFrom(
|
1618
|
-
wandb_settings_pb2.RunMoment(
|
1619
|
-
run=v.run,
|
1620
|
-
value=v.value,
|
1621
|
-
metric=v.metric,
|
1622
|
-
)
|
1623
|
-
)
|
1624
|
-
elif v is None:
|
1625
|
-
# None is the default value for all settings, so we don't need to set it,
|
1626
|
-
# i.e. None means that the value was not set.
|
1627
|
-
pass
|
1628
|
-
else:
|
1629
|
-
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1630
|
-
# TODO: store property sources in the protobuf so that we can reconstruct the
|
1631
|
-
# settings object from the protobuf
|
1632
|
-
return settings
|
1633
|
-
|
1634
|
-
# apply settings from different sources
|
1635
|
-
# TODO(dd): think about doing some|all of that at init
|
1636
|
-
def _apply_settings(
|
1637
|
-
self,
|
1638
|
-
settings: "Settings",
|
1639
|
-
_logger: Optional[_EarlyLogger] = None,
|
1640
|
-
) -> None:
|
1641
|
-
"""Apply settings from a Settings object."""
|
1642
|
-
if _logger is not None:
|
1643
|
-
_logger.info(f"Applying settings from {settings}")
|
1644
|
-
attributes = {
|
1645
|
-
k: v for k, v in settings.__dict__.items() if isinstance(v, Property)
|
1646
|
-
}
|
1647
|
-
# update properties that have deps or are dependent on in the topologically-sorted order
|
1648
|
-
for prop in self.__modification_order:
|
1649
|
-
self.update({prop: attributes[prop]._value}, source=attributes[prop].source)
|
1650
|
-
attributes.pop(prop)
|
1651
|
-
# update the remaining properties
|
1652
|
-
for k, v in attributes.items():
|
1653
|
-
# note that only the same/higher priority settings are propagated
|
1654
|
-
self.update({k: v._value}, source=v.source)
|
839
|
+
The setting exposed to users as `dir=` or `WANDB_DIR` is the `root_dir`.
|
840
|
+
We add the `__stage_dir__` to it to get the full `wandb_dir`
|
841
|
+
"""
|
842
|
+
root_dir = self.root_dir or ""
|
1655
843
|
|
1656
|
-
|
1657
|
-
|
1658
|
-
|
1659
|
-
|
1660
|
-
|
1661
|
-
config: Dict[str, Any] = dict()
|
1662
|
-
for k in parser[section]:
|
1663
|
-
config[k] = parser[section][k]
|
1664
|
-
# TODO (cvp): we didn't do this in the old cli, but it seems necessary
|
1665
|
-
if k == "ignore_globs":
|
1666
|
-
config[k] = config[k].split(",")
|
1667
|
-
return config
|
844
|
+
# We use the hidden version if it already exists, otherwise non-hidden.
|
845
|
+
if os.path.exists(os.path.join(root_dir, ".wandb")):
|
846
|
+
__stage_dir__ = ".wandb" + os.sep
|
847
|
+
else:
|
848
|
+
__stage_dir__ = "wandb" + os.sep
|
1668
849
|
|
1669
|
-
|
1670
|
-
if
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1674
|
-
|
1675
|
-
def _apply_config_files(self, _logger: Optional[_EarlyLogger] = None) -> None:
|
1676
|
-
# TODO(jhr): permit setting of config in system and workspace
|
1677
|
-
if self.settings_system is not None:
|
1678
|
-
if _logger is not None:
|
1679
|
-
_logger.info(f"Loading settings from {self.settings_system}")
|
1680
|
-
self.update(
|
1681
|
-
self._load_config_file(self.settings_system),
|
1682
|
-
source=Source.SYSTEM,
|
850
|
+
path = os.path.join(root_dir, __stage_dir__)
|
851
|
+
if not os.access(root_dir or ".", os.W_OK):
|
852
|
+
termwarn(
|
853
|
+
f"Path {path} wasn't writable, using system temp directory.",
|
854
|
+
repeat=False,
|
1683
855
|
)
|
1684
|
-
|
1685
|
-
|
1686
|
-
_logger.info(f"Loading settings from {self.settings_workspace}")
|
1687
|
-
self.update(
|
1688
|
-
self._load_config_file(self.settings_workspace),
|
1689
|
-
source=Source.WORKSPACE,
|
856
|
+
path = os.path.join(
|
857
|
+
tempfile.gettempdir(), __stage_dir__ or ("wandb" + os.sep)
|
1690
858
|
)
|
1691
859
|
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
1695
|
-
|
1696
|
-
|
860
|
+
return os.path.expanduser(path)
|
861
|
+
|
862
|
+
# Methods to collect and update settings from different sources.
|
863
|
+
#
|
864
|
+
# The Settings class does not track the source of the settings,
|
865
|
+
# so it is up to the developer to ensure that the settings are applied
|
866
|
+
# in the correct order. Most of the updates are done in
|
867
|
+
# wandb/sdk/wandb_setup.py::_WandbSetup__WandbSetup._settings_setup.
|
868
|
+
|
869
|
+
def update_from_system_config_file(self):
|
870
|
+
"""Update settings from the system config file."""
|
871
|
+
if not self.settings_system or not os.path.exists(self.settings_system):
|
872
|
+
return
|
873
|
+
for key, value in self._load_config_file(self.settings_system).items():
|
874
|
+
if value is not None:
|
875
|
+
setattr(self, key, value)
|
876
|
+
|
877
|
+
def update_from_workspace_config_file(self):
|
878
|
+
"""Update settings from the workspace config file."""
|
879
|
+
if not self.settings_workspace or not os.path.exists(self.settings_workspace):
|
880
|
+
return
|
881
|
+
for key, value in self._load_config_file(self.settings_workspace).items():
|
882
|
+
if value is not None:
|
883
|
+
setattr(self, key, value)
|
884
|
+
|
885
|
+
def update_from_env_vars(self, environ: dict[str, Any]):
|
886
|
+
"""Update settings from environment variables."""
|
1697
887
|
env_prefix: str = "WANDB_"
|
888
|
+
private_env_prefix: str = env_prefix + "_"
|
1698
889
|
special_env_var_names = {
|
1699
|
-
"WANDB_DISABLE_SERVICE": "
|
1700
|
-
"WANDB_SERVICE_TRANSPORT": "
|
890
|
+
"WANDB_DISABLE_SERVICE": "x_disable_service",
|
891
|
+
"WANDB_SERVICE_TRANSPORT": "x_service_transport",
|
1701
892
|
"WANDB_DIR": "root_dir",
|
1702
893
|
"WANDB_NAME": "run_name",
|
1703
894
|
"WANDB_NOTES": "run_notes",
|
1704
895
|
"WANDB_TAGS": "run_tags",
|
1705
896
|
"WANDB_JOB_TYPE": "run_job_type",
|
1706
|
-
"WANDB_HTTP_TIMEOUT": "
|
1707
|
-
"WANDB_FILE_PUSHER_TIMEOUT": "
|
897
|
+
"WANDB_HTTP_TIMEOUT": "x_graphql_timeout_seconds",
|
898
|
+
"WANDB_FILE_PUSHER_TIMEOUT": "x_file_transfer_timeout_seconds",
|
1708
899
|
"WANDB_USER_EMAIL": "email",
|
1709
900
|
}
|
1710
901
|
env = dict()
|
@@ -1714,6 +905,8 @@ class Settings(SettingsData):
|
|
1714
905
|
|
1715
906
|
if setting in special_env_var_names:
|
1716
907
|
key = special_env_var_names[setting]
|
908
|
+
elif setting.startswith(private_env_prefix):
|
909
|
+
key = "x_" + setting[len(private_env_prefix) :].lower()
|
1717
910
|
else:
|
1718
911
|
# otherwise, strip the prefix and convert to lowercase
|
1719
912
|
key = setting[len(env_prefix) :].lower()
|
@@ -1722,52 +915,37 @@ class Settings(SettingsData):
|
|
1722
915
|
if key in ("ignore_globs", "run_tags"):
|
1723
916
|
value = value.split(",")
|
1724
917
|
env[key] = value
|
1725
|
-
elif _logger is not None:
|
1726
|
-
_logger.warning(f"Unknown environment variable: {setting}")
|
1727
918
|
|
1728
|
-
|
1729
|
-
|
1730
|
-
|
1731
|
-
)
|
1732
|
-
self.update(env, source=Source.ENV)
|
1733
|
-
|
1734
|
-
def _infer_settings_from_environment(
|
1735
|
-
self, _logger: Optional[_EarlyLogger] = None
|
1736
|
-
) -> None:
|
1737
|
-
"""Modify settings based on environment (for runs and cli)."""
|
1738
|
-
settings: Dict[str, Union[bool, str, Sequence, None]] = dict()
|
1739
|
-
# disable symlinks if on windows (requires admin or developer setup)
|
1740
|
-
settings["symlink"] = True
|
1741
|
-
if self._windows:
|
1742
|
-
settings["symlink"] = False
|
1743
|
-
|
1744
|
-
# TODO(jhr): this needs to be moved last in setting up settings ?
|
1745
|
-
# (dd): loading order does not matter as long as source is set correctly
|
919
|
+
for key, value in env.items():
|
920
|
+
if value is not None:
|
921
|
+
setattr(self, key, value)
|
1746
922
|
|
923
|
+
def update_from_system_environment(self):
|
924
|
+
"""Update settings from the system environment."""
|
1747
925
|
# For code saving, only allow env var override if value from server is true, or
|
1748
926
|
# if no preference was specified.
|
1749
927
|
if (self.save_code is True or self.save_code is None) and (
|
1750
|
-
os.getenv(
|
1751
|
-
or os.getenv(
|
928
|
+
os.getenv(env.SAVE_CODE) is not None
|
929
|
+
or os.getenv(env.DISABLE_CODE) is not None
|
1752
930
|
):
|
1753
|
-
|
931
|
+
self.save_code = env.should_save_code()
|
1754
932
|
|
1755
|
-
|
933
|
+
self.disable_git = env.disable_git()
|
1756
934
|
|
1757
935
|
# Attempt to get notebook information if not already set by the user
|
1758
936
|
if self._jupyter and (self.notebook_name is None or self.notebook_name == ""):
|
1759
937
|
meta = wandb.jupyter.notebook_metadata(self.silent) # type: ignore
|
1760
|
-
|
1761
|
-
|
1762
|
-
|
938
|
+
self.x_jupyter_path = meta.get("path")
|
939
|
+
self.x_jupyter_name = meta.get("name")
|
940
|
+
self.x_jupyter_root = meta.get("root")
|
1763
941
|
elif (
|
1764
942
|
self._jupyter
|
1765
943
|
and self.notebook_name is not None
|
1766
944
|
and os.path.exists(self.notebook_name)
|
1767
945
|
):
|
1768
|
-
|
1769
|
-
|
1770
|
-
|
946
|
+
self.x_jupyter_path = self.notebook_name
|
947
|
+
self.x_jupyter_name = self.notebook_name
|
948
|
+
self.x_jupyter_root = os.getcwd()
|
1771
949
|
elif self._jupyter:
|
1772
950
|
wandb.termwarn(
|
1773
951
|
"WANDB_NOTEBOOK_NAME should be a path to a notebook file, "
|
@@ -1777,244 +955,266 @@ class Settings(SettingsData):
|
|
1777
955
|
# host and username are populated by apply_env_vars if corresponding env
|
1778
956
|
# vars exist -- but if they don't, we'll fill them in here
|
1779
957
|
if self.host is None:
|
1780
|
-
|
958
|
+
self.host = socket.gethostname() # type: ignore
|
1781
959
|
|
1782
960
|
if self.username is None:
|
1783
961
|
try: # type: ignore
|
1784
|
-
|
962
|
+
self.username = getpass.getuser()
|
1785
963
|
except KeyError:
|
1786
964
|
# getuser() could raise KeyError in restricted environments like
|
1787
965
|
# chroot jails or docker containers. Return user id in these cases.
|
1788
|
-
|
966
|
+
self.username = str(os.getuid())
|
1789
967
|
|
1790
968
|
_executable = (
|
1791
|
-
self.
|
1792
|
-
or os.environ.get(
|
969
|
+
self.x_executable
|
970
|
+
or os.environ.get(env._EXECUTABLE)
|
1793
971
|
or sys.executable
|
1794
972
|
or shutil.which("python3")
|
1795
973
|
or "python3"
|
1796
974
|
)
|
1797
|
-
|
1798
|
-
|
1799
|
-
settings["docker"] = wandb.env.get_docker(wandb.util.image_id_from_k8s())
|
975
|
+
self.x_executable = _executable
|
1800
976
|
|
1801
|
-
|
1802
|
-
if os.path.exists("/usr/local/cuda/version.txt"):
|
1803
|
-
with open("/usr/local/cuda/version.txt") as f:
|
1804
|
-
settings["_cuda"] = f.read().split(" ")[-1].strip()
|
1805
|
-
if not self._jupyter:
|
1806
|
-
settings["_args"] = sys.argv[1:]
|
1807
|
-
settings["_os"] = platform.platform(aliased=True)
|
1808
|
-
settings["_python"] = platform.python_version()
|
977
|
+
self.docker = env.get_docker(util.image_id_from_k8s())
|
1809
978
|
|
1810
|
-
|
1811
|
-
|
1812
|
-
|
1813
|
-
)
|
979
|
+
# proceed if not in CLI mode
|
980
|
+
if self.x_cli_only_mode:
|
981
|
+
return
|
1814
982
|
|
1815
|
-
self.
|
983
|
+
program = self.program or self._get_program()
|
1816
984
|
|
1817
|
-
def _infer_run_settings_from_environment(
|
1818
|
-
self,
|
1819
|
-
_logger: Optional[_EarlyLogger] = None,
|
1820
|
-
) -> None:
|
1821
|
-
"""Modify settings based on environment (for runs only)."""
|
1822
|
-
# If there's not already a program file, infer it now.
|
1823
|
-
settings: Dict[str, Union[bool, str, None]] = dict()
|
1824
|
-
program = self.program or _get_program()
|
1825
985
|
if program is not None:
|
1826
986
|
repo = GitRepo()
|
1827
987
|
root = repo.root or os.getcwd()
|
1828
988
|
|
1829
|
-
program_relpath = self.program_relpath or _get_program_relpath(
|
1830
|
-
program,
|
989
|
+
self.program_relpath = self.program_relpath or self._get_program_relpath(
|
990
|
+
program, root
|
1831
991
|
)
|
1832
|
-
settings["program_relpath"] = program_relpath
|
1833
992
|
program_abspath = os.path.abspath(
|
1834
993
|
os.path.join(root, os.path.relpath(os.getcwd(), root), program)
|
1835
994
|
)
|
1836
995
|
if os.path.exists(program_abspath):
|
1837
|
-
|
996
|
+
self.program_abspath = program_abspath
|
1838
997
|
else:
|
1839
998
|
program = "<python with no main file>"
|
1840
999
|
|
1841
|
-
|
1000
|
+
self.program = program
|
1842
1001
|
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1002
|
+
def update_from_dict(self, settings: dict[str, Any]) -> None:
|
1003
|
+
"""Update settings from a dictionary."""
|
1004
|
+
for key, value in dict(settings).items():
|
1005
|
+
if value is not None:
|
1006
|
+
setattr(self, key, value)
|
1847
1007
|
|
1848
|
-
|
1849
|
-
|
1850
|
-
|
1851
|
-
|
1852
|
-
|
1853
|
-
if _logger:
|
1854
|
-
_logger.info(f"Applying setup settings: {_redact_dict(setup_settings)}")
|
1855
|
-
self.update(setup_settings, source=Source.SETUP)
|
1856
|
-
|
1857
|
-
def _apply_user(
|
1858
|
-
self, user_settings: Dict[str, Any], _logger: Optional[_EarlyLogger] = None
|
1859
|
-
) -> None:
|
1860
|
-
if _logger:
|
1861
|
-
_logger.info(f"Applying user settings: {_redact_dict(user_settings)}")
|
1862
|
-
self.update(user_settings, source=Source.USER)
|
1863
|
-
|
1864
|
-
def _apply_init(self, init_settings: Dict[str, Union[str, int, None]]) -> None:
|
1865
|
-
# pop magic from init settings
|
1866
|
-
init_settings.pop("magic", None)
|
1867
|
-
|
1868
|
-
# prevent setting project, entity if in sweep
|
1869
|
-
# TODO(jhr): these should be locked elements in the future
|
1870
|
-
if self.sweep_id:
|
1871
|
-
for key in ("project", "entity", "id"):
|
1872
|
-
val = init_settings.pop(key, None)
|
1873
|
-
if val:
|
1874
|
-
wandb.termwarn(
|
1875
|
-
f"Ignored wandb.init() arg {key} when running a sweep."
|
1876
|
-
)
|
1877
|
-
if self.launch:
|
1878
|
-
if self.project is not None and init_settings.pop("project", None):
|
1879
|
-
wandb.termwarn(
|
1880
|
-
"Project is ignored when running from wandb launch context. "
|
1881
|
-
"Ignored wandb.init() arg project when running running from launch.",
|
1882
|
-
)
|
1883
|
-
for key in ("entity", "id"):
|
1884
|
-
# Init settings cannot override launch settings.
|
1885
|
-
if init_settings.pop(key, None):
|
1886
|
-
wandb.termwarn(
|
1887
|
-
"Project, entity and id are ignored when running from wandb launch context. "
|
1888
|
-
f"Ignored wandb.init() arg {key} when running running from launch.",
|
1889
|
-
)
|
1008
|
+
def update_from_settings(self, settings: Settings) -> None:
|
1009
|
+
"""Update settings from another instance of `Settings`."""
|
1010
|
+
d = {field: getattr(settings, field) for field in settings.model_fields_set}
|
1011
|
+
if d:
|
1012
|
+
self.update_from_dict(d)
|
1890
1013
|
|
1891
|
-
|
1892
|
-
param_map = dict(
|
1893
|
-
name="run_name",
|
1894
|
-
id="run_id",
|
1895
|
-
tags="run_tags",
|
1896
|
-
group="run_group",
|
1897
|
-
job_type="run_job_type",
|
1898
|
-
notes="run_notes",
|
1899
|
-
dir="root_dir",
|
1900
|
-
sweep_id="sweep_id",
|
1901
|
-
)
|
1902
|
-
init_settings = {
|
1903
|
-
param_map.get(k, k): v for k, v in init_settings.items() if v is not None
|
1904
|
-
}
|
1905
|
-
# fun logic to convert the resume init arg
|
1906
|
-
if init_settings.get("resume"):
|
1907
|
-
if isinstance(init_settings["resume"], str):
|
1908
|
-
if init_settings["resume"] not in ("allow", "must", "never", "auto"):
|
1909
|
-
if init_settings.get("run_id") is None:
|
1910
|
-
# TODO: deprecate or don't support
|
1911
|
-
init_settings["run_id"] = init_settings["resume"]
|
1912
|
-
init_settings["resume"] = "allow"
|
1913
|
-
elif init_settings["resume"] is True:
|
1914
|
-
# todo: add deprecation warning, switch to literal strings for resume
|
1915
|
-
init_settings["resume"] = "auto"
|
1916
|
-
|
1917
|
-
# update settings
|
1918
|
-
self.update(init_settings, source=Source.INIT)
|
1919
|
-
self._handle_fork_logic()
|
1920
|
-
self._handle_rewind_logic()
|
1921
|
-
self._handle_resume_logic()
|
1922
|
-
|
1923
|
-
def _handle_fork_logic(self) -> None:
|
1924
|
-
if self.fork_from is None:
|
1925
|
-
return
|
1014
|
+
# Helper methods.
|
1926
1015
|
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1016
|
+
def to_proto(self) -> wandb_settings_pb2.Settings:
|
1017
|
+
"""Generate a protobuf representation of the settings."""
|
1018
|
+
settings_proto = wandb_settings_pb2.Settings()
|
1019
|
+
for k, v in self.model_dump(exclude_none=True).items():
|
1020
|
+
# special case for x_stats_open_metrics_filters
|
1021
|
+
if k == "x_stats_open_metrics_filters":
|
1022
|
+
if isinstance(v, (list, set, tuple)):
|
1023
|
+
setting = getattr(settings_proto, k)
|
1024
|
+
setting.sequence.value.extend(v)
|
1025
|
+
elif isinstance(v, dict):
|
1026
|
+
setting = getattr(settings_proto, k)
|
1027
|
+
for key, value in v.items():
|
1028
|
+
for kk, vv in value.items():
|
1029
|
+
setting.mapping.value[key].value[kk] = vv
|
1030
|
+
else:
|
1031
|
+
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1032
|
+
continue
|
1933
1033
|
|
1934
|
-
|
1935
|
-
|
1936
|
-
|
1034
|
+
if isinstance(v, bool):
|
1035
|
+
getattr(settings_proto, k).CopyFrom(BoolValue(value=v))
|
1036
|
+
elif isinstance(v, int):
|
1037
|
+
getattr(settings_proto, k).CopyFrom(Int32Value(value=v))
|
1038
|
+
elif isinstance(v, float):
|
1039
|
+
getattr(settings_proto, k).CopyFrom(DoubleValue(value=v))
|
1040
|
+
elif isinstance(v, str):
|
1041
|
+
getattr(settings_proto, k).CopyFrom(StringValue(value=v))
|
1042
|
+
elif isinstance(v, (list, set, tuple)):
|
1043
|
+
# we only support sequences of strings for now
|
1044
|
+
sequence = getattr(settings_proto, k)
|
1045
|
+
sequence.value.extend(v)
|
1046
|
+
elif isinstance(v, dict):
|
1047
|
+
mapping = getattr(settings_proto, k)
|
1048
|
+
for key, value in v.items():
|
1049
|
+
# we only support dicts with string values for now
|
1050
|
+
mapping.value[key] = value
|
1051
|
+
elif isinstance(v, RunMoment):
|
1052
|
+
getattr(settings_proto, k).CopyFrom(
|
1053
|
+
wandb_settings_pb2.RunMoment(
|
1054
|
+
run=v.run,
|
1055
|
+
value=v.value,
|
1056
|
+
metric=v.metric,
|
1057
|
+
)
|
1058
|
+
)
|
1059
|
+
elif v is None:
|
1060
|
+
# None means that the setting value was not set.
|
1061
|
+
pass
|
1062
|
+
else:
|
1063
|
+
raise TypeError(f"Unsupported type {type(v)} for setting {k}")
|
1937
1064
|
|
1938
|
-
|
1939
|
-
wandb.termwarn(
|
1940
|
-
"Both `run_id` and `resume_from` have been specified with different ids. "
|
1941
|
-
"`run_id` will be ignored."
|
1942
|
-
)
|
1943
|
-
self.update({"run_id": self.resume_from.run}, source=Source.INIT)
|
1065
|
+
return settings_proto
|
1944
1066
|
|
1945
|
-
def
|
1067
|
+
def handle_resume_logic(self):
|
1068
|
+
"""Handle logic for resuming runs."""
|
1946
1069
|
# handle auto resume logic
|
1947
1070
|
if self.resume == "auto":
|
1948
1071
|
if os.path.exists(self.resume_fname):
|
1949
1072
|
with open(self.resume_fname) as f:
|
1950
1073
|
resume_run_id = json.load(f)["run_id"]
|
1951
1074
|
if self.run_id is None:
|
1952
|
-
self.
|
1075
|
+
self.run_id = resume_run_id
|
1953
1076
|
elif self.run_id != resume_run_id:
|
1954
1077
|
wandb.termwarn(
|
1955
1078
|
"Tried to auto resume run with "
|
1956
1079
|
f"id {resume_run_id} but id {self.run_id} is set.",
|
1957
1080
|
)
|
1081
|
+
if self.run_id is None:
|
1082
|
+
self.run_id = generate_id()
|
1958
1083
|
|
1959
|
-
|
1960
|
-
# persist our run id in case of failure
|
1961
|
-
# check None for mypy
|
1084
|
+
# persist run_id in case of failure
|
1962
1085
|
if self.resume == "auto" and self.resume_fname is not None:
|
1963
1086
|
filesystem.mkdir_exists_ok(self.wandb_dir)
|
1964
1087
|
with open(self.resume_fname, "w") as f:
|
1965
1088
|
f.write(json.dumps({"run_id": self.run_id}))
|
1966
1089
|
|
1967
|
-
def
|
1968
|
-
|
1969
|
-
login_settings: Dict[str, Any],
|
1970
|
-
_logger: Optional[_EarlyLogger] = None,
|
1971
|
-
) -> None:
|
1972
|
-
key_map = {
|
1973
|
-
"key": "api_key",
|
1974
|
-
"host": "base_url",
|
1975
|
-
"timeout": "login_timeout",
|
1976
|
-
}
|
1090
|
+
def handle_sweep_logic(self):
|
1091
|
+
"""Update settings based on sweep context.
|
1977
1092
|
|
1978
|
-
|
1979
|
-
|
1980
|
-
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
1986
|
-
|
1093
|
+
When running a sweep, the project, entity, and run_id are handled externally,
|
1094
|
+
and should be ignored if they are set.
|
1095
|
+
"""
|
1096
|
+
if self.sweep_id is None:
|
1097
|
+
return
|
1098
|
+
|
1099
|
+
for key in ("project", "entity", "run_id"):
|
1100
|
+
value = getattr(self, key)
|
1101
|
+
if value is not None:
|
1102
|
+
wandb.termwarn(f"Ignoring {key} {value!r} when running a sweep.")
|
1103
|
+
setattr(self, key, None)
|
1987
1104
|
|
1988
|
-
|
1989
|
-
|
1105
|
+
def handle_launch_logic(self):
|
1106
|
+
"""Update settings based on launch context.
|
1107
|
+
|
1108
|
+
When running in a launch context, the project, entity, and run_id are handled
|
1109
|
+
externally, and should be ignored if they are set.
|
1110
|
+
"""
|
1111
|
+
if not self.launch:
|
1112
|
+
return
|
1113
|
+
|
1114
|
+
for key in ("project", "entity", "run_id"):
|
1115
|
+
value = getattr(self, key)
|
1116
|
+
if value is not None:
|
1117
|
+
wandb.termwarn(
|
1118
|
+
f"Ignoring {key} {value!r} when running from wandb launch context."
|
1119
|
+
)
|
1120
|
+
setattr(self, key, None)
|
1990
1121
|
|
1991
|
-
|
1992
|
-
|
1993
|
-
|
1122
|
+
@staticmethod
|
1123
|
+
def validate_url(url: str) -> None:
|
1124
|
+
"""Validate a URL string."""
|
1125
|
+
url_validator = SchemaValidator(
|
1126
|
+
core_schema.url_schema(
|
1127
|
+
allowed_schemes=["http", "https"],
|
1128
|
+
strict=True,
|
1129
|
+
)
|
1994
1130
|
)
|
1131
|
+
url_validator.validate_python(url)
|
1995
1132
|
|
1996
|
-
def
|
1997
|
-
|
1998
|
-
|
1999
|
-
|
2000
|
-
|
2001
|
-
|
2002
|
-
|
2003
|
-
|
2004
|
-
|
2005
|
-
|
2006
|
-
|
2007
|
-
|
2008
|
-
|
2009
|
-
|
2010
|
-
|
2011
|
-
|
2012
|
-
|
2013
|
-
|
2014
|
-
|
2015
|
-
|
2016
|
-
|
2017
|
-
|
2018
|
-
|
2019
|
-
|
2020
|
-
self.
|
1133
|
+
def _get_program(self) -> str | None:
|
1134
|
+
"""Get the program that started the current process."""
|
1135
|
+
if not self._jupyter:
|
1136
|
+
# If not in a notebook, try to get the program from the environment
|
1137
|
+
# or the __main__ module for scripts run as `python -m ...`.
|
1138
|
+
program = os.getenv(env.PROGRAM)
|
1139
|
+
if program is not None:
|
1140
|
+
return program
|
1141
|
+
try:
|
1142
|
+
import __main__
|
1143
|
+
|
1144
|
+
if __main__.__spec__ is None:
|
1145
|
+
return __main__.__file__
|
1146
|
+
return f"-m {__main__.__spec__.name}"
|
1147
|
+
except (ImportError, AttributeError):
|
1148
|
+
return None
|
1149
|
+
else:
|
1150
|
+
# If in a notebook, try to get the program from the notebook metadata.
|
1151
|
+
if self.notebook_name:
|
1152
|
+
return self.notebook_name
|
1153
|
+
|
1154
|
+
if not self.x_jupyter_path:
|
1155
|
+
return self.program
|
1156
|
+
|
1157
|
+
if self.x_jupyter_path.startswith("fileId="):
|
1158
|
+
return self.x_jupyter_name
|
1159
|
+
else:
|
1160
|
+
return self.x_jupyter_path
|
1161
|
+
|
1162
|
+
@staticmethod
|
1163
|
+
def _get_program_relpath(program: str, root: str | None = None) -> str | None:
|
1164
|
+
"""Get the relative path to the program from the root directory."""
|
1165
|
+
if not program:
|
1166
|
+
return None
|
1167
|
+
|
1168
|
+
root = root or os.getcwd()
|
1169
|
+
if not root:
|
1170
|
+
return None
|
1171
|
+
|
1172
|
+
full_path_to_program = os.path.join(
|
1173
|
+
root, os.path.relpath(os.getcwd(), root), program
|
1174
|
+
)
|
1175
|
+
if os.path.exists(full_path_to_program):
|
1176
|
+
relative_path = os.path.relpath(full_path_to_program, start=root)
|
1177
|
+
if "../" in relative_path:
|
1178
|
+
return None
|
1179
|
+
return relative_path
|
1180
|
+
|
1181
|
+
return None
|
1182
|
+
|
1183
|
+
@staticmethod
|
1184
|
+
def _load_config_file(file_name: str, section: str = "default") -> dict:
|
1185
|
+
"""Load a config file and return the settings for a given section."""
|
1186
|
+
parser = configparser.ConfigParser()
|
1187
|
+
parser.add_section(section)
|
1188
|
+
parser.read(file_name)
|
1189
|
+
config: dict[str, Any] = dict()
|
1190
|
+
for k in parser[section]:
|
1191
|
+
config[k] = parser[section][k]
|
1192
|
+
if k == "ignore_globs":
|
1193
|
+
config[k] = config[k].split(",")
|
1194
|
+
return config
|
1195
|
+
|
1196
|
+
def _project_url_base(self) -> str:
|
1197
|
+
"""Construct the base URL for the project."""
|
1198
|
+
if not all([self.entity, self.project]):
|
1199
|
+
return ""
|
1200
|
+
|
1201
|
+
app_url = util.app_url(self.base_url)
|
1202
|
+
return f"{app_url}/{quote(self.entity or '')}/{quote(self.project or '')}"
|
1203
|
+
|
1204
|
+
def _get_url_query_string(self) -> str:
|
1205
|
+
"""Construct the query string for project, run, and sweep URLs."""
|
1206
|
+
# TODO: remove dependency on Api()
|
1207
|
+
if Api().settings().get("anonymous") != "true":
|
1208
|
+
return ""
|
1209
|
+
|
1210
|
+
api_key = apikey.api_key(settings=self)
|
1211
|
+
|
1212
|
+
return f"?{urlencode({'apiKey': api_key})}"
|
1213
|
+
|
1214
|
+
@staticmethod
|
1215
|
+
def _runmoment_preprocessor(val: RunMoment | str | None) -> RunMoment | None:
|
1216
|
+
"""Preprocess the setting for forking or resuming a run."""
|
1217
|
+
if isinstance(val, RunMoment) or val is None:
|
1218
|
+
return val
|
1219
|
+
elif isinstance(val, str):
|
1220
|
+
return RunMoment.from_uri(val)
|