wandb 0.19.8__py3-none-any.whl → 0.19.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +5 -1
- wandb/__init__.pyi +15 -8
- wandb/_pydantic/__init__.py +30 -0
- wandb/_pydantic/base.py +148 -0
- wandb/_pydantic/utils.py +66 -0
- wandb/_pydantic/v1_compat.py +284 -0
- wandb/apis/paginator.py +82 -38
- wandb/apis/public/__init__.py +2 -2
- wandb/apis/public/api.py +111 -53
- wandb/apis/public/artifacts.py +387 -639
- wandb/apis/public/automations.py +69 -0
- wandb/apis/public/files.py +2 -2
- wandb/apis/public/integrations.py +168 -0
- wandb/apis/public/projects.py +32 -2
- wandb/apis/public/reports.py +2 -2
- wandb/apis/public/runs.py +19 -11
- wandb/apis/public/utils.py +107 -1
- wandb/automations/__init__.py +81 -0
- wandb/automations/_filters/__init__.py +40 -0
- wandb/automations/_filters/expressions.py +179 -0
- wandb/automations/_filters/operators.py +267 -0
- wandb/automations/_filters/run_metrics.py +183 -0
- wandb/automations/_generated/__init__.py +184 -0
- wandb/automations/_generated/create_filter_trigger.py +21 -0
- wandb/automations/_generated/create_generic_webhook_integration.py +43 -0
- wandb/automations/_generated/delete_trigger.py +19 -0
- wandb/automations/_generated/enums.py +33 -0
- wandb/automations/_generated/fragments.py +343 -0
- wandb/automations/_generated/generic_webhook_integrations_by_entity.py +22 -0
- wandb/automations/_generated/get_triggers.py +24 -0
- wandb/automations/_generated/get_triggers_by_entity.py +24 -0
- wandb/automations/_generated/input_types.py +104 -0
- wandb/automations/_generated/integrations_by_entity.py +22 -0
- wandb/automations/_generated/operations.py +710 -0
- wandb/automations/_generated/slack_integrations_by_entity.py +22 -0
- wandb/automations/_generated/update_filter_trigger.py +21 -0
- wandb/automations/_utils.py +123 -0
- wandb/automations/_validators.py +73 -0
- wandb/automations/actions.py +205 -0
- wandb/automations/automations.py +109 -0
- wandb/automations/events.py +235 -0
- wandb/automations/integrations.py +26 -0
- wandb/automations/scopes.py +76 -0
- wandb/beta/workflows.py +9 -10
- wandb/bin/gpu_stats +0 -0
- wandb/cli/cli.py +3 -3
- wandb/integration/keras/keras.py +2 -1
- wandb/integration/langchain/wandb_tracer.py +2 -1
- wandb/integration/metaflow/metaflow.py +19 -17
- wandb/integration/sacred/__init__.py +1 -1
- wandb/jupyter.py +155 -133
- wandb/old/summary.py +0 -2
- wandb/proto/v3/wandb_internal_pb2.py +297 -292
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +292 -292
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v5/wandb_internal_pb2.py +292 -292
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v6/wandb_base_pb2.py +41 -0
- wandb/proto/v6/wandb_internal_pb2.py +393 -0
- wandb/proto/v6/wandb_server_pb2.py +78 -0
- wandb/proto/v6/wandb_settings_pb2.py +58 -0
- wandb/proto/v6/wandb_telemetry_pb2.py +52 -0
- wandb/proto/wandb_base_pb2.py +2 -0
- wandb/proto/wandb_deprecated.py +10 -0
- wandb/proto/wandb_internal_pb2.py +3 -1
- wandb/proto/wandb_server_pb2.py +2 -0
- wandb/proto/wandb_settings_pb2.py +2 -0
- wandb/proto/wandb_telemetry_pb2.py +2 -0
- wandb/sdk/artifacts/_generated/__init__.py +248 -0
- wandb/sdk/artifacts/_generated/artifact_collection_membership_files.py +43 -0
- wandb/sdk/artifacts/_generated/artifact_version_files.py +36 -0
- wandb/sdk/artifacts/_generated/create_artifact_collection_tag_assignments.py +36 -0
- wandb/sdk/artifacts/_generated/delete_artifact_collection_tag_assignments.py +25 -0
- wandb/sdk/artifacts/_generated/delete_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/delete_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_generated/enums.py +17 -0
- wandb/sdk/artifacts/_generated/fragments.py +186 -0
- wandb/sdk/artifacts/_generated/input_types.py +16 -0
- wandb/sdk/artifacts/_generated/move_artifact_collection.py +35 -0
- wandb/sdk/artifacts/_generated/operations.py +510 -0
- wandb/sdk/artifacts/_generated/project_artifact_collection.py +101 -0
- wandb/sdk/artifacts/_generated/project_artifact_collections.py +33 -0
- wandb/sdk/artifacts/_generated/project_artifact_type.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifact_types.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifacts.py +42 -0
- wandb/sdk/artifacts/_generated/run_input_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/run_output_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/update_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/update_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_graphql_fragments.py +56 -81
- wandb/sdk/artifacts/_validators.py +1 -0
- wandb/sdk/artifacts/artifact.py +110 -49
- wandb/sdk/artifacts/artifact_manifest_entry.py +2 -1
- wandb/sdk/artifacts/artifact_saver.py +16 -2
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +1 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +23 -2
- wandb/sdk/data_types/audio.py +1 -3
- wandb/sdk/data_types/base_types/media.py +13 -7
- wandb/sdk/data_types/base_types/wb_value.py +34 -11
- wandb/sdk/data_types/html.py +36 -9
- wandb/sdk/data_types/image.py +56 -37
- wandb/sdk/data_types/molecule.py +1 -5
- wandb/sdk/data_types/object_3d.py +2 -1
- wandb/sdk/data_types/saved_model.py +7 -9
- wandb/sdk/data_types/table.py +5 -0
- wandb/sdk/data_types/trace_tree.py +2 -0
- wandb/sdk/data_types/utils.py +1 -1
- wandb/sdk/data_types/video.py +15 -30
- wandb/sdk/interface/interface.py +2 -0
- wandb/{apis/public → sdk/internal}/_generated/__init__.py +0 -6
- wandb/{apis/public → sdk/internal}/_generated/server_features_query.py +3 -3
- wandb/sdk/internal/internal_api.py +138 -47
- wandb/sdk/internal/profiler.py +6 -5
- wandb/sdk/internal/run.py +13 -6
- wandb/sdk/internal/sender.py +2 -0
- wandb/sdk/internal/sender_config.py +8 -11
- wandb/sdk/internal/settings_static.py +24 -2
- wandb/sdk/lib/apikey.py +40 -20
- wandb/sdk/lib/asyncio_compat.py +1 -1
- wandb/sdk/lib/deprecate.py +13 -22
- wandb/sdk/lib/disabled.py +2 -1
- wandb/sdk/lib/printer.py +37 -8
- wandb/sdk/lib/printer_asyncio.py +46 -0
- wandb/sdk/lib/redirect.py +10 -5
- wandb/sdk/lib/run_moment.py +4 -6
- wandb/sdk/lib/wb_logging.py +161 -0
- wandb/sdk/service/server_sock.py +19 -14
- wandb/sdk/service/service.py +9 -7
- wandb/sdk/service/streams.py +5 -0
- wandb/sdk/verify/verify.py +6 -3
- wandb/sdk/wandb_config.py +44 -43
- wandb/sdk/wandb_init.py +323 -141
- wandb/sdk/wandb_login.py +13 -4
- wandb/sdk/wandb_metadata.py +107 -91
- wandb/sdk/wandb_run.py +529 -325
- wandb/sdk/wandb_settings.py +422 -202
- wandb/sdk/wandb_setup.py +52 -1
- wandb/util.py +29 -29
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/METADATA +7 -7
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/RECORD +150 -93
- wandb/_globals.py +0 -19
- wandb/apis/public/_generated/base.py +0 -128
- wandb/apis/public/_generated/typing_compat.py +0 -14
- /wandb/{apis/public → sdk/internal}/_generated/enums.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/input_types.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/operations.py +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/WHEEL +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_init.py
CHANGED
@@ -10,6 +10,7 @@ For more on using `wandb.init()`, including code snippets, check out our
|
|
10
10
|
|
11
11
|
from __future__ import annotations
|
12
12
|
|
13
|
+
import contextlib
|
13
14
|
import copy
|
14
15
|
import dataclasses
|
15
16
|
import json
|
@@ -20,7 +21,7 @@ import platform
|
|
20
21
|
import sys
|
21
22
|
import tempfile
|
22
23
|
import time
|
23
|
-
from typing import Any, Literal, Sequence
|
24
|
+
from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, Protocol, Sequence
|
24
25
|
|
25
26
|
if sys.version_info >= (3, 11):
|
26
27
|
from typing import Self
|
@@ -29,24 +30,29 @@ else:
|
|
29
30
|
|
30
31
|
import wandb
|
31
32
|
import wandb.env
|
32
|
-
from wandb import trigger
|
33
|
+
from wandb import env, trigger
|
33
34
|
from wandb.errors import CommError, Error, UsageError
|
34
35
|
from wandb.errors.links import url_registry
|
35
36
|
from wandb.errors.util import ProtobufErrorHandler
|
36
37
|
from wandb.integration import sagemaker
|
37
|
-
from wandb.
|
38
|
+
from wandb.proto.wandb_deprecated import Deprecated
|
39
|
+
from wandb.sdk.lib import ipython as wb_ipython
|
40
|
+
from wandb.sdk.lib import progress, runid, wb_logging
|
38
41
|
from wandb.sdk.lib.paths import StrPath
|
39
42
|
from wandb.util import _is_artifact_representation
|
40
43
|
|
41
44
|
from . import wandb_login, wandb_setup
|
42
45
|
from .backend.backend import Backend
|
43
46
|
from .lib import SummaryDisabled, filesystem, module, paths, printer, telemetry
|
44
|
-
from .lib.deprecate import
|
47
|
+
from .lib.deprecate import deprecate
|
45
48
|
from .mailbox import wait_with_progress
|
46
49
|
from .wandb_helper import parse_config
|
47
50
|
from .wandb_run import Run, TeardownHook, TeardownStage
|
48
51
|
from .wandb_settings import Settings
|
49
52
|
|
53
|
+
if TYPE_CHECKING:
|
54
|
+
import wandb.jupyter
|
55
|
+
|
50
56
|
|
51
57
|
def _huggingface_version() -> str | None:
|
52
58
|
if "transformers" in sys.modules:
|
@@ -122,6 +128,34 @@ class _ConfigParts:
|
|
122
128
|
"""
|
123
129
|
|
124
130
|
|
131
|
+
class _PrinterCallback(Protocol):
|
132
|
+
"""A callback for displaying messages after a printer is configured.
|
133
|
+
|
134
|
+
This is used for a few messages that may be generated before run settings
|
135
|
+
are computed, which are necessary for creating a printer.
|
136
|
+
"""
|
137
|
+
|
138
|
+
def __call__(self, run_printer: printer.Printer) -> None:
|
139
|
+
"""Display information through the given printer."""
|
140
|
+
|
141
|
+
|
142
|
+
def _noop_printer_callback() -> _PrinterCallback:
|
143
|
+
"""A printer callback that does not print anything."""
|
144
|
+
return lambda _: None
|
145
|
+
|
146
|
+
|
147
|
+
def _concat_printer_callbacks(
|
148
|
+
cbs: Iterable[_PrinterCallback],
|
149
|
+
) -> _PrinterCallback:
|
150
|
+
"""Returns a printer callback that runs the given callbacks in order."""
|
151
|
+
|
152
|
+
def do_callbacks(run_printer: printer.Printer) -> None:
|
153
|
+
for cb in cbs:
|
154
|
+
cb(run_printer)
|
155
|
+
|
156
|
+
return do_callbacks
|
157
|
+
|
158
|
+
|
125
159
|
class _WandbInit:
|
126
160
|
def __init__(
|
127
161
|
self,
|
@@ -141,8 +175,7 @@ class _WandbInit:
|
|
141
175
|
self.backend: Backend | None = None
|
142
176
|
|
143
177
|
self._teardown_hooks: list[TeardownHook] = []
|
144
|
-
self.notebook: wandb.jupyter.Notebook | None = None
|
145
|
-
self.printer = printer.new_printer()
|
178
|
+
self.notebook: wandb.jupyter.Notebook | None = None
|
146
179
|
|
147
180
|
self.deprecated_features_used: dict[str, str] = dict()
|
148
181
|
|
@@ -174,20 +207,21 @@ class _WandbInit:
|
|
174
207
|
|
175
208
|
wandb_login._login(
|
176
209
|
anonymous=run_settings.anonymous,
|
210
|
+
host=run_settings.base_url,
|
177
211
|
force=run_settings.force,
|
178
212
|
_disable_warning=True,
|
179
213
|
_silent=run_settings.quiet or run_settings.silent,
|
180
214
|
)
|
181
215
|
|
182
|
-
def warn_env_vars_change_after_setup(self) ->
|
183
|
-
"""Warn if environment variables
|
216
|
+
def warn_env_vars_change_after_setup(self) -> _PrinterCallback:
|
217
|
+
"""Warn if environment variables changed after `wandb.setup()`.
|
184
218
|
|
185
|
-
|
186
|
-
|
219
|
+
Returns:
|
220
|
+
A callback to print any generated warnings.
|
187
221
|
"""
|
188
222
|
singleton = wandb_setup.singleton()
|
189
223
|
if singleton is None:
|
190
|
-
return
|
224
|
+
return _noop_printer_callback()
|
191
225
|
|
192
226
|
exclude_env_vars = {"WANDB_SERVICE", "WANDB_KUBEFLOW_URL"}
|
193
227
|
# check if environment variables have changed
|
@@ -201,26 +235,36 @@ class _WandbInit:
|
|
201
235
|
for k, v in os.environ.items()
|
202
236
|
if k.startswith("WANDB_") and k not in exclude_env_vars
|
203
237
|
}
|
204
|
-
|
205
|
-
|
206
|
-
|
238
|
+
|
239
|
+
if (
|
240
|
+
set(singleton_env.keys()) == set(os_env.keys()) #
|
241
|
+
and set(singleton_env.values()) == set(os_env.values())
|
242
|
+
):
|
243
|
+
return _noop_printer_callback()
|
244
|
+
|
245
|
+
def print_warning(run_printer: printer.Printer) -> None:
|
207
246
|
line = (
|
208
247
|
"Changes to your `wandb` environment variables will be ignored "
|
209
248
|
"because your `wandb` session has already started. "
|
210
249
|
"For more information on how to modify your settings with "
|
211
250
|
"`wandb.init()` arguments, please refer to "
|
212
|
-
f"{
|
251
|
+
f"{run_printer.link(url_registry.url('wandb-init'), 'the W&B docs')}."
|
213
252
|
)
|
214
|
-
|
253
|
+
run_printer.display(line, level="warn")
|
254
|
+
|
255
|
+
return print_warning
|
215
256
|
|
216
257
|
def clear_run_path_if_sweep_or_launch(
|
217
258
|
self,
|
218
259
|
init_settings: Settings,
|
219
|
-
) ->
|
260
|
+
) -> _PrinterCallback:
|
220
261
|
"""Clear project/entity/run_id keys if in a Sweep or a Launch context.
|
221
262
|
|
222
263
|
Args:
|
223
264
|
init_settings: Settings specified in the call to `wandb.init()`.
|
265
|
+
|
266
|
+
Returns:
|
267
|
+
A callback to print any generated warnings.
|
224
268
|
"""
|
225
269
|
when_doing_thing = ""
|
226
270
|
|
@@ -230,13 +274,12 @@ class _WandbInit:
|
|
230
274
|
when_doing_thing = "when running from a wandb launch context"
|
231
275
|
|
232
276
|
if not when_doing_thing:
|
233
|
-
return
|
277
|
+
return _noop_printer_callback()
|
278
|
+
|
279
|
+
warnings = []
|
234
280
|
|
235
281
|
def warn(key: str, value: str) -> None:
|
236
|
-
|
237
|
-
f"Ignoring {key} {value!r} {when_doing_thing}.",
|
238
|
-
level="warn",
|
239
|
-
)
|
282
|
+
warnings.append(f"Ignoring {key} {value!r} {when_doing_thing}.")
|
240
283
|
|
241
284
|
if init_settings.project is not None:
|
242
285
|
warn("project", init_settings.project)
|
@@ -248,16 +291,26 @@ class _WandbInit:
|
|
248
291
|
warn("run_id", init_settings.run_id)
|
249
292
|
init_settings.run_id = None
|
250
293
|
|
251
|
-
|
252
|
-
|
294
|
+
def print_warnings(run_printer: printer.Printer) -> None:
|
295
|
+
for warning in warnings:
|
296
|
+
run_printer.display(warning, level="warn")
|
297
|
+
|
298
|
+
return print_warnings
|
299
|
+
|
300
|
+
def make_run_settings(
|
301
|
+
self,
|
302
|
+
init_settings: Settings,
|
303
|
+
) -> tuple[Settings, _PrinterCallback]:
|
304
|
+
"""Returns the run's settings and any warnings.
|
253
305
|
|
254
306
|
Args:
|
255
307
|
init_settings: Settings passed to `wandb.init()` or set via
|
256
308
|
keyword arguments.
|
257
309
|
"""
|
258
|
-
|
259
|
-
|
260
|
-
|
310
|
+
warning_callbacks: list[_PrinterCallback] = [
|
311
|
+
self.warn_env_vars_change_after_setup(),
|
312
|
+
self.clear_run_path_if_sweep_or_launch(init_settings),
|
313
|
+
]
|
261
314
|
|
262
315
|
# Inherit global settings.
|
263
316
|
settings = self._wl.settings.model_copy()
|
@@ -303,7 +356,7 @@ class _WandbInit:
|
|
303
356
|
label = runid.generate_id()
|
304
357
|
settings.x_label = f"{prefix}-{label}" if prefix else label
|
305
358
|
|
306
|
-
return settings
|
359
|
+
return settings, _concat_printer_callbacks(warning_callbacks)
|
307
360
|
|
308
361
|
def _load_autoresume_run_id(self, resume_file: pathlib.Path) -> str | None:
|
309
362
|
"""Returns the run_id stored in the auto-resume file, if any.
|
@@ -349,6 +402,9 @@ class _WandbInit:
|
|
349
402
|
|
350
403
|
After this, `settings.run_id` is guaranteed to be set.
|
351
404
|
|
405
|
+
If a `resume_from` is provided and `run_id` is not set, initialize
|
406
|
+
`run_id` with the `resume_from` run's `run_id`.
|
407
|
+
|
352
408
|
Args:
|
353
409
|
settings: The run's settings derived from the environment
|
354
410
|
and explicit values passed to `wandb.init()`.
|
@@ -375,7 +431,12 @@ class _WandbInit:
|
|
375
431
|
# If no run ID was inferred, explicitly set, or loaded from an
|
376
432
|
# auto-resume file, then we generate a new ID.
|
377
433
|
if settings.run_id is None:
|
378
|
-
|
434
|
+
# If resume_from is provided and run_id is not already set,
|
435
|
+
# initialize run_id with the value from resume_from.
|
436
|
+
if settings.resume_from:
|
437
|
+
settings.run_id = settings.resume_from.run
|
438
|
+
else:
|
439
|
+
settings.run_id = runid.generate_id()
|
379
440
|
|
380
441
|
if resume_path:
|
381
442
|
self._save_autoresume_run_id(
|
@@ -401,14 +462,17 @@ class _WandbInit:
|
|
401
462
|
Returns:
|
402
463
|
Initial values for the run's config.
|
403
464
|
"""
|
404
|
-
# TODO: remove this once officially deprecated
|
405
465
|
if config_exclude_keys:
|
406
|
-
self.deprecated_features_used["
|
407
|
-
"
|
466
|
+
self.deprecated_features_used["init__config_exclude_keys"] = (
|
467
|
+
"config_exclude_keys is deprecated. Use"
|
468
|
+
" `config=wandb.helper.parse_config(config_object,"
|
469
|
+
" exclude=('key',))` instead."
|
408
470
|
)
|
409
471
|
if config_include_keys:
|
410
|
-
self.deprecated_features_used["
|
411
|
-
"
|
472
|
+
self.deprecated_features_used["init__config_include_keys"] = (
|
473
|
+
"config_include_keys is deprecated. Use"
|
474
|
+
" `config=wandb.helper.parse_config(config_object,"
|
475
|
+
" include=('key',))` instead."
|
412
476
|
)
|
413
477
|
config = parse_config(
|
414
478
|
config or dict(),
|
@@ -498,37 +562,6 @@ class _WandbInit:
|
|
498
562
|
else:
|
499
563
|
config_target.setdefault(k, v)
|
500
564
|
|
501
|
-
def _create_logger(self, log_fname: str) -> logging.Logger:
|
502
|
-
"""Returns a logger configured to write to a file.
|
503
|
-
|
504
|
-
This adds a run_id to the log, in case of multiple processes on the same
|
505
|
-
machine. Currently, there is no way to disable logging after it's
|
506
|
-
enabled.
|
507
|
-
"""
|
508
|
-
handler = logging.FileHandler(log_fname)
|
509
|
-
handler.setLevel(logging.INFO)
|
510
|
-
|
511
|
-
formatter = logging.Formatter(
|
512
|
-
"%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
|
513
|
-
"[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
|
514
|
-
)
|
515
|
-
|
516
|
-
handler.setFormatter(formatter)
|
517
|
-
|
518
|
-
logger = logging.getLogger("wandb")
|
519
|
-
logger.propagate = False
|
520
|
-
logger.addHandler(handler)
|
521
|
-
# TODO: make me configurable
|
522
|
-
logger.setLevel(logging.DEBUG)
|
523
|
-
self._teardown_hooks.append(
|
524
|
-
TeardownHook(
|
525
|
-
lambda: (handler.close(), logger.removeHandler(handler)), # type: ignore
|
526
|
-
TeardownStage.LATE,
|
527
|
-
)
|
528
|
-
)
|
529
|
-
|
530
|
-
return logger
|
531
|
-
|
532
565
|
def _safe_symlink(
|
533
566
|
self, base: str, target: str, name: str, delete: bool = False
|
534
567
|
) -> None:
|
@@ -551,62 +584,73 @@ class _WandbInit:
|
|
551
584
|
except OSError:
|
552
585
|
pass
|
553
586
|
|
554
|
-
def
|
555
|
-
|
556
|
-
|
587
|
+
def _pre_run_cell_hook(self, *args, **kwargs) -> None:
|
588
|
+
"""Hook for the IPython pre_run_cell event.
|
589
|
+
|
590
|
+
This pauses a run, preventing system metrics from being collected
|
591
|
+
the run's runtime from increasing. It also uploads the notebook's code.
|
592
|
+
"""
|
593
|
+
if not self.backend:
|
594
|
+
return
|
557
595
|
|
558
|
-
|
559
|
-
if self.notebook.save_ipynb(): # type: ignore
|
596
|
+
if self.notebook and self.notebook.save_ipynb():
|
560
597
|
assert self.run is not None
|
561
598
|
res = self.run.log_code(root=None)
|
562
|
-
self._logger.info("saved code: %s", res)
|
599
|
+
self._logger.info("saved code: %s", res)
|
600
|
+
|
563
601
|
if self.backend.interface is not None:
|
564
|
-
self._logger.info("pausing backend")
|
602
|
+
self._logger.info("pausing backend")
|
565
603
|
self.backend.interface.publish_pause()
|
566
604
|
|
567
|
-
def
|
568
|
-
|
569
|
-
|
570
|
-
|
605
|
+
def _post_run_cell_hook(self, *args, **kwargs) -> None:
|
606
|
+
"""Hook for the IPython post_run_cell event.
|
607
|
+
|
608
|
+
Resumes collection of system metrics and the run's timer.
|
609
|
+
"""
|
610
|
+
if self.backend is None or self.backend.interface is None:
|
611
|
+
return
|
612
|
+
|
613
|
+
self._logger.info("resuming backend")
|
614
|
+
self.backend.interface.publish_resume()
|
571
615
|
|
572
616
|
def _jupyter_teardown(self) -> None:
|
573
617
|
"""Teardown hooks and display saving, called with wandb.finish."""
|
574
618
|
assert self.notebook
|
575
619
|
ipython = self.notebook.shell
|
576
|
-
|
620
|
+
|
621
|
+
if self.run:
|
622
|
+
self.notebook.save_history(self.run)
|
623
|
+
|
577
624
|
if self.notebook.save_ipynb():
|
578
625
|
assert self.run is not None
|
579
626
|
res = self.run.log_code(root=None)
|
580
|
-
self._logger.info("saved code and history: %s", res)
|
581
|
-
self._logger.info("cleaning up jupyter logic")
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
for hook in ipython.events.callbacks["post_run_cell"]:
|
587
|
-
if "_pause_backend" in hook.__name__:
|
588
|
-
ipython.events.unregister("post_run_cell", hook)
|
627
|
+
self._logger.info("saved code and history: %s", res)
|
628
|
+
self._logger.info("cleaning up jupyter logic")
|
629
|
+
|
630
|
+
ipython.events.unregister("pre_run_cell", self._pre_run_cell_hook)
|
631
|
+
ipython.events.unregister("post_run_cell", self._post_run_cell_hook)
|
632
|
+
|
589
633
|
ipython.display_pub.publish = ipython.display_pub._orig_publish
|
590
634
|
del ipython.display_pub._orig_publish
|
591
635
|
|
592
636
|
def monkeypatch_ipython(self, settings: Settings) -> None:
|
593
637
|
"""Add hooks, and session history saving."""
|
594
|
-
self.notebook = wandb.jupyter.Notebook(settings)
|
638
|
+
self.notebook = wandb.jupyter.Notebook(settings)
|
595
639
|
ipython = self.notebook.shell
|
596
640
|
|
597
641
|
# Monkey patch ipython publish to capture displayed outputs
|
598
642
|
if not hasattr(ipython.display_pub, "_orig_publish"):
|
599
|
-
self._logger.info("configuring jupyter hooks %s", self)
|
643
|
+
self._logger.info("configuring jupyter hooks %s", self)
|
600
644
|
ipython.display_pub._orig_publish = ipython.display_pub.publish
|
601
|
-
# Registering resume and pause hooks
|
602
645
|
|
603
|
-
ipython.events.register("pre_run_cell", self.
|
604
|
-
ipython.events.register("post_run_cell", self.
|
646
|
+
ipython.events.register("pre_run_cell", self._pre_run_cell_hook)
|
647
|
+
ipython.events.register("post_run_cell", self._post_run_cell_hook)
|
648
|
+
|
605
649
|
self._teardown_hooks.append(
|
606
650
|
TeardownHook(self._jupyter_teardown, TeardownStage.EARLY)
|
607
651
|
)
|
608
652
|
|
609
|
-
def publish(data, metadata=None, **kwargs) -> None:
|
653
|
+
def publish(data, metadata=None, **kwargs) -> None:
|
610
654
|
ipython.display_pub._orig_publish(data, metadata=metadata, **kwargs)
|
611
655
|
assert self.notebook is not None
|
612
656
|
self.notebook.save_display(
|
@@ -615,8 +659,14 @@ class _WandbInit:
|
|
615
659
|
|
616
660
|
ipython.display_pub.publish = publish
|
617
661
|
|
618
|
-
|
619
|
-
|
662
|
+
@contextlib.contextmanager
|
663
|
+
def setup_run_log_directory(self, settings: Settings) -> Iterator[None]:
|
664
|
+
"""Set up the run's log directory.
|
665
|
+
|
666
|
+
This is a context manager that closes and unregisters the log handler
|
667
|
+
in case of an uncaught exception, so that future logged messages do not
|
668
|
+
modify this run's log file.
|
669
|
+
"""
|
620
670
|
filesystem.mkdir_exists_ok(os.path.dirname(settings.log_user))
|
621
671
|
filesystem.mkdir_exists_ok(os.path.dirname(settings.log_internal))
|
622
672
|
filesystem.mkdir_exists_ok(os.path.dirname(settings.sync_file))
|
@@ -643,9 +693,41 @@ class _WandbInit:
|
|
643
693
|
delete=True,
|
644
694
|
)
|
645
695
|
|
646
|
-
|
647
|
-
|
648
|
-
|
696
|
+
assert settings.run_id
|
697
|
+
handler = wb_logging.add_file_handler(
|
698
|
+
settings.run_id,
|
699
|
+
pathlib.Path(settings.log_user),
|
700
|
+
)
|
701
|
+
|
702
|
+
if env.is_debug():
|
703
|
+
handler.setLevel(logging.DEBUG)
|
704
|
+
|
705
|
+
disposed = False
|
706
|
+
|
707
|
+
def dispose_handler() -> None:
|
708
|
+
nonlocal disposed
|
709
|
+
|
710
|
+
if not disposed:
|
711
|
+
disposed = True
|
712
|
+
logging.getLogger("wandb").removeHandler(handler)
|
713
|
+
handler.close()
|
714
|
+
|
715
|
+
try:
|
716
|
+
self._teardown_hooks.append(
|
717
|
+
TeardownHook(
|
718
|
+
call=dispose_handler,
|
719
|
+
stage=TeardownStage.LATE,
|
720
|
+
)
|
721
|
+
)
|
722
|
+
|
723
|
+
self._wl._early_logger_flush(logging.getLogger("wandb"))
|
724
|
+
self._logger.info(f"Logging user logs to {settings.log_user}")
|
725
|
+
self._logger.info(f"Logging internal logs to {settings.log_internal}")
|
726
|
+
|
727
|
+
yield
|
728
|
+
except Exception:
|
729
|
+
dispose_handler()
|
730
|
+
raise
|
649
731
|
|
650
732
|
def make_disabled_run(self, config: _ConfigParts) -> Run:
|
651
733
|
"""Returns a Run-like object where all methods are no-ops.
|
@@ -709,6 +791,10 @@ class _WandbInit:
|
|
709
791
|
):
|
710
792
|
setattr(drun, symbol, lambda *_, **__: None) # type: ignore
|
711
793
|
|
794
|
+
# set properties to None
|
795
|
+
for attr in ("url", "project_url", "sweep_url"):
|
796
|
+
setattr(type(drun), attr, property(lambda _: None))
|
797
|
+
|
712
798
|
class _ChainableNoOp:
|
713
799
|
"""An object that allows chaining arbitrary attributes and method calls."""
|
714
800
|
|
@@ -733,7 +819,7 @@ class _WandbInit:
|
|
733
819
|
def __call__(self, *args: Any, **kwargs: Any) -> _ChainableNoOp:
|
734
820
|
return _ChainableNoOp()
|
735
821
|
|
736
|
-
drun.log_artifact = _ChainableNoOpField() # type: ignore
|
822
|
+
drun.log_artifact = _ChainableNoOpField() # type: ignore
|
737
823
|
# attributes
|
738
824
|
drun._start_time = time.time()
|
739
825
|
drun._starting_step = 0
|
@@ -757,7 +843,12 @@ class _WandbInit:
|
|
757
843
|
)
|
758
844
|
return drun
|
759
845
|
|
760
|
-
def init(
|
846
|
+
def init( # noqa: C901
|
847
|
+
self,
|
848
|
+
settings: Settings,
|
849
|
+
config: _ConfigParts,
|
850
|
+
run_printer: printer.Printer,
|
851
|
+
) -> Run:
|
761
852
|
self._logger.info("calling init triggers")
|
762
853
|
trigger.call("on_init")
|
763
854
|
|
@@ -768,18 +859,36 @@ class _WandbInit:
|
|
768
859
|
f"\nconfig: {config.base_no_artifacts}"
|
769
860
|
)
|
770
861
|
|
771
|
-
if
|
772
|
-
if
|
773
|
-
|
774
|
-
wandb.
|
862
|
+
if previous_run := self._wl.most_recent_active_run:
|
863
|
+
if (
|
864
|
+
settings.reinit in (True, "finish_previous")
|
865
|
+
# calling wandb.init() in notebooks finishes previous runs
|
866
|
+
# by default for user convenience.
|
867
|
+
or (settings.reinit == "default" and wb_ipython.in_notebook())
|
868
|
+
):
|
869
|
+
run_printer.display(
|
870
|
+
"Finishing previous runs because reinit is set"
|
871
|
+
f" to {settings.reinit!r}."
|
872
|
+
)
|
873
|
+
self._wl.finish_all_active_runs()
|
874
|
+
|
875
|
+
elif settings.reinit == "create_new":
|
876
|
+
self._logger.info(
|
877
|
+
"wandb.init() called while a run is active,"
|
878
|
+
" and reinit is set to 'create_new', so continuing"
|
879
|
+
)
|
880
|
+
|
775
881
|
else:
|
776
|
-
|
882
|
+
run_printer.display(
|
883
|
+
"wandb.init() called while a run is active and reinit is"
|
884
|
+
f" set to {settings.reinit!r}, so returning the previous"
|
885
|
+
" run."
|
886
|
+
)
|
777
887
|
|
778
|
-
|
779
|
-
with telemetry.context() as tel:
|
888
|
+
with telemetry.context(run=previous_run) as tel:
|
780
889
|
tel.feature.init_return_run = True
|
781
890
|
|
782
|
-
return
|
891
|
+
return previous_run
|
783
892
|
|
784
893
|
self._logger.info("starting backend")
|
785
894
|
|
@@ -872,6 +981,9 @@ class _WandbInit:
|
|
872
981
|
)
|
873
982
|
tel.feature.shared_mode = True
|
874
983
|
|
984
|
+
if settings.x_label:
|
985
|
+
tel.feature.user_provided_label = True
|
986
|
+
|
875
987
|
tel.env.maybe_mp = _maybe_mp_process(backend)
|
876
988
|
|
877
989
|
if not settings.label_disable:
|
@@ -881,10 +993,9 @@ class _WandbInit:
|
|
881
993
|
run._label_probe_main()
|
882
994
|
|
883
995
|
for deprecated_feature, msg in self.deprecated_features_used.items():
|
884
|
-
warning_message = f"`{deprecated_feature}` is deprecated. {msg}"
|
885
996
|
deprecate(
|
886
|
-
field_name=getattr(Deprecated,
|
887
|
-
warning_message=
|
997
|
+
field_name=getattr(Deprecated, deprecated_feature),
|
998
|
+
warning_message=msg,
|
888
999
|
run=run,
|
889
1000
|
)
|
890
1001
|
|
@@ -922,7 +1033,7 @@ class _WandbInit:
|
|
922
1033
|
assert backend.interface
|
923
1034
|
|
924
1035
|
with progress.progress_printer(
|
925
|
-
|
1036
|
+
run_printer,
|
926
1037
|
default_text="Waiting for wandb.init()...",
|
927
1038
|
) as progress_printer:
|
928
1039
|
await progress.loop_printing_operation_stats(
|
@@ -1028,6 +1139,10 @@ class _WandbInit:
|
|
1028
1139
|
run.use_artifact(job_artifact)
|
1029
1140
|
|
1030
1141
|
self.backend = backend
|
1142
|
+
|
1143
|
+
if settings.reinit != "create_new":
|
1144
|
+
_set_global_run(run)
|
1145
|
+
|
1031
1146
|
run._on_start()
|
1032
1147
|
self._logger.info("run started, returning control to user process")
|
1033
1148
|
return run
|
@@ -1104,10 +1219,36 @@ def _attach(
|
|
1104
1219
|
raise UsageError(f"Failed to attach to run: {attach_response.error.message}")
|
1105
1220
|
|
1106
1221
|
run._set_run_obj(attach_response.run)
|
1222
|
+
_set_global_run(run)
|
1107
1223
|
run._on_attach()
|
1108
1224
|
return run
|
1109
1225
|
|
1110
1226
|
|
1227
|
+
def _set_global_run(run: Run) -> None:
|
1228
|
+
"""Set `wandb.run` and point some top-level functions to its methods.
|
1229
|
+
|
1230
|
+
Args:
|
1231
|
+
run: The run to make global.
|
1232
|
+
"""
|
1233
|
+
module.set_global(
|
1234
|
+
run=run,
|
1235
|
+
config=run.config,
|
1236
|
+
log=run.log,
|
1237
|
+
summary=run.summary,
|
1238
|
+
save=run.save,
|
1239
|
+
use_artifact=run.use_artifact,
|
1240
|
+
log_artifact=run.log_artifact,
|
1241
|
+
define_metric=run.define_metric,
|
1242
|
+
alert=run.alert,
|
1243
|
+
watch=run.watch,
|
1244
|
+
unwatch=run.unwatch,
|
1245
|
+
mark_preempting=run.mark_preempting,
|
1246
|
+
log_model=run.log_model,
|
1247
|
+
use_model=run.use_model,
|
1248
|
+
link_model=run.link_model,
|
1249
|
+
)
|
1250
|
+
|
1251
|
+
|
1111
1252
|
def _monkeypatch_openai_gym() -> None:
|
1112
1253
|
"""Patch OpenAI gym to log to the global `wandb.run`."""
|
1113
1254
|
if len(wandb.patched["gym"]) > 0:
|
@@ -1128,6 +1269,29 @@ def _monkeypatch_tensorboard() -> None:
|
|
1128
1269
|
tb_module.patch()
|
1129
1270
|
|
1130
1271
|
|
1272
|
+
def try_create_root_dir(settings: Settings) -> None:
|
1273
|
+
"""Try to create the root directory specified in settings.
|
1274
|
+
|
1275
|
+
If creation fails due to permissions or other errors,
|
1276
|
+
falls back to using the system temp directory.
|
1277
|
+
|
1278
|
+
Args:
|
1279
|
+
settings: The runs settings containing root_dir configuration.
|
1280
|
+
This function may update the root_dir to a temporary directory
|
1281
|
+
if the parent directory is not writable.
|
1282
|
+
"""
|
1283
|
+
try:
|
1284
|
+
if not os.path.exists(settings.root_dir):
|
1285
|
+
os.makedirs(settings.root_dir, exist_ok=True)
|
1286
|
+
except OSError:
|
1287
|
+
temp_dir = tempfile.gettempdir()
|
1288
|
+
wandb.termwarn(
|
1289
|
+
f"Path {settings.root_dir} wasn't writable, using system temp directory {temp_dir}.",
|
1290
|
+
repeat=False,
|
1291
|
+
)
|
1292
|
+
settings.root_dir = temp_dir
|
1293
|
+
|
1294
|
+
|
1131
1295
|
def init( # noqa: C901
|
1132
1296
|
entity: str | None = None,
|
1133
1297
|
project: str | None = None,
|
@@ -1145,7 +1309,16 @@ def init( # noqa: C901
|
|
1145
1309
|
mode: Literal["online", "offline", "disabled"] | None = None,
|
1146
1310
|
force: bool | None = None,
|
1147
1311
|
anonymous: Literal["never", "allow", "must"] | None = None,
|
1148
|
-
reinit:
|
1312
|
+
reinit: (
|
1313
|
+
bool
|
1314
|
+
| Literal[
|
1315
|
+
None,
|
1316
|
+
"default",
|
1317
|
+
"return_previous",
|
1318
|
+
"finish_previous",
|
1319
|
+
"create_new",
|
1320
|
+
]
|
1321
|
+
) = None,
|
1149
1322
|
resume: bool | Literal["allow", "never", "must", "auto"] | None = None,
|
1150
1323
|
resume_from: str | None = None,
|
1151
1324
|
fork_from: str | None = None,
|
@@ -1293,12 +1466,8 @@ def init( # noqa: C901
|
|
1293
1466
|
to view the charts and data in the UI.
|
1294
1467
|
- `"must"`: Forces the run to be logged to an anonymous account, even
|
1295
1468
|
if the user is logged in.
|
1296
|
-
reinit:
|
1297
|
-
|
1298
|
-
exists, calling `wandb.init()` returns the existing run instead of
|
1299
|
-
creating a new one. When `reinit=True`, the active run is finished
|
1300
|
-
before a new run is initialized. In notebook environments, runs are
|
1301
|
-
reinitialized by default unless `reinit` is explicitly set to `False`.
|
1469
|
+
reinit: Shorthand for the "reinit" setting. Determines the behavior of
|
1470
|
+
`wandb.init()` when a run is active.
|
1302
1471
|
resume: Controls the behavior when resuming a run with the specified `id`.
|
1303
1472
|
Available options are:
|
1304
1473
|
- `"allow"`: If a run with the specified `id` exists, it will resume
|
@@ -1425,7 +1594,13 @@ def init( # noqa: C901
|
|
1425
1594
|
wi = _WandbInit(wl, init_telemetry)
|
1426
1595
|
|
1427
1596
|
wi.maybe_login(init_settings)
|
1428
|
-
run_settings = wi.make_run_settings(init_settings)
|
1597
|
+
run_settings, show_warnings = wi.make_run_settings(init_settings)
|
1598
|
+
|
1599
|
+
if isinstance(run_settings.reinit, bool):
|
1600
|
+
wi.deprecated_features_used["run__reinit_bool"] = (
|
1601
|
+
"Using a boolean value for 'reinit' is deprecated."
|
1602
|
+
" Use 'return_previous' or 'finish_previous' instead."
|
1603
|
+
)
|
1429
1604
|
|
1430
1605
|
if run_settings.run_id is not None:
|
1431
1606
|
init_telemetry.feature.set_init_id = True
|
@@ -1437,35 +1612,42 @@ def init( # noqa: C901
|
|
1437
1612
|
init_telemetry.feature.offline = True
|
1438
1613
|
|
1439
1614
|
wi.set_run_id(run_settings)
|
1615
|
+
run_printer = printer.new_printer(run_settings)
|
1616
|
+
show_warnings(run_printer)
|
1440
1617
|
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1618
|
+
with contextlib.ExitStack() as exit_stack:
|
1619
|
+
exit_stack.enter_context(wb_logging.log_to_run(run_settings.run_id))
|
1620
|
+
|
1621
|
+
run_config = wi.make_run_config(
|
1622
|
+
settings=run_settings,
|
1623
|
+
config=config,
|
1624
|
+
config_exclude_keys=config_exclude_keys,
|
1625
|
+
config_include_keys=config_include_keys,
|
1626
|
+
)
|
1627
|
+
|
1628
|
+
if run_settings._noop:
|
1629
|
+
return wi.make_disabled_run(run_config)
|
1447
1630
|
|
1448
|
-
|
1449
|
-
|
1631
|
+
try_create_root_dir(run_settings)
|
1632
|
+
exit_stack.enter_context(wi.setup_run_log_directory(run_settings))
|
1450
1633
|
|
1451
|
-
|
1452
|
-
|
1453
|
-
wi.monkeypatch_ipython(run_settings)
|
1634
|
+
if run_settings._jupyter:
|
1635
|
+
wi.monkeypatch_ipython(run_settings)
|
1454
1636
|
|
1455
|
-
|
1456
|
-
|
1637
|
+
if monitor_gym:
|
1638
|
+
_monkeypatch_openai_gym()
|
1457
1639
|
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
|
1640
|
+
if wandb.patched["tensorboard"]:
|
1641
|
+
# NOTE: The user may have called the patch function directly.
|
1642
|
+
init_telemetry.feature.tensorboard_patch = True
|
1643
|
+
if run_settings.sync_tensorboard:
|
1644
|
+
_monkeypatch_tensorboard()
|
1645
|
+
init_telemetry.feature.tensorboard_sync = True
|
1464
1646
|
|
1465
|
-
|
1466
|
-
|
1647
|
+
if run_settings.x_server_side_derived_summary:
|
1648
|
+
init_telemetry.feature.server_side_derived_summary = True
|
1467
1649
|
|
1468
|
-
|
1650
|
+
return wi.init(run_settings, run_config, run_printer)
|
1469
1651
|
|
1470
1652
|
except KeyboardInterrupt as e:
|
1471
1653
|
if wl:
|