wandb 0.19.8__py3-none-win_amd64.whl → 0.19.10__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +5 -1
- wandb/__init__.pyi +15 -8
- wandb/_pydantic/__init__.py +30 -0
- wandb/_pydantic/base.py +148 -0
- wandb/_pydantic/utils.py +66 -0
- wandb/_pydantic/v1_compat.py +284 -0
- wandb/apis/paginator.py +82 -38
- wandb/apis/public/__init__.py +2 -2
- wandb/apis/public/api.py +111 -53
- wandb/apis/public/artifacts.py +387 -639
- wandb/apis/public/automations.py +69 -0
- wandb/apis/public/files.py +2 -2
- wandb/apis/public/integrations.py +168 -0
- wandb/apis/public/projects.py +32 -2
- wandb/apis/public/reports.py +2 -2
- wandb/apis/public/runs.py +19 -11
- wandb/apis/public/utils.py +107 -1
- wandb/automations/__init__.py +81 -0
- wandb/automations/_filters/__init__.py +40 -0
- wandb/automations/_filters/expressions.py +179 -0
- wandb/automations/_filters/operators.py +267 -0
- wandb/automations/_filters/run_metrics.py +183 -0
- wandb/automations/_generated/__init__.py +184 -0
- wandb/automations/_generated/create_filter_trigger.py +21 -0
- wandb/automations/_generated/create_generic_webhook_integration.py +43 -0
- wandb/automations/_generated/delete_trigger.py +19 -0
- wandb/automations/_generated/enums.py +33 -0
- wandb/automations/_generated/fragments.py +343 -0
- wandb/automations/_generated/generic_webhook_integrations_by_entity.py +22 -0
- wandb/automations/_generated/get_triggers.py +24 -0
- wandb/automations/_generated/get_triggers_by_entity.py +24 -0
- wandb/automations/_generated/input_types.py +104 -0
- wandb/automations/_generated/integrations_by_entity.py +22 -0
- wandb/automations/_generated/operations.py +710 -0
- wandb/automations/_generated/slack_integrations_by_entity.py +22 -0
- wandb/automations/_generated/update_filter_trigger.py +21 -0
- wandb/automations/_utils.py +123 -0
- wandb/automations/_validators.py +73 -0
- wandb/automations/actions.py +205 -0
- wandb/automations/automations.py +109 -0
- wandb/automations/events.py +235 -0
- wandb/automations/integrations.py +26 -0
- wandb/automations/scopes.py +76 -0
- wandb/beta/workflows.py +9 -10
- wandb/bin/gpu_stats.exe +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/cli.py +3 -3
- wandb/integration/keras/keras.py +2 -1
- wandb/integration/langchain/wandb_tracer.py +2 -1
- wandb/integration/metaflow/metaflow.py +19 -17
- wandb/integration/sacred/__init__.py +1 -1
- wandb/jupyter.py +155 -133
- wandb/old/summary.py +0 -2
- wandb/proto/v3/wandb_internal_pb2.py +297 -292
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +292 -292
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v5/wandb_internal_pb2.py +292 -292
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v6/wandb_base_pb2.py +41 -0
- wandb/proto/v6/wandb_internal_pb2.py +393 -0
- wandb/proto/v6/wandb_server_pb2.py +78 -0
- wandb/proto/v6/wandb_settings_pb2.py +58 -0
- wandb/proto/v6/wandb_telemetry_pb2.py +52 -0
- wandb/proto/wandb_base_pb2.py +2 -0
- wandb/proto/wandb_deprecated.py +10 -0
- wandb/proto/wandb_internal_pb2.py +3 -1
- wandb/proto/wandb_server_pb2.py +2 -0
- wandb/proto/wandb_settings_pb2.py +2 -0
- wandb/proto/wandb_telemetry_pb2.py +2 -0
- wandb/sdk/artifacts/_generated/__init__.py +248 -0
- wandb/sdk/artifacts/_generated/artifact_collection_membership_files.py +43 -0
- wandb/sdk/artifacts/_generated/artifact_version_files.py +36 -0
- wandb/sdk/artifacts/_generated/create_artifact_collection_tag_assignments.py +36 -0
- wandb/sdk/artifacts/_generated/delete_artifact_collection_tag_assignments.py +25 -0
- wandb/sdk/artifacts/_generated/delete_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/delete_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_generated/enums.py +17 -0
- wandb/sdk/artifacts/_generated/fragments.py +186 -0
- wandb/sdk/artifacts/_generated/input_types.py +16 -0
- wandb/sdk/artifacts/_generated/move_artifact_collection.py +35 -0
- wandb/sdk/artifacts/_generated/operations.py +510 -0
- wandb/sdk/artifacts/_generated/project_artifact_collection.py +101 -0
- wandb/sdk/artifacts/_generated/project_artifact_collections.py +33 -0
- wandb/sdk/artifacts/_generated/project_artifact_type.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifact_types.py +24 -0
- wandb/sdk/artifacts/_generated/project_artifacts.py +42 -0
- wandb/sdk/artifacts/_generated/run_input_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/run_output_artifacts.py +51 -0
- wandb/sdk/artifacts/_generated/update_artifact_portfolio.py +35 -0
- wandb/sdk/artifacts/_generated/update_artifact_sequence.py +35 -0
- wandb/sdk/artifacts/_graphql_fragments.py +56 -81
- wandb/sdk/artifacts/_validators.py +1 -0
- wandb/sdk/artifacts/artifact.py +110 -49
- wandb/sdk/artifacts/artifact_manifest_entry.py +2 -1
- wandb/sdk/artifacts/artifact_saver.py +16 -2
- wandb/sdk/artifacts/storage_handlers/azure_handler.py +1 -0
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +23 -2
- wandb/sdk/data_types/audio.py +1 -3
- wandb/sdk/data_types/base_types/media.py +13 -7
- wandb/sdk/data_types/base_types/wb_value.py +34 -11
- wandb/sdk/data_types/html.py +36 -9
- wandb/sdk/data_types/image.py +56 -37
- wandb/sdk/data_types/molecule.py +1 -5
- wandb/sdk/data_types/object_3d.py +2 -1
- wandb/sdk/data_types/saved_model.py +7 -9
- wandb/sdk/data_types/table.py +5 -0
- wandb/sdk/data_types/trace_tree.py +2 -0
- wandb/sdk/data_types/utils.py +1 -1
- wandb/sdk/data_types/video.py +15 -30
- wandb/sdk/interface/interface.py +2 -0
- wandb/{apis/public → sdk/internal}/_generated/__init__.py +0 -6
- wandb/{apis/public → sdk/internal}/_generated/server_features_query.py +3 -3
- wandb/sdk/internal/internal_api.py +138 -47
- wandb/sdk/internal/profiler.py +6 -5
- wandb/sdk/internal/run.py +13 -6
- wandb/sdk/internal/sender.py +2 -0
- wandb/sdk/internal/sender_config.py +8 -11
- wandb/sdk/internal/settings_static.py +24 -2
- wandb/sdk/lib/apikey.py +40 -20
- wandb/sdk/lib/asyncio_compat.py +1 -1
- wandb/sdk/lib/deprecate.py +13 -22
- wandb/sdk/lib/disabled.py +2 -1
- wandb/sdk/lib/printer.py +37 -8
- wandb/sdk/lib/printer_asyncio.py +46 -0
- wandb/sdk/lib/redirect.py +10 -5
- wandb/sdk/lib/run_moment.py +4 -6
- wandb/sdk/lib/wb_logging.py +161 -0
- wandb/sdk/service/server_sock.py +19 -14
- wandb/sdk/service/service.py +9 -7
- wandb/sdk/service/streams.py +5 -0
- wandb/sdk/verify/verify.py +6 -3
- wandb/sdk/wandb_config.py +44 -43
- wandb/sdk/wandb_init.py +323 -141
- wandb/sdk/wandb_login.py +13 -4
- wandb/sdk/wandb_metadata.py +107 -91
- wandb/sdk/wandb_run.py +529 -325
- wandb/sdk/wandb_settings.py +422 -202
- wandb/sdk/wandb_setup.py +52 -1
- wandb/util.py +29 -29
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/METADATA +7 -7
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/RECORD +151 -94
- wandb/_globals.py +0 -19
- wandb/apis/public/_generated/base.py +0 -128
- wandb/apis/public/_generated/typing_compat.py +0 -14
- /wandb/{apis/public → sdk/internal}/_generated/enums.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/input_types.py +0 -0
- /wandb/{apis/public → sdk/internal}/_generated/operations.py +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/WHEEL +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/entry_points.txt +0 -0
- {wandb-0.19.8.dist-info → wandb-0.19.10.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -14,13 +14,26 @@ import sys
|
|
14
14
|
import threading
|
15
15
|
import time
|
16
16
|
import traceback
|
17
|
-
import warnings
|
18
17
|
from collections.abc import Mapping
|
19
18
|
from dataclasses import dataclass, field
|
20
19
|
from datetime import datetime, timedelta, timezone
|
21
20
|
from enum import IntEnum
|
22
21
|
from types import TracebackType
|
23
|
-
from typing import
|
22
|
+
from typing import (
|
23
|
+
TYPE_CHECKING,
|
24
|
+
Any,
|
25
|
+
Callable,
|
26
|
+
Literal,
|
27
|
+
NamedTuple,
|
28
|
+
Sequence,
|
29
|
+
TextIO,
|
30
|
+
TypeVar,
|
31
|
+
)
|
32
|
+
|
33
|
+
if sys.version_info < (3, 10):
|
34
|
+
from typing_extensions import Concatenate, ParamSpec
|
35
|
+
else:
|
36
|
+
from typing import Concatenate, ParamSpec
|
24
37
|
|
25
38
|
import requests
|
26
39
|
|
@@ -28,13 +41,13 @@ import wandb
|
|
28
41
|
import wandb.env
|
29
42
|
import wandb.util
|
30
43
|
from wandb import trigger
|
31
|
-
from wandb._globals import _datatypes_set_callback
|
32
44
|
from wandb.apis import internal, public
|
33
45
|
from wandb.apis.public import Api as PublicApi
|
34
46
|
from wandb.errors import CommError, UnsupportedError, UsageError
|
35
47
|
from wandb.errors.links import url_registry
|
36
48
|
from wandb.integration.torch import wandb_torch
|
37
49
|
from wandb.plot import CustomChart, Visualize
|
50
|
+
from wandb.proto.wandb_deprecated import Deprecated
|
38
51
|
from wandb.proto.wandb_internal_pb2 import (
|
39
52
|
MetadataRequest,
|
40
53
|
MetricRecord,
|
@@ -44,7 +57,7 @@ from wandb.proto.wandb_internal_pb2 import (
|
|
44
57
|
)
|
45
58
|
from wandb.sdk.artifacts.artifact import Artifact
|
46
59
|
from wandb.sdk.internal import job_builder
|
47
|
-
from wandb.sdk.lib import asyncio_compat
|
60
|
+
from wandb.sdk.lib import asyncio_compat, wb_logging
|
48
61
|
from wandb.sdk.lib.import_hooks import (
|
49
62
|
register_post_import_hook,
|
50
63
|
unregister_post_import_hook,
|
@@ -62,6 +75,7 @@ from wandb.util import (
|
|
62
75
|
|
63
76
|
from . import wandb_config, wandb_metric, wandb_summary
|
64
77
|
from .artifacts._validators import (
|
78
|
+
MAX_ARTIFACT_METADATA_KEYS,
|
65
79
|
is_artifact_registry_project,
|
66
80
|
validate_aliases,
|
67
81
|
validate_tags,
|
@@ -165,11 +179,13 @@ class RunStatusChecker:
|
|
165
179
|
|
166
180
|
def __init__(
|
167
181
|
self,
|
182
|
+
run_id: str,
|
168
183
|
interface: InterfaceBase,
|
169
184
|
stop_polling_interval: int = 15,
|
170
185
|
retry_polling_interval: int = 5,
|
171
186
|
internal_messages_polling_interval: int = 10,
|
172
187
|
) -> None:
|
188
|
+
self._run_id = run_id
|
173
189
|
self._interface = interface
|
174
190
|
self._stop_polling_interval = stop_polling_interval
|
175
191
|
self._retry_polling_interval = retry_polling_interval
|
@@ -275,19 +291,20 @@ class RunStatusChecker:
|
|
275
291
|
)
|
276
292
|
)
|
277
293
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
self.
|
289
|
-
|
290
|
-
|
294
|
+
with wb_logging.log_to_run(self._run_id):
|
295
|
+
try:
|
296
|
+
self._loop_check_status(
|
297
|
+
lock=self._network_status_lock,
|
298
|
+
set_handle=lambda x: setattr(self, "_network_status_handle", x),
|
299
|
+
timeout=self._retry_polling_interval,
|
300
|
+
request=self._interface.deliver_network_status,
|
301
|
+
process=_process_network_status,
|
302
|
+
)
|
303
|
+
except BrokenPipeError:
|
304
|
+
self._abandon_status_check(
|
305
|
+
self._network_status_lock,
|
306
|
+
self._network_status_handle,
|
307
|
+
)
|
291
308
|
|
292
309
|
def check_stop_status(self) -> None:
|
293
310
|
def _process_stop_status(result: Result) -> None:
|
@@ -299,19 +316,20 @@ class RunStatusChecker:
|
|
299
316
|
interrupt.interrupt_main()
|
300
317
|
return
|
301
318
|
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
self.
|
313
|
-
|
314
|
-
|
319
|
+
with wb_logging.log_to_run(self._run_id):
|
320
|
+
try:
|
321
|
+
self._loop_check_status(
|
322
|
+
lock=self._stop_status_lock,
|
323
|
+
set_handle=lambda x: setattr(self, "_stop_status_handle", x),
|
324
|
+
timeout=self._stop_polling_interval,
|
325
|
+
request=self._interface.deliver_stop_status,
|
326
|
+
process=_process_stop_status,
|
327
|
+
)
|
328
|
+
except BrokenPipeError:
|
329
|
+
self._abandon_status_check(
|
330
|
+
self._stop_status_lock,
|
331
|
+
self._stop_status_handle,
|
332
|
+
)
|
315
333
|
|
316
334
|
def check_internal_messages(self) -> None:
|
317
335
|
def _process_internal_messages(result: Result) -> None:
|
@@ -319,19 +337,20 @@ class RunStatusChecker:
|
|
319
337
|
for msg in internal_messages.messages.warning:
|
320
338
|
wandb.termwarn(msg)
|
321
339
|
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
self.
|
333
|
-
|
334
|
-
|
340
|
+
with wb_logging.log_to_run(self._run_id):
|
341
|
+
try:
|
342
|
+
self._loop_check_status(
|
343
|
+
lock=self._internal_messages_lock,
|
344
|
+
set_handle=lambda x: setattr(self, "_internal_messages_handle", x),
|
345
|
+
timeout=self._internal_messages_polling_interval,
|
346
|
+
request=self._interface.deliver_internal_messages,
|
347
|
+
process=_process_internal_messages,
|
348
|
+
)
|
349
|
+
except BrokenPipeError:
|
350
|
+
self._abandon_status_check(
|
351
|
+
self._internal_messages_lock,
|
352
|
+
self._internal_messages_handle,
|
353
|
+
)
|
335
354
|
|
336
355
|
def stop(self) -> None:
|
337
356
|
self._join_event.set()
|
@@ -355,100 +374,152 @@ class RunStatusChecker:
|
|
355
374
|
self._internal_messages_thread.join()
|
356
375
|
|
357
376
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
# This exception should be caught and fail the execution of the program.
|
388
|
-
cls._is_attaching = ""
|
389
|
-
raise e
|
390
|
-
cls._is_attaching = ""
|
377
|
+
_P = ParamSpec("_P")
|
378
|
+
_T = TypeVar("_T")
|
379
|
+
|
380
|
+
|
381
|
+
def _log_to_run(
|
382
|
+
func: Callable[Concatenate[Run, _P], _T],
|
383
|
+
) -> Callable[Concatenate[Run, _P], _T]:
|
384
|
+
"""Decorate a Run method to set the run ID in the logging context.
|
385
|
+
|
386
|
+
Any logs during the execution of the method go to the run's log file
|
387
|
+
and not to other runs' log files.
|
388
|
+
|
389
|
+
This is meant for use on all public methods and some callbacks. Private
|
390
|
+
methods can be assumed to be called from some public method somewhere.
|
391
|
+
The general rule is to use it on methods that can be called from a
|
392
|
+
context that isn't specific to this run (such as all user code or
|
393
|
+
internal methods that aren't run-specific).
|
394
|
+
"""
|
395
|
+
|
396
|
+
@functools.wraps(func)
|
397
|
+
def wrapper(self: Run, *args, **kwargs) -> _T:
|
398
|
+
# In "attach" usage, many properties of the Run are not initially
|
399
|
+
# populated.
|
400
|
+
if hasattr(self, "_settings"):
|
401
|
+
run_id = self._settings.run_id
|
402
|
+
else:
|
403
|
+
run_id = self._attach_id
|
404
|
+
|
405
|
+
with wb_logging.log_to_run(run_id):
|
391
406
|
return func(self, *args, **kwargs)
|
392
407
|
|
393
|
-
|
408
|
+
return wrapper
|
409
|
+
|
394
410
|
|
395
|
-
|
396
|
-
def _noop_on_finish(cls, message: str = "", only_warn: bool = False) -> Callable:
|
397
|
-
def decorator_fn(func: Callable) -> Callable:
|
398
|
-
@functools.wraps(func)
|
399
|
-
def wrapper_fn(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
400
|
-
if not getattr(self, "_is_finished", False):
|
401
|
-
return func(self, *args, **kwargs)
|
411
|
+
_is_attaching: str = ""
|
402
412
|
|
403
|
-
|
404
|
-
|
405
|
-
|
413
|
+
|
414
|
+
def _attach(
|
415
|
+
func: Callable[Concatenate[Run, _P], _T],
|
416
|
+
) -> Callable[Concatenate[Run, _P], _T]:
|
417
|
+
"""Decorate a Run method to auto-attach when in a new process.
|
418
|
+
|
419
|
+
When in a forked process or using a pickled Run instance, this automatically
|
420
|
+
connects to the service process to "attach" to the existing run.
|
421
|
+
"""
|
422
|
+
|
423
|
+
@functools.wraps(func)
|
424
|
+
def wrapper(self: Run, *args, **kwargs) -> _T:
|
425
|
+
global _is_attaching
|
426
|
+
|
427
|
+
# The _attach_id attribute is only None when running in the "disable
|
428
|
+
# service" mode.
|
429
|
+
#
|
430
|
+
# Since it is set early in `__init__` and included in the run's pickled
|
431
|
+
# state, the attribute always exists.
|
432
|
+
is_using_service = self._attach_id is not None
|
433
|
+
|
434
|
+
# The _attach_pid attribute is not pickled, so it might not exist.
|
435
|
+
# It is set when the run is initialized.
|
436
|
+
attach_pid = getattr(self, "_attach_pid", None)
|
437
|
+
|
438
|
+
if is_using_service and attach_pid != os.getpid():
|
439
|
+
if _is_attaching:
|
440
|
+
raise RuntimeError(
|
441
|
+
f"Trying to attach `{func.__name__}`"
|
442
|
+
f" while in the middle of attaching `{_is_attaching}`"
|
406
443
|
)
|
407
|
-
resolved_message = message or default_message
|
408
|
-
if only_warn:
|
409
|
-
warnings.warn(resolved_message, UserWarning, stacklevel=2)
|
410
|
-
else:
|
411
|
-
raise UsageError(resolved_message)
|
412
|
-
|
413
|
-
return wrapper_fn
|
414
|
-
|
415
|
-
return decorator_fn
|
416
|
-
|
417
|
-
@classmethod
|
418
|
-
def _noop(cls, func: Callable) -> Callable:
|
419
|
-
@functools.wraps(func)
|
420
|
-
def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
421
|
-
# `_attach_id` is only assigned in service hence for all service cases
|
422
|
-
# it will be a passthrough. We don't pickle non-service so again a way
|
423
|
-
# to see that we are in non-service case
|
424
|
-
if getattr(self, "_attach_id", None) is None:
|
425
|
-
# `_init_pid` is only assigned in __init__ (this will be constant check for mp):
|
426
|
-
# - for non-fork case the object is shared through pickling,
|
427
|
-
# and we don't pickle non-service so will be None
|
428
|
-
# - for fork case the new process share mem space hence the value would be of parent process.
|
429
|
-
_init_pid = getattr(self, "_init_pid", None)
|
430
|
-
if _init_pid != os.getpid():
|
431
|
-
message = (
|
432
|
-
f"`{func.__name__}` ignored (called from pid={os.getpid()}, "
|
433
|
-
f"`init` called from pid={_init_pid}). "
|
434
|
-
f"See: {url_registry.url('multiprocess')}"
|
435
|
-
)
|
436
|
-
# - if this process was pickled in non-service case,
|
437
|
-
# we ignore the attributes (since pickle is not supported)
|
438
|
-
# - for fork case will use the settings of the parent process
|
439
|
-
# - only point of inconsistent behavior from forked and non-forked cases
|
440
|
-
settings = getattr(self, "_settings", None)
|
441
|
-
if settings and settings.strict:
|
442
|
-
wandb.termerror(message, repeat=False)
|
443
|
-
raise UnsupportedError(
|
444
|
-
f"`{func.__name__}` does not support multiprocessing"
|
445
|
-
)
|
446
|
-
wandb.termwarn(message, repeat=False)
|
447
|
-
return cls.Dummy()
|
448
444
|
|
445
|
+
_is_attaching = func.__name__
|
446
|
+
try:
|
447
|
+
wandb._attach(run=self) # type: ignore
|
448
|
+
finally:
|
449
|
+
_is_attaching = ""
|
450
|
+
|
451
|
+
return func(self, *args, **kwargs)
|
452
|
+
|
453
|
+
return wrapper
|
454
|
+
|
455
|
+
|
456
|
+
def _raise_if_finished(
|
457
|
+
func: Callable[Concatenate[Run, _P], _T],
|
458
|
+
) -> Callable[Concatenate[Run, _P], _T]:
|
459
|
+
"""Decorate a Run method to raise an error after the run is finished."""
|
460
|
+
|
461
|
+
@functools.wraps(func)
|
462
|
+
def wrapper_fn(self: Run, *args, **kwargs) -> _T:
|
463
|
+
if not getattr(self, "_is_finished", False):
|
449
464
|
return func(self, *args, **kwargs)
|
450
465
|
|
451
|
-
|
466
|
+
message = (
|
467
|
+
f"Run ({self.id}) is finished. The call to"
|
468
|
+
f" `{func.__name__}` will be ignored."
|
469
|
+
f" Please make sure that you are using an active run."
|
470
|
+
)
|
471
|
+
|
472
|
+
raise UsageError(message)
|
473
|
+
|
474
|
+
return wrapper_fn
|
475
|
+
|
476
|
+
|
477
|
+
def _noop_if_forked_with_no_service(
|
478
|
+
func: Callable[Concatenate[Run, _P], None],
|
479
|
+
) -> Callable[Concatenate[Run, _P], None]:
|
480
|
+
"""Do nothing if called in a forked process and service is disabled.
|
481
|
+
|
482
|
+
Disabling the service is a very old and barely supported setting.
|
483
|
+
"""
|
484
|
+
|
485
|
+
@functools.wraps(func)
|
486
|
+
def wrapper(self: Run, *args, **kwargs) -> None:
|
487
|
+
# The _attach_id attribute is only None when running in the "disable
|
488
|
+
# service" mode.
|
489
|
+
#
|
490
|
+
# Since it is set early in `__init__` and included in the run's pickled
|
491
|
+
# state, the attribute always exists.
|
492
|
+
is_using_service = self._attach_id is not None
|
493
|
+
|
494
|
+
# This is the PID in which the Run object was constructed. The attribute
|
495
|
+
# always exists because it is set early in `__init__` and is included
|
496
|
+
# in the pickled state in `__getstate__` and `__setstate__`.
|
497
|
+
#
|
498
|
+
# It is not equal to the current PID if the process was forked or if
|
499
|
+
# the Run object was pickled and sent to another process.
|
500
|
+
init_pid = self._init_pid
|
501
|
+
|
502
|
+
if is_using_service or init_pid == os.getpid():
|
503
|
+
return func(self, *args, **kwargs)
|
504
|
+
|
505
|
+
message = (
|
506
|
+
f"`{func.__name__}` ignored (called from pid={os.getpid()},"
|
507
|
+
f" `init` called from pid={init_pid})."
|
508
|
+
f" See: {url_registry.url('multiprocess')}"
|
509
|
+
)
|
510
|
+
|
511
|
+
# This attribute may not exist because it is not included in the run's
|
512
|
+
# pickled state.
|
513
|
+
settings = getattr(self, "_settings", None)
|
514
|
+
if settings and settings.strict:
|
515
|
+
wandb.termerror(message, repeat=False)
|
516
|
+
raise UnsupportedError(
|
517
|
+
f"`{func.__name__}` does not support multiprocessing"
|
518
|
+
)
|
519
|
+
wandb.termwarn(message, repeat=False)
|
520
|
+
return None
|
521
|
+
|
522
|
+
return wrapper
|
452
523
|
|
453
524
|
|
454
525
|
@dataclass
|
@@ -572,6 +643,7 @@ class Run:
|
|
572
643
|
) -> None:
|
573
644
|
# pid is set, so we know if this run object was initialized by this process
|
574
645
|
self._init_pid = os.getpid()
|
646
|
+
self._attach_id = None
|
575
647
|
|
576
648
|
if settings._noop:
|
577
649
|
# TODO: properly handle setting for disabled mode
|
@@ -613,14 +685,13 @@ class Run:
|
|
613
685
|
|
614
686
|
self._step = 0
|
615
687
|
self._starting_step = 0
|
688
|
+
self._start_runtime = 0
|
616
689
|
# TODO: eventually would be nice to make this configurable using self._settings._start_time
|
617
690
|
# need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
|
618
691
|
# does the total time get calculated right (not as 2 days and 15 minutes)?
|
619
692
|
self._start_time = time.time()
|
620
693
|
|
621
|
-
|
622
|
-
|
623
|
-
self._printer = printer.new_printer()
|
694
|
+
self._printer = printer.new_printer(settings)
|
624
695
|
|
625
696
|
self._torch_history: wandb_torch.TorchHistory | None = None # type: ignore
|
626
697
|
|
@@ -742,8 +813,9 @@ class Run:
|
|
742
813
|
self._unique_launch_artifact_sequence_names[sequence_name] = item
|
743
814
|
|
744
815
|
def _telemetry_callback(self, telem_obj: telemetry.TelemetryRecord) -> None:
|
745
|
-
if not hasattr(self, "_telemetry_obj"):
|
816
|
+
if not hasattr(self, "_telemetry_obj") or self._is_finished:
|
746
817
|
return
|
818
|
+
|
747
819
|
self._telemetry_obj.MergeFrom(telem_obj)
|
748
820
|
self._telemetry_obj_dirty = True
|
749
821
|
self._telemetry_flush()
|
@@ -811,42 +883,46 @@ class Run:
|
|
811
883
|
return self._torch_history
|
812
884
|
|
813
885
|
@property
|
814
|
-
@
|
886
|
+
@_log_to_run
|
887
|
+
@_attach
|
815
888
|
def settings(self) -> Settings:
|
816
889
|
"""A frozen copy of run's Settings object."""
|
817
890
|
return self._settings.model_copy(deep=True)
|
818
891
|
|
819
892
|
@property
|
820
|
-
@
|
893
|
+
@_log_to_run
|
894
|
+
@_attach
|
821
895
|
def dir(self) -> str:
|
822
896
|
"""The directory where files associated with the run are saved."""
|
823
897
|
return self._settings.files_dir
|
824
898
|
|
825
899
|
@property
|
826
|
-
@
|
900
|
+
@_log_to_run
|
901
|
+
@_attach
|
827
902
|
def config(self) -> wandb_config.Config:
|
828
903
|
"""Config object associated with this run."""
|
829
904
|
return self._config
|
830
905
|
|
831
906
|
@property
|
832
|
-
@
|
907
|
+
@_log_to_run
|
908
|
+
@_attach
|
833
909
|
def config_static(self) -> wandb_config.ConfigStatic:
|
834
910
|
return wandb_config.ConfigStatic(self._config)
|
835
911
|
|
836
912
|
@property
|
837
|
-
@
|
913
|
+
@_log_to_run
|
914
|
+
@_attach
|
838
915
|
def name(self) -> str | None:
|
839
916
|
"""Display name of the run.
|
840
917
|
|
841
918
|
Display names are not guaranteed to be unique and may be descriptive.
|
842
919
|
By default, they are randomly generated.
|
843
920
|
"""
|
844
|
-
|
845
|
-
return self._settings.run_name
|
846
|
-
return None
|
921
|
+
return self._settings.run_name
|
847
922
|
|
848
923
|
@name.setter
|
849
|
-
@
|
924
|
+
@_log_to_run
|
925
|
+
@_raise_if_finished
|
850
926
|
def name(self, name: str) -> None:
|
851
927
|
with telemetry.context(run=self) as tel:
|
852
928
|
tel.feature.set_run_name = True
|
@@ -855,30 +931,34 @@ class Run:
|
|
855
931
|
self._backend.interface.publish_run(self)
|
856
932
|
|
857
933
|
@property
|
858
|
-
@
|
934
|
+
@_log_to_run
|
935
|
+
@_attach
|
859
936
|
def notes(self) -> str | None:
|
860
937
|
"""Notes associated with the run, if there are any.
|
861
938
|
|
862
|
-
Notes can be a multiline string and can also use markdown and latex
|
863
|
-
inside `$$`, like `$x + 3$`.
|
939
|
+
Notes can be a multiline string and can also use markdown and latex
|
940
|
+
equations inside `$$`, like `$x + 3$`.
|
864
941
|
"""
|
865
942
|
return self._settings.run_notes
|
866
943
|
|
867
944
|
@notes.setter
|
868
|
-
@
|
945
|
+
@_log_to_run
|
946
|
+
@_raise_if_finished
|
869
947
|
def notes(self, notes: str) -> None:
|
870
948
|
self._settings.run_notes = notes
|
871
949
|
if self._backend and self._backend.interface:
|
872
950
|
self._backend.interface.publish_run(self)
|
873
951
|
|
874
952
|
@property
|
875
|
-
@
|
953
|
+
@_log_to_run
|
954
|
+
@_attach
|
876
955
|
def tags(self) -> tuple | None:
|
877
956
|
"""Tags associated with the run, if there are any."""
|
878
957
|
return self._settings.run_tags or ()
|
879
958
|
|
880
959
|
@tags.setter
|
881
|
-
@
|
960
|
+
@_log_to_run
|
961
|
+
@_raise_if_finished
|
882
962
|
def tags(self, tags: Sequence) -> None:
|
883
963
|
with telemetry.context(run=self) as tel:
|
884
964
|
tel.feature.set_run_tags = True
|
@@ -887,17 +967,18 @@ class Run:
|
|
887
967
|
self._backend.interface.publish_run(self)
|
888
968
|
|
889
969
|
@property
|
890
|
-
@
|
970
|
+
@_log_to_run
|
971
|
+
@_attach
|
891
972
|
def id(self) -> str:
|
892
973
|
"""Identifier for this run."""
|
893
|
-
|
894
|
-
assert self._settings.run_id is not None
|
974
|
+
assert self._settings.run_id is not None
|
895
975
|
return self._settings.run_id
|
896
976
|
|
897
977
|
@property
|
898
|
-
@
|
978
|
+
@_log_to_run
|
979
|
+
@_attach
|
899
980
|
def sweep_id(self) -> str | None:
|
900
|
-
"""
|
981
|
+
"""Identifier for the sweep associated with the run, if there is one."""
|
901
982
|
return self._settings.sweep_id
|
902
983
|
|
903
984
|
def _get_path(self) -> str:
|
@@ -912,7 +993,8 @@ class Run:
|
|
912
993
|
)
|
913
994
|
|
914
995
|
@property
|
915
|
-
@
|
996
|
+
@_log_to_run
|
997
|
+
@_attach
|
916
998
|
def path(self) -> str:
|
917
999
|
"""Path to the run.
|
918
1000
|
|
@@ -922,25 +1004,29 @@ class Run:
|
|
922
1004
|
return self._get_path()
|
923
1005
|
|
924
1006
|
@property
|
925
|
-
@
|
1007
|
+
@_log_to_run
|
1008
|
+
@_attach
|
926
1009
|
def start_time(self) -> float:
|
927
1010
|
"""Unix timestamp (in seconds) of when the run started."""
|
928
1011
|
return self._start_time
|
929
1012
|
|
930
1013
|
@property
|
931
|
-
@
|
1014
|
+
@_log_to_run
|
1015
|
+
@_attach
|
932
1016
|
def starting_step(self) -> int:
|
933
1017
|
"""The first step of the run."""
|
934
1018
|
return self._starting_step
|
935
1019
|
|
936
1020
|
@property
|
937
|
-
@
|
1021
|
+
@_log_to_run
|
1022
|
+
@_attach
|
938
1023
|
def resumed(self) -> bool:
|
939
1024
|
"""True if the run was resumed, False otherwise."""
|
940
1025
|
return self._settings.resumed
|
941
1026
|
|
942
1027
|
@property
|
943
|
-
@
|
1028
|
+
@_log_to_run
|
1029
|
+
@_attach
|
944
1030
|
def step(self) -> int:
|
945
1031
|
"""Current value of the step.
|
946
1032
|
|
@@ -949,31 +1035,35 @@ class Run:
|
|
949
1035
|
return self._step
|
950
1036
|
|
951
1037
|
@property
|
952
|
-
@
|
1038
|
+
@_log_to_run
|
1039
|
+
@_attach
|
953
1040
|
def mode(self) -> str:
|
954
1041
|
"""For compatibility with `0.9.x` and earlier, deprecate eventually."""
|
955
|
-
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
|
1042
|
+
deprecate.deprecate(
|
1043
|
+
field_name=Deprecated.run__mode,
|
1044
|
+
warning_message=(
|
1045
|
+
"The mode property of wandb.run is deprecated "
|
1046
|
+
"and will be removed in a future release."
|
1047
|
+
),
|
1048
|
+
run=self,
|
1049
|
+
)
|
963
1050
|
return "dryrun" if self._settings._offline else "run"
|
964
1051
|
|
965
1052
|
@property
|
966
|
-
@
|
1053
|
+
@_log_to_run
|
1054
|
+
@_attach
|
967
1055
|
def offline(self) -> bool:
|
968
1056
|
return self._settings._offline
|
969
1057
|
|
970
1058
|
@property
|
971
|
-
@
|
1059
|
+
@_log_to_run
|
1060
|
+
@_attach
|
972
1061
|
def disabled(self) -> bool:
|
973
1062
|
return self._settings._noop
|
974
1063
|
|
975
1064
|
@property
|
976
|
-
@
|
1065
|
+
@_log_to_run
|
1066
|
+
@_attach
|
977
1067
|
def group(self) -> str:
|
978
1068
|
"""Name of the group associated with the run.
|
979
1069
|
|
@@ -987,22 +1077,68 @@ class Run:
|
|
987
1077
|
return self._settings.run_group or ""
|
988
1078
|
|
989
1079
|
@property
|
990
|
-
@
|
1080
|
+
@_log_to_run
|
1081
|
+
@_attach
|
991
1082
|
def job_type(self) -> str:
|
992
1083
|
return self._settings.run_job_type or ""
|
993
1084
|
|
994
1085
|
def project_name(self) -> str:
|
995
|
-
|
996
|
-
|
1086
|
+
"""Name of the W&B project associated with the run.
|
1087
|
+
|
1088
|
+
Note: this method is deprecated and will be removed in a future release.
|
1089
|
+
Please use `run.project` instead.
|
1090
|
+
"""
|
1091
|
+
deprecate.deprecate(
|
1092
|
+
field_name=Deprecated.run__project_name,
|
1093
|
+
warning_message=(
|
1094
|
+
"The project_name method is deprecated and will be removed in a"
|
1095
|
+
" future release. Please use `run.project` instead."
|
1096
|
+
),
|
1097
|
+
)
|
1098
|
+
return self.project
|
997
1099
|
|
998
1100
|
@property
|
999
|
-
@
|
1101
|
+
@_log_to_run
|
1102
|
+
@_attach
|
1000
1103
|
def project(self) -> str:
|
1001
1104
|
"""Name of the W&B project associated with the run."""
|
1002
|
-
|
1105
|
+
assert self._settings.project is not None
|
1106
|
+
return self._settings.project
|
1107
|
+
|
1108
|
+
@_log_to_run
|
1109
|
+
def get_project_url(self) -> str | None:
|
1110
|
+
"""URL of the W&B project associated with the run, if there is one.
|
1003
1111
|
|
1004
|
-
|
1005
|
-
|
1112
|
+
Offline runs do not have a project URL.
|
1113
|
+
|
1114
|
+
Note: this method is deprecated and will be removed in a future release.
|
1115
|
+
Please use `run.project_url` instead.
|
1116
|
+
"""
|
1117
|
+
deprecate.deprecate(
|
1118
|
+
field_name=Deprecated.run__get_project_url,
|
1119
|
+
warning_message=(
|
1120
|
+
"The get_project_url method is deprecated and will be removed in a"
|
1121
|
+
" future release. Please use `run.project_url` instead."
|
1122
|
+
),
|
1123
|
+
)
|
1124
|
+
return self.project_url
|
1125
|
+
|
1126
|
+
@property
|
1127
|
+
@_log_to_run
|
1128
|
+
@_attach
|
1129
|
+
def project_url(self) -> str | None:
|
1130
|
+
"""URL of the W&B project associated with the run, if there is one.
|
1131
|
+
|
1132
|
+
Offline runs do not have a project URL.
|
1133
|
+
"""
|
1134
|
+
if self._settings._offline:
|
1135
|
+
wandb.termwarn("URL not available in offline run")
|
1136
|
+
return None
|
1137
|
+
return self._settings.project_url
|
1138
|
+
|
1139
|
+
@_raise_if_finished
|
1140
|
+
@_log_to_run
|
1141
|
+
@_attach
|
1006
1142
|
def log_code(
|
1007
1143
|
self,
|
1008
1144
|
root: str | None = ".",
|
@@ -1088,25 +1224,59 @@ class Run:
|
|
1088
1224
|
|
1089
1225
|
return self._log_artifact(art)
|
1090
1226
|
|
1091
|
-
|
1092
|
-
|
1227
|
+
@_log_to_run
|
1228
|
+
def get_sweep_url(self) -> str | None:
|
1229
|
+
"""The URL of the sweep associated with the run, if there is one.
|
1093
1230
|
|
1094
|
-
Offline runs
|
1231
|
+
Offline runs do not have a sweep URL.
|
1232
|
+
|
1233
|
+
Note: this method is deprecated and will be removed in a future release.
|
1234
|
+
Please use `run.sweep_url` instead.
|
1095
1235
|
"""
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1236
|
+
deprecate.deprecate(
|
1237
|
+
field_name=Deprecated.run__get_sweep_url,
|
1238
|
+
warning_message=(
|
1239
|
+
"The get_sweep_url method is deprecated and will be removed in a"
|
1240
|
+
" future release. Please use `run.sweep_url` instead."
|
1241
|
+
),
|
1242
|
+
)
|
1243
|
+
return self.sweep_url
|
1100
1244
|
|
1101
|
-
|
1102
|
-
|
1245
|
+
@property
|
1246
|
+
@_attach
|
1247
|
+
def sweep_url(self) -> str | None:
|
1248
|
+
"""URL of the sweep associated with the run, if there is one.
|
1249
|
+
|
1250
|
+
Offline runs do not have a sweep URL.
|
1251
|
+
"""
|
1103
1252
|
if self._settings._offline:
|
1104
1253
|
wandb.termwarn("URL not available in offline run")
|
1105
1254
|
return None
|
1106
1255
|
return self._settings.sweep_url
|
1107
1256
|
|
1257
|
+
@_log_to_run
|
1108
1258
|
def get_url(self) -> str | None:
|
1109
|
-
"""
|
1259
|
+
"""URL of the W&B run, if there is one.
|
1260
|
+
|
1261
|
+
Offline runs do not have a URL.
|
1262
|
+
|
1263
|
+
Note: this method is deprecated and will be removed in a future release.
|
1264
|
+
Please use `run.url` instead.
|
1265
|
+
"""
|
1266
|
+
deprecate.deprecate(
|
1267
|
+
field_name=Deprecated.run__get_url,
|
1268
|
+
warning_message=(
|
1269
|
+
"The get_url method is deprecated and will be removed in a"
|
1270
|
+
" future release. Please use `run.url` instead."
|
1271
|
+
),
|
1272
|
+
)
|
1273
|
+
return self.url
|
1274
|
+
|
1275
|
+
@property
|
1276
|
+
@_log_to_run
|
1277
|
+
@_attach
|
1278
|
+
def url(self) -> str | None:
|
1279
|
+
"""The url for the W&B run, if there is one.
|
1110
1280
|
|
1111
1281
|
Offline runs will not have a url.
|
1112
1282
|
"""
|
@@ -1116,13 +1286,8 @@ class Run:
|
|
1116
1286
|
return self._settings.run_url
|
1117
1287
|
|
1118
1288
|
@property
|
1119
|
-
@
|
1120
|
-
|
1121
|
-
"""The W&B url associated with the run."""
|
1122
|
-
return self.get_url()
|
1123
|
-
|
1124
|
-
@property
|
1125
|
-
@_run_decorator._attach
|
1289
|
+
@_log_to_run
|
1290
|
+
@_attach
|
1126
1291
|
def entity(self) -> str:
|
1127
1292
|
"""The name of the W&B entity associated with the run.
|
1128
1293
|
|
@@ -1216,7 +1381,8 @@ class Run:
|
|
1216
1381
|
if lines:
|
1217
1382
|
self._label_probe_lines(lines)
|
1218
1383
|
|
1219
|
-
@
|
1384
|
+
@_log_to_run
|
1385
|
+
@_attach
|
1220
1386
|
def display(self, height: int = 420, hidden: bool = False) -> bool:
|
1221
1387
|
"""Display this run in jupyter."""
|
1222
1388
|
if self._settings.silent:
|
@@ -1235,7 +1401,8 @@ class Run:
|
|
1235
1401
|
wandb.termwarn(".display() only works in jupyter environments")
|
1236
1402
|
return False
|
1237
1403
|
|
1238
|
-
@
|
1404
|
+
@_log_to_run
|
1405
|
+
@_attach
|
1239
1406
|
def to_html(self, height: int = 420, hidden: bool = False) -> str:
|
1240
1407
|
"""Generate HTML containing an iframe displaying the current run."""
|
1241
1408
|
url = self._settings.run_url + "?jupyter=true"
|
@@ -1251,7 +1418,8 @@ class Run:
|
|
1251
1418
|
) -> dict[str, str]:
|
1252
1419
|
return {"text/html": self.to_html(hidden=True)}
|
1253
1420
|
|
1254
|
-
@
|
1421
|
+
@_log_to_run
|
1422
|
+
@_raise_if_finished
|
1255
1423
|
def _config_callback(
|
1256
1424
|
self,
|
1257
1425
|
key: tuple[str, ...] | str | None = None,
|
@@ -1262,6 +1430,7 @@ class Run:
|
|
1262
1430
|
if self._backend and self._backend.interface:
|
1263
1431
|
self._backend.interface.publish_config(key=key, val=val, data=data)
|
1264
1432
|
|
1433
|
+
@_log_to_run
|
1265
1434
|
def _config_artifact_callback(
|
1266
1435
|
self, key: str, val: str | Artifact | dict
|
1267
1436
|
) -> Artifact:
|
@@ -1272,6 +1441,8 @@ class Run:
|
|
1272
1441
|
assert isinstance(val, dict)
|
1273
1442
|
public_api = self._public_api()
|
1274
1443
|
artifact = Artifact._from_id(val["id"], public_api.client)
|
1444
|
+
|
1445
|
+
assert artifact
|
1275
1446
|
return self.use_artifact(artifact, use_as=key)
|
1276
1447
|
elif _is_artifact_string(val):
|
1277
1448
|
# this will never fail, but is required to make mypy happy
|
@@ -1290,6 +1461,7 @@ class Run:
|
|
1290
1461
|
# in the future we'll need to support using artifacts from
|
1291
1462
|
# different instances of wandb.
|
1292
1463
|
|
1464
|
+
assert artifact
|
1293
1465
|
return self.use_artifact(artifact, use_as=key)
|
1294
1466
|
elif _is_artifact_object(val):
|
1295
1467
|
return self.use_artifact(val, use_as=key)
|
@@ -1301,13 +1473,15 @@ class Run:
|
|
1301
1473
|
def _set_config_wandb(self, key: str, val: Any) -> None:
|
1302
1474
|
self._config_callback(key=("_wandb", key), val=val)
|
1303
1475
|
|
1304
|
-
@
|
1476
|
+
@_log_to_run
|
1477
|
+
@_raise_if_finished
|
1305
1478
|
def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
|
1306
1479
|
with telemetry.context(run=self) as tel:
|
1307
1480
|
tel.feature.set_summary = True
|
1308
1481
|
if self._backend and self._backend.interface:
|
1309
1482
|
self._backend.interface.publish_summary(self, summary_record)
|
1310
1483
|
|
1484
|
+
@_log_to_run
|
1311
1485
|
def _summary_get_current_summary_callback(self) -> dict[str, Any]:
|
1312
1486
|
if self._is_finished:
|
1313
1487
|
# TODO: WB-18420: fetch summary from backend and stage it before run is finished
|
@@ -1325,11 +1499,22 @@ class Run:
|
|
1325
1499
|
get_summary_response = result.response.get_summary_response
|
1326
1500
|
return proto_util.dict_from_proto_list(get_summary_response.item)
|
1327
1501
|
|
1502
|
+
@_log_to_run
|
1328
1503
|
def _metric_callback(self, metric_record: MetricRecord) -> None:
|
1329
1504
|
if self._backend and self._backend.interface:
|
1330
1505
|
self._backend.interface._publish_metric(metric_record)
|
1331
1506
|
|
1332
|
-
|
1507
|
+
@_log_to_run
|
1508
|
+
def _publish_file(self, fname: str) -> None:
|
1509
|
+
"""Mark a run file to be uploaded with the run.
|
1510
|
+
|
1511
|
+
This is a W&B-internal function: it can be used by other internal
|
1512
|
+
wandb code.
|
1513
|
+
|
1514
|
+
Args:
|
1515
|
+
fname: The path to the file in the run's files directory, relative
|
1516
|
+
to the run's files directory.
|
1517
|
+
"""
|
1333
1518
|
if not self._backend or not self._backend.interface:
|
1334
1519
|
return
|
1335
1520
|
files: FilesDict = dict(files=[(GlobStr(fname), "now")])
|
@@ -1399,6 +1584,7 @@ class Run:
|
|
1399
1584
|
|
1400
1585
|
return data
|
1401
1586
|
|
1587
|
+
@_log_to_run
|
1402
1588
|
def _partial_history_callback(
|
1403
1589
|
self,
|
1404
1590
|
data: dict[str, Any],
|
@@ -1423,12 +1609,14 @@ class Run:
|
|
1423
1609
|
publish_step=not_using_tensorboard,
|
1424
1610
|
)
|
1425
1611
|
|
1612
|
+
@_log_to_run
|
1426
1613
|
def _console_callback(self, name: str, data: str) -> None:
|
1427
1614
|
# logger.info("console callback: %s, %s", name, data)
|
1428
1615
|
if self._backend and self._backend.interface:
|
1429
1616
|
self._backend.interface.publish_output(name, data)
|
1430
1617
|
|
1431
|
-
@
|
1618
|
+
@_log_to_run
|
1619
|
+
@_raise_if_finished
|
1432
1620
|
def _console_raw_callback(self, name: str, data: str) -> None:
|
1433
1621
|
# logger.info("console callback: %s, %s", name, data)
|
1434
1622
|
|
@@ -1443,6 +1631,7 @@ class Run:
|
|
1443
1631
|
if self._backend and self._backend.interface:
|
1444
1632
|
self._backend.interface.publish_output_raw(name, data)
|
1445
1633
|
|
1634
|
+
@_log_to_run
|
1446
1635
|
def _tensorboard_callback(
|
1447
1636
|
self, logdir: str, save: bool = True, root_logdir: str = ""
|
1448
1637
|
) -> None:
|
@@ -1473,12 +1662,12 @@ class Run:
|
|
1473
1662
|
if run_obj.start_time:
|
1474
1663
|
self._start_time = run_obj.start_time.ToMicroseconds() / 1e6
|
1475
1664
|
|
1665
|
+
if run_obj.runtime:
|
1666
|
+
self._start_runtime = run_obj.runtime
|
1667
|
+
|
1476
1668
|
# Grab the config from resuming
|
1477
1669
|
if run_obj.config:
|
1478
1670
|
c_dict = config_util.dict_no_value_from_proto_list(run_obj.config.update)
|
1479
|
-
# TODO: Windows throws a wild error when this is set...
|
1480
|
-
if "_wandb" in c_dict:
|
1481
|
-
del c_dict["_wandb"]
|
1482
1671
|
# We update the config object here without triggering the callback
|
1483
1672
|
self._config._update(c_dict, allow_val_change=True, ignore_locked=True)
|
1484
1673
|
# Update the summary, this will trigger an un-needed graphql request :(
|
@@ -1610,9 +1799,10 @@ class Run:
|
|
1610
1799
|
if (step is None and commit is None) or commit:
|
1611
1800
|
self._step += 1
|
1612
1801
|
|
1613
|
-
@
|
1614
|
-
@
|
1615
|
-
@
|
1802
|
+
@_log_to_run
|
1803
|
+
@_noop_if_forked_with_no_service
|
1804
|
+
@_raise_if_finished
|
1805
|
+
@_attach
|
1616
1806
|
def log(
|
1617
1807
|
self,
|
1618
1808
|
data: dict[str, Any],
|
@@ -1858,10 +2048,11 @@ class Run:
|
|
1858
2048
|
|
1859
2049
|
if sync is not None:
|
1860
2050
|
deprecate.deprecate(
|
1861
|
-
field_name=
|
2051
|
+
field_name=Deprecated.run__log_sync,
|
1862
2052
|
warning_message=(
|
1863
2053
|
"`sync` argument is deprecated and does not affect the behaviour of `wandb.log`"
|
1864
2054
|
),
|
2055
|
+
run=self,
|
1865
2056
|
)
|
1866
2057
|
if self._settings._shared and step is not None:
|
1867
2058
|
wandb.termwarn(
|
@@ -1872,8 +2063,9 @@ class Run:
|
|
1872
2063
|
)
|
1873
2064
|
self._log(data=data, step=step, commit=commit)
|
1874
2065
|
|
1875
|
-
@
|
1876
|
-
@
|
2066
|
+
@_log_to_run
|
2067
|
+
@_raise_if_finished
|
2068
|
+
@_attach
|
1877
2069
|
def save(
|
1878
2070
|
self,
|
1879
2071
|
glob_str: str | os.PathLike | None = None,
|
@@ -1930,11 +2122,12 @@ class Run:
|
|
1930
2122
|
if glob_str is None:
|
1931
2123
|
# noop for historical reasons, run.save() may be called in legacy code
|
1932
2124
|
deprecate.deprecate(
|
1933
|
-
field_name=
|
2125
|
+
field_name=Deprecated.run__save_no_args,
|
1934
2126
|
warning_message=(
|
1935
2127
|
"Calling wandb.run.save without any arguments is deprecated."
|
1936
2128
|
"Changes to attributes are automatically persisted."
|
1937
2129
|
),
|
2130
|
+
run=self,
|
1938
2131
|
)
|
1939
2132
|
return True
|
1940
2133
|
|
@@ -2057,7 +2250,8 @@ class Run:
|
|
2057
2250
|
|
2058
2251
|
return [str(f) for f in globbed_files]
|
2059
2252
|
|
2060
|
-
@
|
2253
|
+
@_log_to_run
|
2254
|
+
@_attach
|
2061
2255
|
def restore(
|
2062
2256
|
self,
|
2063
2257
|
name: str,
|
@@ -2072,8 +2266,9 @@ class Run:
|
|
2072
2266
|
root or self._settings.files_dir,
|
2073
2267
|
)
|
2074
2268
|
|
2075
|
-
@
|
2076
|
-
@
|
2269
|
+
@_log_to_run
|
2270
|
+
@_noop_if_forked_with_no_service
|
2271
|
+
@_attach
|
2077
2272
|
def finish(
|
2078
2273
|
self,
|
2079
2274
|
exit_code: int | None = None,
|
@@ -2097,18 +2292,25 @@ class Run:
|
|
2097
2292
|
"""
|
2098
2293
|
if quiet is not None:
|
2099
2294
|
deprecate.deprecate(
|
2100
|
-
field_name=
|
2295
|
+
field_name=Deprecated.run__finish_quiet,
|
2101
2296
|
warning_message=(
|
2102
2297
|
"The `quiet` argument to `wandb.run.finish()` is deprecated, "
|
2103
2298
|
"use `wandb.Settings(quiet=...)` to set this instead."
|
2104
2299
|
),
|
2300
|
+
run=self,
|
2105
2301
|
)
|
2106
2302
|
return self._finish(exit_code)
|
2107
2303
|
|
2304
|
+
@_log_to_run
|
2108
2305
|
def _finish(
|
2109
2306
|
self,
|
2110
2307
|
exit_code: int | None = None,
|
2111
2308
|
) -> None:
|
2309
|
+
if self._is_finished:
|
2310
|
+
return
|
2311
|
+
|
2312
|
+
assert self._wl
|
2313
|
+
|
2112
2314
|
logger.info(f"finishing run {self._get_path()}")
|
2113
2315
|
with telemetry.context(run=self) as tel:
|
2114
2316
|
tel.feature.finish = True
|
@@ -2122,6 +2324,7 @@ class Run:
|
|
2122
2324
|
# Early-stage hooks may use methods that require _is_finished
|
2123
2325
|
# to be False, so we set this after running those hooks.
|
2124
2326
|
self._is_finished = True
|
2327
|
+
self._wl.remove_active_run(self)
|
2125
2328
|
|
2126
2329
|
try:
|
2127
2330
|
self._atexit_cleanup(exit_code=exit_code)
|
@@ -2137,29 +2340,32 @@ class Run:
|
|
2137
2340
|
#
|
2138
2341
|
# TODO: Why not do this in _atexit_cleanup()?
|
2139
2342
|
if self._settings.run_id:
|
2140
|
-
assert self._wl
|
2141
2343
|
service = self._wl.assert_service()
|
2142
2344
|
service.inform_finish(run_id=self._settings.run_id)
|
2143
2345
|
|
2144
2346
|
finally:
|
2145
|
-
|
2347
|
+
if wandb.run is self:
|
2348
|
+
module.unset_globals()
|
2146
2349
|
wandb._sentry.end_session()
|
2147
2350
|
|
2148
|
-
@
|
2149
|
-
@
|
2351
|
+
@_log_to_run
|
2352
|
+
@_noop_if_forked_with_no_service
|
2353
|
+
@_attach
|
2150
2354
|
def join(self, exit_code: int | None = None) -> None:
|
2151
2355
|
"""Deprecated alias for `finish()` - use finish instead."""
|
2152
2356
|
if hasattr(self, "_telemetry_obj"):
|
2153
2357
|
deprecate.deprecate(
|
2154
|
-
field_name=
|
2358
|
+
field_name=Deprecated.run__join,
|
2155
2359
|
warning_message=(
|
2156
2360
|
"wandb.run.join() is deprecated, please use wandb.run.finish()."
|
2157
2361
|
),
|
2362
|
+
run=self,
|
2158
2363
|
)
|
2159
2364
|
self._finish(exit_code=exit_code)
|
2160
2365
|
|
2161
|
-
@
|
2162
|
-
@
|
2366
|
+
@_log_to_run
|
2367
|
+
@_raise_if_finished
|
2368
|
+
@_attach
|
2163
2369
|
def status(
|
2164
2370
|
self,
|
2165
2371
|
) -> RunStatus:
|
@@ -2191,25 +2397,6 @@ class Run:
|
|
2191
2397
|
}
|
2192
2398
|
self._config_callback(val=config, key=("_wandb", "visualize", visualize_key))
|
2193
2399
|
|
2194
|
-
def _set_globals(self) -> None:
|
2195
|
-
module.set_global(
|
2196
|
-
run=self,
|
2197
|
-
config=self.config,
|
2198
|
-
log=self.log,
|
2199
|
-
summary=self.summary,
|
2200
|
-
save=self.save,
|
2201
|
-
use_artifact=self.use_artifact,
|
2202
|
-
log_artifact=self.log_artifact,
|
2203
|
-
define_metric=self.define_metric,
|
2204
|
-
alert=self.alert,
|
2205
|
-
watch=self.watch,
|
2206
|
-
unwatch=self.unwatch,
|
2207
|
-
mark_preempting=self.mark_preempting,
|
2208
|
-
log_model=self.log_model,
|
2209
|
-
use_model=self.use_model,
|
2210
|
-
link_model=self.link_model,
|
2211
|
-
)
|
2212
|
-
|
2213
2400
|
def _redirect(
|
2214
2401
|
self,
|
2215
2402
|
stdout_slave_fd: int | None,
|
@@ -2248,6 +2435,7 @@ class Run:
|
|
2248
2435
|
lambda data: self._console_callback("stdout", data),
|
2249
2436
|
self._output_writer.write, # type: ignore
|
2250
2437
|
],
|
2438
|
+
flush_periodically=(self._settings.mode == "online"),
|
2251
2439
|
)
|
2252
2440
|
err_redir = redirect.Redirect(
|
2253
2441
|
src="stderr",
|
@@ -2255,6 +2443,7 @@ class Run:
|
|
2255
2443
|
lambda data: self._console_callback("stderr", data),
|
2256
2444
|
self._output_writer.write, # type: ignore
|
2257
2445
|
],
|
2446
|
+
flush_periodically=(self._settings.mode == "online"),
|
2258
2447
|
)
|
2259
2448
|
if os.name == "nt":
|
2260
2449
|
|
@@ -2280,6 +2469,7 @@ class Run:
|
|
2280
2469
|
lambda data: self._console_callback("stdout", data),
|
2281
2470
|
self._output_writer.write, # type: ignore
|
2282
2471
|
],
|
2472
|
+
flush_periodically=(self._settings.mode == "online"),
|
2283
2473
|
)
|
2284
2474
|
err_redir = redirect.StreamWrapper(
|
2285
2475
|
src="stderr",
|
@@ -2287,6 +2477,7 @@ class Run:
|
|
2287
2477
|
lambda data: self._console_callback("stderr", data),
|
2288
2478
|
self._output_writer.write, # type: ignore
|
2289
2479
|
],
|
2480
|
+
flush_periodically=(self._settings.mode == "online"),
|
2290
2481
|
)
|
2291
2482
|
elif console == "wrap_raw":
|
2292
2483
|
logger.info("Wrapping output streams.")
|
@@ -2389,13 +2580,7 @@ class Run:
|
|
2389
2580
|
self._output_writer = None
|
2390
2581
|
|
2391
2582
|
def _on_start(self) -> None:
|
2392
|
-
|
2393
|
-
# (we want to do the set globals after attach)
|
2394
|
-
# TODO(console) However _console_start calls Redirect that uses `wandb.run` hence breaks
|
2395
|
-
# TODO(jupyter) However _header calls _header_run_info that uses wandb.jupyter that uses
|
2396
|
-
# `wandb.run` and hence breaks
|
2397
|
-
self._set_globals()
|
2398
|
-
self._header(settings=self._settings, printer=self._printer)
|
2583
|
+
self._header()
|
2399
2584
|
|
2400
2585
|
if self._settings.save_code and self._settings.code_dir is not None:
|
2401
2586
|
self.log_code(self._settings.code_dir)
|
@@ -2410,7 +2595,9 @@ class Run:
|
|
2410
2595
|
self._backend.interface.publish_python_packages(working_set())
|
2411
2596
|
|
2412
2597
|
if self._backend and self._backend.interface and not self._settings._offline:
|
2598
|
+
assert self._settings.run_id
|
2413
2599
|
self._run_status_checker = RunStatusChecker(
|
2600
|
+
self._settings.run_id,
|
2414
2601
|
interface=self._backend.interface,
|
2415
2602
|
)
|
2416
2603
|
self._run_status_checker.start()
|
@@ -2423,7 +2610,6 @@ class Run:
|
|
2423
2610
|
with telemetry.context(run=self) as tel:
|
2424
2611
|
tel.feature.attach = True
|
2425
2612
|
|
2426
|
-
self._set_globals()
|
2427
2613
|
self._is_attached = True
|
2428
2614
|
self._on_ready()
|
2429
2615
|
|
@@ -2455,6 +2641,9 @@ class Run:
|
|
2455
2641
|
|
2456
2642
|
def _on_ready(self) -> None:
|
2457
2643
|
"""Event triggered when run is ready for the user."""
|
2644
|
+
assert self._wl
|
2645
|
+
self._wl.add_active_run(self)
|
2646
|
+
|
2458
2647
|
self._register_telemetry_import_hooks()
|
2459
2648
|
|
2460
2649
|
# start reporting any telemetry changes
|
@@ -2546,6 +2735,7 @@ class Run:
|
|
2546
2735
|
docker_image_name=docker_image_name,
|
2547
2736
|
)
|
2548
2737
|
|
2738
|
+
assert job_artifact
|
2549
2739
|
artifact = self.log_artifact(job_artifact)
|
2550
2740
|
|
2551
2741
|
if not artifact:
|
@@ -2657,8 +2847,9 @@ class Run:
|
|
2657
2847
|
for module_name in import_telemetry_set:
|
2658
2848
|
unregister_post_import_hook(module_name, run_id)
|
2659
2849
|
|
2660
|
-
@
|
2661
|
-
@
|
2850
|
+
@_log_to_run
|
2851
|
+
@_raise_if_finished
|
2852
|
+
@_attach
|
2662
2853
|
def define_metric(
|
2663
2854
|
self,
|
2664
2855
|
name: str,
|
@@ -2697,14 +2888,14 @@ class Run:
|
|
2697
2888
|
"""
|
2698
2889
|
if summary and "copy" in summary:
|
2699
2890
|
deprecate.deprecate(
|
2700
|
-
|
2891
|
+
Deprecated.run__define_metric_copy,
|
2701
2892
|
"define_metric(summary='copy') is deprecated and will be removed.",
|
2702
2893
|
self,
|
2703
2894
|
)
|
2704
2895
|
|
2705
2896
|
if (summary and "best" in summary) or goal is not None:
|
2706
2897
|
deprecate.deprecate(
|
2707
|
-
|
2898
|
+
Deprecated.run__define_metric_best_goal,
|
2708
2899
|
"define_metric(summary='best', goal=...) is deprecated and will be removed. "
|
2709
2900
|
"Use define_metric(summary='min') or define_metric(summary='max') instead.",
|
2710
2901
|
self,
|
@@ -2798,7 +2989,8 @@ class Run:
|
|
2798
2989
|
m._commit()
|
2799
2990
|
return m
|
2800
2991
|
|
2801
|
-
@
|
2992
|
+
@_log_to_run
|
2993
|
+
@_attach
|
2802
2994
|
def watch(
|
2803
2995
|
self,
|
2804
2996
|
models: torch.nn.Module | Sequence[torch.nn.Module],
|
@@ -2835,7 +3027,8 @@ class Run:
|
|
2835
3027
|
"""
|
2836
3028
|
wandb.sdk._watch(self, models, criterion, log, log_freq, idx, log_graph)
|
2837
3029
|
|
2838
|
-
@
|
3030
|
+
@_log_to_run
|
3031
|
+
@_attach
|
2839
3032
|
def unwatch(
|
2840
3033
|
self, models: torch.nn.Module | Sequence[torch.nn.Module] | None = None
|
2841
3034
|
) -> None:
|
@@ -2885,8 +3078,9 @@ class Run:
|
|
2885
3078
|
def _detach(self) -> None:
|
2886
3079
|
pass
|
2887
3080
|
|
2888
|
-
@
|
2889
|
-
@
|
3081
|
+
@_log_to_run
|
3082
|
+
@_raise_if_finished
|
3083
|
+
@_attach
|
2890
3084
|
def link_artifact(
|
2891
3085
|
self,
|
2892
3086
|
artifact: Artifact,
|
@@ -2950,8 +3144,9 @@ class Run:
|
|
2950
3144
|
if response.error_message:
|
2951
3145
|
wandb.termerror(response.error_message)
|
2952
3146
|
|
2953
|
-
@
|
2954
|
-
@
|
3147
|
+
@_log_to_run
|
3148
|
+
@_raise_if_finished
|
3149
|
+
@_attach
|
2955
3150
|
def use_artifact(
|
2956
3151
|
self,
|
2957
3152
|
artifact_or_name: str | Artifact,
|
@@ -3021,6 +3216,8 @@ class Run:
|
|
3021
3216
|
artifact.id,
|
3022
3217
|
entity_name=self._settings.entity,
|
3023
3218
|
project_name=self._settings.project,
|
3219
|
+
artifact_entity_name=artifact.entity,
|
3220
|
+
artifact_project_name=artifact.project,
|
3024
3221
|
use_as=use_as or artifact_or_name,
|
3025
3222
|
)
|
3026
3223
|
else:
|
@@ -3053,7 +3250,10 @@ class Run:
|
|
3053
3250
|
)
|
3054
3251
|
artifact._use_as = use_as or artifact.name
|
3055
3252
|
api.use_artifact(
|
3056
|
-
artifact.id,
|
3253
|
+
artifact.id,
|
3254
|
+
use_as=use_as or artifact._use_as or artifact.name,
|
3255
|
+
artifact_entity_name=artifact.entity,
|
3256
|
+
artifact_project_name=artifact.project,
|
3057
3257
|
)
|
3058
3258
|
else:
|
3059
3259
|
raise ValueError(
|
@@ -3064,8 +3264,9 @@ class Run:
|
|
3064
3264
|
self._backend.interface.publish_use_artifact(artifact)
|
3065
3265
|
return artifact
|
3066
3266
|
|
3067
|
-
@
|
3068
|
-
@
|
3267
|
+
@_log_to_run
|
3268
|
+
@_raise_if_finished
|
3269
|
+
@_attach
|
3069
3270
|
def log_artifact(
|
3070
3271
|
self,
|
3071
3272
|
artifact_or_path: Artifact | StrPath,
|
@@ -3106,8 +3307,9 @@ class Run:
|
|
3106
3307
|
tags=tags,
|
3107
3308
|
)
|
3108
3309
|
|
3109
|
-
@
|
3110
|
-
@
|
3310
|
+
@_log_to_run
|
3311
|
+
@_raise_if_finished
|
3312
|
+
@_attach
|
3111
3313
|
def upsert_artifact(
|
3112
3314
|
self,
|
3113
3315
|
artifact_or_path: Artifact | str,
|
@@ -3160,8 +3362,9 @@ class Run:
|
|
3160
3362
|
finalize=False,
|
3161
3363
|
)
|
3162
3364
|
|
3163
|
-
@
|
3164
|
-
@
|
3365
|
+
@_log_to_run
|
3366
|
+
@_raise_if_finished
|
3367
|
+
@_attach
|
3165
3368
|
def finish_artifact(
|
3166
3369
|
self,
|
3167
3370
|
artifact_or_path: Artifact | str,
|
@@ -3244,6 +3447,12 @@ class Run:
|
|
3244
3447
|
artifact, aliases = self._prepare_artifact(
|
3245
3448
|
artifact_or_path, name, type, aliases
|
3246
3449
|
)
|
3450
|
+
|
3451
|
+
if len(artifact.metadata) > MAX_ARTIFACT_METADATA_KEYS:
|
3452
|
+
raise ValueError(
|
3453
|
+
f"Artifact must not have more than {MAX_ARTIFACT_METADATA_KEYS} metadata keys."
|
3454
|
+
)
|
3455
|
+
|
3247
3456
|
artifact.distributed_id = distributed_id
|
3248
3457
|
self._assert_can_log_artifact(artifact)
|
3249
3458
|
if self._backend and self._backend.interface:
|
@@ -3355,8 +3564,9 @@ class Run:
|
|
3355
3564
|
artifact.finalize()
|
3356
3565
|
return artifact, _resolve_aliases(aliases)
|
3357
3566
|
|
3358
|
-
@
|
3359
|
-
@
|
3567
|
+
@_log_to_run
|
3568
|
+
@_raise_if_finished
|
3569
|
+
@_attach
|
3360
3570
|
def log_model(
|
3361
3571
|
self,
|
3362
3572
|
path: StrPath,
|
@@ -3406,8 +3616,9 @@ class Run:
|
|
3406
3616
|
artifact_or_path=path, name=name, type="model", aliases=aliases
|
3407
3617
|
)
|
3408
3618
|
|
3409
|
-
@
|
3410
|
-
@
|
3619
|
+
@_log_to_run
|
3620
|
+
@_raise_if_finished
|
3621
|
+
@_attach
|
3411
3622
|
def use_model(self, name: str) -> FilePathStr:
|
3412
3623
|
"""Download the files logged in a model artifact 'name'.
|
3413
3624
|
|
@@ -3447,6 +3658,10 @@ class Run:
|
|
3447
3658
|
Returns:
|
3448
3659
|
path: (str) path to downloaded model artifact file(s).
|
3449
3660
|
"""
|
3661
|
+
if self._settings._offline:
|
3662
|
+
# Downloading artifacts is not supported when offline.
|
3663
|
+
raise RuntimeError("`use_model` not supported in offline mode.")
|
3664
|
+
|
3450
3665
|
artifact = self.use_artifact(artifact_or_name=name)
|
3451
3666
|
if "model" not in str(artifact.type.lower()):
|
3452
3667
|
raise AssertionError(
|
@@ -3463,8 +3678,9 @@ class Run:
|
|
3463
3678
|
return FilePathStr(os.path.join(path, dir_list[0]))
|
3464
3679
|
return path
|
3465
3680
|
|
3466
|
-
@
|
3467
|
-
@
|
3681
|
+
@_log_to_run
|
3682
|
+
@_raise_if_finished
|
3683
|
+
@_attach
|
3468
3684
|
def link_model(
|
3469
3685
|
self,
|
3470
3686
|
path: StrPath,
|
@@ -3564,8 +3780,9 @@ class Run:
|
|
3564
3780
|
)
|
3565
3781
|
self.link_artifact(artifact=artifact, target_path=target_path, aliases=aliases)
|
3566
3782
|
|
3567
|
-
@
|
3568
|
-
@
|
3783
|
+
@_log_to_run
|
3784
|
+
@_raise_if_finished
|
3785
|
+
@_attach
|
3569
3786
|
def alert(
|
3570
3787
|
self,
|
3571
3788
|
title: str,
|
@@ -3615,8 +3832,9 @@ class Run:
|
|
3615
3832
|
self._finish(exit_code=exit_code)
|
3616
3833
|
return not exception_raised
|
3617
3834
|
|
3618
|
-
@
|
3619
|
-
@
|
3835
|
+
@_log_to_run
|
3836
|
+
@_raise_if_finished
|
3837
|
+
@_attach
|
3620
3838
|
def mark_preempting(self) -> None:
|
3621
3839
|
"""Mark this run as preempting.
|
3622
3840
|
|
@@ -3626,8 +3844,9 @@ class Run:
|
|
3626
3844
|
self._backend.interface.publish_preempting()
|
3627
3845
|
|
3628
3846
|
@property
|
3629
|
-
@
|
3630
|
-
@
|
3847
|
+
@_log_to_run
|
3848
|
+
@_raise_if_finished
|
3849
|
+
@_attach
|
3631
3850
|
def _system_metrics(self) -> dict[str, list[tuple[datetime, float]]]:
|
3632
3851
|
"""Returns a dictionary of system metrics.
|
3633
3852
|
|
@@ -3674,8 +3893,9 @@ class Run:
|
|
3674
3893
|
return {}
|
3675
3894
|
|
3676
3895
|
@property
|
3677
|
-
@
|
3678
|
-
@
|
3896
|
+
@_log_to_run
|
3897
|
+
@_attach
|
3898
|
+
@_raise_if_finished
|
3679
3899
|
def _metadata(self) -> Metadata | None:
|
3680
3900
|
"""The metadata associated with this run.
|
3681
3901
|
|
@@ -3712,8 +3932,9 @@ class Run:
|
|
3712
3932
|
|
3713
3933
|
return None
|
3714
3934
|
|
3715
|
-
@
|
3716
|
-
@
|
3935
|
+
@_log_to_run
|
3936
|
+
@_raise_if_finished
|
3937
|
+
@_attach
|
3717
3938
|
def _metadata_callback(
|
3718
3939
|
self,
|
3719
3940
|
metadata: MetadataRequest,
|
@@ -3733,59 +3954,41 @@ class Run:
|
|
3733
3954
|
# ------------------------------------------------------------------------------
|
3734
3955
|
# HEADER
|
3735
3956
|
# ------------------------------------------------------------------------------
|
3736
|
-
|
3737
|
-
|
3738
|
-
|
3739
|
-
|
3740
|
-
*,
|
3741
|
-
settings: Settings,
|
3742
|
-
printer: printer.Printer,
|
3743
|
-
) -> None:
|
3744
|
-
Run._header_wandb_version_info(settings=settings, printer=printer)
|
3745
|
-
Run._header_sync_info(settings=settings, printer=printer)
|
3746
|
-
Run._header_run_info(settings=settings, printer=printer)
|
3957
|
+
def _header(self) -> None:
|
3958
|
+
self._header_wandb_version_info()
|
3959
|
+
self._header_sync_info()
|
3960
|
+
self._header_run_info()
|
3747
3961
|
|
3748
|
-
|
3749
|
-
|
3750
|
-
*,
|
3751
|
-
settings: Settings,
|
3752
|
-
printer: printer.Printer,
|
3753
|
-
) -> None:
|
3754
|
-
if settings.quiet or settings.silent:
|
3962
|
+
def _header_wandb_version_info(self) -> None:
|
3963
|
+
if self._settings.quiet or self._settings.silent:
|
3755
3964
|
return
|
3756
3965
|
|
3757
3966
|
# TODO: add this to a higher verbosity level
|
3758
|
-
|
3967
|
+
self._printer.display(f"Tracking run with wandb version {wandb.__version__}")
|
3759
3968
|
|
3760
|
-
|
3761
|
-
|
3762
|
-
|
3763
|
-
settings: Settings,
|
3764
|
-
printer: printer.Printer,
|
3765
|
-
) -> None:
|
3766
|
-
if settings._offline:
|
3767
|
-
printer.display(
|
3969
|
+
def _header_sync_info(self) -> None:
|
3970
|
+
if self._settings._offline:
|
3971
|
+
self._printer.display(
|
3768
3972
|
[
|
3769
|
-
f"W&B syncing is set to {
|
3770
|
-
f"Run {
|
3771
|
-
"
|
3973
|
+
f"W&B syncing is set to {self._printer.code('`offline`')}"
|
3974
|
+
f" in this directory. Run {self._printer.code('`wandb online`')}"
|
3975
|
+
f" or set {self._printer.code('WANDB_MODE=online')}"
|
3976
|
+
" to enable cloud syncing.",
|
3772
3977
|
]
|
3773
3978
|
)
|
3774
3979
|
else:
|
3775
|
-
|
3776
|
-
|
3980
|
+
sync_dir = self._settings.sync_dir
|
3981
|
+
info = [f"Run data is saved locally in {self._printer.files(sync_dir)}"]
|
3982
|
+
if not self._printer.supports_html:
|
3777
3983
|
info.append(
|
3778
|
-
f"Run {
|
3984
|
+
f"Run {self._printer.code('`wandb offline`')} to turn off syncing."
|
3779
3985
|
)
|
3780
|
-
if not
|
3781
|
-
|
3986
|
+
if not self._settings.quiet and not self._settings.silent:
|
3987
|
+
self._printer.display(info)
|
3988
|
+
|
3989
|
+
def _header_run_info(self) -> None:
|
3990
|
+
settings, printer = self._settings, self._printer
|
3782
3991
|
|
3783
|
-
@staticmethod
|
3784
|
-
def _header_run_info(
|
3785
|
-
*,
|
3786
|
-
settings: Settings,
|
3787
|
-
printer: printer.Printer,
|
3788
|
-
) -> None:
|
3789
3992
|
if settings._offline or settings.silent:
|
3790
3993
|
return
|
3791
3994
|
|
@@ -3803,12 +4006,13 @@ class Run:
|
|
3803
4006
|
return
|
3804
4007
|
|
3805
4008
|
if printer.supports_html:
|
3806
|
-
|
4009
|
+
import wandb.jupyter
|
4010
|
+
|
4011
|
+
if not wandb.jupyter.display_if_magic_is_used(self):
|
3807
4012
|
run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
|
3808
4013
|
project_line, sweep_line = "", ""
|
3809
4014
|
|
3810
|
-
|
3811
|
-
if not wandb.jupyter.quiet(): # type: ignore
|
4015
|
+
if not settings.quiet:
|
3812
4016
|
doc_html = printer.link(url_registry.url("developer-guide"), "docs")
|
3813
4017
|
|
3814
4018
|
project_html = printer.link(project_url, "Weights & Biases")
|