wandb 0.18.3__py3-none-macosx_11_0_arm64.whl → 0.18.5__py3-none-macosx_11_0_arm64.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +16 -7
- wandb/__init__.pyi +96 -63
- wandb/analytics/sentry.py +91 -88
- wandb/apis/public/api.py +18 -4
- wandb/apis/public/runs.py +53 -2
- wandb/bin/gpu_stats +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/beta.py +178 -0
- wandb/cli/cli.py +5 -171
- wandb/data_types.py +3 -0
- wandb/env.py +74 -73
- wandb/errors/term.py +300 -43
- wandb/proto/v3/wandb_internal_pb2.py +263 -223
- wandb/proto/v3/wandb_server_pb2.py +57 -37
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_internal_pb2.py +226 -218
- wandb/proto/v4/wandb_server_pb2.py +41 -37
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_internal_pb2.py +226 -218
- wandb/proto/v5/wandb_server_pb2.py +41 -37
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/sdk/__init__.py +3 -3
- wandb/sdk/artifacts/_validators.py +41 -8
- wandb/sdk/artifacts/artifact.py +32 -1
- wandb/sdk/artifacts/artifact_file_cache.py +1 -2
- wandb/sdk/data_types/_dtypes.py +7 -3
- wandb/sdk/data_types/video.py +15 -6
- wandb/sdk/interface/interface.py +2 -0
- wandb/sdk/internal/internal_api.py +126 -5
- wandb/sdk/internal/sender.py +16 -3
- wandb/sdk/launch/inputs/internal.py +1 -1
- wandb/sdk/lib/module.py +12 -0
- wandb/sdk/lib/printer.py +291 -105
- wandb/sdk/lib/progress.py +274 -0
- wandb/sdk/service/streams.py +21 -11
- wandb/sdk/wandb_init.py +58 -54
- wandb/sdk/wandb_run.py +380 -454
- wandb/sdk/wandb_settings.py +2 -0
- wandb/sdk/wandb_watch.py +17 -11
- wandb/util.py +6 -2
- {wandb-0.18.3.dist-info → wandb-0.18.5.dist-info}/METADATA +4 -3
- {wandb-0.18.3.dist-info → wandb-0.18.5.dist-info}/RECORD +45 -43
- wandb/bin/apple_gpu_stats +0 -0
- {wandb-0.18.3.dist-info → wandb-0.18.5.dist-info}/WHEEL +0 -0
- {wandb-0.18.3.dist-info → wandb-0.18.5.dist-info}/entry_points.txt +0 -0
- {wandb-0.18.3.dist-info → wandb-0.18.5.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
1
3
|
import _thread as thread
|
2
4
|
import atexit
|
3
5
|
import functools
|
@@ -18,20 +20,12 @@ from dataclasses import dataclass, field
|
|
18
20
|
from datetime import datetime, timedelta, timezone
|
19
21
|
from enum import IntEnum
|
20
22
|
from types import TracebackType
|
21
|
-
from typing import
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
NamedTuple,
|
28
|
-
Optional,
|
29
|
-
Sequence,
|
30
|
-
TextIO,
|
31
|
-
Tuple,
|
32
|
-
Type,
|
33
|
-
Union,
|
34
|
-
)
|
23
|
+
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Sequence, TextIO
|
24
|
+
|
25
|
+
if sys.version_info < (3, 8):
|
26
|
+
from typing_extensions import Literal
|
27
|
+
else:
|
28
|
+
from typing import Literal
|
35
29
|
|
36
30
|
import requests
|
37
31
|
|
@@ -69,7 +63,11 @@ from wandb.util import (
|
|
69
63
|
)
|
70
64
|
|
71
65
|
from . import wandb_config, wandb_metric, wandb_summary
|
72
|
-
from .artifacts._validators import
|
66
|
+
from .artifacts._validators import (
|
67
|
+
is_artifact_registry_project,
|
68
|
+
validate_aliases,
|
69
|
+
validate_tags,
|
70
|
+
)
|
73
71
|
from .data_types._dtypes import TypeRegistry
|
74
72
|
from .interface.interface import FilesDict, GlobStr, InterfaceBase, PolicyName
|
75
73
|
from .interface.summary_record import SummaryRecord
|
@@ -80,6 +78,8 @@ from .lib import (
|
|
80
78
|
filesystem,
|
81
79
|
ipython,
|
82
80
|
module,
|
81
|
+
printer,
|
82
|
+
progress,
|
83
83
|
proto_util,
|
84
84
|
redirect,
|
85
85
|
telemetry,
|
@@ -87,7 +87,6 @@ from .lib import (
|
|
87
87
|
from .lib.exit_hooks import ExitHooks
|
88
88
|
from .lib.gitlib import GitRepo
|
89
89
|
from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
|
90
|
-
from .lib.printer import get_printer
|
91
90
|
from .lib.proto_util import message_to_dict
|
92
91
|
from .lib.reporting import Reporter
|
93
92
|
from .lib.wburls import wburls
|
@@ -101,26 +100,27 @@ if TYPE_CHECKING:
|
|
101
100
|
else:
|
102
101
|
from typing_extensions import TypedDict
|
103
102
|
|
103
|
+
import torch # type: ignore [import-not-found]
|
104
|
+
|
104
105
|
import wandb.apis.public
|
105
106
|
import wandb.sdk.backend.backend
|
106
107
|
import wandb.sdk.interface.interface_queue
|
108
|
+
from wandb.data_types import Table
|
107
109
|
from wandb.proto.wandb_internal_pb2 import (
|
108
110
|
GetSummaryResponse,
|
109
111
|
InternalMessagesResponse,
|
110
112
|
SampledHistoryResponse,
|
111
113
|
)
|
112
114
|
|
113
|
-
from .lib.printer import PrinterJupyter, PrinterTerm
|
114
|
-
|
115
115
|
class GitSourceDict(TypedDict):
|
116
116
|
remote: str
|
117
117
|
commit: str
|
118
|
-
entrypoint:
|
118
|
+
entrypoint: list[str]
|
119
119
|
args: Sequence[str]
|
120
120
|
|
121
121
|
class ArtifactSourceDict(TypedDict):
|
122
122
|
artifact: str
|
123
|
-
entrypoint:
|
123
|
+
entrypoint: list[str]
|
124
124
|
args: Sequence[str]
|
125
125
|
|
126
126
|
class ImageSourceDict(TypedDict):
|
@@ -130,10 +130,10 @@ if TYPE_CHECKING:
|
|
130
130
|
class JobSourceDict(TypedDict, total=False):
|
131
131
|
_version: str
|
132
132
|
source_type: str
|
133
|
-
source:
|
134
|
-
input_types:
|
135
|
-
output_types:
|
136
|
-
runtime:
|
133
|
+
source: GitSourceDict | ArtifactSourceDict | ImageSourceDict
|
134
|
+
input_types: dict[str, Any]
|
135
|
+
output_types: dict[str, Any]
|
136
|
+
runtime: str | None
|
137
137
|
|
138
138
|
|
139
139
|
logger = logging.getLogger("wandb")
|
@@ -160,11 +160,11 @@ class RunStatusChecker:
|
|
160
160
|
"""
|
161
161
|
|
162
162
|
_stop_status_lock: threading.Lock
|
163
|
-
_stop_status_handle:
|
163
|
+
_stop_status_handle: MailboxHandle | None
|
164
164
|
_network_status_lock: threading.Lock
|
165
|
-
_network_status_handle:
|
165
|
+
_network_status_handle: MailboxHandle | None
|
166
166
|
_internal_messages_lock: threading.Lock
|
167
|
-
_internal_messages_handle:
|
167
|
+
_internal_messages_handle: MailboxHandle | None
|
168
168
|
|
169
169
|
def __init__(
|
170
170
|
self,
|
@@ -212,7 +212,7 @@ class RunStatusChecker:
|
|
212
212
|
@staticmethod
|
213
213
|
def _abandon_status_check(
|
214
214
|
lock: threading.Lock,
|
215
|
-
handle:
|
215
|
+
handle: MailboxHandle | None,
|
216
216
|
):
|
217
217
|
with lock:
|
218
218
|
if handle:
|
@@ -227,7 +227,7 @@ class RunStatusChecker:
|
|
227
227
|
request: Any,
|
228
228
|
process: Any,
|
229
229
|
) -> None:
|
230
|
-
local_handle:
|
230
|
+
local_handle: MailboxHandle | None = None
|
231
231
|
join_requested = False
|
232
232
|
while not join_requested:
|
233
233
|
time_probe = time.monotonic()
|
@@ -365,7 +365,7 @@ class _run_decorator: # noqa: N801
|
|
365
365
|
@classmethod
|
366
366
|
def _attach(cls, func: Callable) -> Callable:
|
367
367
|
@functools.wraps(func)
|
368
|
-
def wrapper(self:
|
368
|
+
def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
369
369
|
# * `_attach_id` is only assigned in service hence for all non-service cases
|
370
370
|
# it will be a passthrough.
|
371
371
|
# * `_attach_pid` is only assigned in _init (using _attach_pid guarantees single attach):
|
@@ -398,7 +398,7 @@ class _run_decorator: # noqa: N801
|
|
398
398
|
def _noop_on_finish(cls, message: str = "", only_warn: bool = False) -> Callable:
|
399
399
|
def decorator_fn(func: Callable) -> Callable:
|
400
400
|
@functools.wraps(func)
|
401
|
-
def wrapper_fn(self:
|
401
|
+
def wrapper_fn(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
402
402
|
if not getattr(self, "_is_finished", False):
|
403
403
|
return func(self, *args, **kwargs)
|
404
404
|
|
@@ -419,7 +419,7 @@ class _run_decorator: # noqa: N801
|
|
419
419
|
@classmethod
|
420
420
|
def _noop(cls, func: Callable) -> Callable:
|
421
421
|
@functools.wraps(func)
|
422
|
-
def wrapper(self:
|
422
|
+
def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
423
423
|
# `_attach_id` is only assigned in service hence for all service cases
|
424
424
|
# it will be a passthrough. We don't pickle non-service so again a way
|
425
425
|
# to see that we are in non-service case
|
@@ -458,7 +458,7 @@ class _run_decorator: # noqa: N801
|
|
458
458
|
class RunStatus:
|
459
459
|
sync_items_total: int = field(default=0)
|
460
460
|
sync_items_pending: int = field(default=0)
|
461
|
-
sync_time:
|
461
|
+
sync_time: datetime | None = field(default=None)
|
462
462
|
|
463
463
|
|
464
464
|
class Run:
|
@@ -528,67 +528,65 @@ class Run:
|
|
528
528
|
_telemetry_obj_dirty: bool
|
529
529
|
_telemetry_obj_flushed: bytes
|
530
530
|
|
531
|
-
_teardown_hooks:
|
532
|
-
_tags:
|
531
|
+
_teardown_hooks: list[TeardownHook]
|
532
|
+
_tags: tuple[Any, ...] | None
|
533
533
|
|
534
|
-
_entity:
|
535
|
-
_project:
|
536
|
-
_group:
|
537
|
-
_job_type:
|
538
|
-
_name:
|
539
|
-
_notes:
|
540
|
-
_sweep_id:
|
534
|
+
_entity: str | None
|
535
|
+
_project: str | None
|
536
|
+
_group: str | None
|
537
|
+
_job_type: str | None
|
538
|
+
_name: str | None
|
539
|
+
_notes: str | None
|
540
|
+
_sweep_id: str | None
|
541
541
|
|
542
|
-
_run_obj:
|
542
|
+
_run_obj: RunRecord | None
|
543
543
|
# Use string literal annotation because of type reference loop
|
544
|
-
_backend:
|
545
|
-
_internal_run_interface:
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
_output_writer: Optional["filesystem.CRDedupedFile"]
|
555
|
-
_quiet: Optional[bool]
|
544
|
+
_backend: wandb.sdk.backend.backend.Backend | None
|
545
|
+
_internal_run_interface: wandb.sdk.interface.interface_queue.InterfaceQueue | None
|
546
|
+
_wl: _WandbSetup | None
|
547
|
+
|
548
|
+
_out_redir: redirect.RedirectBase | None
|
549
|
+
_err_redir: redirect.RedirectBase | None
|
550
|
+
_redirect_cb: Callable[[str, str], None] | None
|
551
|
+
_redirect_raw_cb: Callable[[str, str], None] | None
|
552
|
+
_output_writer: filesystem.CRDedupedFile | None
|
553
|
+
_quiet: bool | None
|
556
554
|
|
557
555
|
_atexit_cleanup_called: bool
|
558
|
-
_hooks:
|
559
|
-
_exit_code:
|
556
|
+
_hooks: ExitHooks | None
|
557
|
+
_exit_code: int | None
|
560
558
|
|
561
|
-
_run_status_checker:
|
559
|
+
_run_status_checker: RunStatusChecker | None
|
562
560
|
|
563
|
-
_sampled_history:
|
564
|
-
_final_summary:
|
565
|
-
_poll_exit_handle:
|
566
|
-
_poll_exit_response:
|
567
|
-
_internal_messages_response:
|
561
|
+
_sampled_history: SampledHistoryResponse | None
|
562
|
+
_final_summary: GetSummaryResponse | None
|
563
|
+
_poll_exit_handle: MailboxHandle | None
|
564
|
+
_poll_exit_response: PollExitResponse | None
|
565
|
+
_internal_messages_response: InternalMessagesResponse | None
|
568
566
|
|
569
|
-
_stdout_slave_fd:
|
570
|
-
_stderr_slave_fd:
|
571
|
-
_artifact_slots:
|
567
|
+
_stdout_slave_fd: int | None
|
568
|
+
_stderr_slave_fd: int | None
|
569
|
+
_artifact_slots: list[str]
|
572
570
|
|
573
571
|
_init_pid: int
|
574
572
|
_attach_pid: int
|
575
|
-
_iface_pid:
|
576
|
-
_iface_port:
|
573
|
+
_iface_pid: int | None
|
574
|
+
_iface_port: int | None
|
577
575
|
|
578
|
-
_attach_id:
|
576
|
+
_attach_id: str | None
|
579
577
|
_is_attached: bool
|
580
578
|
_is_finished: bool
|
581
579
|
_settings: Settings
|
582
580
|
|
583
|
-
_launch_artifacts:
|
584
|
-
_printer:
|
581
|
+
_launch_artifacts: dict[str, Any] | None
|
582
|
+
_printer: printer.Printer
|
585
583
|
|
586
584
|
def __init__(
|
587
585
|
self,
|
588
586
|
settings: Settings,
|
589
|
-
config:
|
590
|
-
sweep_config:
|
591
|
-
launch_config:
|
587
|
+
config: dict[str, Any] | None = None,
|
588
|
+
sweep_config: dict[str, Any] | None = None,
|
589
|
+
launch_config: dict[str, Any] | None = None,
|
592
590
|
) -> None:
|
593
591
|
# pid is set, so we know if this run object was initialized by this process
|
594
592
|
self._init_pid = os.getpid()
|
@@ -608,9 +606,9 @@ class Run:
|
|
608
606
|
def _init(
|
609
607
|
self,
|
610
608
|
settings: Settings,
|
611
|
-
config:
|
612
|
-
sweep_config:
|
613
|
-
launch_config:
|
609
|
+
config: dict[str, Any] | None = None,
|
610
|
+
sweep_config: dict[str, Any] | None = None,
|
611
|
+
launch_config: dict[str, Any] | None = None,
|
614
612
|
) -> None:
|
615
613
|
self._settings = settings
|
616
614
|
|
@@ -626,7 +624,7 @@ class Run:
|
|
626
624
|
)
|
627
625
|
self.summary._set_update_callback(self._summary_update_callback)
|
628
626
|
self._step = 0
|
629
|
-
self._torch_history:
|
627
|
+
self._torch_history: wandb_torch.TorchHistory | None = None # type: ignore
|
630
628
|
|
631
629
|
# todo: eventually would be nice to make this configurable using self._settings._start_time
|
632
630
|
# need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
|
@@ -635,9 +633,9 @@ class Run:
|
|
635
633
|
|
636
634
|
_datatypes_set_callback(self._datatypes_callback)
|
637
635
|
|
638
|
-
self._printer = get_printer(self._settings._jupyter)
|
636
|
+
self._printer = printer.get_printer(self._settings._jupyter)
|
639
637
|
self._wl = None
|
640
|
-
self._reporter:
|
638
|
+
self._reporter: Reporter | None = None
|
641
639
|
|
642
640
|
self._entity = None
|
643
641
|
self._project = None
|
@@ -663,7 +661,7 @@ class Run:
|
|
663
661
|
self._quiet = self._settings.quiet
|
664
662
|
|
665
663
|
self._output_writer = None
|
666
|
-
self._used_artifact_slots:
|
664
|
+
self._used_artifact_slots: dict[str, str] = {}
|
667
665
|
|
668
666
|
# Returned from backend request_run(), set from wandb_init?
|
669
667
|
self._run_obj = None
|
@@ -699,8 +697,8 @@ class Run:
|
|
699
697
|
config = config or dict()
|
700
698
|
wandb_key = "_wandb"
|
701
699
|
config.setdefault(wandb_key, dict())
|
702
|
-
self._launch_artifact_mapping:
|
703
|
-
self._unique_launch_artifact_sequence_names:
|
700
|
+
self._launch_artifact_mapping: dict[str, Any] = {}
|
701
|
+
self._unique_launch_artifact_sequence_names: dict[str, Any] = {}
|
704
702
|
if self._settings.save_code and self._settings.program_relpath:
|
705
703
|
config[wandb_key]["code_path"] = LogicalPath(
|
706
704
|
os.path.join("code", self._settings.program_relpath)
|
@@ -766,7 +764,7 @@ class Run:
|
|
766
764
|
def _handle_launch_artifact_overrides(self) -> None:
|
767
765
|
if self._settings.launch and (os.environ.get("WANDB_ARTIFACTS") is not None):
|
768
766
|
try:
|
769
|
-
artifacts:
|
767
|
+
artifacts: dict[str, Any] = json.loads(
|
770
768
|
os.environ.get("WANDB_ARTIFACTS", "{}")
|
771
769
|
)
|
772
770
|
except (ValueError, SyntaxError):
|
@@ -786,7 +784,7 @@ class Run:
|
|
786
784
|
artifacts = launch_config.get("overrides").get("artifacts")
|
787
785
|
self._initialize_launch_artifact_maps(artifacts)
|
788
786
|
|
789
|
-
def _initialize_launch_artifact_maps(self, artifacts:
|
787
|
+
def _initialize_launch_artifact_maps(self, artifacts: dict[str, Any]) -> None:
|
790
788
|
for key, item in artifacts.items():
|
791
789
|
self._launch_artifact_mapping[key] = item
|
792
790
|
artifact_sequence_tuple_or_slot = key.split(":")
|
@@ -893,7 +891,7 @@ class Run:
|
|
893
891
|
except Exception:
|
894
892
|
wandb.termwarn("Cannot find valid git repo associated with this directory.")
|
895
893
|
|
896
|
-
def __deepcopy__(self, memo:
|
894
|
+
def __deepcopy__(self, memo: dict[int, Any]) -> Run:
|
897
895
|
return self
|
898
896
|
|
899
897
|
def __getstate__(self) -> Any:
|
@@ -927,7 +925,7 @@ class Run:
|
|
927
925
|
self.__dict__.update(state)
|
928
926
|
|
929
927
|
@property
|
930
|
-
def _torch(self) ->
|
928
|
+
def _torch(self) -> wandb_torch.TorchHistory: # type: ignore
|
931
929
|
if self._torch_history is None:
|
932
930
|
self._torch_history = wandb_torch.TorchHistory() # type: ignore
|
933
931
|
return self._torch_history
|
@@ -959,7 +957,7 @@ class Run:
|
|
959
957
|
|
960
958
|
@property
|
961
959
|
@_run_decorator._attach
|
962
|
-
def name(self) ->
|
960
|
+
def name(self) -> str | None:
|
963
961
|
"""Display name of the run.
|
964
962
|
|
965
963
|
Display names are not guaranteed to be unique and may be descriptive.
|
@@ -982,7 +980,7 @@ class Run:
|
|
982
980
|
|
983
981
|
@property
|
984
982
|
@_run_decorator._attach
|
985
|
-
def notes(self) ->
|
983
|
+
def notes(self) -> str | None:
|
986
984
|
"""Notes associated with the run, if there are any.
|
987
985
|
|
988
986
|
Notes can be a multiline string and can also use markdown and latex equations
|
@@ -1003,7 +1001,7 @@ class Run:
|
|
1003
1001
|
|
1004
1002
|
@property
|
1005
1003
|
@_run_decorator._attach
|
1006
|
-
def tags(self) ->
|
1004
|
+
def tags(self) -> tuple | None:
|
1007
1005
|
"""Tags associated with the run, if there are any."""
|
1008
1006
|
if self._tags:
|
1009
1007
|
return self._tags
|
@@ -1030,7 +1028,7 @@ class Run:
|
|
1030
1028
|
|
1031
1029
|
@property
|
1032
1030
|
@_run_decorator._attach
|
1033
|
-
def sweep_id(self) ->
|
1031
|
+
def sweep_id(self) -> str | None:
|
1034
1032
|
"""ID of the sweep associated with the run, if there is one."""
|
1035
1033
|
if not self._run_obj:
|
1036
1034
|
return None
|
@@ -1149,15 +1147,13 @@ class Run:
|
|
1149
1147
|
@_run_decorator._attach
|
1150
1148
|
def log_code(
|
1151
1149
|
self,
|
1152
|
-
root:
|
1153
|
-
name:
|
1154
|
-
include_fn:
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
] = filenames.exclude_wandb_fn,
|
1160
|
-
) -> Optional[Artifact]:
|
1150
|
+
root: str | None = ".",
|
1151
|
+
name: str | None = None,
|
1152
|
+
include_fn: Callable[[str, str], bool]
|
1153
|
+
| Callable[[str], bool] = _is_py_requirements_or_dockerfile,
|
1154
|
+
exclude_fn: Callable[[str, str], bool]
|
1155
|
+
| Callable[[str], bool] = filenames.exclude_wandb_fn,
|
1156
|
+
) -> Artifact | None:
|
1161
1157
|
"""Save the current state of your code to a W&B Artifact.
|
1162
1158
|
|
1163
1159
|
By default, it walks the current directory and logs all files that end with `.py`.
|
@@ -1230,7 +1226,7 @@ class Run:
|
|
1230
1226
|
|
1231
1227
|
return self._log_artifact(art)
|
1232
1228
|
|
1233
|
-
def get_url(self) ->
|
1229
|
+
def get_url(self) -> str | None:
|
1234
1230
|
"""Return the url for the W&B run, if there is one.
|
1235
1231
|
|
1236
1232
|
Offline runs will not have a url.
|
@@ -1240,7 +1236,7 @@ class Run:
|
|
1240
1236
|
return None
|
1241
1237
|
return self._settings.run_url
|
1242
1238
|
|
1243
|
-
def get_project_url(self) ->
|
1239
|
+
def get_project_url(self) -> str | None:
|
1244
1240
|
"""Return the url for the W&B project associated with the run, if there is one.
|
1245
1241
|
|
1246
1242
|
Offline runs will not have a project url.
|
@@ -1250,7 +1246,7 @@ class Run:
|
|
1250
1246
|
return None
|
1251
1247
|
return self._settings.project_url
|
1252
1248
|
|
1253
|
-
def get_sweep_url(self) ->
|
1249
|
+
def get_sweep_url(self) -> str | None:
|
1254
1250
|
"""Return the url for the sweep associated with the run, if there is one."""
|
1255
1251
|
if self._settings._offline:
|
1256
1252
|
wandb.termwarn("URL not available in offline run")
|
@@ -1259,7 +1255,7 @@ class Run:
|
|
1259
1255
|
|
1260
1256
|
@property
|
1261
1257
|
@_run_decorator._attach
|
1262
|
-
def url(self) ->
|
1258
|
+
def url(self) -> str | None:
|
1263
1259
|
"""The W&B url associated with the run."""
|
1264
1260
|
return self.get_url()
|
1265
1261
|
|
@@ -1274,9 +1270,9 @@ class Run:
|
|
1274
1270
|
|
1275
1271
|
def _label_internal(
|
1276
1272
|
self,
|
1277
|
-
code:
|
1278
|
-
repo:
|
1279
|
-
code_version:
|
1273
|
+
code: str | None = None,
|
1274
|
+
repo: str | None = None,
|
1275
|
+
code_version: str | None = None,
|
1280
1276
|
) -> None:
|
1281
1277
|
with telemetry.context(run=self) as tel:
|
1282
1278
|
if code and RE_LABEL.match(code):
|
@@ -1288,9 +1284,9 @@ class Run:
|
|
1288
1284
|
|
1289
1285
|
def _label(
|
1290
1286
|
self,
|
1291
|
-
code:
|
1292
|
-
repo:
|
1293
|
-
code_version:
|
1287
|
+
code: str | None = None,
|
1288
|
+
repo: str | None = None,
|
1289
|
+
code_version: str | None = None,
|
1294
1290
|
**kwargs: str,
|
1295
1291
|
) -> None:
|
1296
1292
|
if self._settings.label_disable:
|
@@ -1312,7 +1308,7 @@ class Run:
|
|
1312
1308
|
# update telemetry in the backend immediately for _label() callers
|
1313
1309
|
self._telemetry_flush()
|
1314
1310
|
|
1315
|
-
def _label_probe_lines(self, lines:
|
1311
|
+
def _label_probe_lines(self, lines: list[str]) -> None:
|
1316
1312
|
if not lines:
|
1317
1313
|
return
|
1318
1314
|
parsed = telemetry._parse_label_lines(lines)
|
@@ -1380,23 +1376,23 @@ class Run:
|
|
1380
1376
|
return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
|
1381
1377
|
|
1382
1378
|
def _repr_mimebundle_(
|
1383
|
-
self, include:
|
1384
|
-
) ->
|
1379
|
+
self, include: Any | None = None, exclude: Any | None = None
|
1380
|
+
) -> dict[str, str]:
|
1385
1381
|
return {"text/html": self.to_html(hidden=True)}
|
1386
1382
|
|
1387
1383
|
@_run_decorator._noop_on_finish()
|
1388
1384
|
def _config_callback(
|
1389
1385
|
self,
|
1390
|
-
key:
|
1391
|
-
val:
|
1392
|
-
data:
|
1386
|
+
key: tuple[str, ...] | str | None = None,
|
1387
|
+
val: Any | None = None,
|
1388
|
+
data: dict[str, object] | None = None,
|
1393
1389
|
) -> None:
|
1394
1390
|
logger.info(f"config_cb {key} {val} {data}")
|
1395
1391
|
if self._backend and self._backend.interface:
|
1396
1392
|
self._backend.interface.publish_config(key=key, val=val, data=data)
|
1397
1393
|
|
1398
1394
|
def _config_artifact_callback(
|
1399
|
-
self, key: str, val:
|
1395
|
+
self, key: str, val: str | Artifact | dict
|
1400
1396
|
) -> Artifact:
|
1401
1397
|
# artifacts can look like dicts as they are passed into the run config
|
1402
1398
|
# since the run config stores them on the backend as a dict with fields shown
|
@@ -1447,7 +1443,7 @@ class Run:
|
|
1447
1443
|
# line = "Waiting for run.summary data..."
|
1448
1444
|
# self._printer.display(line)
|
1449
1445
|
|
1450
|
-
def _summary_get_current_summary_callback(self) ->
|
1446
|
+
def _summary_get_current_summary_callback(self) -> dict[str, Any]:
|
1451
1447
|
if self._is_finished:
|
1452
1448
|
# TODO: WB-18420: fetch summary from backend and stage it before run is finished
|
1453
1449
|
wandb.termwarn("Summary data not available in finished run")
|
@@ -1474,7 +1470,7 @@ class Run:
|
|
1474
1470
|
files: FilesDict = dict(files=[(GlobStr(glob.escape(fname)), "now")])
|
1475
1471
|
self._backend.interface.publish_files(files)
|
1476
1472
|
|
1477
|
-
def _visualization_hack(self, row:
|
1473
|
+
def _visualization_hack(self, row: dict[str, Any]) -> dict[str, Any]:
|
1478
1474
|
# TODO(jhr): move visualize hack somewhere else
|
1479
1475
|
chart_keys = set()
|
1480
1476
|
split_table_set = set()
|
@@ -1511,9 +1507,9 @@ class Run:
|
|
1511
1507
|
|
1512
1508
|
def _partial_history_callback(
|
1513
1509
|
self,
|
1514
|
-
row:
|
1515
|
-
step:
|
1516
|
-
commit:
|
1510
|
+
row: dict[str, Any],
|
1511
|
+
step: int | None = None,
|
1512
|
+
commit: bool | None = None,
|
1517
1513
|
) -> None:
|
1518
1514
|
row = row.copy()
|
1519
1515
|
if row:
|
@@ -1560,19 +1556,19 @@ class Run:
|
|
1560
1556
|
def _set_library(self, library: _WandbSetup) -> None:
|
1561
1557
|
self._wl = library
|
1562
1558
|
|
1563
|
-
def _set_backend(self, backend:
|
1559
|
+
def _set_backend(self, backend: wandb.sdk.backend.backend.Backend) -> None:
|
1564
1560
|
self._backend = backend
|
1565
1561
|
|
1566
1562
|
def _set_internal_run_interface(
|
1567
1563
|
self,
|
1568
|
-
interface:
|
1564
|
+
interface: wandb.sdk.interface.interface_queue.InterfaceQueue,
|
1569
1565
|
) -> None:
|
1570
1566
|
self._internal_run_interface = interface
|
1571
1567
|
|
1572
1568
|
def _set_reporter(self, reporter: Reporter) -> None:
|
1573
1569
|
self._reporter = reporter
|
1574
1570
|
|
1575
|
-
def _set_teardown_hooks(self, hooks:
|
1571
|
+
def _set_teardown_hooks(self, hooks: list[TeardownHook]) -> None:
|
1576
1572
|
self._teardown_hooks = hooks
|
1577
1573
|
|
1578
1574
|
def _set_run_obj(self, run_obj: RunRecord) -> None:
|
@@ -1610,7 +1606,7 @@ class Run:
|
|
1610
1606
|
)
|
1611
1607
|
|
1612
1608
|
def _add_singleton(
|
1613
|
-
self, data_type: str, key: str, value:
|
1609
|
+
self, data_type: str, key: str, value: dict[int | str, str]
|
1614
1610
|
) -> None:
|
1615
1611
|
"""Store a singleton item to wandb config.
|
1616
1612
|
|
@@ -1640,9 +1636,9 @@ class Run:
|
|
1640
1636
|
|
1641
1637
|
def _log(
|
1642
1638
|
self,
|
1643
|
-
data:
|
1644
|
-
step:
|
1645
|
-
commit:
|
1639
|
+
data: dict[str, Any],
|
1640
|
+
step: int | None = None,
|
1641
|
+
commit: bool | None = None,
|
1646
1642
|
) -> None:
|
1647
1643
|
if not isinstance(data, Mapping):
|
1648
1644
|
raise ValueError("wandb.log must be passed a dictionary")
|
@@ -1681,10 +1677,10 @@ class Run:
|
|
1681
1677
|
@_run_decorator._attach
|
1682
1678
|
def log(
|
1683
1679
|
self,
|
1684
|
-
data:
|
1685
|
-
step:
|
1686
|
-
commit:
|
1687
|
-
sync:
|
1680
|
+
data: dict[str, Any],
|
1681
|
+
step: int | None = None,
|
1682
|
+
commit: bool | None = None,
|
1683
|
+
sync: bool | None = None,
|
1688
1684
|
) -> None:
|
1689
1685
|
"""Upload run data.
|
1690
1686
|
|
@@ -1937,10 +1933,10 @@ class Run:
|
|
1937
1933
|
@_run_decorator._attach
|
1938
1934
|
def save(
|
1939
1935
|
self,
|
1940
|
-
glob_str:
|
1941
|
-
base_path:
|
1936
|
+
glob_str: str | os.PathLike | None = None,
|
1937
|
+
base_path: str | os.PathLike | None = None,
|
1942
1938
|
policy: PolicyName = "live",
|
1943
|
-
) ->
|
1939
|
+
) -> bool | list[str]:
|
1944
1940
|
"""Sync one or more files to W&B.
|
1945
1941
|
|
1946
1942
|
Relative paths are relative to the current working directory.
|
@@ -2045,8 +2041,8 @@ class Run:
|
|
2045
2041
|
self,
|
2046
2042
|
glob_path: pathlib.PurePath,
|
2047
2043
|
base_path: pathlib.PurePath,
|
2048
|
-
policy:
|
2049
|
-
) ->
|
2044
|
+
policy: PolicyName,
|
2045
|
+
) -> list[str]:
|
2050
2046
|
# Can't use is_relative_to() because that's added in Python 3.9,
|
2051
2047
|
# but we support down to Python 3.7.
|
2052
2048
|
if not str(glob_path).startswith(str(base_path)):
|
@@ -2129,10 +2125,10 @@ class Run:
|
|
2129
2125
|
def restore(
|
2130
2126
|
self,
|
2131
2127
|
name: str,
|
2132
|
-
run_path:
|
2128
|
+
run_path: str | None = None,
|
2133
2129
|
replace: bool = False,
|
2134
|
-
root:
|
2135
|
-
) ->
|
2130
|
+
root: str | None = None,
|
2131
|
+
) -> None | TextIO:
|
2136
2132
|
return restore(
|
2137
2133
|
name,
|
2138
2134
|
run_path or self._get_path(),
|
@@ -2142,9 +2138,7 @@ class Run:
|
|
2142
2138
|
|
2143
2139
|
@_run_decorator._noop
|
2144
2140
|
@_run_decorator._attach
|
2145
|
-
def finish(
|
2146
|
-
self, exit_code: Optional[int] = None, quiet: Optional[bool] = None
|
2147
|
-
) -> None:
|
2141
|
+
def finish(self, exit_code: int | None = None, quiet: bool | None = None) -> None:
|
2148
2142
|
"""Mark a run as finished, and finish uploading all data.
|
2149
2143
|
|
2150
2144
|
This is used when creating multiple runs in the same process. We automatically
|
@@ -2158,8 +2152,8 @@ class Run:
|
|
2158
2152
|
|
2159
2153
|
def _finish(
|
2160
2154
|
self,
|
2161
|
-
exit_code:
|
2162
|
-
quiet:
|
2155
|
+
exit_code: int | None = None,
|
2156
|
+
quiet: bool | None = None,
|
2163
2157
|
) -> None:
|
2164
2158
|
logger.info(f"finishing run {self._get_path()}")
|
2165
2159
|
with telemetry.context(run=self) as tel:
|
@@ -2210,7 +2204,7 @@ class Run:
|
|
2210
2204
|
|
2211
2205
|
@_run_decorator._noop
|
2212
2206
|
@_run_decorator._attach
|
2213
|
-
def join(self, exit_code:
|
2207
|
+
def join(self, exit_code: int | None = None) -> None:
|
2214
2208
|
"""Deprecated alias for `finish()` - use finish instead."""
|
2215
2209
|
if hasattr(self, "_telemetry_obj"):
|
2216
2210
|
deprecate.deprecate(
|
@@ -2249,10 +2243,10 @@ class Run:
|
|
2249
2243
|
@staticmethod
|
2250
2244
|
def plot_table(
|
2251
2245
|
vega_spec_name: str,
|
2252
|
-
data_table:
|
2253
|
-
fields:
|
2254
|
-
string_fields:
|
2255
|
-
split_table:
|
2246
|
+
data_table: Table,
|
2247
|
+
fields: dict[str, Any],
|
2248
|
+
string_fields: dict[str, Any] | None = None,
|
2249
|
+
split_table: bool | None = False,
|
2256
2250
|
) -> CustomChart:
|
2257
2251
|
"""Create a custom plot on a table.
|
2258
2252
|
|
@@ -2264,6 +2258,8 @@ class Run:
|
|
2264
2258
|
visualization needs
|
2265
2259
|
string_fields: a dict that provides values for any string constants
|
2266
2260
|
the custom visualization needs
|
2261
|
+
split_table: a boolean that indicates whether the table should be in
|
2262
|
+
a separate section in the UI
|
2267
2263
|
"""
|
2268
2264
|
return custom_chart(
|
2269
2265
|
vega_spec_name, data_table, fields, string_fields or {}, split_table
|
@@ -2290,6 +2286,8 @@ class Run:
|
|
2290
2286
|
define_metric=self.define_metric,
|
2291
2287
|
plot_table=self.plot_table,
|
2292
2288
|
alert=self.alert,
|
2289
|
+
watch=self.watch,
|
2290
|
+
unwatch=self.unwatch,
|
2293
2291
|
mark_preempting=self.mark_preempting,
|
2294
2292
|
log_model=self.log_model,
|
2295
2293
|
use_model=self.use_model,
|
@@ -2298,9 +2296,9 @@ class Run:
|
|
2298
2296
|
|
2299
2297
|
def _redirect(
|
2300
2298
|
self,
|
2301
|
-
stdout_slave_fd:
|
2302
|
-
stderr_slave_fd:
|
2303
|
-
console:
|
2299
|
+
stdout_slave_fd: int | None,
|
2300
|
+
stderr_slave_fd: int | None,
|
2301
|
+
console: str | None = None,
|
2304
2302
|
) -> None:
|
2305
2303
|
if console is None:
|
2306
2304
|
console = self._settings.console
|
@@ -2415,7 +2413,7 @@ class Run:
|
|
2415
2413
|
self._err_redir.uninstall()
|
2416
2414
|
logger.info("restore done")
|
2417
2415
|
|
2418
|
-
def _atexit_cleanup(self, exit_code:
|
2416
|
+
def _atexit_cleanup(self, exit_code: int | None = None) -> None:
|
2419
2417
|
if self._backend is None:
|
2420
2418
|
logger.warning("process exited without backend configured")
|
2421
2419
|
return
|
@@ -2526,7 +2524,7 @@ class Run:
|
|
2526
2524
|
self,
|
2527
2525
|
) -> None:
|
2528
2526
|
def _telemetry_import_hook(
|
2529
|
-
run:
|
2527
|
+
run: Run,
|
2530
2528
|
module: Any,
|
2531
2529
|
) -> None:
|
2532
2530
|
with telemetry.context(run=run) as tel:
|
@@ -2569,7 +2567,7 @@ class Run:
|
|
2569
2567
|
|
2570
2568
|
StagedLaunchInputs().apply(self)
|
2571
2569
|
|
2572
|
-
def _make_job_source_reqs(self) ->
|
2570
|
+
def _make_job_source_reqs(self) -> tuple[list[str], dict[str, Any], dict[str, Any]]:
|
2573
2571
|
from wandb.util import working_set
|
2574
2572
|
|
2575
2573
|
installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
|
@@ -2581,10 +2579,10 @@ class Run:
|
|
2581
2579
|
def _construct_job_artifact(
|
2582
2580
|
self,
|
2583
2581
|
name: str,
|
2584
|
-
source_dict:
|
2585
|
-
installed_packages_list:
|
2586
|
-
patch_path:
|
2587
|
-
) ->
|
2582
|
+
source_dict: JobSourceDict,
|
2583
|
+
installed_packages_list: list[str],
|
2584
|
+
patch_path: os.PathLike | None = None,
|
2585
|
+
) -> Artifact:
|
2588
2586
|
job_artifact = job_builder.JobArtifact(name)
|
2589
2587
|
if patch_path and os.path.exists(patch_path):
|
2590
2588
|
job_artifact.add_file(FilePathStr(str(patch_path)), "diff.patch")
|
@@ -2597,12 +2595,12 @@ class Run:
|
|
2597
2595
|
|
2598
2596
|
def _create_image_job(
|
2599
2597
|
self,
|
2600
|
-
input_types:
|
2601
|
-
output_types:
|
2602
|
-
installed_packages_list:
|
2603
|
-
docker_image_name:
|
2604
|
-
args:
|
2605
|
-
) ->
|
2598
|
+
input_types: dict[str, Any],
|
2599
|
+
output_types: dict[str, Any],
|
2600
|
+
installed_packages_list: list[str],
|
2601
|
+
docker_image_name: str | None = None,
|
2602
|
+
args: list[str] | None = None,
|
2603
|
+
) -> Artifact | None:
|
2606
2604
|
docker_image_name = docker_image_name or os.getenv("WANDB_DOCKER")
|
2607
2605
|
|
2608
2606
|
if not docker_image_name:
|
@@ -2625,7 +2623,7 @@ class Run:
|
|
2625
2623
|
return job_artifact
|
2626
2624
|
|
2627
2625
|
def _log_job_artifact_with_image(
|
2628
|
-
self, docker_image_name: str, args:
|
2626
|
+
self, docker_image_name: str, args: list[str] | None = None
|
2629
2627
|
) -> Artifact:
|
2630
2628
|
packages, in_types, out_types = self._make_job_source_reqs()
|
2631
2629
|
job_artifact = self._create_image_job(
|
@@ -2654,16 +2652,20 @@ class Run:
|
|
2654
2652
|
handle = self._backend.interface.deliver_poll_exit()
|
2655
2653
|
probe_handle.set_mailbox_handle(handle)
|
2656
2654
|
|
2657
|
-
def _on_progress_exit(
|
2655
|
+
def _on_progress_exit(
|
2656
|
+
self,
|
2657
|
+
progress_printer: progress.ProgressPrinter,
|
2658
|
+
progress_handle: MailboxProgress,
|
2659
|
+
) -> None:
|
2658
2660
|
probe_handles = progress_handle.get_probe_handles()
|
2659
|
-
|
2661
|
+
if not probe_handles or len(probe_handles) != 1:
|
2662
|
+
return
|
2660
2663
|
|
2661
2664
|
result = probe_handles[0].get_probe_result()
|
2662
2665
|
if not result:
|
2663
2666
|
return
|
2664
|
-
|
2665
|
-
|
2666
|
-
)
|
2667
|
+
|
2668
|
+
progress_printer.update([result.response.poll_exit_response])
|
2667
2669
|
|
2668
2670
|
def _on_finish(self) -> None:
|
2669
2671
|
trigger.call("on_finished")
|
@@ -2678,16 +2680,28 @@ class Run:
|
|
2678
2680
|
exit_handle = self._backend.interface.deliver_exit(self._exit_code)
|
2679
2681
|
exit_handle.add_probe(on_probe=self._on_probe_exit)
|
2680
2682
|
|
2681
|
-
|
2682
|
-
|
2683
|
+
with progress.progress_printer(
|
2684
|
+
self._printer,
|
2685
|
+
self._settings,
|
2686
|
+
) as progress_printer:
|
2687
|
+
# Wait for the run to complete.
|
2688
|
+
_ = exit_handle.wait(
|
2689
|
+
timeout=-1,
|
2690
|
+
on_progress=functools.partial(
|
2691
|
+
self._on_progress_exit,
|
2692
|
+
progress_printer,
|
2693
|
+
),
|
2694
|
+
)
|
2683
2695
|
|
2696
|
+
# Print some final statistics.
|
2684
2697
|
poll_exit_handle = self._backend.interface.deliver_poll_exit()
|
2685
|
-
# wait for them, it's ok to do this serially but this can be improved
|
2686
2698
|
result = poll_exit_handle.wait(timeout=-1)
|
2687
2699
|
assert result
|
2688
|
-
|
2689
|
-
|
2700
|
+
progress.print_sync_dedupe_stats(
|
2701
|
+
self._printer,
|
2702
|
+
result.response.poll_exit_response,
|
2690
2703
|
)
|
2704
|
+
|
2691
2705
|
self._poll_exit_response = result.response.poll_exit_response
|
2692
2706
|
internal_messages_handle = self._backend.interface.deliver_internal_messages()
|
2693
2707
|
result = internal_messages_handle.wait(timeout=-1)
|
@@ -2728,12 +2742,12 @@ class Run:
|
|
2728
2742
|
def define_metric(
|
2729
2743
|
self,
|
2730
2744
|
name: str,
|
2731
|
-
step_metric:
|
2732
|
-
step_sync:
|
2733
|
-
hidden:
|
2734
|
-
summary:
|
2735
|
-
goal:
|
2736
|
-
overwrite:
|
2745
|
+
step_metric: str | wandb_metric.Metric | None = None,
|
2746
|
+
step_sync: bool | None = None,
|
2747
|
+
hidden: bool | None = None,
|
2748
|
+
summary: str | None = None,
|
2749
|
+
goal: str | None = None,
|
2750
|
+
overwrite: bool | None = None,
|
2737
2751
|
) -> wandb_metric.Metric:
|
2738
2752
|
"""Customize metrics logged with `wandb.log()`.
|
2739
2753
|
|
@@ -2789,12 +2803,12 @@ class Run:
|
|
2789
2803
|
def _define_metric(
|
2790
2804
|
self,
|
2791
2805
|
name: str,
|
2792
|
-
step_metric:
|
2793
|
-
step_sync:
|
2794
|
-
hidden:
|
2795
|
-
summary:
|
2796
|
-
goal:
|
2797
|
-
overwrite:
|
2806
|
+
step_metric: str | wandb_metric.Metric | None = None,
|
2807
|
+
step_sync: bool | None = None,
|
2808
|
+
hidden: bool | None = None,
|
2809
|
+
summary: str | None = None,
|
2810
|
+
goal: str | None = None,
|
2811
|
+
overwrite: bool | None = None,
|
2798
2812
|
) -> wandb_metric.Metric:
|
2799
2813
|
if not name:
|
2800
2814
|
raise wandb.Error("define_metric() requires non-empty name argument")
|
@@ -2820,7 +2834,7 @@ class Run:
|
|
2820
2834
|
raise wandb.Error(
|
2821
2835
|
f"Unhandled define_metric() arg: name (glob suffixes only): {name}"
|
2822
2836
|
)
|
2823
|
-
summary_ops:
|
2837
|
+
summary_ops: Sequence[str] | None = None
|
2824
2838
|
if summary:
|
2825
2839
|
summary_items = [s.lower() for s in summary.split(",")]
|
2826
2840
|
summary_ops = []
|
@@ -2833,7 +2847,7 @@ class Run:
|
|
2833
2847
|
with telemetry.context(run=self) as tel:
|
2834
2848
|
tel.feature.metric_summary = True
|
2835
2849
|
# TODO: deprecate goal
|
2836
|
-
goal_cleaned:
|
2850
|
+
goal_cleaned: str | None = None
|
2837
2851
|
if goal is not None:
|
2838
2852
|
goal_cleaned = goal[:3].lower()
|
2839
2853
|
valid_goal = {"min", "max"}
|
@@ -2864,26 +2878,57 @@ class Run:
|
|
2864
2878
|
m._commit()
|
2865
2879
|
return m
|
2866
2880
|
|
2867
|
-
# TODO(jhr): annotate this
|
2868
2881
|
@_run_decorator._attach
|
2869
|
-
def watch(
|
2882
|
+
def watch(
|
2870
2883
|
self,
|
2871
|
-
models,
|
2872
|
-
criterion=None,
|
2873
|
-
log="gradients",
|
2874
|
-
log_freq=
|
2875
|
-
idx=None,
|
2876
|
-
log_graph=False,
|
2884
|
+
models: torch.nn.Module | Sequence[torch.nn.Module],
|
2885
|
+
criterion: torch.F | None = None,
|
2886
|
+
log: Literal["gradients", "parameters", "all"] | None = "gradients",
|
2887
|
+
log_freq: int = 1000,
|
2888
|
+
idx: int | None = None,
|
2889
|
+
log_graph: bool = False,
|
2877
2890
|
) -> None:
|
2878
|
-
|
2891
|
+
"""Hooks into the given PyTorch model(s) to monitor gradients and the model's computational graph.
|
2892
|
+
|
2893
|
+
This function can track parameters, gradients, or both during training. It should be
|
2894
|
+
extended to support arbitrary machine learning models in the future.
|
2895
|
+
|
2896
|
+
Args:
|
2897
|
+
models (Union[torch.nn.Module, Sequence[torch.nn.Module]]):
|
2898
|
+
A single model or a sequence of models to be monitored.
|
2899
|
+
criterion (Optional[torch.F]):
|
2900
|
+
The loss function being optimized (optional).
|
2901
|
+
log (Optional[Literal["gradients", "parameters", "all"]]):
|
2902
|
+
Specifies whether to log "gradients", "parameters", or "all".
|
2903
|
+
Set to None to disable logging. (default="gradients")
|
2904
|
+
log_freq (int):
|
2905
|
+
Frequency (in batches) to log gradients and parameters. (default=1000)
|
2906
|
+
idx (Optional[int]):
|
2907
|
+
Index used when tracking multiple models with `wandb.watch`. (default=None)
|
2908
|
+
log_graph (bool):
|
2909
|
+
Whether to log the model's computational graph. (default=False)
|
2910
|
+
|
2911
|
+
Raises:
|
2912
|
+
ValueError:
|
2913
|
+
If `wandb.init` has not been called or if any of the models are not instances
|
2914
|
+
of `torch.nn.Module`.
|
2915
|
+
"""
|
2916
|
+
wandb.sdk._watch(self, models, criterion, log, log_freq, idx, log_graph)
|
2879
2917
|
|
2880
|
-
# TODO(jhr): annotate this
|
2881
2918
|
@_run_decorator._attach
|
2882
|
-
def unwatch(
|
2883
|
-
|
2919
|
+
def unwatch(
|
2920
|
+
self, models: torch.nn.Module | Sequence[torch.nn.Module] | None = None
|
2921
|
+
) -> None:
|
2922
|
+
"""Remove pytorch model topology, gradient and parameter hooks.
|
2923
|
+
|
2924
|
+
Args:
|
2925
|
+
models (torch.nn.Module | Sequence[torch.nn.Module]):
|
2926
|
+
Optional list of pytorch models that have had watch called on them
|
2927
|
+
"""
|
2928
|
+
wandb.sdk._unwatch(self, models=models)
|
2884
2929
|
|
2885
2930
|
# TODO(kdg): remove all artifact swapping logic
|
2886
|
-
def _swap_artifact_name(self, artifact_name: str, use_as:
|
2931
|
+
def _swap_artifact_name(self, artifact_name: str, use_as: str | None) -> str:
|
2887
2932
|
artifact_key_string = use_as or artifact_name
|
2888
2933
|
replacement_artifact_info = self._launch_artifact_mapping.get(
|
2889
2934
|
artifact_key_string
|
@@ -2926,7 +2971,7 @@ class Run:
|
|
2926
2971
|
self,
|
2927
2972
|
artifact: Artifact,
|
2928
2973
|
target_path: str,
|
2929
|
-
aliases:
|
2974
|
+
aliases: list[str] | None = None,
|
2930
2975
|
) -> None:
|
2931
2976
|
"""Link the given artifact to a portfolio (a promoted collection of artifacts).
|
2932
2977
|
|
@@ -2961,6 +3006,12 @@ class Run:
|
|
2961
3006
|
# Wait until the artifact is committed before trying to link it.
|
2962
3007
|
artifact.wait()
|
2963
3008
|
|
3009
|
+
organization = ""
|
3010
|
+
if is_artifact_registry_project(project):
|
3011
|
+
organization = entity
|
3012
|
+
# In a Registry linking, the entity is used to fetch the organization of the artifact
|
3013
|
+
# therefore the source artifact's entity is passed to the backend
|
3014
|
+
entity = artifact._source_entity
|
2964
3015
|
handle = self._backend.interface.deliver_link_artifact(
|
2965
3016
|
self,
|
2966
3017
|
artifact,
|
@@ -2968,6 +3019,7 @@ class Run:
|
|
2968
3019
|
aliases,
|
2969
3020
|
entity,
|
2970
3021
|
project,
|
3022
|
+
organization,
|
2971
3023
|
)
|
2972
3024
|
if artifact._ttl_duration_seconds is not None:
|
2973
3025
|
wandb.termwarn(
|
@@ -2985,10 +3037,10 @@ class Run:
|
|
2985
3037
|
@_run_decorator._attach
|
2986
3038
|
def use_artifact(
|
2987
3039
|
self,
|
2988
|
-
artifact_or_name:
|
2989
|
-
type:
|
2990
|
-
aliases:
|
2991
|
-
use_as:
|
3040
|
+
artifact_or_name: str | Artifact,
|
3041
|
+
type: str | None = None,
|
3042
|
+
aliases: list[str] | None = None,
|
3043
|
+
use_as: str | None = None,
|
2992
3044
|
) -> Artifact:
|
2993
3045
|
"""Declare an artifact as an input to a run.
|
2994
3046
|
|
@@ -2996,8 +3048,9 @@ class Run:
|
|
2996
3048
|
|
2997
3049
|
Arguments:
|
2998
3050
|
artifact_or_name: (str or Artifact) An artifact name.
|
2999
|
-
May be prefixed with entity/project/.
|
3000
|
-
|
3051
|
+
May be prefixed with project/ or entity/project/.
|
3052
|
+
If no entity is specified in the name, the Run or API setting's entity is used.
|
3053
|
+
Valid names can be in the following forms:
|
3001
3054
|
- name:version
|
3002
3055
|
- name:alias
|
3003
3056
|
You can also pass an Artifact object created by calling `wandb.Artifact`
|
@@ -3094,11 +3147,11 @@ class Run:
|
|
3094
3147
|
@_run_decorator._attach
|
3095
3148
|
def log_artifact(
|
3096
3149
|
self,
|
3097
|
-
artifact_or_path:
|
3098
|
-
name:
|
3099
|
-
type:
|
3100
|
-
aliases:
|
3101
|
-
tags:
|
3150
|
+
artifact_or_path: Artifact | StrPath,
|
3151
|
+
name: str | None = None,
|
3152
|
+
type: str | None = None,
|
3153
|
+
aliases: list[str] | None = None,
|
3154
|
+
tags: list[str] | None = None,
|
3102
3155
|
) -> Artifact:
|
3103
3156
|
"""Declare an artifact as an output of a run.
|
3104
3157
|
|
@@ -3136,11 +3189,11 @@ class Run:
|
|
3136
3189
|
@_run_decorator._attach
|
3137
3190
|
def upsert_artifact(
|
3138
3191
|
self,
|
3139
|
-
artifact_or_path:
|
3140
|
-
name:
|
3141
|
-
type:
|
3142
|
-
aliases:
|
3143
|
-
distributed_id:
|
3192
|
+
artifact_or_path: Artifact | str,
|
3193
|
+
name: str | None = None,
|
3194
|
+
type: str | None = None,
|
3195
|
+
aliases: list[str] | None = None,
|
3196
|
+
distributed_id: str | None = None,
|
3144
3197
|
) -> Artifact:
|
3145
3198
|
"""Declare (or append to) a non-finalized artifact as output of a run.
|
3146
3199
|
|
@@ -3190,11 +3243,11 @@ class Run:
|
|
3190
3243
|
@_run_decorator._attach
|
3191
3244
|
def finish_artifact(
|
3192
3245
|
self,
|
3193
|
-
artifact_or_path:
|
3194
|
-
name:
|
3195
|
-
type:
|
3196
|
-
aliases:
|
3197
|
-
distributed_id:
|
3246
|
+
artifact_or_path: Artifact | str,
|
3247
|
+
name: str | None = None,
|
3248
|
+
type: str | None = None,
|
3249
|
+
aliases: list[str] | None = None,
|
3250
|
+
distributed_id: str | None = None,
|
3198
3251
|
) -> Artifact:
|
3199
3252
|
"""Finishes a non-finalized artifact as output of a run.
|
3200
3253
|
|
@@ -3242,12 +3295,12 @@ class Run:
|
|
3242
3295
|
|
3243
3296
|
def _log_artifact(
|
3244
3297
|
self,
|
3245
|
-
artifact_or_path:
|
3246
|
-
name:
|
3247
|
-
type:
|
3248
|
-
aliases:
|
3249
|
-
tags:
|
3250
|
-
distributed_id:
|
3298
|
+
artifact_or_path: Artifact | StrPath,
|
3299
|
+
name: str | None = None,
|
3300
|
+
type: str | None = None,
|
3301
|
+
aliases: list[str] | None = None,
|
3302
|
+
tags: list[str] | None = None,
|
3303
|
+
distributed_id: str | None = None,
|
3251
3304
|
finalize: bool = True,
|
3252
3305
|
is_user_created: bool = False,
|
3253
3306
|
use_after_commit: bool = False,
|
@@ -3308,7 +3361,7 @@ class Run:
|
|
3308
3361
|
)
|
3309
3362
|
return artifact
|
3310
3363
|
|
3311
|
-
def _public_api(self, overrides:
|
3364
|
+
def _public_api(self, overrides: dict[str, str] | None = None) -> PublicApi:
|
3312
3365
|
overrides = {"run": self._run_id}
|
3313
3366
|
if not (self._settings._offline or self._run_obj is None):
|
3314
3367
|
overrides["entity"] = self._run_obj.entity
|
@@ -3349,11 +3402,11 @@ class Run:
|
|
3349
3402
|
|
3350
3403
|
def _prepare_artifact(
|
3351
3404
|
self,
|
3352
|
-
artifact_or_path:
|
3353
|
-
name:
|
3354
|
-
type:
|
3355
|
-
aliases:
|
3356
|
-
) ->
|
3405
|
+
artifact_or_path: Artifact | StrPath,
|
3406
|
+
name: str | None = None,
|
3407
|
+
type: str | None = None,
|
3408
|
+
aliases: list[str] | None = None,
|
3409
|
+
) -> tuple[Artifact, list[str]]:
|
3357
3410
|
if isinstance(artifact_or_path, (str, os.PathLike)):
|
3358
3411
|
name = name or f"run-{self._run_id}-{os.path.basename(artifact_or_path)}"
|
3359
3412
|
artifact = wandb.Artifact(name, type or "unspecified")
|
@@ -3384,8 +3437,8 @@ class Run:
|
|
3384
3437
|
def log_model(
|
3385
3438
|
self,
|
3386
3439
|
path: StrPath,
|
3387
|
-
name:
|
3388
|
-
aliases:
|
3440
|
+
name: str | None = None,
|
3441
|
+
aliases: list[str] | None = None,
|
3389
3442
|
) -> None:
|
3390
3443
|
"""Logs a model artifact containing the contents inside the 'path' to a run and marks it as an output to this run.
|
3391
3444
|
|
@@ -3489,8 +3542,8 @@ class Run:
|
|
3489
3542
|
self,
|
3490
3543
|
path: StrPath,
|
3491
3544
|
registered_model_name: str,
|
3492
|
-
name:
|
3493
|
-
aliases:
|
3545
|
+
name: str | None = None,
|
3546
|
+
aliases: list[str] | None = None,
|
3494
3547
|
) -> None:
|
3495
3548
|
"""Log a model artifact version and link it to a registered model in the model registry.
|
3496
3549
|
|
@@ -3583,8 +3636,8 @@ class Run:
|
|
3583
3636
|
self,
|
3584
3637
|
title: str,
|
3585
3638
|
text: str,
|
3586
|
-
level:
|
3587
|
-
wait_duration:
|
3639
|
+
level: str | AlertLevel | None = None,
|
3640
|
+
wait_duration: int | float | timedelta | None = None,
|
3588
3641
|
) -> None:
|
3589
3642
|
"""Launch an alert with the given title and text.
|
3590
3643
|
|
@@ -3612,12 +3665,12 @@ class Run:
|
|
3612
3665
|
if self._backend and self._backend.interface:
|
3613
3666
|
self._backend.interface.publish_alert(title, text, level_str, wait_duration)
|
3614
3667
|
|
3615
|
-
def __enter__(self) ->
|
3668
|
+
def __enter__(self) -> Run:
|
3616
3669
|
return self
|
3617
3670
|
|
3618
3671
|
def __exit__(
|
3619
3672
|
self,
|
3620
|
-
exc_type:
|
3673
|
+
exc_type: type[BaseException],
|
3621
3674
|
exc_val: BaseException,
|
3622
3675
|
exc_tb: TracebackType,
|
3623
3676
|
) -> bool:
|
@@ -3641,7 +3694,7 @@ class Run:
|
|
3641
3694
|
@property
|
3642
3695
|
@_run_decorator._noop_on_finish()
|
3643
3696
|
@_run_decorator._attach
|
3644
|
-
def _system_metrics(self) ->
|
3697
|
+
def _system_metrics(self) -> dict[str, list[tuple[datetime, float]]]:
|
3645
3698
|
"""Returns a dictionary of system metrics.
|
3646
3699
|
|
3647
3700
|
Returns:
|
@@ -3650,7 +3703,7 @@ class Run:
|
|
3650
3703
|
|
3651
3704
|
def pb_to_dict(
|
3652
3705
|
system_metrics_pb: wandb.proto.wandb_internal_pb2.GetSystemMetricsResponse,
|
3653
|
-
) ->
|
3706
|
+
) -> dict[str, list[tuple[datetime, float]]]:
|
3654
3707
|
res = {}
|
3655
3708
|
|
3656
3709
|
for metric, records in system_metrics_pb.system_metrics.items():
|
@@ -3691,8 +3744,8 @@ class Run:
|
|
3691
3744
|
@staticmethod
|
3692
3745
|
def _header(
|
3693
3746
|
*,
|
3694
|
-
settings:
|
3695
|
-
printer:
|
3747
|
+
settings: Settings,
|
3748
|
+
printer: printer.Printer,
|
3696
3749
|
) -> None:
|
3697
3750
|
Run._header_wandb_version_info(settings=settings, printer=printer)
|
3698
3751
|
Run._header_sync_info(settings=settings, printer=printer)
|
@@ -3701,22 +3754,20 @@ class Run:
|
|
3701
3754
|
@staticmethod
|
3702
3755
|
def _header_wandb_version_info(
|
3703
3756
|
*,
|
3704
|
-
settings:
|
3705
|
-
printer:
|
3757
|
+
settings: Settings,
|
3758
|
+
printer: printer.Printer,
|
3706
3759
|
) -> None:
|
3707
3760
|
if settings.quiet or settings.silent:
|
3708
3761
|
return
|
3709
3762
|
|
3710
3763
|
# TODO: add this to a higher verbosity level
|
3711
|
-
printer.display(
|
3712
|
-
f"Tracking run with wandb version {wandb.__version__}", off=False
|
3713
|
-
)
|
3764
|
+
printer.display(f"Tracking run with wandb version {wandb.__version__}")
|
3714
3765
|
|
3715
3766
|
@staticmethod
|
3716
3767
|
def _header_sync_info(
|
3717
3768
|
*,
|
3718
|
-
settings:
|
3719
|
-
printer:
|
3769
|
+
settings: Settings,
|
3770
|
+
printer: printer.Printer,
|
3720
3771
|
) -> None:
|
3721
3772
|
if settings._offline:
|
3722
3773
|
printer.display(
|
@@ -3728,17 +3779,18 @@ class Run:
|
|
3728
3779
|
)
|
3729
3780
|
else:
|
3730
3781
|
info = [f"Run data is saved locally in {printer.files(settings.sync_dir)}"]
|
3731
|
-
if not printer.
|
3782
|
+
if not printer.supports_html:
|
3732
3783
|
info.append(
|
3733
3784
|
f"Run {printer.code('`wandb offline`')} to turn off syncing."
|
3734
3785
|
)
|
3735
|
-
|
3786
|
+
if not settings.quiet and not settings.silent:
|
3787
|
+
printer.display(info)
|
3736
3788
|
|
3737
3789
|
@staticmethod
|
3738
3790
|
def _header_run_info(
|
3739
3791
|
*,
|
3740
|
-
settings:
|
3741
|
-
printer:
|
3792
|
+
settings: Settings,
|
3793
|
+
printer: printer.Printer,
|
3742
3794
|
) -> None:
|
3743
3795
|
if settings._offline or settings.silent:
|
3744
3796
|
return
|
@@ -3756,7 +3808,7 @@ class Run:
|
|
3756
3808
|
if not run_name:
|
3757
3809
|
return
|
3758
3810
|
|
3759
|
-
if printer.
|
3811
|
+
if printer.supports_html:
|
3760
3812
|
if not wandb.jupyter.maybe_display(): # type: ignore
|
3761
3813
|
run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
|
3762
3814
|
project_line, sweep_line = "", ""
|
@@ -3775,10 +3827,8 @@ class Run:
|
|
3775
3827
|
[f"{run_state_str} {run_line} {project_line}", sweep_line],
|
3776
3828
|
)
|
3777
3829
|
|
3778
|
-
|
3779
|
-
printer.display(
|
3780
|
-
f"{run_state_str} {printer.name(run_name)}", off=not run_name
|
3781
|
-
)
|
3830
|
+
elif run_name:
|
3831
|
+
printer.display(f"{run_state_str} {printer.name(run_name)}")
|
3782
3832
|
|
3783
3833
|
if not settings.quiet:
|
3784
3834
|
# TODO: add verbosity levels and add this to higher levels
|
@@ -3794,11 +3844,13 @@ class Run:
|
|
3794
3844
|
)
|
3795
3845
|
|
3796
3846
|
# TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
|
3797
|
-
if Api().api.settings().get("anonymous") == "true":
|
3847
|
+
if run_name and Api().api.settings().get("anonymous") == "true":
|
3798
3848
|
printer.display(
|
3799
|
-
|
3849
|
+
(
|
3850
|
+
"Do NOT share these links with anyone."
|
3851
|
+
" They can be used to claim your runs."
|
3852
|
+
),
|
3800
3853
|
level="warn",
|
3801
|
-
off=not run_name,
|
3802
3854
|
)
|
3803
3855
|
|
3804
3856
|
# ------------------------------------------------------------------------------
|
@@ -3808,15 +3860,15 @@ class Run:
|
|
3808
3860
|
# with the service execution path that doesn't have access to the run instance
|
3809
3861
|
@staticmethod
|
3810
3862
|
def _footer(
|
3811
|
-
sampled_history:
|
3812
|
-
final_summary:
|
3813
|
-
poll_exit_response:
|
3814
|
-
internal_messages_response:
|
3815
|
-
reporter:
|
3816
|
-
quiet:
|
3863
|
+
sampled_history: SampledHistoryResponse | None = None,
|
3864
|
+
final_summary: GetSummaryResponse | None = None,
|
3865
|
+
poll_exit_response: PollExitResponse | None = None,
|
3866
|
+
internal_messages_response: InternalMessagesResponse | None = None,
|
3867
|
+
reporter: Reporter | None = None,
|
3868
|
+
quiet: bool | None = None,
|
3817
3869
|
*,
|
3818
|
-
settings:
|
3819
|
-
printer:
|
3870
|
+
settings: Settings,
|
3871
|
+
printer: printer.Printer,
|
3820
3872
|
) -> None:
|
3821
3873
|
Run._footer_history_summary_info(
|
3822
3874
|
history=sampled_history,
|
@@ -3848,177 +3900,51 @@ class Run:
|
|
3848
3900
|
reporter=reporter, quiet=quiet, settings=settings, printer=printer
|
3849
3901
|
)
|
3850
3902
|
|
3851
|
-
# fixme: Temporary hack until we move to rich which allows multiple spinners
|
3852
3903
|
@staticmethod
|
3853
|
-
def
|
3854
|
-
|
3855
|
-
|
3856
|
-
] = None,
|
3857
|
-
*,
|
3858
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3859
|
-
) -> None:
|
3860
|
-
if not poll_exit_responses:
|
3861
|
-
return
|
3862
|
-
if isinstance(poll_exit_responses, PollExitResponse):
|
3863
|
-
Run._footer_single_run_file_pusher_status_info(
|
3864
|
-
poll_exit_responses, printer=printer
|
3865
|
-
)
|
3866
|
-
elif isinstance(poll_exit_responses, list):
|
3867
|
-
poll_exit_responses_list = poll_exit_responses
|
3868
|
-
assert all(
|
3869
|
-
response is None or isinstance(response, PollExitResponse)
|
3870
|
-
for response in poll_exit_responses_list
|
3871
|
-
)
|
3872
|
-
if len(poll_exit_responses_list) == 0:
|
3873
|
-
return
|
3874
|
-
elif len(poll_exit_responses_list) == 1:
|
3875
|
-
Run._footer_single_run_file_pusher_status_info(
|
3876
|
-
poll_exit_responses_list[0], printer=printer
|
3877
|
-
)
|
3878
|
-
else:
|
3879
|
-
Run._footer_multiple_runs_file_pusher_status_info(
|
3880
|
-
poll_exit_responses_list, printer=printer
|
3881
|
-
)
|
3882
|
-
else:
|
3883
|
-
logger.error(
|
3884
|
-
f"Got the type `{type(poll_exit_responses)}` for `poll_exit_responses`. "
|
3885
|
-
"Expected either None, PollExitResponse or a List[Union[PollExitResponse, None]]"
|
3886
|
-
)
|
3887
|
-
|
3888
|
-
@staticmethod
|
3889
|
-
def _footer_single_run_file_pusher_status_info(
|
3890
|
-
poll_exit_response: Optional[PollExitResponse] = None,
|
3904
|
+
def _footer_sync_info(
|
3905
|
+
poll_exit_response: PollExitResponse | None = None,
|
3906
|
+
quiet: bool | None = None,
|
3891
3907
|
*,
|
3892
|
-
|
3908
|
+
settings: Settings,
|
3909
|
+
printer: printer.Printer,
|
3893
3910
|
) -> None:
|
3894
|
-
|
3895
|
-
if not poll_exit_response:
|
3911
|
+
if settings.silent:
|
3896
3912
|
return
|
3897
3913
|
|
3898
|
-
|
3899
|
-
|
3900
|
-
megabyte = wandb.util.POW_2_BYTES[2][1]
|
3901
|
-
line = (
|
3902
|
-
f"{stats.uploaded_bytes / megabyte:.3f} MB"
|
3903
|
-
f" of {stats.total_bytes / megabyte:.3f} MB uploaded"
|
3904
|
-
)
|
3905
|
-
if stats.deduped_bytes > 0:
|
3906
|
-
line += f" ({stats.deduped_bytes / megabyte:.3f} MB deduped)"
|
3907
|
-
line += "\r"
|
3908
|
-
|
3909
|
-
if stats.total_bytes > 0:
|
3910
|
-
printer.progress_update(line, stats.uploaded_bytes / stats.total_bytes)
|
3911
|
-
else:
|
3912
|
-
printer.progress_update(line, 1.0)
|
3913
|
-
|
3914
|
-
if poll_exit_response.done:
|
3915
|
-
printer.progress_close()
|
3916
|
-
|
3917
|
-
if stats.total_bytes > 0:
|
3918
|
-
dedupe_fraction = stats.deduped_bytes / float(stats.total_bytes)
|
3919
|
-
else:
|
3920
|
-
dedupe_fraction = 0
|
3921
|
-
|
3922
|
-
if stats.deduped_bytes > 0.01:
|
3914
|
+
if settings._offline:
|
3915
|
+
if not quiet and not settings.quiet:
|
3923
3916
|
printer.display(
|
3924
|
-
|
3917
|
+
[
|
3918
|
+
"You can sync this run to the cloud by running:",
|
3919
|
+
printer.code(f"wandb sync {settings.sync_dir}"),
|
3920
|
+
],
|
3925
3921
|
)
|
3926
|
-
|
3927
|
-
@staticmethod
|
3928
|
-
def _footer_multiple_runs_file_pusher_status_info(
|
3929
|
-
poll_exit_responses: List[Optional[PollExitResponse]],
|
3930
|
-
*,
|
3931
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3932
|
-
) -> None:
|
3933
|
-
# todo: is this same as settings._offline?
|
3934
|
-
if not all(poll_exit_responses):
|
3935
3922
|
return
|
3936
3923
|
|
3937
|
-
|
3938
|
-
|
3939
|
-
|
3940
|
-
|
3941
|
-
response.file_counts.wandb_count,
|
3942
|
-
response.file_counts.media_count,
|
3943
|
-
response.file_counts.artifact_count,
|
3944
|
-
response.file_counts.other_count,
|
3945
|
-
]
|
3924
|
+
info = []
|
3925
|
+
if settings.run_name and settings.run_url:
|
3926
|
+
info.append(
|
3927
|
+
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
|
3946
3928
|
)
|
3947
|
-
|
3948
|
-
|
3949
|
-
|
3950
|
-
uploaded = sum(
|
3951
|
-
response.pusher_stats.uploaded_bytes
|
3952
|
-
for response in poll_exit_responses
|
3953
|
-
if response is not None and response.pusher_stats is not None
|
3954
|
-
)
|
3955
|
-
total = sum(
|
3956
|
-
response.pusher_stats.total_bytes
|
3957
|
-
for response in poll_exit_responses
|
3958
|
-
if response is not None and response.pusher_stats is not None
|
3959
|
-
)
|
3960
|
-
|
3961
|
-
line = (
|
3962
|
-
f"Processing {len(poll_exit_responses)} runs with {total_files} files "
|
3963
|
-
f"({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r"
|
3964
|
-
)
|
3965
|
-
# line = "{}{:<{max_len}}\r".format(line, " ", max_len=(80 - len(line)))
|
3966
|
-
printer.progress_update(line) # type:ignore[call-arg]
|
3967
|
-
|
3968
|
-
done = all(
|
3969
|
-
[
|
3970
|
-
poll_exit_response.done
|
3971
|
-
for poll_exit_response in poll_exit_responses
|
3972
|
-
if poll_exit_response
|
3973
|
-
]
|
3974
|
-
)
|
3975
|
-
if done:
|
3976
|
-
printer.progress_close()
|
3977
|
-
|
3978
|
-
@staticmethod
|
3979
|
-
def _footer_sync_info(
|
3980
|
-
poll_exit_response: Optional[PollExitResponse] = None,
|
3981
|
-
quiet: Optional[bool] = None,
|
3982
|
-
*,
|
3983
|
-
settings: "Settings",
|
3984
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3985
|
-
) -> None:
|
3986
|
-
if settings.silent:
|
3987
|
-
return
|
3988
|
-
|
3989
|
-
if settings._offline:
|
3990
|
-
printer.display(
|
3991
|
-
[
|
3992
|
-
"You can sync this run to the cloud by running:",
|
3993
|
-
printer.code(f"wandb sync {settings.sync_dir}"),
|
3994
|
-
],
|
3995
|
-
off=(quiet or settings.quiet),
|
3929
|
+
if settings.project_url:
|
3930
|
+
info.append(
|
3931
|
+
f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
|
3996
3932
|
)
|
3997
|
-
|
3998
|
-
info
|
3999
|
-
|
4000
|
-
|
4001
|
-
|
4002
|
-
)
|
4003
|
-
|
4004
|
-
|
4005
|
-
f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
|
4006
|
-
)
|
4007
|
-
if poll_exit_response and poll_exit_response.file_counts:
|
4008
|
-
logger.info("logging synced files")
|
4009
|
-
file_counts = poll_exit_response.file_counts
|
4010
|
-
info.append(
|
4011
|
-
f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
|
4012
|
-
f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
|
4013
|
-
)
|
4014
|
-
printer.display(info)
|
3933
|
+
if poll_exit_response and poll_exit_response.file_counts:
|
3934
|
+
logger.info("logging synced files")
|
3935
|
+
file_counts = poll_exit_response.file_counts
|
3936
|
+
info.append(
|
3937
|
+
f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
|
3938
|
+
f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
|
3939
|
+
)
|
3940
|
+
printer.display(info)
|
4015
3941
|
|
4016
3942
|
@staticmethod
|
4017
3943
|
def _footer_log_dir_info(
|
4018
|
-
quiet:
|
3944
|
+
quiet: bool | None = None,
|
4019
3945
|
*,
|
4020
|
-
settings:
|
4021
|
-
printer:
|
3946
|
+
settings: Settings,
|
3947
|
+
printer: printer.Printer,
|
4022
3948
|
) -> None:
|
4023
3949
|
if (quiet or settings.quiet) or settings.silent:
|
4024
3950
|
return
|
@@ -4032,12 +3958,12 @@ class Run:
|
|
4032
3958
|
|
4033
3959
|
@staticmethod
|
4034
3960
|
def _footer_history_summary_info(
|
4035
|
-
history:
|
4036
|
-
summary:
|
4037
|
-
quiet:
|
3961
|
+
history: SampledHistoryResponse | None = None,
|
3962
|
+
summary: GetSummaryResponse | None = None,
|
3963
|
+
quiet: bool | None = None,
|
4038
3964
|
*,
|
4039
|
-
settings:
|
4040
|
-
printer:
|
3965
|
+
settings: Settings,
|
3966
|
+
printer: printer.Printer,
|
4041
3967
|
) -> None:
|
4042
3968
|
if (quiet or settings.quiet) or settings.silent:
|
4043
3969
|
return
|
@@ -4103,11 +4029,11 @@ class Run:
|
|
4103
4029
|
|
4104
4030
|
@staticmethod
|
4105
4031
|
def _footer_internal_messages(
|
4106
|
-
internal_messages_response:
|
4107
|
-
quiet:
|
4032
|
+
internal_messages_response: InternalMessagesResponse | None = None,
|
4033
|
+
quiet: bool | None = None,
|
4108
4034
|
*,
|
4109
|
-
settings:
|
4110
|
-
printer:
|
4035
|
+
settings: Settings,
|
4036
|
+
printer: printer.Printer,
|
4111
4037
|
) -> None:
|
4112
4038
|
if (quiet or settings.quiet) or settings.silent:
|
4113
4039
|
return
|
@@ -4121,9 +4047,9 @@ class Run:
|
|
4121
4047
|
@staticmethod
|
4122
4048
|
def _footer_notify_wandb_core(
|
4123
4049
|
*,
|
4124
|
-
quiet:
|
4125
|
-
settings:
|
4126
|
-
printer:
|
4050
|
+
quiet: bool | None = None,
|
4051
|
+
settings: Settings,
|
4052
|
+
printer: printer.Printer,
|
4127
4053
|
) -> None:
|
4128
4054
|
"""Prints a message advertising the upcoming core release."""
|
4129
4055
|
if quiet or not settings._require_legacy_service:
|
@@ -4138,11 +4064,11 @@ class Run:
|
|
4138
4064
|
|
4139
4065
|
@staticmethod
|
4140
4066
|
def _footer_reporter_warn_err(
|
4141
|
-
reporter:
|
4142
|
-
quiet:
|
4067
|
+
reporter: Reporter | None = None,
|
4068
|
+
quiet: bool | None = None,
|
4143
4069
|
*,
|
4144
|
-
settings:
|
4145
|
-
printer:
|
4070
|
+
settings: Settings,
|
4071
|
+
printer: printer.Printer,
|
4146
4072
|
) -> None:
|
4147
4073
|
if (quiet or settings.quiet) or settings.silent:
|
4148
4074
|
return
|
@@ -4168,10 +4094,10 @@ class Run:
|
|
4168
4094
|
# We define this outside of the run context to support restoring before init
|
4169
4095
|
def restore(
|
4170
4096
|
name: str,
|
4171
|
-
run_path:
|
4097
|
+
run_path: str | None = None,
|
4172
4098
|
replace: bool = False,
|
4173
|
-
root:
|
4174
|
-
) ->
|
4099
|
+
root: str | None = None,
|
4100
|
+
) -> None | TextIO:
|
4175
4101
|
"""Download the specified file from cloud storage.
|
4176
4102
|
|
4177
4103
|
File is placed into the current directory or run directory.
|
@@ -4229,7 +4155,7 @@ except AttributeError:
|
|
4229
4155
|
pass
|
4230
4156
|
|
4231
4157
|
|
4232
|
-
def finish(exit_code:
|
4158
|
+
def finish(exit_code: int | None = None, quiet: bool | None = None) -> None:
|
4233
4159
|
"""Mark a run as finished, and finish uploading all data.
|
4234
4160
|
|
4235
4161
|
This is used when creating multiple runs in the same process.
|