wandb 0.18.2__py3-none-win32.whl → 0.18.4__py3-none-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +16 -7
- wandb/__init__.pyi +96 -63
- wandb/analytics/sentry.py +91 -88
- wandb/apis/public/api.py +18 -4
- wandb/apis/public/runs.py +53 -2
- wandb/bin/gpu_stats.exe +0 -0
- wandb/bin/wandb-core +0 -0
- wandb/cli/beta.py +178 -0
- wandb/cli/cli.py +5 -171
- wandb/data_types.py +3 -0
- wandb/env.py +74 -73
- wandb/errors/term.py +300 -43
- wandb/proto/v3/wandb_internal_pb2.py +271 -221
- wandb/proto/v3/wandb_server_pb2.py +57 -37
- wandb/proto/v3/wandb_settings_pb2.py +2 -2
- wandb/proto/v4/wandb_internal_pb2.py +226 -216
- wandb/proto/v4/wandb_server_pb2.py +41 -37
- wandb/proto/v4/wandb_settings_pb2.py +2 -2
- wandb/proto/v5/wandb_internal_pb2.py +226 -216
- wandb/proto/v5/wandb_server_pb2.py +41 -37
- wandb/proto/v5/wandb_settings_pb2.py +2 -2
- wandb/sdk/__init__.py +3 -3
- wandb/sdk/artifacts/_validators.py +41 -8
- wandb/sdk/artifacts/artifact.py +35 -4
- wandb/sdk/artifacts/artifact_file_cache.py +1 -2
- wandb/sdk/data_types/_dtypes.py +7 -3
- wandb/sdk/data_types/video.py +15 -6
- wandb/sdk/interface/interface.py +2 -0
- wandb/sdk/internal/internal_api.py +122 -5
- wandb/sdk/internal/sender.py +16 -3
- wandb/sdk/launch/inputs/internal.py +1 -1
- wandb/sdk/lib/module.py +12 -0
- wandb/sdk/lib/printer.py +291 -105
- wandb/sdk/lib/progress.py +274 -0
- wandb/sdk/service/streams.py +21 -11
- wandb/sdk/wandb_init.py +59 -54
- wandb/sdk/wandb_run.py +413 -480
- wandb/sdk/wandb_settings.py +2 -0
- wandb/sdk/wandb_watch.py +17 -11
- wandb/util.py +6 -2
- {wandb-0.18.2.dist-info → wandb-0.18.4.dist-info}/METADATA +5 -4
- {wandb-0.18.2.dist-info → wandb-0.18.4.dist-info}/RECORD +45 -42
- {wandb-0.18.2.dist-info → wandb-0.18.4.dist-info}/WHEEL +0 -0
- {wandb-0.18.2.dist-info → wandb-0.18.4.dist-info}/entry_points.txt +0 -0
- {wandb-0.18.2.dist-info → wandb-0.18.4.dist-info}/licenses/LICENSE +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
1
3
|
import _thread as thread
|
2
4
|
import atexit
|
3
5
|
import functools
|
@@ -18,20 +20,12 @@ from dataclasses import dataclass, field
|
|
18
20
|
from datetime import datetime, timedelta, timezone
|
19
21
|
from enum import IntEnum
|
20
22
|
from types import TracebackType
|
21
|
-
from typing import
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
NamedTuple,
|
28
|
-
Optional,
|
29
|
-
Sequence,
|
30
|
-
TextIO,
|
31
|
-
Tuple,
|
32
|
-
Type,
|
33
|
-
Union,
|
34
|
-
)
|
23
|
+
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Sequence, TextIO
|
24
|
+
|
25
|
+
if sys.version_info < (3, 8):
|
26
|
+
from typing_extensions import Literal
|
27
|
+
else:
|
28
|
+
from typing import Literal
|
35
29
|
|
36
30
|
import requests
|
37
31
|
|
@@ -69,7 +63,11 @@ from wandb.util import (
|
|
69
63
|
)
|
70
64
|
|
71
65
|
from . import wandb_config, wandb_metric, wandb_summary
|
72
|
-
from .artifacts._validators import
|
66
|
+
from .artifacts._validators import (
|
67
|
+
is_artifact_registry_project,
|
68
|
+
validate_aliases,
|
69
|
+
validate_tags,
|
70
|
+
)
|
73
71
|
from .data_types._dtypes import TypeRegistry
|
74
72
|
from .interface.interface import FilesDict, GlobStr, InterfaceBase, PolicyName
|
75
73
|
from .interface.summary_record import SummaryRecord
|
@@ -80,6 +78,8 @@ from .lib import (
|
|
80
78
|
filesystem,
|
81
79
|
ipython,
|
82
80
|
module,
|
81
|
+
printer,
|
82
|
+
progress,
|
83
83
|
proto_util,
|
84
84
|
redirect,
|
85
85
|
telemetry,
|
@@ -87,7 +87,6 @@ from .lib import (
|
|
87
87
|
from .lib.exit_hooks import ExitHooks
|
88
88
|
from .lib.gitlib import GitRepo
|
89
89
|
from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
|
90
|
-
from .lib.printer import get_printer
|
91
90
|
from .lib.proto_util import message_to_dict
|
92
91
|
from .lib.reporting import Reporter
|
93
92
|
from .lib.wburls import wburls
|
@@ -101,26 +100,27 @@ if TYPE_CHECKING:
|
|
101
100
|
else:
|
102
101
|
from typing_extensions import TypedDict
|
103
102
|
|
103
|
+
import torch # type: ignore [import-not-found]
|
104
|
+
|
104
105
|
import wandb.apis.public
|
105
106
|
import wandb.sdk.backend.backend
|
106
107
|
import wandb.sdk.interface.interface_queue
|
108
|
+
from wandb.data_types import Table
|
107
109
|
from wandb.proto.wandb_internal_pb2 import (
|
108
110
|
GetSummaryResponse,
|
109
111
|
InternalMessagesResponse,
|
110
112
|
SampledHistoryResponse,
|
111
113
|
)
|
112
114
|
|
113
|
-
from .lib.printer import PrinterJupyter, PrinterTerm
|
114
|
-
|
115
115
|
class GitSourceDict(TypedDict):
|
116
116
|
remote: str
|
117
117
|
commit: str
|
118
|
-
entrypoint:
|
118
|
+
entrypoint: list[str]
|
119
119
|
args: Sequence[str]
|
120
120
|
|
121
121
|
class ArtifactSourceDict(TypedDict):
|
122
122
|
artifact: str
|
123
|
-
entrypoint:
|
123
|
+
entrypoint: list[str]
|
124
124
|
args: Sequence[str]
|
125
125
|
|
126
126
|
class ImageSourceDict(TypedDict):
|
@@ -130,10 +130,10 @@ if TYPE_CHECKING:
|
|
130
130
|
class JobSourceDict(TypedDict, total=False):
|
131
131
|
_version: str
|
132
132
|
source_type: str
|
133
|
-
source:
|
134
|
-
input_types:
|
135
|
-
output_types:
|
136
|
-
runtime:
|
133
|
+
source: GitSourceDict | ArtifactSourceDict | ImageSourceDict
|
134
|
+
input_types: dict[str, Any]
|
135
|
+
output_types: dict[str, Any]
|
136
|
+
runtime: str | None
|
137
137
|
|
138
138
|
|
139
139
|
logger = logging.getLogger("wandb")
|
@@ -160,11 +160,11 @@ class RunStatusChecker:
|
|
160
160
|
"""
|
161
161
|
|
162
162
|
_stop_status_lock: threading.Lock
|
163
|
-
_stop_status_handle:
|
163
|
+
_stop_status_handle: MailboxHandle | None
|
164
164
|
_network_status_lock: threading.Lock
|
165
|
-
_network_status_handle:
|
165
|
+
_network_status_handle: MailboxHandle | None
|
166
166
|
_internal_messages_lock: threading.Lock
|
167
|
-
_internal_messages_handle:
|
167
|
+
_internal_messages_handle: MailboxHandle | None
|
168
168
|
|
169
169
|
def __init__(
|
170
170
|
self,
|
@@ -212,7 +212,7 @@ class RunStatusChecker:
|
|
212
212
|
@staticmethod
|
213
213
|
def _abandon_status_check(
|
214
214
|
lock: threading.Lock,
|
215
|
-
handle:
|
215
|
+
handle: MailboxHandle | None,
|
216
216
|
):
|
217
217
|
with lock:
|
218
218
|
if handle:
|
@@ -227,7 +227,7 @@ class RunStatusChecker:
|
|
227
227
|
request: Any,
|
228
228
|
process: Any,
|
229
229
|
) -> None:
|
230
|
-
local_handle:
|
230
|
+
local_handle: MailboxHandle | None = None
|
231
231
|
join_requested = False
|
232
232
|
while not join_requested:
|
233
233
|
time_probe = time.monotonic()
|
@@ -365,7 +365,7 @@ class _run_decorator: # noqa: N801
|
|
365
365
|
@classmethod
|
366
366
|
def _attach(cls, func: Callable) -> Callable:
|
367
367
|
@functools.wraps(func)
|
368
|
-
def wrapper(self:
|
368
|
+
def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
369
369
|
# * `_attach_id` is only assigned in service hence for all non-service cases
|
370
370
|
# it will be a passthrough.
|
371
371
|
# * `_attach_pid` is only assigned in _init (using _attach_pid guarantees single attach):
|
@@ -398,7 +398,7 @@ class _run_decorator: # noqa: N801
|
|
398
398
|
def _noop_on_finish(cls, message: str = "", only_warn: bool = False) -> Callable:
|
399
399
|
def decorator_fn(func: Callable) -> Callable:
|
400
400
|
@functools.wraps(func)
|
401
|
-
def wrapper_fn(self:
|
401
|
+
def wrapper_fn(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
402
402
|
if not getattr(self, "_is_finished", False):
|
403
403
|
return func(self, *args, **kwargs)
|
404
404
|
|
@@ -419,7 +419,7 @@ class _run_decorator: # noqa: N801
|
|
419
419
|
@classmethod
|
420
420
|
def _noop(cls, func: Callable) -> Callable:
|
421
421
|
@functools.wraps(func)
|
422
|
-
def wrapper(self:
|
422
|
+
def wrapper(self: type[Run], *args: Any, **kwargs: Any) -> Any:
|
423
423
|
# `_attach_id` is only assigned in service hence for all service cases
|
424
424
|
# it will be a passthrough. We don't pickle non-service so again a way
|
425
425
|
# to see that we are in non-service case
|
@@ -458,7 +458,7 @@ class _run_decorator: # noqa: N801
|
|
458
458
|
class RunStatus:
|
459
459
|
sync_items_total: int = field(default=0)
|
460
460
|
sync_items_pending: int = field(default=0)
|
461
|
-
sync_time:
|
461
|
+
sync_time: datetime | None = field(default=None)
|
462
462
|
|
463
463
|
|
464
464
|
class Run:
|
@@ -528,67 +528,65 @@ class Run:
|
|
528
528
|
_telemetry_obj_dirty: bool
|
529
529
|
_telemetry_obj_flushed: bytes
|
530
530
|
|
531
|
-
_teardown_hooks:
|
532
|
-
_tags:
|
531
|
+
_teardown_hooks: list[TeardownHook]
|
532
|
+
_tags: tuple[Any, ...] | None
|
533
533
|
|
534
|
-
_entity:
|
535
|
-
_project:
|
536
|
-
_group:
|
537
|
-
_job_type:
|
538
|
-
_name:
|
539
|
-
_notes:
|
540
|
-
_sweep_id:
|
534
|
+
_entity: str | None
|
535
|
+
_project: str | None
|
536
|
+
_group: str | None
|
537
|
+
_job_type: str | None
|
538
|
+
_name: str | None
|
539
|
+
_notes: str | None
|
540
|
+
_sweep_id: str | None
|
541
541
|
|
542
|
-
_run_obj:
|
542
|
+
_run_obj: RunRecord | None
|
543
543
|
# Use string literal annotation because of type reference loop
|
544
|
-
_backend:
|
545
|
-
_internal_run_interface:
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
_output_writer: Optional["filesystem.CRDedupedFile"]
|
555
|
-
_quiet: Optional[bool]
|
544
|
+
_backend: wandb.sdk.backend.backend.Backend | None
|
545
|
+
_internal_run_interface: wandb.sdk.interface.interface_queue.InterfaceQueue | None
|
546
|
+
_wl: _WandbSetup | None
|
547
|
+
|
548
|
+
_out_redir: redirect.RedirectBase | None
|
549
|
+
_err_redir: redirect.RedirectBase | None
|
550
|
+
_redirect_cb: Callable[[str, str], None] | None
|
551
|
+
_redirect_raw_cb: Callable[[str, str], None] | None
|
552
|
+
_output_writer: filesystem.CRDedupedFile | None
|
553
|
+
_quiet: bool | None
|
556
554
|
|
557
555
|
_atexit_cleanup_called: bool
|
558
|
-
_hooks:
|
559
|
-
_exit_code:
|
556
|
+
_hooks: ExitHooks | None
|
557
|
+
_exit_code: int | None
|
560
558
|
|
561
|
-
_run_status_checker:
|
559
|
+
_run_status_checker: RunStatusChecker | None
|
562
560
|
|
563
|
-
_sampled_history:
|
564
|
-
_final_summary:
|
565
|
-
_poll_exit_handle:
|
566
|
-
_poll_exit_response:
|
567
|
-
_internal_messages_response:
|
561
|
+
_sampled_history: SampledHistoryResponse | None
|
562
|
+
_final_summary: GetSummaryResponse | None
|
563
|
+
_poll_exit_handle: MailboxHandle | None
|
564
|
+
_poll_exit_response: PollExitResponse | None
|
565
|
+
_internal_messages_response: InternalMessagesResponse | None
|
568
566
|
|
569
|
-
_stdout_slave_fd:
|
570
|
-
_stderr_slave_fd:
|
571
|
-
_artifact_slots:
|
567
|
+
_stdout_slave_fd: int | None
|
568
|
+
_stderr_slave_fd: int | None
|
569
|
+
_artifact_slots: list[str]
|
572
570
|
|
573
571
|
_init_pid: int
|
574
572
|
_attach_pid: int
|
575
|
-
_iface_pid:
|
576
|
-
_iface_port:
|
573
|
+
_iface_pid: int | None
|
574
|
+
_iface_port: int | None
|
577
575
|
|
578
|
-
_attach_id:
|
576
|
+
_attach_id: str | None
|
579
577
|
_is_attached: bool
|
580
578
|
_is_finished: bool
|
581
579
|
_settings: Settings
|
582
580
|
|
583
|
-
_launch_artifacts:
|
584
|
-
_printer:
|
581
|
+
_launch_artifacts: dict[str, Any] | None
|
582
|
+
_printer: printer.Printer
|
585
583
|
|
586
584
|
def __init__(
|
587
585
|
self,
|
588
586
|
settings: Settings,
|
589
|
-
config:
|
590
|
-
sweep_config:
|
591
|
-
launch_config:
|
587
|
+
config: dict[str, Any] | None = None,
|
588
|
+
sweep_config: dict[str, Any] | None = None,
|
589
|
+
launch_config: dict[str, Any] | None = None,
|
592
590
|
) -> None:
|
593
591
|
# pid is set, so we know if this run object was initialized by this process
|
594
592
|
self._init_pid = os.getpid()
|
@@ -608,9 +606,9 @@ class Run:
|
|
608
606
|
def _init(
|
609
607
|
self,
|
610
608
|
settings: Settings,
|
611
|
-
config:
|
612
|
-
sweep_config:
|
613
|
-
launch_config:
|
609
|
+
config: dict[str, Any] | None = None,
|
610
|
+
sweep_config: dict[str, Any] | None = None,
|
611
|
+
launch_config: dict[str, Any] | None = None,
|
614
612
|
) -> None:
|
615
613
|
self._settings = settings
|
616
614
|
|
@@ -626,7 +624,7 @@ class Run:
|
|
626
624
|
)
|
627
625
|
self.summary._set_update_callback(self._summary_update_callback)
|
628
626
|
self._step = 0
|
629
|
-
self._torch_history:
|
627
|
+
self._torch_history: wandb_torch.TorchHistory | None = None # type: ignore
|
630
628
|
|
631
629
|
# todo: eventually would be nice to make this configurable using self._settings._start_time
|
632
630
|
# need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
|
@@ -635,9 +633,9 @@ class Run:
|
|
635
633
|
|
636
634
|
_datatypes_set_callback(self._datatypes_callback)
|
637
635
|
|
638
|
-
self._printer = get_printer(self._settings._jupyter)
|
636
|
+
self._printer = printer.get_printer(self._settings._jupyter)
|
639
637
|
self._wl = None
|
640
|
-
self._reporter:
|
638
|
+
self._reporter: Reporter | None = None
|
641
639
|
|
642
640
|
self._entity = None
|
643
641
|
self._project = None
|
@@ -663,7 +661,7 @@ class Run:
|
|
663
661
|
self._quiet = self._settings.quiet
|
664
662
|
|
665
663
|
self._output_writer = None
|
666
|
-
self._used_artifact_slots:
|
664
|
+
self._used_artifact_slots: dict[str, str] = {}
|
667
665
|
|
668
666
|
# Returned from backend request_run(), set from wandb_init?
|
669
667
|
self._run_obj = None
|
@@ -699,8 +697,8 @@ class Run:
|
|
699
697
|
config = config or dict()
|
700
698
|
wandb_key = "_wandb"
|
701
699
|
config.setdefault(wandb_key, dict())
|
702
|
-
self._launch_artifact_mapping:
|
703
|
-
self._unique_launch_artifact_sequence_names:
|
700
|
+
self._launch_artifact_mapping: dict[str, Any] = {}
|
701
|
+
self._unique_launch_artifact_sequence_names: dict[str, Any] = {}
|
704
702
|
if self._settings.save_code and self._settings.program_relpath:
|
705
703
|
config[wandb_key]["code_path"] = LogicalPath(
|
706
704
|
os.path.join("code", self._settings.program_relpath)
|
@@ -766,7 +764,7 @@ class Run:
|
|
766
764
|
def _handle_launch_artifact_overrides(self) -> None:
|
767
765
|
if self._settings.launch and (os.environ.get("WANDB_ARTIFACTS") is not None):
|
768
766
|
try:
|
769
|
-
artifacts:
|
767
|
+
artifacts: dict[str, Any] = json.loads(
|
770
768
|
os.environ.get("WANDB_ARTIFACTS", "{}")
|
771
769
|
)
|
772
770
|
except (ValueError, SyntaxError):
|
@@ -786,7 +784,7 @@ class Run:
|
|
786
784
|
artifacts = launch_config.get("overrides").get("artifacts")
|
787
785
|
self._initialize_launch_artifact_maps(artifacts)
|
788
786
|
|
789
|
-
def _initialize_launch_artifact_maps(self, artifacts:
|
787
|
+
def _initialize_launch_artifact_maps(self, artifacts: dict[str, Any]) -> None:
|
790
788
|
for key, item in artifacts.items():
|
791
789
|
self._launch_artifact_mapping[key] = item
|
792
790
|
artifact_sequence_tuple_or_slot = key.split(":")
|
@@ -893,7 +891,7 @@ class Run:
|
|
893
891
|
except Exception:
|
894
892
|
wandb.termwarn("Cannot find valid git repo associated with this directory.")
|
895
893
|
|
896
|
-
def __deepcopy__(self, memo:
|
894
|
+
def __deepcopy__(self, memo: dict[int, Any]) -> Run:
|
897
895
|
return self
|
898
896
|
|
899
897
|
def __getstate__(self) -> Any:
|
@@ -927,7 +925,7 @@ class Run:
|
|
927
925
|
self.__dict__.update(state)
|
928
926
|
|
929
927
|
@property
|
930
|
-
def _torch(self) ->
|
928
|
+
def _torch(self) -> wandb_torch.TorchHistory: # type: ignore
|
931
929
|
if self._torch_history is None:
|
932
930
|
self._torch_history = wandb_torch.TorchHistory() # type: ignore
|
933
931
|
return self._torch_history
|
@@ -959,7 +957,7 @@ class Run:
|
|
959
957
|
|
960
958
|
@property
|
961
959
|
@_run_decorator._attach
|
962
|
-
def name(self) ->
|
960
|
+
def name(self) -> str | None:
|
963
961
|
"""Display name of the run.
|
964
962
|
|
965
963
|
Display names are not guaranteed to be unique and may be descriptive.
|
@@ -982,7 +980,7 @@ class Run:
|
|
982
980
|
|
983
981
|
@property
|
984
982
|
@_run_decorator._attach
|
985
|
-
def notes(self) ->
|
983
|
+
def notes(self) -> str | None:
|
986
984
|
"""Notes associated with the run, if there are any.
|
987
985
|
|
988
986
|
Notes can be a multiline string and can also use markdown and latex equations
|
@@ -1003,7 +1001,7 @@ class Run:
|
|
1003
1001
|
|
1004
1002
|
@property
|
1005
1003
|
@_run_decorator._attach
|
1006
|
-
def tags(self) ->
|
1004
|
+
def tags(self) -> tuple | None:
|
1007
1005
|
"""Tags associated with the run, if there are any."""
|
1008
1006
|
if self._tags:
|
1009
1007
|
return self._tags
|
@@ -1030,7 +1028,7 @@ class Run:
|
|
1030
1028
|
|
1031
1029
|
@property
|
1032
1030
|
@_run_decorator._attach
|
1033
|
-
def sweep_id(self) ->
|
1031
|
+
def sweep_id(self) -> str | None:
|
1034
1032
|
"""ID of the sweep associated with the run, if there is one."""
|
1035
1033
|
if not self._run_obj:
|
1036
1034
|
return None
|
@@ -1090,6 +1088,7 @@ class Run:
|
|
1090
1088
|
return self._step
|
1091
1089
|
|
1092
1090
|
def project_name(self) -> str:
|
1091
|
+
# TODO: deprecate this in favor of project
|
1093
1092
|
return self._run_obj.project if self._run_obj else ""
|
1094
1093
|
|
1095
1094
|
@property
|
@@ -1148,15 +1147,13 @@ class Run:
|
|
1148
1147
|
@_run_decorator._attach
|
1149
1148
|
def log_code(
|
1150
1149
|
self,
|
1151
|
-
root:
|
1152
|
-
name:
|
1153
|
-
include_fn:
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
] = filenames.exclude_wandb_fn,
|
1159
|
-
) -> Optional[Artifact]:
|
1150
|
+
root: str | None = ".",
|
1151
|
+
name: str | None = None,
|
1152
|
+
include_fn: Callable[[str, str], bool]
|
1153
|
+
| Callable[[str], bool] = _is_py_requirements_or_dockerfile,
|
1154
|
+
exclude_fn: Callable[[str, str], bool]
|
1155
|
+
| Callable[[str], bool] = filenames.exclude_wandb_fn,
|
1156
|
+
) -> Artifact | None:
|
1160
1157
|
"""Save the current state of your code to a W&B Artifact.
|
1161
1158
|
|
1162
1159
|
By default, it walks the current directory and logs all files that end with `.py`.
|
@@ -1229,7 +1226,7 @@ class Run:
|
|
1229
1226
|
|
1230
1227
|
return self._log_artifact(art)
|
1231
1228
|
|
1232
|
-
def get_url(self) ->
|
1229
|
+
def get_url(self) -> str | None:
|
1233
1230
|
"""Return the url for the W&B run, if there is one.
|
1234
1231
|
|
1235
1232
|
Offline runs will not have a url.
|
@@ -1239,7 +1236,7 @@ class Run:
|
|
1239
1236
|
return None
|
1240
1237
|
return self._settings.run_url
|
1241
1238
|
|
1242
|
-
def get_project_url(self) ->
|
1239
|
+
def get_project_url(self) -> str | None:
|
1243
1240
|
"""Return the url for the W&B project associated with the run, if there is one.
|
1244
1241
|
|
1245
1242
|
Offline runs will not have a project url.
|
@@ -1249,7 +1246,7 @@ class Run:
|
|
1249
1246
|
return None
|
1250
1247
|
return self._settings.project_url
|
1251
1248
|
|
1252
|
-
def get_sweep_url(self) ->
|
1249
|
+
def get_sweep_url(self) -> str | None:
|
1253
1250
|
"""Return the url for the sweep associated with the run, if there is one."""
|
1254
1251
|
if self._settings._offline:
|
1255
1252
|
wandb.termwarn("URL not available in offline run")
|
@@ -1258,7 +1255,7 @@ class Run:
|
|
1258
1255
|
|
1259
1256
|
@property
|
1260
1257
|
@_run_decorator._attach
|
1261
|
-
def url(self) ->
|
1258
|
+
def url(self) -> str | None:
|
1262
1259
|
"""The W&B url associated with the run."""
|
1263
1260
|
return self.get_url()
|
1264
1261
|
|
@@ -1273,9 +1270,9 @@ class Run:
|
|
1273
1270
|
|
1274
1271
|
def _label_internal(
|
1275
1272
|
self,
|
1276
|
-
code:
|
1277
|
-
repo:
|
1278
|
-
code_version:
|
1273
|
+
code: str | None = None,
|
1274
|
+
repo: str | None = None,
|
1275
|
+
code_version: str | None = None,
|
1279
1276
|
) -> None:
|
1280
1277
|
with telemetry.context(run=self) as tel:
|
1281
1278
|
if code and RE_LABEL.match(code):
|
@@ -1287,9 +1284,9 @@ class Run:
|
|
1287
1284
|
|
1288
1285
|
def _label(
|
1289
1286
|
self,
|
1290
|
-
code:
|
1291
|
-
repo:
|
1292
|
-
code_version:
|
1287
|
+
code: str | None = None,
|
1288
|
+
repo: str | None = None,
|
1289
|
+
code_version: str | None = None,
|
1293
1290
|
**kwargs: str,
|
1294
1291
|
) -> None:
|
1295
1292
|
if self._settings.label_disable:
|
@@ -1311,7 +1308,7 @@ class Run:
|
|
1311
1308
|
# update telemetry in the backend immediately for _label() callers
|
1312
1309
|
self._telemetry_flush()
|
1313
1310
|
|
1314
|
-
def _label_probe_lines(self, lines:
|
1311
|
+
def _label_probe_lines(self, lines: list[str]) -> None:
|
1315
1312
|
if not lines:
|
1316
1313
|
return
|
1317
1314
|
parsed = telemetry._parse_label_lines(lines)
|
@@ -1379,23 +1376,23 @@ class Run:
|
|
1379
1376
|
return prefix + f"<iframe src={url!r} style={style!r}></iframe>"
|
1380
1377
|
|
1381
1378
|
def _repr_mimebundle_(
|
1382
|
-
self, include:
|
1383
|
-
) ->
|
1379
|
+
self, include: Any | None = None, exclude: Any | None = None
|
1380
|
+
) -> dict[str, str]:
|
1384
1381
|
return {"text/html": self.to_html(hidden=True)}
|
1385
1382
|
|
1386
1383
|
@_run_decorator._noop_on_finish()
|
1387
1384
|
def _config_callback(
|
1388
1385
|
self,
|
1389
|
-
key:
|
1390
|
-
val:
|
1391
|
-
data:
|
1386
|
+
key: tuple[str, ...] | str | None = None,
|
1387
|
+
val: Any | None = None,
|
1388
|
+
data: dict[str, object] | None = None,
|
1392
1389
|
) -> None:
|
1393
1390
|
logger.info(f"config_cb {key} {val} {data}")
|
1394
1391
|
if self._backend and self._backend.interface:
|
1395
1392
|
self._backend.interface.publish_config(key=key, val=val, data=data)
|
1396
1393
|
|
1397
1394
|
def _config_artifact_callback(
|
1398
|
-
self, key: str, val:
|
1395
|
+
self, key: str, val: str | Artifact | dict
|
1399
1396
|
) -> Artifact:
|
1400
1397
|
# artifacts can look like dicts as they are passed into the run config
|
1401
1398
|
# since the run config stores them on the backend as a dict with fields shown
|
@@ -1446,7 +1443,7 @@ class Run:
|
|
1446
1443
|
# line = "Waiting for run.summary data..."
|
1447
1444
|
# self._printer.display(line)
|
1448
1445
|
|
1449
|
-
def _summary_get_current_summary_callback(self) ->
|
1446
|
+
def _summary_get_current_summary_callback(self) -> dict[str, Any]:
|
1450
1447
|
if self._is_finished:
|
1451
1448
|
# TODO: WB-18420: fetch summary from backend and stage it before run is finished
|
1452
1449
|
wandb.termwarn("Summary data not available in finished run")
|
@@ -1473,7 +1470,7 @@ class Run:
|
|
1473
1470
|
files: FilesDict = dict(files=[(GlobStr(glob.escape(fname)), "now")])
|
1474
1471
|
self._backend.interface.publish_files(files)
|
1475
1472
|
|
1476
|
-
def _visualization_hack(self, row:
|
1473
|
+
def _visualization_hack(self, row: dict[str, Any]) -> dict[str, Any]:
|
1477
1474
|
# TODO(jhr): move visualize hack somewhere else
|
1478
1475
|
chart_keys = set()
|
1479
1476
|
split_table_set = set()
|
@@ -1510,9 +1507,9 @@ class Run:
|
|
1510
1507
|
|
1511
1508
|
def _partial_history_callback(
|
1512
1509
|
self,
|
1513
|
-
row:
|
1514
|
-
step:
|
1515
|
-
commit:
|
1510
|
+
row: dict[str, Any],
|
1511
|
+
step: int | None = None,
|
1512
|
+
commit: bool | None = None,
|
1516
1513
|
) -> None:
|
1517
1514
|
row = row.copy()
|
1518
1515
|
if row:
|
@@ -1559,19 +1556,19 @@ class Run:
|
|
1559
1556
|
def _set_library(self, library: _WandbSetup) -> None:
|
1560
1557
|
self._wl = library
|
1561
1558
|
|
1562
|
-
def _set_backend(self, backend:
|
1559
|
+
def _set_backend(self, backend: wandb.sdk.backend.backend.Backend) -> None:
|
1563
1560
|
self._backend = backend
|
1564
1561
|
|
1565
1562
|
def _set_internal_run_interface(
|
1566
1563
|
self,
|
1567
|
-
interface:
|
1564
|
+
interface: wandb.sdk.interface.interface_queue.InterfaceQueue,
|
1568
1565
|
) -> None:
|
1569
1566
|
self._internal_run_interface = interface
|
1570
1567
|
|
1571
1568
|
def _set_reporter(self, reporter: Reporter) -> None:
|
1572
1569
|
self._reporter = reporter
|
1573
1570
|
|
1574
|
-
def _set_teardown_hooks(self, hooks:
|
1571
|
+
def _set_teardown_hooks(self, hooks: list[TeardownHook]) -> None:
|
1575
1572
|
self._teardown_hooks = hooks
|
1576
1573
|
|
1577
1574
|
def _set_run_obj(self, run_obj: RunRecord) -> None:
|
@@ -1609,7 +1606,7 @@ class Run:
|
|
1609
1606
|
)
|
1610
1607
|
|
1611
1608
|
def _add_singleton(
|
1612
|
-
self, data_type: str, key: str, value:
|
1609
|
+
self, data_type: str, key: str, value: dict[int | str, str]
|
1613
1610
|
) -> None:
|
1614
1611
|
"""Store a singleton item to wandb config.
|
1615
1612
|
|
@@ -1639,9 +1636,9 @@ class Run:
|
|
1639
1636
|
|
1640
1637
|
def _log(
|
1641
1638
|
self,
|
1642
|
-
data:
|
1643
|
-
step:
|
1644
|
-
commit:
|
1639
|
+
data: dict[str, Any],
|
1640
|
+
step: int | None = None,
|
1641
|
+
commit: bool | None = None,
|
1645
1642
|
) -> None:
|
1646
1643
|
if not isinstance(data, Mapping):
|
1647
1644
|
raise ValueError("wandb.log must be passed a dictionary")
|
@@ -1680,10 +1677,10 @@ class Run:
|
|
1680
1677
|
@_run_decorator._attach
|
1681
1678
|
def log(
|
1682
1679
|
self,
|
1683
|
-
data:
|
1684
|
-
step:
|
1685
|
-
commit:
|
1686
|
-
sync:
|
1680
|
+
data: dict[str, Any],
|
1681
|
+
step: int | None = None,
|
1682
|
+
commit: bool | None = None,
|
1683
|
+
sync: bool | None = None,
|
1687
1684
|
) -> None:
|
1688
1685
|
"""Upload run data.
|
1689
1686
|
|
@@ -1936,10 +1933,10 @@ class Run:
|
|
1936
1933
|
@_run_decorator._attach
|
1937
1934
|
def save(
|
1938
1935
|
self,
|
1939
|
-
glob_str:
|
1940
|
-
base_path:
|
1936
|
+
glob_str: str | os.PathLike | None = None,
|
1937
|
+
base_path: str | os.PathLike | None = None,
|
1941
1938
|
policy: PolicyName = "live",
|
1942
|
-
) ->
|
1939
|
+
) -> bool | list[str]:
|
1943
1940
|
"""Sync one or more files to W&B.
|
1944
1941
|
|
1945
1942
|
Relative paths are relative to the current working directory.
|
@@ -2044,8 +2041,8 @@ class Run:
|
|
2044
2041
|
self,
|
2045
2042
|
glob_path: pathlib.PurePath,
|
2046
2043
|
base_path: pathlib.PurePath,
|
2047
|
-
policy:
|
2048
|
-
) ->
|
2044
|
+
policy: PolicyName,
|
2045
|
+
) -> list[str]:
|
2049
2046
|
# Can't use is_relative_to() because that's added in Python 3.9,
|
2050
2047
|
# but we support down to Python 3.7.
|
2051
2048
|
if not str(glob_path).startswith(str(base_path)):
|
@@ -2128,10 +2125,10 @@ class Run:
|
|
2128
2125
|
def restore(
|
2129
2126
|
self,
|
2130
2127
|
name: str,
|
2131
|
-
run_path:
|
2128
|
+
run_path: str | None = None,
|
2132
2129
|
replace: bool = False,
|
2133
|
-
root:
|
2134
|
-
) ->
|
2130
|
+
root: str | None = None,
|
2131
|
+
) -> None | TextIO:
|
2135
2132
|
return restore(
|
2136
2133
|
name,
|
2137
2134
|
run_path or self._get_path(),
|
@@ -2141,9 +2138,7 @@ class Run:
|
|
2141
2138
|
|
2142
2139
|
@_run_decorator._noop
|
2143
2140
|
@_run_decorator._attach
|
2144
|
-
def finish(
|
2145
|
-
self, exit_code: Optional[int] = None, quiet: Optional[bool] = None
|
2146
|
-
) -> None:
|
2141
|
+
def finish(self, exit_code: int | None = None, quiet: bool | None = None) -> None:
|
2147
2142
|
"""Mark a run as finished, and finish uploading all data.
|
2148
2143
|
|
2149
2144
|
This is used when creating multiple runs in the same process. We automatically
|
@@ -2157,8 +2152,8 @@ class Run:
|
|
2157
2152
|
|
2158
2153
|
def _finish(
|
2159
2154
|
self,
|
2160
|
-
exit_code:
|
2161
|
-
quiet:
|
2155
|
+
exit_code: int | None = None,
|
2156
|
+
quiet: bool | None = None,
|
2162
2157
|
) -> None:
|
2163
2158
|
logger.info(f"finishing run {self._get_path()}")
|
2164
2159
|
with telemetry.context(run=self) as tel:
|
@@ -2209,7 +2204,7 @@ class Run:
|
|
2209
2204
|
|
2210
2205
|
@_run_decorator._noop
|
2211
2206
|
@_run_decorator._attach
|
2212
|
-
def join(self, exit_code:
|
2207
|
+
def join(self, exit_code: int | None = None) -> None:
|
2213
2208
|
"""Deprecated alias for `finish()` - use finish instead."""
|
2214
2209
|
if hasattr(self, "_telemetry_obj"):
|
2215
2210
|
deprecate.deprecate(
|
@@ -2248,10 +2243,10 @@ class Run:
|
|
2248
2243
|
@staticmethod
|
2249
2244
|
def plot_table(
|
2250
2245
|
vega_spec_name: str,
|
2251
|
-
data_table:
|
2252
|
-
fields:
|
2253
|
-
string_fields:
|
2254
|
-
split_table:
|
2246
|
+
data_table: Table,
|
2247
|
+
fields: dict[str, Any],
|
2248
|
+
string_fields: dict[str, Any] | None = None,
|
2249
|
+
split_table: bool | None = False,
|
2255
2250
|
) -> CustomChart:
|
2256
2251
|
"""Create a custom plot on a table.
|
2257
2252
|
|
@@ -2263,6 +2258,8 @@ class Run:
|
|
2263
2258
|
visualization needs
|
2264
2259
|
string_fields: a dict that provides values for any string constants
|
2265
2260
|
the custom visualization needs
|
2261
|
+
split_table: a boolean that indicates whether the table should be in
|
2262
|
+
a separate section in the UI
|
2266
2263
|
"""
|
2267
2264
|
return custom_chart(
|
2268
2265
|
vega_spec_name, data_table, fields, string_fields or {}, split_table
|
@@ -2289,6 +2286,8 @@ class Run:
|
|
2289
2286
|
define_metric=self.define_metric,
|
2290
2287
|
plot_table=self.plot_table,
|
2291
2288
|
alert=self.alert,
|
2289
|
+
watch=self.watch,
|
2290
|
+
unwatch=self.unwatch,
|
2292
2291
|
mark_preempting=self.mark_preempting,
|
2293
2292
|
log_model=self.log_model,
|
2294
2293
|
use_model=self.use_model,
|
@@ -2297,9 +2296,9 @@ class Run:
|
|
2297
2296
|
|
2298
2297
|
def _redirect(
|
2299
2298
|
self,
|
2300
|
-
stdout_slave_fd:
|
2301
|
-
stderr_slave_fd:
|
2302
|
-
console:
|
2299
|
+
stdout_slave_fd: int | None,
|
2300
|
+
stderr_slave_fd: int | None,
|
2301
|
+
console: str | None = None,
|
2303
2302
|
) -> None:
|
2304
2303
|
if console is None:
|
2305
2304
|
console = self._settings.console
|
@@ -2414,7 +2413,7 @@ class Run:
|
|
2414
2413
|
self._err_redir.uninstall()
|
2415
2414
|
logger.info("restore done")
|
2416
2415
|
|
2417
|
-
def _atexit_cleanup(self, exit_code:
|
2416
|
+
def _atexit_cleanup(self, exit_code: int | None = None) -> None:
|
2418
2417
|
if self._backend is None:
|
2419
2418
|
logger.warning("process exited without backend configured")
|
2420
2419
|
return
|
@@ -2525,7 +2524,7 @@ class Run:
|
|
2525
2524
|
self,
|
2526
2525
|
) -> None:
|
2527
2526
|
def _telemetry_import_hook(
|
2528
|
-
run:
|
2527
|
+
run: Run,
|
2529
2528
|
module: Any,
|
2530
2529
|
) -> None:
|
2531
2530
|
with telemetry.context(run=run) as tel:
|
@@ -2568,7 +2567,7 @@ class Run:
|
|
2568
2567
|
|
2569
2568
|
StagedLaunchInputs().apply(self)
|
2570
2569
|
|
2571
|
-
def _make_job_source_reqs(self) ->
|
2570
|
+
def _make_job_source_reqs(self) -> tuple[list[str], dict[str, Any], dict[str, Any]]:
|
2572
2571
|
from wandb.util import working_set
|
2573
2572
|
|
2574
2573
|
installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
|
@@ -2580,10 +2579,10 @@ class Run:
|
|
2580
2579
|
def _construct_job_artifact(
|
2581
2580
|
self,
|
2582
2581
|
name: str,
|
2583
|
-
source_dict:
|
2584
|
-
installed_packages_list:
|
2585
|
-
patch_path:
|
2586
|
-
) ->
|
2582
|
+
source_dict: JobSourceDict,
|
2583
|
+
installed_packages_list: list[str],
|
2584
|
+
patch_path: os.PathLike | None = None,
|
2585
|
+
) -> Artifact:
|
2587
2586
|
job_artifact = job_builder.JobArtifact(name)
|
2588
2587
|
if patch_path and os.path.exists(patch_path):
|
2589
2588
|
job_artifact.add_file(FilePathStr(str(patch_path)), "diff.patch")
|
@@ -2596,12 +2595,12 @@ class Run:
|
|
2596
2595
|
|
2597
2596
|
def _create_image_job(
|
2598
2597
|
self,
|
2599
|
-
input_types:
|
2600
|
-
output_types:
|
2601
|
-
installed_packages_list:
|
2602
|
-
docker_image_name:
|
2603
|
-
args:
|
2604
|
-
) ->
|
2598
|
+
input_types: dict[str, Any],
|
2599
|
+
output_types: dict[str, Any],
|
2600
|
+
installed_packages_list: list[str],
|
2601
|
+
docker_image_name: str | None = None,
|
2602
|
+
args: list[str] | None = None,
|
2603
|
+
) -> Artifact | None:
|
2605
2604
|
docker_image_name = docker_image_name or os.getenv("WANDB_DOCKER")
|
2606
2605
|
|
2607
2606
|
if not docker_image_name:
|
@@ -2624,7 +2623,7 @@ class Run:
|
|
2624
2623
|
return job_artifact
|
2625
2624
|
|
2626
2625
|
def _log_job_artifact_with_image(
|
2627
|
-
self, docker_image_name: str, args:
|
2626
|
+
self, docker_image_name: str, args: list[str] | None = None
|
2628
2627
|
) -> Artifact:
|
2629
2628
|
packages, in_types, out_types = self._make_job_source_reqs()
|
2630
2629
|
job_artifact = self._create_image_job(
|
@@ -2653,16 +2652,20 @@ class Run:
|
|
2653
2652
|
handle = self._backend.interface.deliver_poll_exit()
|
2654
2653
|
probe_handle.set_mailbox_handle(handle)
|
2655
2654
|
|
2656
|
-
def _on_progress_exit(
|
2655
|
+
def _on_progress_exit(
|
2656
|
+
self,
|
2657
|
+
progress_printer: progress.ProgressPrinter,
|
2658
|
+
progress_handle: MailboxProgress,
|
2659
|
+
) -> None:
|
2657
2660
|
probe_handles = progress_handle.get_probe_handles()
|
2658
|
-
|
2661
|
+
if not probe_handles or len(probe_handles) != 1:
|
2662
|
+
return
|
2659
2663
|
|
2660
2664
|
result = probe_handles[0].get_probe_result()
|
2661
2665
|
if not result:
|
2662
2666
|
return
|
2663
|
-
|
2664
|
-
|
2665
|
-
)
|
2667
|
+
|
2668
|
+
progress_printer.update([result.response.poll_exit_response])
|
2666
2669
|
|
2667
2670
|
def _on_finish(self) -> None:
|
2668
2671
|
trigger.call("on_finished")
|
@@ -2677,16 +2680,28 @@ class Run:
|
|
2677
2680
|
exit_handle = self._backend.interface.deliver_exit(self._exit_code)
|
2678
2681
|
exit_handle.add_probe(on_probe=self._on_probe_exit)
|
2679
2682
|
|
2680
|
-
|
2681
|
-
|
2683
|
+
with progress.progress_printer(
|
2684
|
+
self._printer,
|
2685
|
+
self._settings,
|
2686
|
+
) as progress_printer:
|
2687
|
+
# Wait for the run to complete.
|
2688
|
+
_ = exit_handle.wait(
|
2689
|
+
timeout=-1,
|
2690
|
+
on_progress=functools.partial(
|
2691
|
+
self._on_progress_exit,
|
2692
|
+
progress_printer,
|
2693
|
+
),
|
2694
|
+
)
|
2682
2695
|
|
2696
|
+
# Print some final statistics.
|
2683
2697
|
poll_exit_handle = self._backend.interface.deliver_poll_exit()
|
2684
|
-
# wait for them, it's ok to do this serially but this can be improved
|
2685
2698
|
result = poll_exit_handle.wait(timeout=-1)
|
2686
2699
|
assert result
|
2687
|
-
|
2688
|
-
|
2700
|
+
progress.print_sync_dedupe_stats(
|
2701
|
+
self._printer,
|
2702
|
+
result.response.poll_exit_response,
|
2689
2703
|
)
|
2704
|
+
|
2690
2705
|
self._poll_exit_response = result.response.poll_exit_response
|
2691
2706
|
internal_messages_handle = self._backend.interface.deliver_internal_messages()
|
2692
2707
|
result = internal_messages_handle.wait(timeout=-1)
|
@@ -2727,12 +2742,12 @@ class Run:
|
|
2727
2742
|
def define_metric(
|
2728
2743
|
self,
|
2729
2744
|
name: str,
|
2730
|
-
step_metric:
|
2731
|
-
step_sync:
|
2732
|
-
hidden:
|
2733
|
-
summary:
|
2734
|
-
goal:
|
2735
|
-
overwrite:
|
2745
|
+
step_metric: str | wandb_metric.Metric | None = None,
|
2746
|
+
step_sync: bool | None = None,
|
2747
|
+
hidden: bool | None = None,
|
2748
|
+
summary: str | None = None,
|
2749
|
+
goal: str | None = None,
|
2750
|
+
overwrite: bool | None = None,
|
2736
2751
|
) -> wandb_metric.Metric:
|
2737
2752
|
"""Customize metrics logged with `wandb.log()`.
|
2738
2753
|
|
@@ -2788,12 +2803,12 @@ class Run:
|
|
2788
2803
|
def _define_metric(
|
2789
2804
|
self,
|
2790
2805
|
name: str,
|
2791
|
-
step_metric:
|
2792
|
-
step_sync:
|
2793
|
-
hidden:
|
2794
|
-
summary:
|
2795
|
-
goal:
|
2796
|
-
overwrite:
|
2806
|
+
step_metric: str | wandb_metric.Metric | None = None,
|
2807
|
+
step_sync: bool | None = None,
|
2808
|
+
hidden: bool | None = None,
|
2809
|
+
summary: str | None = None,
|
2810
|
+
goal: str | None = None,
|
2811
|
+
overwrite: bool | None = None,
|
2797
2812
|
) -> wandb_metric.Metric:
|
2798
2813
|
if not name:
|
2799
2814
|
raise wandb.Error("define_metric() requires non-empty name argument")
|
@@ -2819,7 +2834,7 @@ class Run:
|
|
2819
2834
|
raise wandb.Error(
|
2820
2835
|
f"Unhandled define_metric() arg: name (glob suffixes only): {name}"
|
2821
2836
|
)
|
2822
|
-
summary_ops:
|
2837
|
+
summary_ops: Sequence[str] | None = None
|
2823
2838
|
if summary:
|
2824
2839
|
summary_items = [s.lower() for s in summary.split(",")]
|
2825
2840
|
summary_ops = []
|
@@ -2832,7 +2847,7 @@ class Run:
|
|
2832
2847
|
with telemetry.context(run=self) as tel:
|
2833
2848
|
tel.feature.metric_summary = True
|
2834
2849
|
# TODO: deprecate goal
|
2835
|
-
goal_cleaned:
|
2850
|
+
goal_cleaned: str | None = None
|
2836
2851
|
if goal is not None:
|
2837
2852
|
goal_cleaned = goal[:3].lower()
|
2838
2853
|
valid_goal = {"min", "max"}
|
@@ -2863,26 +2878,57 @@ class Run:
|
|
2863
2878
|
m._commit()
|
2864
2879
|
return m
|
2865
2880
|
|
2866
|
-
# TODO(jhr): annotate this
|
2867
2881
|
@_run_decorator._attach
|
2868
|
-
def watch(
|
2882
|
+
def watch(
|
2869
2883
|
self,
|
2870
|
-
models,
|
2871
|
-
criterion=None,
|
2872
|
-
log="gradients",
|
2873
|
-
log_freq=
|
2874
|
-
idx=None,
|
2875
|
-
log_graph=False,
|
2884
|
+
models: torch.nn.Module | Sequence[torch.nn.Module],
|
2885
|
+
criterion: torch.F | None = None,
|
2886
|
+
log: Literal["gradients", "parameters", "all"] | None = "gradients",
|
2887
|
+
log_freq: int = 1000,
|
2888
|
+
idx: int | None = None,
|
2889
|
+
log_graph: bool = False,
|
2876
2890
|
) -> None:
|
2877
|
-
|
2891
|
+
"""Hooks into the given PyTorch model(s) to monitor gradients and the model's computational graph.
|
2892
|
+
|
2893
|
+
This function can track parameters, gradients, or both during training. It should be
|
2894
|
+
extended to support arbitrary machine learning models in the future.
|
2895
|
+
|
2896
|
+
Args:
|
2897
|
+
models (Union[torch.nn.Module, Sequence[torch.nn.Module]]):
|
2898
|
+
A single model or a sequence of models to be monitored.
|
2899
|
+
criterion (Optional[torch.F]):
|
2900
|
+
The loss function being optimized (optional).
|
2901
|
+
log (Optional[Literal["gradients", "parameters", "all"]]):
|
2902
|
+
Specifies whether to log "gradients", "parameters", or "all".
|
2903
|
+
Set to None to disable logging. (default="gradients")
|
2904
|
+
log_freq (int):
|
2905
|
+
Frequency (in batches) to log gradients and parameters. (default=1000)
|
2906
|
+
idx (Optional[int]):
|
2907
|
+
Index used when tracking multiple models with `wandb.watch`. (default=None)
|
2908
|
+
log_graph (bool):
|
2909
|
+
Whether to log the model's computational graph. (default=False)
|
2910
|
+
|
2911
|
+
Raises:
|
2912
|
+
ValueError:
|
2913
|
+
If `wandb.init` has not been called or if any of the models are not instances
|
2914
|
+
of `torch.nn.Module`.
|
2915
|
+
"""
|
2916
|
+
wandb.sdk._watch(self, models, criterion, log, log_freq, idx, log_graph)
|
2878
2917
|
|
2879
|
-
# TODO(jhr): annotate this
|
2880
2918
|
@_run_decorator._attach
|
2881
|
-
def unwatch(
|
2882
|
-
|
2919
|
+
def unwatch(
|
2920
|
+
self, models: torch.nn.Module | Sequence[torch.nn.Module] | None = None
|
2921
|
+
) -> None:
|
2922
|
+
"""Remove pytorch model topology, gradient and parameter hooks.
|
2923
|
+
|
2924
|
+
Args:
|
2925
|
+
models (torch.nn.Module | Sequence[torch.nn.Module]):
|
2926
|
+
Optional list of pytorch models that have had watch called on them
|
2927
|
+
"""
|
2928
|
+
wandb.sdk._unwatch(self, models=models)
|
2883
2929
|
|
2884
2930
|
# TODO(kdg): remove all artifact swapping logic
|
2885
|
-
def _swap_artifact_name(self, artifact_name: str, use_as:
|
2931
|
+
def _swap_artifact_name(self, artifact_name: str, use_as: str | None) -> str:
|
2886
2932
|
artifact_key_string = use_as or artifact_name
|
2887
2933
|
replacement_artifact_info = self._launch_artifact_mapping.get(
|
2888
2934
|
artifact_key_string
|
@@ -2925,7 +2971,7 @@ class Run:
|
|
2925
2971
|
self,
|
2926
2972
|
artifact: Artifact,
|
2927
2973
|
target_path: str,
|
2928
|
-
aliases:
|
2974
|
+
aliases: list[str] | None = None,
|
2929
2975
|
) -> None:
|
2930
2976
|
"""Link the given artifact to a portfolio (a promoted collection of artifacts).
|
2931
2977
|
|
@@ -2947,41 +2993,54 @@ class Run:
|
|
2947
2993
|
if aliases is None:
|
2948
2994
|
aliases = []
|
2949
2995
|
|
2950
|
-
if self._backend
|
2951
|
-
|
2952
|
-
|
2953
|
-
|
2954
|
-
|
2955
|
-
|
2956
|
-
|
2957
|
-
|
2958
|
-
|
2959
|
-
|
2960
|
-
|
2961
|
-
|
2962
|
-
|
2963
|
-
|
2964
|
-
|
2965
|
-
|
2966
|
-
|
2967
|
-
|
2968
|
-
|
2969
|
-
|
2970
|
-
|
2971
|
-
|
2972
|
-
|
2973
|
-
|
2974
|
-
|
2975
|
-
|
2996
|
+
if not self._backend or not self._backend.interface:
|
2997
|
+
return
|
2998
|
+
|
2999
|
+
if artifact.is_draft() and not artifact._is_draft_save_started():
|
3000
|
+
artifact = self._log_artifact(artifact)
|
3001
|
+
|
3002
|
+
if self._settings._offline:
|
3003
|
+
# TODO: implement offline mode + sync
|
3004
|
+
raise NotImplementedError
|
3005
|
+
|
3006
|
+
# Wait until the artifact is committed before trying to link it.
|
3007
|
+
artifact.wait()
|
3008
|
+
|
3009
|
+
organization = ""
|
3010
|
+
if is_artifact_registry_project(project):
|
3011
|
+
organization = entity
|
3012
|
+
# In a Registry linking, the entity is used to fetch the organization of the artifact
|
3013
|
+
# therefore the source artifact's entity is passed to the backend
|
3014
|
+
entity = artifact._source_entity
|
3015
|
+
handle = self._backend.interface.deliver_link_artifact(
|
3016
|
+
self,
|
3017
|
+
artifact,
|
3018
|
+
portfolio,
|
3019
|
+
aliases,
|
3020
|
+
entity,
|
3021
|
+
project,
|
3022
|
+
organization,
|
3023
|
+
)
|
3024
|
+
if artifact._ttl_duration_seconds is not None:
|
3025
|
+
wandb.termwarn(
|
3026
|
+
"Artifact TTL will be disabled for source artifacts that are linked to portfolios."
|
3027
|
+
)
|
3028
|
+
result = handle.wait(timeout=-1)
|
3029
|
+
if result is None:
|
3030
|
+
handle.abandon()
|
3031
|
+
else:
|
3032
|
+
response = result.response.link_artifact_response
|
3033
|
+
if response.error_message:
|
3034
|
+
wandb.termerror(response.error_message)
|
2976
3035
|
|
2977
3036
|
@_run_decorator._noop_on_finish()
|
2978
3037
|
@_run_decorator._attach
|
2979
3038
|
def use_artifact(
|
2980
3039
|
self,
|
2981
|
-
artifact_or_name:
|
2982
|
-
type:
|
2983
|
-
aliases:
|
2984
|
-
use_as:
|
3040
|
+
artifact_or_name: str | Artifact,
|
3041
|
+
type: str | None = None,
|
3042
|
+
aliases: list[str] | None = None,
|
3043
|
+
use_as: str | None = None,
|
2985
3044
|
) -> Artifact:
|
2986
3045
|
"""Declare an artifact as an input to a run.
|
2987
3046
|
|
@@ -2989,8 +3048,9 @@ class Run:
|
|
2989
3048
|
|
2990
3049
|
Arguments:
|
2991
3050
|
artifact_or_name: (str or Artifact) An artifact name.
|
2992
|
-
May be prefixed with entity/project/.
|
2993
|
-
|
3051
|
+
May be prefixed with project/ or entity/project/.
|
3052
|
+
If no entity is specified in the name, the Run or API setting's entity is used.
|
3053
|
+
Valid names can be in the following forms:
|
2994
3054
|
- name:version
|
2995
3055
|
- name:alias
|
2996
3056
|
You can also pass an Artifact object created by calling `wandb.Artifact`
|
@@ -3087,11 +3147,11 @@ class Run:
|
|
3087
3147
|
@_run_decorator._attach
|
3088
3148
|
def log_artifact(
|
3089
3149
|
self,
|
3090
|
-
artifact_or_path:
|
3091
|
-
name:
|
3092
|
-
type:
|
3093
|
-
aliases:
|
3094
|
-
tags:
|
3150
|
+
artifact_or_path: Artifact | StrPath,
|
3151
|
+
name: str | None = None,
|
3152
|
+
type: str | None = None,
|
3153
|
+
aliases: list[str] | None = None,
|
3154
|
+
tags: list[str] | None = None,
|
3095
3155
|
) -> Artifact:
|
3096
3156
|
"""Declare an artifact as an output of a run.
|
3097
3157
|
|
@@ -3129,11 +3189,11 @@ class Run:
|
|
3129
3189
|
@_run_decorator._attach
|
3130
3190
|
def upsert_artifact(
|
3131
3191
|
self,
|
3132
|
-
artifact_or_path:
|
3133
|
-
name:
|
3134
|
-
type:
|
3135
|
-
aliases:
|
3136
|
-
distributed_id:
|
3192
|
+
artifact_or_path: Artifact | str,
|
3193
|
+
name: str | None = None,
|
3194
|
+
type: str | None = None,
|
3195
|
+
aliases: list[str] | None = None,
|
3196
|
+
distributed_id: str | None = None,
|
3137
3197
|
) -> Artifact:
|
3138
3198
|
"""Declare (or append to) a non-finalized artifact as output of a run.
|
3139
3199
|
|
@@ -3183,11 +3243,11 @@ class Run:
|
|
3183
3243
|
@_run_decorator._attach
|
3184
3244
|
def finish_artifact(
|
3185
3245
|
self,
|
3186
|
-
artifact_or_path:
|
3187
|
-
name:
|
3188
|
-
type:
|
3189
|
-
aliases:
|
3190
|
-
distributed_id:
|
3246
|
+
artifact_or_path: Artifact | str,
|
3247
|
+
name: str | None = None,
|
3248
|
+
type: str | None = None,
|
3249
|
+
aliases: list[str] | None = None,
|
3250
|
+
distributed_id: str | None = None,
|
3191
3251
|
) -> Artifact:
|
3192
3252
|
"""Finishes a non-finalized artifact as output of a run.
|
3193
3253
|
|
@@ -3235,12 +3295,12 @@ class Run:
|
|
3235
3295
|
|
3236
3296
|
def _log_artifact(
|
3237
3297
|
self,
|
3238
|
-
artifact_or_path:
|
3239
|
-
name:
|
3240
|
-
type:
|
3241
|
-
aliases:
|
3242
|
-
tags:
|
3243
|
-
distributed_id:
|
3298
|
+
artifact_or_path: Artifact | StrPath,
|
3299
|
+
name: str | None = None,
|
3300
|
+
type: str | None = None,
|
3301
|
+
aliases: list[str] | None = None,
|
3302
|
+
tags: list[str] | None = None,
|
3303
|
+
distributed_id: str | None = None,
|
3244
3304
|
finalize: bool = True,
|
3245
3305
|
is_user_created: bool = False,
|
3246
3306
|
use_after_commit: bool = False,
|
@@ -3301,7 +3361,7 @@ class Run:
|
|
3301
3361
|
)
|
3302
3362
|
return artifact
|
3303
3363
|
|
3304
|
-
def _public_api(self, overrides:
|
3364
|
+
def _public_api(self, overrides: dict[str, str] | None = None) -> PublicApi:
|
3305
3365
|
overrides = {"run": self._run_id}
|
3306
3366
|
if not (self._settings._offline or self._run_obj is None):
|
3307
3367
|
overrides["entity"] = self._run_obj.entity
|
@@ -3342,11 +3402,11 @@ class Run:
|
|
3342
3402
|
|
3343
3403
|
def _prepare_artifact(
|
3344
3404
|
self,
|
3345
|
-
artifact_or_path:
|
3346
|
-
name:
|
3347
|
-
type:
|
3348
|
-
aliases:
|
3349
|
-
) ->
|
3405
|
+
artifact_or_path: Artifact | StrPath,
|
3406
|
+
name: str | None = None,
|
3407
|
+
type: str | None = None,
|
3408
|
+
aliases: list[str] | None = None,
|
3409
|
+
) -> tuple[Artifact, list[str]]:
|
3350
3410
|
if isinstance(artifact_or_path, (str, os.PathLike)):
|
3351
3411
|
name = name or f"run-{self._run_id}-{os.path.basename(artifact_or_path)}"
|
3352
3412
|
artifact = wandb.Artifact(name, type or "unspecified")
|
@@ -3377,8 +3437,8 @@ class Run:
|
|
3377
3437
|
def log_model(
|
3378
3438
|
self,
|
3379
3439
|
path: StrPath,
|
3380
|
-
name:
|
3381
|
-
aliases:
|
3440
|
+
name: str | None = None,
|
3441
|
+
aliases: list[str] | None = None,
|
3382
3442
|
) -> None:
|
3383
3443
|
"""Logs a model artifact containing the contents inside the 'path' to a run and marks it as an output to this run.
|
3384
3444
|
|
@@ -3482,8 +3542,8 @@ class Run:
|
|
3482
3542
|
self,
|
3483
3543
|
path: StrPath,
|
3484
3544
|
registered_model_name: str,
|
3485
|
-
name:
|
3486
|
-
aliases:
|
3545
|
+
name: str | None = None,
|
3546
|
+
aliases: list[str] | None = None,
|
3487
3547
|
) -> None:
|
3488
3548
|
"""Log a model artifact version and link it to a registered model in the model registry.
|
3489
3549
|
|
@@ -3576,8 +3636,8 @@ class Run:
|
|
3576
3636
|
self,
|
3577
3637
|
title: str,
|
3578
3638
|
text: str,
|
3579
|
-
level:
|
3580
|
-
wait_duration:
|
3639
|
+
level: str | AlertLevel | None = None,
|
3640
|
+
wait_duration: int | float | timedelta | None = None,
|
3581
3641
|
) -> None:
|
3582
3642
|
"""Launch an alert with the given title and text.
|
3583
3643
|
|
@@ -3605,12 +3665,12 @@ class Run:
|
|
3605
3665
|
if self._backend and self._backend.interface:
|
3606
3666
|
self._backend.interface.publish_alert(title, text, level_str, wait_duration)
|
3607
3667
|
|
3608
|
-
def __enter__(self) ->
|
3668
|
+
def __enter__(self) -> Run:
|
3609
3669
|
return self
|
3610
3670
|
|
3611
3671
|
def __exit__(
|
3612
3672
|
self,
|
3613
|
-
exc_type:
|
3673
|
+
exc_type: type[BaseException],
|
3614
3674
|
exc_val: BaseException,
|
3615
3675
|
exc_tb: TracebackType,
|
3616
3676
|
) -> bool:
|
@@ -3634,7 +3694,7 @@ class Run:
|
|
3634
3694
|
@property
|
3635
3695
|
@_run_decorator._noop_on_finish()
|
3636
3696
|
@_run_decorator._attach
|
3637
|
-
def _system_metrics(self) ->
|
3697
|
+
def _system_metrics(self) -> dict[str, list[tuple[datetime, float]]]:
|
3638
3698
|
"""Returns a dictionary of system metrics.
|
3639
3699
|
|
3640
3700
|
Returns:
|
@@ -3643,7 +3703,7 @@ class Run:
|
|
3643
3703
|
|
3644
3704
|
def pb_to_dict(
|
3645
3705
|
system_metrics_pb: wandb.proto.wandb_internal_pb2.GetSystemMetricsResponse,
|
3646
|
-
) ->
|
3706
|
+
) -> dict[str, list[tuple[datetime, float]]]:
|
3647
3707
|
res = {}
|
3648
3708
|
|
3649
3709
|
for metric, records in system_metrics_pb.system_metrics.items():
|
@@ -3684,8 +3744,8 @@ class Run:
|
|
3684
3744
|
@staticmethod
|
3685
3745
|
def _header(
|
3686
3746
|
*,
|
3687
|
-
settings:
|
3688
|
-
printer:
|
3747
|
+
settings: Settings,
|
3748
|
+
printer: printer.Printer,
|
3689
3749
|
) -> None:
|
3690
3750
|
Run._header_wandb_version_info(settings=settings, printer=printer)
|
3691
3751
|
Run._header_sync_info(settings=settings, printer=printer)
|
@@ -3694,22 +3754,20 @@ class Run:
|
|
3694
3754
|
@staticmethod
|
3695
3755
|
def _header_wandb_version_info(
|
3696
3756
|
*,
|
3697
|
-
settings:
|
3698
|
-
printer:
|
3757
|
+
settings: Settings,
|
3758
|
+
printer: printer.Printer,
|
3699
3759
|
) -> None:
|
3700
3760
|
if settings.quiet or settings.silent:
|
3701
3761
|
return
|
3702
3762
|
|
3703
3763
|
# TODO: add this to a higher verbosity level
|
3704
|
-
printer.display(
|
3705
|
-
f"Tracking run with wandb version {wandb.__version__}", off=False
|
3706
|
-
)
|
3764
|
+
printer.display(f"Tracking run with wandb version {wandb.__version__}")
|
3707
3765
|
|
3708
3766
|
@staticmethod
|
3709
3767
|
def _header_sync_info(
|
3710
3768
|
*,
|
3711
|
-
settings:
|
3712
|
-
printer:
|
3769
|
+
settings: Settings,
|
3770
|
+
printer: printer.Printer,
|
3713
3771
|
) -> None:
|
3714
3772
|
if settings._offline:
|
3715
3773
|
printer.display(
|
@@ -3721,17 +3779,18 @@ class Run:
|
|
3721
3779
|
)
|
3722
3780
|
else:
|
3723
3781
|
info = [f"Run data is saved locally in {printer.files(settings.sync_dir)}"]
|
3724
|
-
if not printer.
|
3782
|
+
if not printer.supports_html:
|
3725
3783
|
info.append(
|
3726
3784
|
f"Run {printer.code('`wandb offline`')} to turn off syncing."
|
3727
3785
|
)
|
3728
|
-
|
3786
|
+
if not settings.quiet and not settings.silent:
|
3787
|
+
printer.display(info)
|
3729
3788
|
|
3730
3789
|
@staticmethod
|
3731
3790
|
def _header_run_info(
|
3732
3791
|
*,
|
3733
|
-
settings:
|
3734
|
-
printer:
|
3792
|
+
settings: Settings,
|
3793
|
+
printer: printer.Printer,
|
3735
3794
|
) -> None:
|
3736
3795
|
if settings._offline or settings.silent:
|
3737
3796
|
return
|
@@ -3749,7 +3808,7 @@ class Run:
|
|
3749
3808
|
if not run_name:
|
3750
3809
|
return
|
3751
3810
|
|
3752
|
-
if printer.
|
3811
|
+
if printer.supports_html:
|
3753
3812
|
if not wandb.jupyter.maybe_display(): # type: ignore
|
3754
3813
|
run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
|
3755
3814
|
project_line, sweep_line = "", ""
|
@@ -3768,10 +3827,8 @@ class Run:
|
|
3768
3827
|
[f"{run_state_str} {run_line} {project_line}", sweep_line],
|
3769
3828
|
)
|
3770
3829
|
|
3771
|
-
|
3772
|
-
printer.display(
|
3773
|
-
f"{run_state_str} {printer.name(run_name)}", off=not run_name
|
3774
|
-
)
|
3830
|
+
elif run_name:
|
3831
|
+
printer.display(f"{run_state_str} {printer.name(run_name)}")
|
3775
3832
|
|
3776
3833
|
if not settings.quiet:
|
3777
3834
|
# TODO: add verbosity levels and add this to higher levels
|
@@ -3787,11 +3844,13 @@ class Run:
|
|
3787
3844
|
)
|
3788
3845
|
|
3789
3846
|
# TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
|
3790
|
-
if Api().api.settings().get("anonymous") == "true":
|
3847
|
+
if run_name and Api().api.settings().get("anonymous") == "true":
|
3791
3848
|
printer.display(
|
3792
|
-
|
3849
|
+
(
|
3850
|
+
"Do NOT share these links with anyone."
|
3851
|
+
" They can be used to claim your runs."
|
3852
|
+
),
|
3793
3853
|
level="warn",
|
3794
|
-
off=not run_name,
|
3795
3854
|
)
|
3796
3855
|
|
3797
3856
|
# ------------------------------------------------------------------------------
|
@@ -3801,15 +3860,15 @@ class Run:
|
|
3801
3860
|
# with the service execution path that doesn't have access to the run instance
|
3802
3861
|
@staticmethod
|
3803
3862
|
def _footer(
|
3804
|
-
sampled_history:
|
3805
|
-
final_summary:
|
3806
|
-
poll_exit_response:
|
3807
|
-
internal_messages_response:
|
3808
|
-
reporter:
|
3809
|
-
quiet:
|
3863
|
+
sampled_history: SampledHistoryResponse | None = None,
|
3864
|
+
final_summary: GetSummaryResponse | None = None,
|
3865
|
+
poll_exit_response: PollExitResponse | None = None,
|
3866
|
+
internal_messages_response: InternalMessagesResponse | None = None,
|
3867
|
+
reporter: Reporter | None = None,
|
3868
|
+
quiet: bool | None = None,
|
3810
3869
|
*,
|
3811
|
-
settings:
|
3812
|
-
printer:
|
3870
|
+
settings: Settings,
|
3871
|
+
printer: printer.Printer,
|
3813
3872
|
) -> None:
|
3814
3873
|
Run._footer_history_summary_info(
|
3815
3874
|
history=sampled_history,
|
@@ -3841,177 +3900,51 @@ class Run:
|
|
3841
3900
|
reporter=reporter, quiet=quiet, settings=settings, printer=printer
|
3842
3901
|
)
|
3843
3902
|
|
3844
|
-
# fixme: Temporary hack until we move to rich which allows multiple spinners
|
3845
3903
|
@staticmethod
|
3846
|
-
def
|
3847
|
-
|
3848
|
-
|
3849
|
-
] = None,
|
3850
|
-
*,
|
3851
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3852
|
-
) -> None:
|
3853
|
-
if not poll_exit_responses:
|
3854
|
-
return
|
3855
|
-
if isinstance(poll_exit_responses, PollExitResponse):
|
3856
|
-
Run._footer_single_run_file_pusher_status_info(
|
3857
|
-
poll_exit_responses, printer=printer
|
3858
|
-
)
|
3859
|
-
elif isinstance(poll_exit_responses, list):
|
3860
|
-
poll_exit_responses_list = poll_exit_responses
|
3861
|
-
assert all(
|
3862
|
-
response is None or isinstance(response, PollExitResponse)
|
3863
|
-
for response in poll_exit_responses_list
|
3864
|
-
)
|
3865
|
-
if len(poll_exit_responses_list) == 0:
|
3866
|
-
return
|
3867
|
-
elif len(poll_exit_responses_list) == 1:
|
3868
|
-
Run._footer_single_run_file_pusher_status_info(
|
3869
|
-
poll_exit_responses_list[0], printer=printer
|
3870
|
-
)
|
3871
|
-
else:
|
3872
|
-
Run._footer_multiple_runs_file_pusher_status_info(
|
3873
|
-
poll_exit_responses_list, printer=printer
|
3874
|
-
)
|
3875
|
-
else:
|
3876
|
-
logger.error(
|
3877
|
-
f"Got the type `{type(poll_exit_responses)}` for `poll_exit_responses`. "
|
3878
|
-
"Expected either None, PollExitResponse or a List[Union[PollExitResponse, None]]"
|
3879
|
-
)
|
3880
|
-
|
3881
|
-
@staticmethod
|
3882
|
-
def _footer_single_run_file_pusher_status_info(
|
3883
|
-
poll_exit_response: Optional[PollExitResponse] = None,
|
3904
|
+
def _footer_sync_info(
|
3905
|
+
poll_exit_response: PollExitResponse | None = None,
|
3906
|
+
quiet: bool | None = None,
|
3884
3907
|
*,
|
3885
|
-
|
3908
|
+
settings: Settings,
|
3909
|
+
printer: printer.Printer,
|
3886
3910
|
) -> None:
|
3887
|
-
|
3888
|
-
if not poll_exit_response:
|
3911
|
+
if settings.silent:
|
3889
3912
|
return
|
3890
3913
|
|
3891
|
-
|
3892
|
-
|
3893
|
-
megabyte = wandb.util.POW_2_BYTES[2][1]
|
3894
|
-
line = (
|
3895
|
-
f"{stats.uploaded_bytes / megabyte:.3f} MB"
|
3896
|
-
f" of {stats.total_bytes / megabyte:.3f} MB uploaded"
|
3897
|
-
)
|
3898
|
-
if stats.deduped_bytes > 0:
|
3899
|
-
line += f" ({stats.deduped_bytes / megabyte:.3f} MB deduped)"
|
3900
|
-
line += "\r"
|
3901
|
-
|
3902
|
-
if stats.total_bytes > 0:
|
3903
|
-
printer.progress_update(line, stats.uploaded_bytes / stats.total_bytes)
|
3904
|
-
else:
|
3905
|
-
printer.progress_update(line, 1.0)
|
3906
|
-
|
3907
|
-
if poll_exit_response.done:
|
3908
|
-
printer.progress_close()
|
3909
|
-
|
3910
|
-
if stats.total_bytes > 0:
|
3911
|
-
dedupe_fraction = stats.deduped_bytes / float(stats.total_bytes)
|
3912
|
-
else:
|
3913
|
-
dedupe_fraction = 0
|
3914
|
-
|
3915
|
-
if stats.deduped_bytes > 0.01:
|
3914
|
+
if settings._offline:
|
3915
|
+
if not quiet and not settings.quiet:
|
3916
3916
|
printer.display(
|
3917
|
-
|
3917
|
+
[
|
3918
|
+
"You can sync this run to the cloud by running:",
|
3919
|
+
printer.code(f"wandb sync {settings.sync_dir}"),
|
3920
|
+
],
|
3918
3921
|
)
|
3919
|
-
|
3920
|
-
@staticmethod
|
3921
|
-
def _footer_multiple_runs_file_pusher_status_info(
|
3922
|
-
poll_exit_responses: List[Optional[PollExitResponse]],
|
3923
|
-
*,
|
3924
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3925
|
-
) -> None:
|
3926
|
-
# todo: is this same as settings._offline?
|
3927
|
-
if not all(poll_exit_responses):
|
3928
3922
|
return
|
3929
3923
|
|
3930
|
-
|
3931
|
-
|
3932
|
-
|
3933
|
-
|
3934
|
-
response.file_counts.wandb_count,
|
3935
|
-
response.file_counts.media_count,
|
3936
|
-
response.file_counts.artifact_count,
|
3937
|
-
response.file_counts.other_count,
|
3938
|
-
]
|
3924
|
+
info = []
|
3925
|
+
if settings.run_name and settings.run_url:
|
3926
|
+
info.append(
|
3927
|
+
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
|
3939
3928
|
)
|
3940
|
-
|
3941
|
-
|
3942
|
-
|
3943
|
-
uploaded = sum(
|
3944
|
-
response.pusher_stats.uploaded_bytes
|
3945
|
-
for response in poll_exit_responses
|
3946
|
-
if response is not None and response.pusher_stats is not None
|
3947
|
-
)
|
3948
|
-
total = sum(
|
3949
|
-
response.pusher_stats.total_bytes
|
3950
|
-
for response in poll_exit_responses
|
3951
|
-
if response is not None and response.pusher_stats is not None
|
3952
|
-
)
|
3953
|
-
|
3954
|
-
line = (
|
3955
|
-
f"Processing {len(poll_exit_responses)} runs with {total_files} files "
|
3956
|
-
f"({uploaded/megabyte :.2f} MB/{total/megabyte :.2f} MB)\r"
|
3957
|
-
)
|
3958
|
-
# line = "{}{:<{max_len}}\r".format(line, " ", max_len=(80 - len(line)))
|
3959
|
-
printer.progress_update(line) # type:ignore[call-arg]
|
3960
|
-
|
3961
|
-
done = all(
|
3962
|
-
[
|
3963
|
-
poll_exit_response.done
|
3964
|
-
for poll_exit_response in poll_exit_responses
|
3965
|
-
if poll_exit_response
|
3966
|
-
]
|
3967
|
-
)
|
3968
|
-
if done:
|
3969
|
-
printer.progress_close()
|
3970
|
-
|
3971
|
-
@staticmethod
|
3972
|
-
def _footer_sync_info(
|
3973
|
-
poll_exit_response: Optional[PollExitResponse] = None,
|
3974
|
-
quiet: Optional[bool] = None,
|
3975
|
-
*,
|
3976
|
-
settings: "Settings",
|
3977
|
-
printer: Union["PrinterTerm", "PrinterJupyter"],
|
3978
|
-
) -> None:
|
3979
|
-
if settings.silent:
|
3980
|
-
return
|
3981
|
-
|
3982
|
-
if settings._offline:
|
3983
|
-
printer.display(
|
3984
|
-
[
|
3985
|
-
"You can sync this run to the cloud by running:",
|
3986
|
-
printer.code(f"wandb sync {settings.sync_dir}"),
|
3987
|
-
],
|
3988
|
-
off=(quiet or settings.quiet),
|
3929
|
+
if settings.project_url:
|
3930
|
+
info.append(
|
3931
|
+
f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
|
3989
3932
|
)
|
3990
|
-
|
3991
|
-
info
|
3992
|
-
|
3993
|
-
|
3994
|
-
|
3995
|
-
)
|
3996
|
-
|
3997
|
-
|
3998
|
-
f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
|
3999
|
-
)
|
4000
|
-
if poll_exit_response and poll_exit_response.file_counts:
|
4001
|
-
logger.info("logging synced files")
|
4002
|
-
file_counts = poll_exit_response.file_counts
|
4003
|
-
info.append(
|
4004
|
-
f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
|
4005
|
-
f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
|
4006
|
-
)
|
4007
|
-
printer.display(info)
|
3933
|
+
if poll_exit_response and poll_exit_response.file_counts:
|
3934
|
+
logger.info("logging synced files")
|
3935
|
+
file_counts = poll_exit_response.file_counts
|
3936
|
+
info.append(
|
3937
|
+
f"Synced {file_counts.wandb_count} W&B file(s), {file_counts.media_count} media file(s), "
|
3938
|
+
f"{file_counts.artifact_count} artifact file(s) and {file_counts.other_count} other file(s)",
|
3939
|
+
)
|
3940
|
+
printer.display(info)
|
4008
3941
|
|
4009
3942
|
@staticmethod
|
4010
3943
|
def _footer_log_dir_info(
|
4011
|
-
quiet:
|
3944
|
+
quiet: bool | None = None,
|
4012
3945
|
*,
|
4013
|
-
settings:
|
4014
|
-
printer:
|
3946
|
+
settings: Settings,
|
3947
|
+
printer: printer.Printer,
|
4015
3948
|
) -> None:
|
4016
3949
|
if (quiet or settings.quiet) or settings.silent:
|
4017
3950
|
return
|
@@ -4025,12 +3958,12 @@ class Run:
|
|
4025
3958
|
|
4026
3959
|
@staticmethod
|
4027
3960
|
def _footer_history_summary_info(
|
4028
|
-
history:
|
4029
|
-
summary:
|
4030
|
-
quiet:
|
3961
|
+
history: SampledHistoryResponse | None = None,
|
3962
|
+
summary: GetSummaryResponse | None = None,
|
3963
|
+
quiet: bool | None = None,
|
4031
3964
|
*,
|
4032
|
-
settings:
|
4033
|
-
printer:
|
3965
|
+
settings: Settings,
|
3966
|
+
printer: printer.Printer,
|
4034
3967
|
) -> None:
|
4035
3968
|
if (quiet or settings.quiet) or settings.silent:
|
4036
3969
|
return
|
@@ -4096,11 +4029,11 @@ class Run:
|
|
4096
4029
|
|
4097
4030
|
@staticmethod
|
4098
4031
|
def _footer_internal_messages(
|
4099
|
-
internal_messages_response:
|
4100
|
-
quiet:
|
4032
|
+
internal_messages_response: InternalMessagesResponse | None = None,
|
4033
|
+
quiet: bool | None = None,
|
4101
4034
|
*,
|
4102
|
-
settings:
|
4103
|
-
printer:
|
4035
|
+
settings: Settings,
|
4036
|
+
printer: printer.Printer,
|
4104
4037
|
) -> None:
|
4105
4038
|
if (quiet or settings.quiet) or settings.silent:
|
4106
4039
|
return
|
@@ -4114,9 +4047,9 @@ class Run:
|
|
4114
4047
|
@staticmethod
|
4115
4048
|
def _footer_notify_wandb_core(
|
4116
4049
|
*,
|
4117
|
-
quiet:
|
4118
|
-
settings:
|
4119
|
-
printer:
|
4050
|
+
quiet: bool | None = None,
|
4051
|
+
settings: Settings,
|
4052
|
+
printer: printer.Printer,
|
4120
4053
|
) -> None:
|
4121
4054
|
"""Prints a message advertising the upcoming core release."""
|
4122
4055
|
if quiet or not settings._require_legacy_service:
|
@@ -4131,11 +4064,11 @@ class Run:
|
|
4131
4064
|
|
4132
4065
|
@staticmethod
|
4133
4066
|
def _footer_reporter_warn_err(
|
4134
|
-
reporter:
|
4135
|
-
quiet:
|
4067
|
+
reporter: Reporter | None = None,
|
4068
|
+
quiet: bool | None = None,
|
4136
4069
|
*,
|
4137
|
-
settings:
|
4138
|
-
printer:
|
4070
|
+
settings: Settings,
|
4071
|
+
printer: printer.Printer,
|
4139
4072
|
) -> None:
|
4140
4073
|
if (quiet or settings.quiet) or settings.silent:
|
4141
4074
|
return
|
@@ -4161,10 +4094,10 @@ class Run:
|
|
4161
4094
|
# We define this outside of the run context to support restoring before init
|
4162
4095
|
def restore(
|
4163
4096
|
name: str,
|
4164
|
-
run_path:
|
4097
|
+
run_path: str | None = None,
|
4165
4098
|
replace: bool = False,
|
4166
|
-
root:
|
4167
|
-
) ->
|
4099
|
+
root: str | None = None,
|
4100
|
+
) -> None | TextIO:
|
4168
4101
|
"""Download the specified file from cloud storage.
|
4169
4102
|
|
4170
4103
|
File is placed into the current directory or run directory.
|
@@ -4222,7 +4155,7 @@ except AttributeError:
|
|
4222
4155
|
pass
|
4223
4156
|
|
4224
4157
|
|
4225
|
-
def finish(exit_code:
|
4158
|
+
def finish(exit_code: int | None = None, quiet: bool | None = None) -> None:
|
4226
4159
|
"""Mark a run as finished, and finish uploading all data.
|
4227
4160
|
|
4228
4161
|
This is used when creating multiple runs in the same process.
|