wandb 0.16.3__py3-none-any.whl → 0.16.5__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +2 -2
- wandb/agents/pyagent.py +1 -1
- wandb/apis/importers/__init__.py +1 -4
- wandb/apis/importers/internals/internal.py +386 -0
- wandb/apis/importers/internals/protocols.py +125 -0
- wandb/apis/importers/internals/util.py +78 -0
- wandb/apis/importers/mlflow.py +125 -88
- wandb/apis/importers/validation.py +108 -0
- wandb/apis/importers/wandb.py +1604 -0
- wandb/apis/public/api.py +7 -10
- wandb/apis/public/artifacts.py +38 -0
- wandb/apis/public/files.py +11 -2
- wandb/apis/reports/v2/__init__.py +0 -19
- wandb/apis/reports/v2/expr_parsing.py +0 -1
- wandb/apis/reports/v2/interface.py +15 -18
- wandb/apis/reports/v2/internal.py +12 -45
- wandb/cli/cli.py +52 -55
- wandb/integration/gym/__init__.py +2 -1
- wandb/integration/keras/callbacks/model_checkpoint.py +1 -1
- wandb/integration/keras/keras.py +6 -4
- wandb/integration/kfp/kfp_patch.py +2 -2
- wandb/integration/openai/fine_tuning.py +1 -2
- wandb/integration/ultralytics/callback.py +0 -1
- wandb/proto/v3/wandb_internal_pb2.py +332 -312
- wandb/proto/v3/wandb_settings_pb2.py +13 -3
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +316 -312
- wandb/proto/v4/wandb_settings_pb2.py +5 -3
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/sdk/artifacts/artifact.py +75 -31
- wandb/sdk/artifacts/artifact_manifest.py +5 -2
- wandb/sdk/artifacts/artifact_manifest_entry.py +6 -1
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +8 -2
- wandb/sdk/artifacts/artifact_saver.py +19 -47
- wandb/sdk/artifacts/storage_handler.py +2 -1
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +22 -9
- wandb/sdk/artifacts/storage_policy.py +4 -1
- wandb/sdk/data_types/base_types/wb_value.py +1 -1
- wandb/sdk/data_types/image.py +2 -2
- wandb/sdk/interface/interface.py +49 -13
- wandb/sdk/interface/interface_shared.py +17 -11
- wandb/sdk/internal/file_stream.py +20 -1
- wandb/sdk/internal/handler.py +1 -4
- wandb/sdk/internal/internal_api.py +3 -1
- wandb/sdk/internal/job_builder.py +49 -19
- wandb/sdk/internal/profiler.py +1 -1
- wandb/sdk/internal/sender.py +96 -124
- wandb/sdk/internal/sender_config.py +197 -0
- wandb/sdk/internal/settings_static.py +9 -0
- wandb/sdk/internal/system/system_info.py +5 -3
- wandb/sdk/internal/update.py +1 -1
- wandb/sdk/launch/_launch.py +3 -3
- wandb/sdk/launch/_launch_add.py +28 -29
- wandb/sdk/launch/_project_spec.py +148 -136
- wandb/sdk/launch/agent/agent.py +3 -7
- wandb/sdk/launch/agent/config.py +0 -27
- wandb/sdk/launch/builder/build.py +54 -28
- wandb/sdk/launch/builder/docker_builder.py +4 -15
- wandb/sdk/launch/builder/kaniko_builder.py +72 -45
- wandb/sdk/launch/create_job.py +6 -40
- wandb/sdk/launch/loader.py +10 -0
- wandb/sdk/launch/registry/anon.py +29 -0
- wandb/sdk/launch/registry/local_registry.py +4 -1
- wandb/sdk/launch/runner/kubernetes_runner.py +20 -2
- wandb/sdk/launch/runner/local_container.py +15 -10
- wandb/sdk/launch/runner/sagemaker_runner.py +1 -1
- wandb/sdk/launch/sweeps/scheduler.py +11 -3
- wandb/sdk/launch/utils.py +14 -0
- wandb/sdk/lib/__init__.py +2 -5
- wandb/sdk/lib/_settings_toposort_generated.py +4 -1
- wandb/sdk/lib/apikey.py +0 -5
- wandb/sdk/lib/config_util.py +0 -31
- wandb/sdk/lib/filesystem.py +11 -1
- wandb/sdk/lib/run_moment.py +72 -0
- wandb/sdk/service/service.py +7 -2
- wandb/sdk/service/streams.py +1 -6
- wandb/sdk/verify/verify.py +2 -1
- wandb/sdk/wandb_init.py +12 -1
- wandb/sdk/wandb_login.py +43 -26
- wandb/sdk/wandb_run.py +164 -110
- wandb/sdk/wandb_settings.py +58 -16
- wandb/testing/relay.py +5 -6
- wandb/util.py +50 -7
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/METADATA +8 -1
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/RECORD +89 -82
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/WHEEL +1 -1
- wandb/apis/importers/base.py +0 -400
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/LICENSE +0 -0
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/entry_points.txt +0 -0
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -6,6 +6,7 @@ import json
|
|
6
6
|
import logging
|
7
7
|
import numbers
|
8
8
|
import os
|
9
|
+
import pathlib
|
9
10
|
import re
|
10
11
|
import sys
|
11
12
|
import threading
|
@@ -42,7 +43,6 @@ from wandb.apis import internal, public
|
|
42
43
|
from wandb.apis.internal import Api
|
43
44
|
from wandb.apis.public import Api as PublicApi
|
44
45
|
from wandb.proto.wandb_internal_pb2 import (
|
45
|
-
JobInfoResponse,
|
46
46
|
MetricRecord,
|
47
47
|
PollExitResponse,
|
48
48
|
Result,
|
@@ -531,7 +531,6 @@ class Run:
|
|
531
531
|
_check_version: Optional["CheckVersionResponse"]
|
532
532
|
_sampled_history: Optional["SampledHistoryResponse"]
|
533
533
|
_final_summary: Optional["GetSummaryResponse"]
|
534
|
-
_job_info: Optional["JobInfoResponse"]
|
535
534
|
_poll_exit_handle: Optional[MailboxHandle]
|
536
535
|
_poll_exit_response: Optional[PollExitResponse]
|
537
536
|
_server_info_response: Optional[ServerInfoResponse]
|
@@ -642,7 +641,6 @@ class Run:
|
|
642
641
|
self._server_info_response = None
|
643
642
|
self._internal_messages_response = None
|
644
643
|
self._poll_exit_handle = None
|
645
|
-
self._job_info = None
|
646
644
|
|
647
645
|
# Initialize telemetry object
|
648
646
|
self._telemetry_obj = telemetry.TelemetryRecord()
|
@@ -673,6 +671,12 @@ class Run:
|
|
673
671
|
os.path.join("code", self._settings.program_relpath)
|
674
672
|
)
|
675
673
|
|
674
|
+
if self._settings.fork_from is not None:
|
675
|
+
config[wandb_key]["branch_point"] = {
|
676
|
+
"run_id": self._settings.fork_from.run,
|
677
|
+
"step": self._settings.fork_from.value,
|
678
|
+
}
|
679
|
+
|
676
680
|
self._config._update(config, ignore_locked=True)
|
677
681
|
|
678
682
|
if sweep_config:
|
@@ -734,7 +738,7 @@ class Run:
|
|
734
738
|
and self._settings.launch_config_path
|
735
739
|
and os.path.exists(self._settings.launch_config_path)
|
736
740
|
):
|
737
|
-
self.
|
741
|
+
self.save(self._settings.launch_config_path)
|
738
742
|
with open(self._settings.launch_config_path) as fp:
|
739
743
|
launch_config = json.loads(fp.read())
|
740
744
|
if launch_config.get("overrides", {}).get("artifacts") is not None:
|
@@ -1385,6 +1389,8 @@ class Run:
|
|
1385
1389
|
|
1386
1390
|
@_run_decorator._noop_on_finish()
|
1387
1391
|
def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
|
1392
|
+
with telemetry.context(run=self) as tel:
|
1393
|
+
tel.feature.set_summary = True
|
1388
1394
|
if self._backend and self._backend.interface:
|
1389
1395
|
self._backend.interface.publish_summary(summary_record)
|
1390
1396
|
|
@@ -1811,6 +1817,10 @@ class Run:
|
|
1811
1817
|
ValueError: if invalid data is passed
|
1812
1818
|
|
1813
1819
|
"""
|
1820
|
+
if step is not None:
|
1821
|
+
with telemetry.context(run=self) as tel:
|
1822
|
+
tel.feature.set_step_log = True
|
1823
|
+
|
1814
1824
|
if sync is not None:
|
1815
1825
|
deprecate.deprecate(
|
1816
1826
|
field_name=deprecate.Deprecated.run__log_sync,
|
@@ -1831,20 +1841,53 @@ class Run:
|
|
1831
1841
|
@_run_decorator._attach
|
1832
1842
|
def save(
|
1833
1843
|
self,
|
1834
|
-
glob_str: Optional[str] = None,
|
1835
|
-
base_path: Optional[str] = None,
|
1844
|
+
glob_str: Optional[Union[str, os.PathLike]] = None,
|
1845
|
+
base_path: Optional[Union[str, os.PathLike]] = None,
|
1836
1846
|
policy: "PolicyName" = "live",
|
1837
1847
|
) -> Union[bool, List[str]]:
|
1838
|
-
"""
|
1848
|
+
"""Sync one or more files to W&B.
|
1849
|
+
|
1850
|
+
Relative paths are relative to the current working directory.
|
1851
|
+
|
1852
|
+
A Unix glob, such as "myfiles/*", is expanded at the time `save` is
|
1853
|
+
called regardless of the `policy`. In particular, new files are not
|
1854
|
+
picked up automatically.
|
1855
|
+
|
1856
|
+
A `base_path` may be provided to control the directory structure of
|
1857
|
+
uploaded files. It should be a prefix of `glob_str`, and the direcotry
|
1858
|
+
structure beneath it is preserved. It's best understood through
|
1859
|
+
examples:
|
1860
|
+
|
1861
|
+
```
|
1862
|
+
wandb.save("these/are/myfiles/*")
|
1863
|
+
# => Saves files in a "these/are/myfiles/" folder in the run.
|
1864
|
+
|
1865
|
+
wandb.save("these/are/myfiles/*", base_path="these")
|
1866
|
+
# => Saves files in an "are/myfiles/" folder in the run.
|
1867
|
+
|
1868
|
+
wandb.save("/User/username/Documents/run123/*.txt")
|
1869
|
+
# => Saves files in a "run123/" folder in the run.
|
1870
|
+
|
1871
|
+
wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
|
1872
|
+
# => Saves files in a "username/Documents/run123/" folder in the run.
|
1873
|
+
|
1874
|
+
wandb.save("files/*/saveme.txt")
|
1875
|
+
# => Saves each "saveme.txt" file in an appropriate subdirectory
|
1876
|
+
# of "files/".
|
1877
|
+
```
|
1839
1878
|
|
1840
1879
|
Arguments:
|
1841
|
-
glob_str:
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1880
|
+
glob_str: A relative or absolute path or Unix glob.
|
1881
|
+
base_path: A path to use to infer a directory structure; see examples.
|
1882
|
+
policy: One of `live`, `now`, or `end`.
|
1883
|
+
* live: upload the file as it changes, overwriting the previous version
|
1884
|
+
* now: upload the file once now
|
1885
|
+
* end: upload file when the run ends
|
1886
|
+
|
1887
|
+
Returns:
|
1888
|
+
Paths to the symlinks created for the matched files.
|
1889
|
+
|
1890
|
+
For historical reasons, this may return a boolean in legacy code.
|
1848
1891
|
"""
|
1849
1892
|
if glob_str is None:
|
1850
1893
|
# noop for historical reasons, run.save() may be called in legacy code
|
@@ -1857,77 +1900,116 @@ class Run:
|
|
1857
1900
|
)
|
1858
1901
|
return True
|
1859
1902
|
|
1860
|
-
|
1903
|
+
if isinstance(glob_str, bytes):
|
1904
|
+
# Preserved for backward compatibility: allow bytes inputs.
|
1905
|
+
glob_str = glob_str.decode("utf-8")
|
1906
|
+
if isinstance(glob_str, str) and (
|
1907
|
+
glob_str.startswith("gs://") or glob_str.startswith("s3://")
|
1908
|
+
):
|
1909
|
+
# Provide a better error message for a common misuse.
|
1910
|
+
wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
|
1911
|
+
return []
|
1912
|
+
glob_path = pathlib.Path(glob_str)
|
1913
|
+
|
1914
|
+
if base_path is not None:
|
1915
|
+
base_path = pathlib.Path(base_path)
|
1916
|
+
elif not glob_path.is_absolute():
|
1917
|
+
base_path = pathlib.Path(".")
|
1918
|
+
else:
|
1919
|
+
# Absolute glob paths with no base path get special handling.
|
1920
|
+
wandb.termwarn(
|
1921
|
+
"Saving files without folders. If you want to preserve "
|
1922
|
+
"subdirectories pass base_path to wandb.save, i.e. "
|
1923
|
+
'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
|
1924
|
+
repeat=False,
|
1925
|
+
)
|
1926
|
+
base_path = glob_path.resolve().parent.parent
|
1861
1927
|
|
1862
|
-
def _save(
|
1863
|
-
self,
|
1864
|
-
glob_str: Optional[str] = None,
|
1865
|
-
base_path: Optional[str] = None,
|
1866
|
-
policy: "PolicyName" = "live",
|
1867
|
-
) -> Union[bool, List[str]]:
|
1868
1928
|
if policy not in ("live", "end", "now"):
|
1869
1929
|
raise ValueError(
|
1870
|
-
'Only "live" "end" and "now" policies are currently supported.'
|
1930
|
+
'Only "live", "end" and "now" policies are currently supported.'
|
1871
1931
|
)
|
1872
|
-
if isinstance(glob_str, bytes):
|
1873
|
-
glob_str = glob_str.decode("utf-8")
|
1874
|
-
if not isinstance(glob_str, str):
|
1875
|
-
raise ValueError("Must call wandb.save(glob_str) with glob_str a str")
|
1876
1932
|
|
1877
|
-
|
1878
|
-
|
1879
|
-
|
1880
|
-
|
1881
|
-
|
1882
|
-
|
1883
|
-
|
1884
|
-
|
1885
|
-
|
1886
|
-
|
1887
|
-
|
1888
|
-
|
1889
|
-
|
1890
|
-
|
1933
|
+
resolved_glob_path = glob_path.resolve()
|
1934
|
+
resolved_base_path = base_path.resolve()
|
1935
|
+
|
1936
|
+
return self._save(
|
1937
|
+
resolved_glob_path,
|
1938
|
+
resolved_base_path,
|
1939
|
+
policy,
|
1940
|
+
)
|
1941
|
+
|
1942
|
+
def _save(
|
1943
|
+
self,
|
1944
|
+
glob_path: pathlib.Path,
|
1945
|
+
base_path: pathlib.Path,
|
1946
|
+
policy: "PolicyName",
|
1947
|
+
) -> List[str]:
|
1948
|
+
# Can't use is_relative_to() because that's added in Python 3.9,
|
1949
|
+
# but we support down to Python 3.7.
|
1950
|
+
if not str(glob_path).startswith(str(base_path)):
|
1951
|
+
raise ValueError("Glob may not walk above the base path")
|
1952
|
+
|
1953
|
+
if glob_path == base_path:
|
1954
|
+
raise ValueError("Glob cannot be the same as the base path")
|
1955
|
+
|
1956
|
+
relative_glob = glob_path.relative_to(base_path)
|
1957
|
+
if relative_glob.parts[0] == "*":
|
1958
|
+
raise ValueError("Glob may not start with '*' relative to the base path")
|
1959
|
+
relative_glob_str = GlobStr(str(relative_glob))
|
1891
1960
|
|
1892
1961
|
with telemetry.context(run=self) as tel:
|
1893
1962
|
tel.feature.save = True
|
1894
1963
|
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
1898
|
-
|
1899
|
-
|
1900
|
-
|
1901
|
-
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
1908
|
-
|
1909
|
-
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
|
1964
|
+
# Paths to the symlinks created for the globbed files.
|
1965
|
+
wandb_files = [
|
1966
|
+
str(path)
|
1967
|
+
for path in pathlib.Path(
|
1968
|
+
self._settings.files_dir,
|
1969
|
+
).glob(relative_glob_str)
|
1970
|
+
]
|
1971
|
+
|
1972
|
+
had_symlinked_files = len(wandb_files) > 0
|
1973
|
+
is_star_glob = "*" in relative_glob_str
|
1974
|
+
|
1975
|
+
# The base_path may itself be a glob, so we can't do
|
1976
|
+
# base_path.glob(relative_glob_str)
|
1977
|
+
for path_str in glob.glob(str(base_path / relative_glob_str)):
|
1978
|
+
path = pathlib.Path(path_str).absolute()
|
1979
|
+
|
1980
|
+
# We can't use relative_to() because base_path may be a glob.
|
1981
|
+
saved_path = pathlib.Path(*path.parts[len(base_path.parts) :])
|
1982
|
+
|
1983
|
+
wandb_path = pathlib.Path(self._settings.files_dir, saved_path)
|
1984
|
+
|
1985
|
+
wandb_files.append(str(wandb_path))
|
1986
|
+
wandb_path.parent.mkdir(parents=True, exist_ok=True)
|
1987
|
+
|
1988
|
+
# Delete the symlink if it exists.
|
1989
|
+
try:
|
1990
|
+
wandb_path.unlink()
|
1991
|
+
except FileNotFoundError:
|
1992
|
+
# In Python 3.8, we would pass missing_ok=True, but as of now
|
1993
|
+
# we support down to Python 3.7.
|
1994
|
+
pass
|
1995
|
+
|
1996
|
+
wandb_path.symlink_to(path)
|
1997
|
+
|
1998
|
+
# Inform users that new files aren't detected automatically.
|
1999
|
+
if not had_symlinked_files and is_star_glob:
|
2000
|
+
file_str = f"{len(wandb_files)} file"
|
2001
|
+
if len(wandb_files) > 1:
|
1919
2002
|
file_str += "s"
|
1920
2003
|
wandb.termwarn(
|
1921
|
-
|
1922
|
-
|
1923
|
-
"call wandb.save again to sync new files."
|
1924
|
-
)
|
1925
|
-
% file_str
|
2004
|
+
f"Symlinked {file_str} into the W&B run directory, "
|
2005
|
+
"call wandb.save again to sync new files."
|
1926
2006
|
)
|
1927
|
-
|
2007
|
+
|
2008
|
+
files_dict: FilesDict = {"files": [(relative_glob_str, policy)]}
|
1928
2009
|
if self._backend and self._backend.interface:
|
1929
2010
|
self._backend.interface.publish_files(files_dict)
|
1930
|
-
|
2011
|
+
|
2012
|
+
return wandb_files
|
1931
2013
|
|
1932
2014
|
@_run_decorator._attach
|
1933
2015
|
def restore(
|
@@ -2286,14 +2368,12 @@ class Run:
|
|
2286
2368
|
|
2287
2369
|
if self._settings._save_requirements:
|
2288
2370
|
if self._backend and self._backend.interface:
|
2289
|
-
import
|
2371
|
+
from wandb.util import working_set
|
2290
2372
|
|
2291
2373
|
logger.debug(
|
2292
2374
|
"Saving list of pip packages installed into the current environment"
|
2293
2375
|
)
|
2294
|
-
self._backend.interface.publish_python_packages(
|
2295
|
-
pkg_resources.working_set
|
2296
|
-
)
|
2376
|
+
self._backend.interface.publish_python_packages(working_set())
|
2297
2377
|
|
2298
2378
|
if self._backend and self._backend.interface and not self._settings._offline:
|
2299
2379
|
self._run_status_checker = RunStatusChecker(
|
@@ -2353,11 +2433,9 @@ class Run:
|
|
2353
2433
|
os.remove(self._settings.resume_fname)
|
2354
2434
|
|
2355
2435
|
def _make_job_source_reqs(self) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]:
|
2356
|
-
import
|
2436
|
+
from wandb.util import working_set
|
2357
2437
|
|
2358
|
-
installed_packages_list = sorted(
|
2359
|
-
f"{d.key}=={d.version}" for d in iter(pkg_resources.working_set)
|
2360
|
-
)
|
2438
|
+
installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
|
2361
2439
|
input_types = TypeRegistry.type_of(self.config.as_dict()).to_json()
|
2362
2440
|
output_types = TypeRegistry.type_of(self.summary._as_dict()).to_json()
|
2363
2441
|
|
@@ -2428,8 +2506,6 @@ class Run:
|
|
2428
2506
|
else:
|
2429
2507
|
return artifact
|
2430
2508
|
|
2431
|
-
# Add a recurring callback (probe) to poll the backend process
|
2432
|
-
# for its status using the "poll_exit" message.
|
2433
2509
|
def _on_probe_exit(self, probe_handle: MailboxProbe) -> None:
|
2434
2510
|
handle = probe_handle.get_mailbox_handle()
|
2435
2511
|
if handle:
|
@@ -2441,8 +2517,6 @@ class Run:
|
|
2441
2517
|
handle = self._backend.interface.deliver_poll_exit()
|
2442
2518
|
probe_handle.set_mailbox_handle(handle)
|
2443
2519
|
|
2444
|
-
# Handles the progress message from the backend process and prints
|
2445
|
-
# the current status to the terminal footer
|
2446
2520
|
def _on_progress_exit(self, progress_handle: MailboxProgress) -> None:
|
2447
2521
|
probe_handles = progress_handle.get_probe_handles()
|
2448
2522
|
assert probe_handles and len(probe_handles) == 1
|
@@ -2493,7 +2567,6 @@ class Run:
|
|
2493
2567
|
sampled_history_handle = (
|
2494
2568
|
self._backend.interface.deliver_request_sampled_history()
|
2495
2569
|
)
|
2496
|
-
job_info_handle = self._backend.interface.deliver_request_job_info()
|
2497
2570
|
|
2498
2571
|
result = server_info_handle.wait(timeout=-1)
|
2499
2572
|
assert result
|
@@ -2507,10 +2580,6 @@ class Run:
|
|
2507
2580
|
assert result
|
2508
2581
|
self._final_summary = result.response.get_summary_response
|
2509
2582
|
|
2510
|
-
result = job_info_handle.wait(timeout=-1)
|
2511
|
-
assert result
|
2512
|
-
self._job_info = result.response.job_info_response
|
2513
|
-
|
2514
2583
|
if self._backend:
|
2515
2584
|
self._backend.cleanup()
|
2516
2585
|
|
@@ -2533,7 +2602,6 @@ class Run:
|
|
2533
2602
|
server_info_response=self._server_info_response,
|
2534
2603
|
check_version_response=self._check_version,
|
2535
2604
|
internal_messages_response=self._internal_messages_response,
|
2536
|
-
job_info=self._job_info,
|
2537
2605
|
reporter=self._reporter,
|
2538
2606
|
quiet=self._quiet,
|
2539
2607
|
settings=self._settings,
|
@@ -2730,9 +2798,6 @@ class Run:
|
|
2730
2798
|
if self._backend and self._backend.interface:
|
2731
2799
|
if artifact.is_draft() and not artifact._is_draft_save_started():
|
2732
2800
|
artifact = self._log_artifact(artifact)
|
2733
|
-
# artifact logging is async, wait until the artifact is committed
|
2734
|
-
# before trying to link it
|
2735
|
-
artifact.wait()
|
2736
2801
|
if not self._settings._offline:
|
2737
2802
|
self._backend.interface.publish_link_artifact(
|
2738
2803
|
self,
|
@@ -2769,7 +2834,6 @@ class Run:
|
|
2769
2834
|
can be in the following forms:
|
2770
2835
|
- name:version
|
2771
2836
|
- name:alias
|
2772
|
-
- digest
|
2773
2837
|
You can also pass an Artifact object created by calling `wandb.Artifact`
|
2774
2838
|
type: (str, optional) The type of artifact to use.
|
2775
2839
|
aliases: (list, optional) Aliases to apply to this artifact
|
@@ -3034,7 +3098,7 @@ class Run:
|
|
3034
3098
|
self._assert_can_log_artifact(artifact)
|
3035
3099
|
if self._backend and self._backend.interface:
|
3036
3100
|
if not self._settings._offline:
|
3037
|
-
|
3101
|
+
future = self._backend.interface.communicate_artifact(
|
3038
3102
|
self,
|
3039
3103
|
artifact,
|
3040
3104
|
aliases,
|
@@ -3043,9 +3107,7 @@ class Run:
|
|
3043
3107
|
is_user_created=is_user_created,
|
3044
3108
|
use_after_commit=use_after_commit,
|
3045
3109
|
)
|
3046
|
-
|
3047
|
-
handle.add_progress(self._on_progress_exit)
|
3048
|
-
artifact._set_save_handle(handle, self._public_api().client)
|
3110
|
+
artifact._set_save_future(future, self._public_api().client)
|
3049
3111
|
else:
|
3050
3112
|
self._backend.interface.publish_artifact(
|
3051
3113
|
self,
|
@@ -3199,7 +3261,6 @@ class Run:
|
|
3199
3261
|
can be in the following forms:
|
3200
3262
|
- model_artifact_name:version
|
3201
3263
|
- model_artifact_name:alias
|
3202
|
-
- model_artifact_name:digest.
|
3203
3264
|
|
3204
3265
|
Examples:
|
3205
3266
|
```python
|
@@ -3524,7 +3585,7 @@ class Run:
|
|
3524
3585
|
if settings._offline or settings.silent:
|
3525
3586
|
return
|
3526
3587
|
|
3527
|
-
|
3588
|
+
workspace_url = f"{settings.run_url}/workspace"
|
3528
3589
|
project_url = settings.project_url
|
3529
3590
|
sweep_url = settings.sweep_url
|
3530
3591
|
|
@@ -3535,7 +3596,7 @@ class Run:
|
|
3535
3596
|
|
3536
3597
|
if printer._html:
|
3537
3598
|
if not wandb.jupyter.maybe_display():
|
3538
|
-
run_line = f"<strong>{printer.link(
|
3599
|
+
run_line = f"<strong>{printer.link(workspace_url, run_name)}</strong>"
|
3539
3600
|
project_line, sweep_line = "", ""
|
3540
3601
|
|
3541
3602
|
# TODO(settings): make settings the source of truth
|
@@ -3567,7 +3628,7 @@ class Run:
|
|
3567
3628
|
f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
|
3568
3629
|
)
|
3569
3630
|
printer.display(
|
3570
|
-
f'{printer.emoji("rocket")} View run at {printer.link(
|
3631
|
+
f'{printer.emoji("rocket")} View run at {printer.link(workspace_url)}',
|
3571
3632
|
)
|
3572
3633
|
|
3573
3634
|
# TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
|
@@ -3591,7 +3652,6 @@ class Run:
|
|
3591
3652
|
server_info_response: Optional[ServerInfoResponse] = None,
|
3592
3653
|
check_version_response: Optional["CheckVersionResponse"] = None,
|
3593
3654
|
internal_messages_response: Optional["InternalMessagesResponse"] = None,
|
3594
|
-
job_info: Optional["JobInfoResponse"] = None,
|
3595
3655
|
reporter: Optional[Reporter] = None,
|
3596
3656
|
quiet: Optional[bool] = None,
|
3597
3657
|
*,
|
@@ -3608,7 +3668,6 @@ class Run:
|
|
3608
3668
|
|
3609
3669
|
Run._footer_sync_info(
|
3610
3670
|
poll_exit_response=poll_exit_response,
|
3611
|
-
job_info=job_info,
|
3612
3671
|
quiet=quiet,
|
3613
3672
|
settings=settings,
|
3614
3673
|
printer=printer,
|
@@ -3793,7 +3852,6 @@ class Run:
|
|
3793
3852
|
@staticmethod
|
3794
3853
|
def _footer_sync_info(
|
3795
3854
|
poll_exit_response: Optional[PollExitResponse] = None,
|
3796
|
-
job_info: Optional["JobInfoResponse"] = None,
|
3797
3855
|
quiet: Optional[bool] = None,
|
3798
3856
|
*,
|
3799
3857
|
settings: "Settings",
|
@@ -3813,14 +3871,10 @@ class Run:
|
|
3813
3871
|
else:
|
3814
3872
|
info = []
|
3815
3873
|
if settings.run_name and settings.run_url:
|
3874
|
+
run_workspace = f"{settings.run_url}/workspace"
|
3816
3875
|
info = [
|
3817
|
-
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(
|
3876
|
+
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(run_workspace)}"
|
3818
3877
|
]
|
3819
|
-
if job_info and job_info.version and job_info.sequenceId:
|
3820
|
-
link = f"{settings.project_url}/jobs/{job_info.sequenceId}/version_details/{job_info.version}"
|
3821
|
-
info.append(
|
3822
|
-
f"{printer.emoji('lightning')} View job at {printer.link(link)}",
|
3823
|
-
)
|
3824
3878
|
if poll_exit_response and poll_exit_response.file_counts:
|
3825
3879
|
logger.info("logging synced files")
|
3826
3880
|
file_counts = poll_exit_response.file_counts
|
wandb/sdk/wandb_settings.py
CHANGED
@@ -45,6 +45,7 @@ from wandb.proto import wandb_settings_pb2
|
|
45
45
|
from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
|
46
46
|
from wandb.sdk.lib import filesystem
|
47
47
|
from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
|
48
|
+
from wandb.sdk.lib.run_moment import RunMoment
|
48
49
|
from wandb.sdk.wandb_setup import _EarlyLogger
|
49
50
|
|
50
51
|
from .lib import apikey
|
@@ -160,6 +161,14 @@ def _get_program() -> Optional[str]:
|
|
160
161
|
return None
|
161
162
|
|
162
163
|
|
164
|
+
def _runmoment_preprocessor(val: Any) -> Optional[RunMoment]:
|
165
|
+
if isinstance(val, RunMoment) or val is None:
|
166
|
+
return val
|
167
|
+
elif isinstance(val, str):
|
168
|
+
return RunMoment.from_uri(val)
|
169
|
+
raise UsageError(f"Could not parse value {val} as a RunMoment.")
|
170
|
+
|
171
|
+
|
163
172
|
def _get_program_relpath(
|
164
173
|
program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
|
165
174
|
) -> Optional[str]:
|
@@ -291,13 +300,14 @@ class SettingsData:
|
|
291
300
|
_aws_lambda: bool
|
292
301
|
_async_upload_concurrency_limit: int
|
293
302
|
_cli_only_mode: bool # Avoid running any code specific for runs
|
303
|
+
_code_path_local: str
|
294
304
|
_colab: bool
|
295
305
|
# _config_dict: Config
|
296
306
|
_cuda: str
|
297
307
|
_disable_meta: bool # Do not collect system metadata
|
298
308
|
_disable_service: (
|
299
|
-
bool
|
300
|
-
)
|
309
|
+
bool # Disable wandb-service, spin up internal process the old way
|
310
|
+
)
|
301
311
|
_disable_setproctitle: bool # Do not use setproctitle on internal process
|
302
312
|
_disable_stats: bool # Do not collect system metrics
|
303
313
|
_disable_viewer: bool # Prevent early viewer query
|
@@ -354,20 +364,18 @@ class SettingsData:
|
|
354
364
|
_stats_sample_rate_seconds: float
|
355
365
|
_stats_samples_to_average: int
|
356
366
|
_stats_join_assets: (
|
357
|
-
bool
|
358
|
-
)
|
367
|
+
bool # join metrics from different assets before sending to backend
|
368
|
+
)
|
359
369
|
_stats_neuron_monitor_config_path: (
|
360
|
-
str
|
361
|
-
)
|
370
|
+
str # path to place config file for neuron-monitor (AWS Trainium)
|
371
|
+
)
|
362
372
|
_stats_open_metrics_endpoints: Mapping[str, str] # open metrics endpoint names/urls
|
363
373
|
# open metrics filters in one of the two formats:
|
364
374
|
# - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
|
365
375
|
# - ("metric regex pattern 1", "metric regex pattern 2", ...)
|
366
376
|
_stats_open_metrics_filters: Union[Sequence[str], Mapping[str, Mapping[str, str]]]
|
367
377
|
_stats_disk_paths: Sequence[str] # paths to monitor disk usage
|
368
|
-
_stats_buffer_size:
|
369
|
-
int
|
370
|
-
) # number of consolidated samples to buffer before flushing, available in run obj
|
378
|
+
_stats_buffer_size: int # number of consolidated samples to buffer before flushing, available in run obj
|
371
379
|
_tmp_code_dir: str
|
372
380
|
_tracelog: str
|
373
381
|
_unsaved_keys: Sequence[str]
|
@@ -392,6 +400,7 @@ class SettingsData:
|
|
392
400
|
entity: str
|
393
401
|
files_dir: str
|
394
402
|
force: bool
|
403
|
+
fork_from: Optional[RunMoment]
|
395
404
|
git_commit: str
|
396
405
|
git_remote: str
|
397
406
|
git_remote_url: str
|
@@ -619,6 +628,10 @@ class Settings(SettingsData):
|
|
619
628
|
"hook": lambda _: is_aws_lambda(),
|
620
629
|
"auto_hook": True,
|
621
630
|
},
|
631
|
+
_code_path_local={
|
632
|
+
"hook": lambda _: _get_program_relpath(self.program),
|
633
|
+
"auto_hook": True,
|
634
|
+
},
|
622
635
|
_colab={
|
623
636
|
"hook": lambda _: "google.colab" in sys.modules,
|
624
637
|
"auto_hook": True,
|
@@ -800,6 +813,10 @@ class Settings(SettingsData):
|
|
800
813
|
),
|
801
814
|
},
|
802
815
|
force={"preprocessor": _str_as_bool},
|
816
|
+
fork_from={
|
817
|
+
"value": None,
|
818
|
+
"preprocessor": _runmoment_preprocessor,
|
819
|
+
},
|
803
820
|
git_remote={"value": "origin"},
|
804
821
|
heartbeat_seconds={"value": 30},
|
805
822
|
ignore_globs={
|
@@ -1570,6 +1587,14 @@ class Settings(SettingsData):
|
|
1570
1587
|
for key, value in v.items():
|
1571
1588
|
# we only support dicts with string values for now
|
1572
1589
|
mapping.value[key] = value
|
1590
|
+
elif isinstance(v, RunMoment):
|
1591
|
+
getattr(settings, k).CopyFrom(
|
1592
|
+
wandb_settings_pb2.RunMoment(
|
1593
|
+
run=v.run,
|
1594
|
+
value=v.value,
|
1595
|
+
metric=v.metric,
|
1596
|
+
)
|
1597
|
+
)
|
1573
1598
|
elif v is None:
|
1574
1599
|
# None is the default value for all settings, so we don't need to set it,
|
1575
1600
|
# i.e. None means that the value was not set.
|
@@ -1892,16 +1917,33 @@ class Settings(SettingsData):
|
|
1892
1917
|
f.write(json.dumps({"run_id": self.run_id}))
|
1893
1918
|
|
1894
1919
|
def _apply_login(
|
1895
|
-
self,
|
1920
|
+
self,
|
1921
|
+
login_settings: Dict[str, Any],
|
1922
|
+
_logger: Optional[_EarlyLogger] = None,
|
1896
1923
|
) -> None:
|
1897
|
-
|
1924
|
+
key_map = {
|
1925
|
+
"key": "api_key",
|
1926
|
+
"host": "base_url",
|
1927
|
+
"timeout": "login_timeout",
|
1928
|
+
}
|
1929
|
+
|
1930
|
+
# Rename keys and keep only the non-None values.
|
1931
|
+
#
|
1932
|
+
# The input keys are parameters to wandb.login(), but we use different
|
1933
|
+
# names for some of them in Settings.
|
1898
1934
|
login_settings = {
|
1899
|
-
|
1935
|
+
key_map.get(key, key): value
|
1936
|
+
for key, value in login_settings.items()
|
1937
|
+
if value is not None
|
1900
1938
|
}
|
1901
|
-
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1939
|
+
|
1940
|
+
if _logger:
|
1941
|
+
_logger.info(f"Applying login settings: {_redact_dict(login_settings)}")
|
1942
|
+
|
1943
|
+
self.update(
|
1944
|
+
login_settings,
|
1945
|
+
source=Source.LOGIN,
|
1946
|
+
)
|
1905
1947
|
|
1906
1948
|
def _apply_run_start(self, run_start_settings: Dict[str, Any]) -> None:
|
1907
1949
|
# This dictionary maps from the "run message dict" to relevant fields in settings
|