wandb 0.16.3__py3-none-any.whl → 0.16.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wandb/__init__.py +2 -2
- wandb/agents/pyagent.py +1 -1
- wandb/apis/importers/__init__.py +1 -4
- wandb/apis/importers/internals/internal.py +386 -0
- wandb/apis/importers/internals/protocols.py +125 -0
- wandb/apis/importers/internals/util.py +78 -0
- wandb/apis/importers/mlflow.py +125 -88
- wandb/apis/importers/validation.py +108 -0
- wandb/apis/importers/wandb.py +1604 -0
- wandb/apis/public/api.py +7 -10
- wandb/apis/public/artifacts.py +38 -0
- wandb/apis/public/files.py +11 -2
- wandb/apis/reports/v2/__init__.py +0 -19
- wandb/apis/reports/v2/expr_parsing.py +0 -1
- wandb/apis/reports/v2/interface.py +15 -18
- wandb/apis/reports/v2/internal.py +12 -45
- wandb/cli/cli.py +52 -55
- wandb/integration/gym/__init__.py +2 -1
- wandb/integration/keras/callbacks/model_checkpoint.py +1 -1
- wandb/integration/keras/keras.py +6 -4
- wandb/integration/kfp/kfp_patch.py +2 -2
- wandb/integration/openai/fine_tuning.py +1 -2
- wandb/integration/ultralytics/callback.py +0 -1
- wandb/proto/v3/wandb_internal_pb2.py +332 -312
- wandb/proto/v3/wandb_settings_pb2.py +13 -3
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_internal_pb2.py +316 -312
- wandb/proto/v4/wandb_settings_pb2.py +5 -3
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/sdk/artifacts/artifact.py +75 -31
- wandb/sdk/artifacts/artifact_manifest.py +5 -2
- wandb/sdk/artifacts/artifact_manifest_entry.py +6 -1
- wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +8 -2
- wandb/sdk/artifacts/artifact_saver.py +19 -47
- wandb/sdk/artifacts/storage_handler.py +2 -1
- wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +22 -9
- wandb/sdk/artifacts/storage_policy.py +4 -1
- wandb/sdk/data_types/base_types/wb_value.py +1 -1
- wandb/sdk/data_types/image.py +2 -2
- wandb/sdk/interface/interface.py +49 -13
- wandb/sdk/interface/interface_shared.py +17 -11
- wandb/sdk/internal/file_stream.py +20 -1
- wandb/sdk/internal/handler.py +1 -4
- wandb/sdk/internal/internal_api.py +3 -1
- wandb/sdk/internal/job_builder.py +49 -19
- wandb/sdk/internal/profiler.py +1 -1
- wandb/sdk/internal/sender.py +96 -124
- wandb/sdk/internal/sender_config.py +197 -0
- wandb/sdk/internal/settings_static.py +9 -0
- wandb/sdk/internal/system/system_info.py +5 -3
- wandb/sdk/internal/update.py +1 -1
- wandb/sdk/launch/_launch.py +3 -3
- wandb/sdk/launch/_launch_add.py +28 -29
- wandb/sdk/launch/_project_spec.py +148 -136
- wandb/sdk/launch/agent/agent.py +3 -7
- wandb/sdk/launch/agent/config.py +0 -27
- wandb/sdk/launch/builder/build.py +54 -28
- wandb/sdk/launch/builder/docker_builder.py +4 -15
- wandb/sdk/launch/builder/kaniko_builder.py +72 -45
- wandb/sdk/launch/create_job.py +6 -40
- wandb/sdk/launch/loader.py +10 -0
- wandb/sdk/launch/registry/anon.py +29 -0
- wandb/sdk/launch/registry/local_registry.py +4 -1
- wandb/sdk/launch/runner/kubernetes_runner.py +20 -2
- wandb/sdk/launch/runner/local_container.py +15 -10
- wandb/sdk/launch/runner/sagemaker_runner.py +1 -1
- wandb/sdk/launch/sweeps/scheduler.py +11 -3
- wandb/sdk/launch/utils.py +14 -0
- wandb/sdk/lib/__init__.py +2 -5
- wandb/sdk/lib/_settings_toposort_generated.py +4 -1
- wandb/sdk/lib/apikey.py +0 -5
- wandb/sdk/lib/config_util.py +0 -31
- wandb/sdk/lib/filesystem.py +11 -1
- wandb/sdk/lib/run_moment.py +72 -0
- wandb/sdk/service/service.py +7 -2
- wandb/sdk/service/streams.py +1 -6
- wandb/sdk/verify/verify.py +2 -1
- wandb/sdk/wandb_init.py +12 -1
- wandb/sdk/wandb_login.py +43 -26
- wandb/sdk/wandb_run.py +164 -110
- wandb/sdk/wandb_settings.py +58 -16
- wandb/testing/relay.py +5 -6
- wandb/util.py +50 -7
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/METADATA +8 -1
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/RECORD +89 -82
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/WHEEL +1 -1
- wandb/apis/importers/base.py +0 -400
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/LICENSE +0 -0
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/entry_points.txt +0 -0
- {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_run.py
CHANGED
@@ -6,6 +6,7 @@ import json
|
|
6
6
|
import logging
|
7
7
|
import numbers
|
8
8
|
import os
|
9
|
+
import pathlib
|
9
10
|
import re
|
10
11
|
import sys
|
11
12
|
import threading
|
@@ -42,7 +43,6 @@ from wandb.apis import internal, public
|
|
42
43
|
from wandb.apis.internal import Api
|
43
44
|
from wandb.apis.public import Api as PublicApi
|
44
45
|
from wandb.proto.wandb_internal_pb2 import (
|
45
|
-
JobInfoResponse,
|
46
46
|
MetricRecord,
|
47
47
|
PollExitResponse,
|
48
48
|
Result,
|
@@ -531,7 +531,6 @@ class Run:
|
|
531
531
|
_check_version: Optional["CheckVersionResponse"]
|
532
532
|
_sampled_history: Optional["SampledHistoryResponse"]
|
533
533
|
_final_summary: Optional["GetSummaryResponse"]
|
534
|
-
_job_info: Optional["JobInfoResponse"]
|
535
534
|
_poll_exit_handle: Optional[MailboxHandle]
|
536
535
|
_poll_exit_response: Optional[PollExitResponse]
|
537
536
|
_server_info_response: Optional[ServerInfoResponse]
|
@@ -642,7 +641,6 @@ class Run:
|
|
642
641
|
self._server_info_response = None
|
643
642
|
self._internal_messages_response = None
|
644
643
|
self._poll_exit_handle = None
|
645
|
-
self._job_info = None
|
646
644
|
|
647
645
|
# Initialize telemetry object
|
648
646
|
self._telemetry_obj = telemetry.TelemetryRecord()
|
@@ -673,6 +671,12 @@ class Run:
|
|
673
671
|
os.path.join("code", self._settings.program_relpath)
|
674
672
|
)
|
675
673
|
|
674
|
+
if self._settings.fork_from is not None:
|
675
|
+
config[wandb_key]["branch_point"] = {
|
676
|
+
"run_id": self._settings.fork_from.run,
|
677
|
+
"step": self._settings.fork_from.value,
|
678
|
+
}
|
679
|
+
|
676
680
|
self._config._update(config, ignore_locked=True)
|
677
681
|
|
678
682
|
if sweep_config:
|
@@ -734,7 +738,7 @@ class Run:
|
|
734
738
|
and self._settings.launch_config_path
|
735
739
|
and os.path.exists(self._settings.launch_config_path)
|
736
740
|
):
|
737
|
-
self.
|
741
|
+
self.save(self._settings.launch_config_path)
|
738
742
|
with open(self._settings.launch_config_path) as fp:
|
739
743
|
launch_config = json.loads(fp.read())
|
740
744
|
if launch_config.get("overrides", {}).get("artifacts") is not None:
|
@@ -1385,6 +1389,8 @@ class Run:
|
|
1385
1389
|
|
1386
1390
|
@_run_decorator._noop_on_finish()
|
1387
1391
|
def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
|
1392
|
+
with telemetry.context(run=self) as tel:
|
1393
|
+
tel.feature.set_summary = True
|
1388
1394
|
if self._backend and self._backend.interface:
|
1389
1395
|
self._backend.interface.publish_summary(summary_record)
|
1390
1396
|
|
@@ -1811,6 +1817,10 @@ class Run:
|
|
1811
1817
|
ValueError: if invalid data is passed
|
1812
1818
|
|
1813
1819
|
"""
|
1820
|
+
if step is not None:
|
1821
|
+
with telemetry.context(run=self) as tel:
|
1822
|
+
tel.feature.set_step_log = True
|
1823
|
+
|
1814
1824
|
if sync is not None:
|
1815
1825
|
deprecate.deprecate(
|
1816
1826
|
field_name=deprecate.Deprecated.run__log_sync,
|
@@ -1831,20 +1841,53 @@ class Run:
|
|
1831
1841
|
@_run_decorator._attach
|
1832
1842
|
def save(
|
1833
1843
|
self,
|
1834
|
-
glob_str: Optional[str] = None,
|
1835
|
-
base_path: Optional[str] = None,
|
1844
|
+
glob_str: Optional[Union[str, os.PathLike]] = None,
|
1845
|
+
base_path: Optional[Union[str, os.PathLike]] = None,
|
1836
1846
|
policy: "PolicyName" = "live",
|
1837
1847
|
) -> Union[bool, List[str]]:
|
1838
|
-
"""
|
1848
|
+
"""Sync one or more files to W&B.
|
1849
|
+
|
1850
|
+
Relative paths are relative to the current working directory.
|
1851
|
+
|
1852
|
+
A Unix glob, such as "myfiles/*", is expanded at the time `save` is
|
1853
|
+
called regardless of the `policy`. In particular, new files are not
|
1854
|
+
picked up automatically.
|
1855
|
+
|
1856
|
+
A `base_path` may be provided to control the directory structure of
|
1857
|
+
uploaded files. It should be a prefix of `glob_str`, and the direcotry
|
1858
|
+
structure beneath it is preserved. It's best understood through
|
1859
|
+
examples:
|
1860
|
+
|
1861
|
+
```
|
1862
|
+
wandb.save("these/are/myfiles/*")
|
1863
|
+
# => Saves files in a "these/are/myfiles/" folder in the run.
|
1864
|
+
|
1865
|
+
wandb.save("these/are/myfiles/*", base_path="these")
|
1866
|
+
# => Saves files in an "are/myfiles/" folder in the run.
|
1867
|
+
|
1868
|
+
wandb.save("/User/username/Documents/run123/*.txt")
|
1869
|
+
# => Saves files in a "run123/" folder in the run.
|
1870
|
+
|
1871
|
+
wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
|
1872
|
+
# => Saves files in a "username/Documents/run123/" folder in the run.
|
1873
|
+
|
1874
|
+
wandb.save("files/*/saveme.txt")
|
1875
|
+
# => Saves each "saveme.txt" file in an appropriate subdirectory
|
1876
|
+
# of "files/".
|
1877
|
+
```
|
1839
1878
|
|
1840
1879
|
Arguments:
|
1841
|
-
glob_str:
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1880
|
+
glob_str: A relative or absolute path or Unix glob.
|
1881
|
+
base_path: A path to use to infer a directory structure; see examples.
|
1882
|
+
policy: One of `live`, `now`, or `end`.
|
1883
|
+
* live: upload the file as it changes, overwriting the previous version
|
1884
|
+
* now: upload the file once now
|
1885
|
+
* end: upload file when the run ends
|
1886
|
+
|
1887
|
+
Returns:
|
1888
|
+
Paths to the symlinks created for the matched files.
|
1889
|
+
|
1890
|
+
For historical reasons, this may return a boolean in legacy code.
|
1848
1891
|
"""
|
1849
1892
|
if glob_str is None:
|
1850
1893
|
# noop for historical reasons, run.save() may be called in legacy code
|
@@ -1857,77 +1900,116 @@ class Run:
|
|
1857
1900
|
)
|
1858
1901
|
return True
|
1859
1902
|
|
1860
|
-
|
1903
|
+
if isinstance(glob_str, bytes):
|
1904
|
+
# Preserved for backward compatibility: allow bytes inputs.
|
1905
|
+
glob_str = glob_str.decode("utf-8")
|
1906
|
+
if isinstance(glob_str, str) and (
|
1907
|
+
glob_str.startswith("gs://") or glob_str.startswith("s3://")
|
1908
|
+
):
|
1909
|
+
# Provide a better error message for a common misuse.
|
1910
|
+
wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
|
1911
|
+
return []
|
1912
|
+
glob_path = pathlib.Path(glob_str)
|
1913
|
+
|
1914
|
+
if base_path is not None:
|
1915
|
+
base_path = pathlib.Path(base_path)
|
1916
|
+
elif not glob_path.is_absolute():
|
1917
|
+
base_path = pathlib.Path(".")
|
1918
|
+
else:
|
1919
|
+
# Absolute glob paths with no base path get special handling.
|
1920
|
+
wandb.termwarn(
|
1921
|
+
"Saving files without folders. If you want to preserve "
|
1922
|
+
"subdirectories pass base_path to wandb.save, i.e. "
|
1923
|
+
'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
|
1924
|
+
repeat=False,
|
1925
|
+
)
|
1926
|
+
base_path = glob_path.resolve().parent.parent
|
1861
1927
|
|
1862
|
-
def _save(
|
1863
|
-
self,
|
1864
|
-
glob_str: Optional[str] = None,
|
1865
|
-
base_path: Optional[str] = None,
|
1866
|
-
policy: "PolicyName" = "live",
|
1867
|
-
) -> Union[bool, List[str]]:
|
1868
1928
|
if policy not in ("live", "end", "now"):
|
1869
1929
|
raise ValueError(
|
1870
|
-
'Only "live" "end" and "now" policies are currently supported.'
|
1930
|
+
'Only "live", "end" and "now" policies are currently supported.'
|
1871
1931
|
)
|
1872
|
-
if isinstance(glob_str, bytes):
|
1873
|
-
glob_str = glob_str.decode("utf-8")
|
1874
|
-
if not isinstance(glob_str, str):
|
1875
|
-
raise ValueError("Must call wandb.save(glob_str) with glob_str a str")
|
1876
1932
|
|
1877
|
-
|
1878
|
-
|
1879
|
-
|
1880
|
-
|
1881
|
-
|
1882
|
-
|
1883
|
-
|
1884
|
-
|
1885
|
-
|
1886
|
-
|
1887
|
-
|
1888
|
-
|
1889
|
-
|
1890
|
-
|
1933
|
+
resolved_glob_path = glob_path.resolve()
|
1934
|
+
resolved_base_path = base_path.resolve()
|
1935
|
+
|
1936
|
+
return self._save(
|
1937
|
+
resolved_glob_path,
|
1938
|
+
resolved_base_path,
|
1939
|
+
policy,
|
1940
|
+
)
|
1941
|
+
|
1942
|
+
def _save(
|
1943
|
+
self,
|
1944
|
+
glob_path: pathlib.Path,
|
1945
|
+
base_path: pathlib.Path,
|
1946
|
+
policy: "PolicyName",
|
1947
|
+
) -> List[str]:
|
1948
|
+
# Can't use is_relative_to() because that's added in Python 3.9,
|
1949
|
+
# but we support down to Python 3.7.
|
1950
|
+
if not str(glob_path).startswith(str(base_path)):
|
1951
|
+
raise ValueError("Glob may not walk above the base path")
|
1952
|
+
|
1953
|
+
if glob_path == base_path:
|
1954
|
+
raise ValueError("Glob cannot be the same as the base path")
|
1955
|
+
|
1956
|
+
relative_glob = glob_path.relative_to(base_path)
|
1957
|
+
if relative_glob.parts[0] == "*":
|
1958
|
+
raise ValueError("Glob may not start with '*' relative to the base path")
|
1959
|
+
relative_glob_str = GlobStr(str(relative_glob))
|
1891
1960
|
|
1892
1961
|
with telemetry.context(run=self) as tel:
|
1893
1962
|
tel.feature.save = True
|
1894
1963
|
|
1895
|
-
|
1896
|
-
|
1897
|
-
|
1898
|
-
|
1899
|
-
|
1900
|
-
|
1901
|
-
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1905
|
-
|
1906
|
-
|
1907
|
-
|
1908
|
-
|
1909
|
-
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
|
1964
|
+
# Paths to the symlinks created for the globbed files.
|
1965
|
+
wandb_files = [
|
1966
|
+
str(path)
|
1967
|
+
for path in pathlib.Path(
|
1968
|
+
self._settings.files_dir,
|
1969
|
+
).glob(relative_glob_str)
|
1970
|
+
]
|
1971
|
+
|
1972
|
+
had_symlinked_files = len(wandb_files) > 0
|
1973
|
+
is_star_glob = "*" in relative_glob_str
|
1974
|
+
|
1975
|
+
# The base_path may itself be a glob, so we can't do
|
1976
|
+
# base_path.glob(relative_glob_str)
|
1977
|
+
for path_str in glob.glob(str(base_path / relative_glob_str)):
|
1978
|
+
path = pathlib.Path(path_str).absolute()
|
1979
|
+
|
1980
|
+
# We can't use relative_to() because base_path may be a glob.
|
1981
|
+
saved_path = pathlib.Path(*path.parts[len(base_path.parts) :])
|
1982
|
+
|
1983
|
+
wandb_path = pathlib.Path(self._settings.files_dir, saved_path)
|
1984
|
+
|
1985
|
+
wandb_files.append(str(wandb_path))
|
1986
|
+
wandb_path.parent.mkdir(parents=True, exist_ok=True)
|
1987
|
+
|
1988
|
+
# Delete the symlink if it exists.
|
1989
|
+
try:
|
1990
|
+
wandb_path.unlink()
|
1991
|
+
except FileNotFoundError:
|
1992
|
+
# In Python 3.8, we would pass missing_ok=True, but as of now
|
1993
|
+
# we support down to Python 3.7.
|
1994
|
+
pass
|
1995
|
+
|
1996
|
+
wandb_path.symlink_to(path)
|
1997
|
+
|
1998
|
+
# Inform users that new files aren't detected automatically.
|
1999
|
+
if not had_symlinked_files and is_star_glob:
|
2000
|
+
file_str = f"{len(wandb_files)} file"
|
2001
|
+
if len(wandb_files) > 1:
|
1919
2002
|
file_str += "s"
|
1920
2003
|
wandb.termwarn(
|
1921
|
-
|
1922
|
-
|
1923
|
-
"call wandb.save again to sync new files."
|
1924
|
-
)
|
1925
|
-
% file_str
|
2004
|
+
f"Symlinked {file_str} into the W&B run directory, "
|
2005
|
+
"call wandb.save again to sync new files."
|
1926
2006
|
)
|
1927
|
-
|
2007
|
+
|
2008
|
+
files_dict: FilesDict = {"files": [(relative_glob_str, policy)]}
|
1928
2009
|
if self._backend and self._backend.interface:
|
1929
2010
|
self._backend.interface.publish_files(files_dict)
|
1930
|
-
|
2011
|
+
|
2012
|
+
return wandb_files
|
1931
2013
|
|
1932
2014
|
@_run_decorator._attach
|
1933
2015
|
def restore(
|
@@ -2286,14 +2368,12 @@ class Run:
|
|
2286
2368
|
|
2287
2369
|
if self._settings._save_requirements:
|
2288
2370
|
if self._backend and self._backend.interface:
|
2289
|
-
import
|
2371
|
+
from wandb.util import working_set
|
2290
2372
|
|
2291
2373
|
logger.debug(
|
2292
2374
|
"Saving list of pip packages installed into the current environment"
|
2293
2375
|
)
|
2294
|
-
self._backend.interface.publish_python_packages(
|
2295
|
-
pkg_resources.working_set
|
2296
|
-
)
|
2376
|
+
self._backend.interface.publish_python_packages(working_set())
|
2297
2377
|
|
2298
2378
|
if self._backend and self._backend.interface and not self._settings._offline:
|
2299
2379
|
self._run_status_checker = RunStatusChecker(
|
@@ -2353,11 +2433,9 @@ class Run:
|
|
2353
2433
|
os.remove(self._settings.resume_fname)
|
2354
2434
|
|
2355
2435
|
def _make_job_source_reqs(self) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]:
|
2356
|
-
import
|
2436
|
+
from wandb.util import working_set
|
2357
2437
|
|
2358
|
-
installed_packages_list = sorted(
|
2359
|
-
f"{d.key}=={d.version}" for d in iter(pkg_resources.working_set)
|
2360
|
-
)
|
2438
|
+
installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
|
2361
2439
|
input_types = TypeRegistry.type_of(self.config.as_dict()).to_json()
|
2362
2440
|
output_types = TypeRegistry.type_of(self.summary._as_dict()).to_json()
|
2363
2441
|
|
@@ -2428,8 +2506,6 @@ class Run:
|
|
2428
2506
|
else:
|
2429
2507
|
return artifact
|
2430
2508
|
|
2431
|
-
# Add a recurring callback (probe) to poll the backend process
|
2432
|
-
# for its status using the "poll_exit" message.
|
2433
2509
|
def _on_probe_exit(self, probe_handle: MailboxProbe) -> None:
|
2434
2510
|
handle = probe_handle.get_mailbox_handle()
|
2435
2511
|
if handle:
|
@@ -2441,8 +2517,6 @@ class Run:
|
|
2441
2517
|
handle = self._backend.interface.deliver_poll_exit()
|
2442
2518
|
probe_handle.set_mailbox_handle(handle)
|
2443
2519
|
|
2444
|
-
# Handles the progress message from the backend process and prints
|
2445
|
-
# the current status to the terminal footer
|
2446
2520
|
def _on_progress_exit(self, progress_handle: MailboxProgress) -> None:
|
2447
2521
|
probe_handles = progress_handle.get_probe_handles()
|
2448
2522
|
assert probe_handles and len(probe_handles) == 1
|
@@ -2493,7 +2567,6 @@ class Run:
|
|
2493
2567
|
sampled_history_handle = (
|
2494
2568
|
self._backend.interface.deliver_request_sampled_history()
|
2495
2569
|
)
|
2496
|
-
job_info_handle = self._backend.interface.deliver_request_job_info()
|
2497
2570
|
|
2498
2571
|
result = server_info_handle.wait(timeout=-1)
|
2499
2572
|
assert result
|
@@ -2507,10 +2580,6 @@ class Run:
|
|
2507
2580
|
assert result
|
2508
2581
|
self._final_summary = result.response.get_summary_response
|
2509
2582
|
|
2510
|
-
result = job_info_handle.wait(timeout=-1)
|
2511
|
-
assert result
|
2512
|
-
self._job_info = result.response.job_info_response
|
2513
|
-
|
2514
2583
|
if self._backend:
|
2515
2584
|
self._backend.cleanup()
|
2516
2585
|
|
@@ -2533,7 +2602,6 @@ class Run:
|
|
2533
2602
|
server_info_response=self._server_info_response,
|
2534
2603
|
check_version_response=self._check_version,
|
2535
2604
|
internal_messages_response=self._internal_messages_response,
|
2536
|
-
job_info=self._job_info,
|
2537
2605
|
reporter=self._reporter,
|
2538
2606
|
quiet=self._quiet,
|
2539
2607
|
settings=self._settings,
|
@@ -2730,9 +2798,6 @@ class Run:
|
|
2730
2798
|
if self._backend and self._backend.interface:
|
2731
2799
|
if artifact.is_draft() and not artifact._is_draft_save_started():
|
2732
2800
|
artifact = self._log_artifact(artifact)
|
2733
|
-
# artifact logging is async, wait until the artifact is committed
|
2734
|
-
# before trying to link it
|
2735
|
-
artifact.wait()
|
2736
2801
|
if not self._settings._offline:
|
2737
2802
|
self._backend.interface.publish_link_artifact(
|
2738
2803
|
self,
|
@@ -2769,7 +2834,6 @@ class Run:
|
|
2769
2834
|
can be in the following forms:
|
2770
2835
|
- name:version
|
2771
2836
|
- name:alias
|
2772
|
-
- digest
|
2773
2837
|
You can also pass an Artifact object created by calling `wandb.Artifact`
|
2774
2838
|
type: (str, optional) The type of artifact to use.
|
2775
2839
|
aliases: (list, optional) Aliases to apply to this artifact
|
@@ -3034,7 +3098,7 @@ class Run:
|
|
3034
3098
|
self._assert_can_log_artifact(artifact)
|
3035
3099
|
if self._backend and self._backend.interface:
|
3036
3100
|
if not self._settings._offline:
|
3037
|
-
|
3101
|
+
future = self._backend.interface.communicate_artifact(
|
3038
3102
|
self,
|
3039
3103
|
artifact,
|
3040
3104
|
aliases,
|
@@ -3043,9 +3107,7 @@ class Run:
|
|
3043
3107
|
is_user_created=is_user_created,
|
3044
3108
|
use_after_commit=use_after_commit,
|
3045
3109
|
)
|
3046
|
-
|
3047
|
-
handle.add_progress(self._on_progress_exit)
|
3048
|
-
artifact._set_save_handle(handle, self._public_api().client)
|
3110
|
+
artifact._set_save_future(future, self._public_api().client)
|
3049
3111
|
else:
|
3050
3112
|
self._backend.interface.publish_artifact(
|
3051
3113
|
self,
|
@@ -3199,7 +3261,6 @@ class Run:
|
|
3199
3261
|
can be in the following forms:
|
3200
3262
|
- model_artifact_name:version
|
3201
3263
|
- model_artifact_name:alias
|
3202
|
-
- model_artifact_name:digest.
|
3203
3264
|
|
3204
3265
|
Examples:
|
3205
3266
|
```python
|
@@ -3524,7 +3585,7 @@ class Run:
|
|
3524
3585
|
if settings._offline or settings.silent:
|
3525
3586
|
return
|
3526
3587
|
|
3527
|
-
|
3588
|
+
workspace_url = f"{settings.run_url}/workspace"
|
3528
3589
|
project_url = settings.project_url
|
3529
3590
|
sweep_url = settings.sweep_url
|
3530
3591
|
|
@@ -3535,7 +3596,7 @@ class Run:
|
|
3535
3596
|
|
3536
3597
|
if printer._html:
|
3537
3598
|
if not wandb.jupyter.maybe_display():
|
3538
|
-
run_line = f"<strong>{printer.link(
|
3599
|
+
run_line = f"<strong>{printer.link(workspace_url, run_name)}</strong>"
|
3539
3600
|
project_line, sweep_line = "", ""
|
3540
3601
|
|
3541
3602
|
# TODO(settings): make settings the source of truth
|
@@ -3567,7 +3628,7 @@ class Run:
|
|
3567
3628
|
f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
|
3568
3629
|
)
|
3569
3630
|
printer.display(
|
3570
|
-
f'{printer.emoji("rocket")} View run at {printer.link(
|
3631
|
+
f'{printer.emoji("rocket")} View run at {printer.link(workspace_url)}',
|
3571
3632
|
)
|
3572
3633
|
|
3573
3634
|
# TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
|
@@ -3591,7 +3652,6 @@ class Run:
|
|
3591
3652
|
server_info_response: Optional[ServerInfoResponse] = None,
|
3592
3653
|
check_version_response: Optional["CheckVersionResponse"] = None,
|
3593
3654
|
internal_messages_response: Optional["InternalMessagesResponse"] = None,
|
3594
|
-
job_info: Optional["JobInfoResponse"] = None,
|
3595
3655
|
reporter: Optional[Reporter] = None,
|
3596
3656
|
quiet: Optional[bool] = None,
|
3597
3657
|
*,
|
@@ -3608,7 +3668,6 @@ class Run:
|
|
3608
3668
|
|
3609
3669
|
Run._footer_sync_info(
|
3610
3670
|
poll_exit_response=poll_exit_response,
|
3611
|
-
job_info=job_info,
|
3612
3671
|
quiet=quiet,
|
3613
3672
|
settings=settings,
|
3614
3673
|
printer=printer,
|
@@ -3793,7 +3852,6 @@ class Run:
|
|
3793
3852
|
@staticmethod
|
3794
3853
|
def _footer_sync_info(
|
3795
3854
|
poll_exit_response: Optional[PollExitResponse] = None,
|
3796
|
-
job_info: Optional["JobInfoResponse"] = None,
|
3797
3855
|
quiet: Optional[bool] = None,
|
3798
3856
|
*,
|
3799
3857
|
settings: "Settings",
|
@@ -3813,14 +3871,10 @@ class Run:
|
|
3813
3871
|
else:
|
3814
3872
|
info = []
|
3815
3873
|
if settings.run_name and settings.run_url:
|
3874
|
+
run_workspace = f"{settings.run_url}/workspace"
|
3816
3875
|
info = [
|
3817
|
-
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(
|
3876
|
+
f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(run_workspace)}"
|
3818
3877
|
]
|
3819
|
-
if job_info and job_info.version and job_info.sequenceId:
|
3820
|
-
link = f"{settings.project_url}/jobs/{job_info.sequenceId}/version_details/{job_info.version}"
|
3821
|
-
info.append(
|
3822
|
-
f"{printer.emoji('lightning')} View job at {printer.link(link)}",
|
3823
|
-
)
|
3824
3878
|
if poll_exit_response and poll_exit_response.file_counts:
|
3825
3879
|
logger.info("logging synced files")
|
3826
3880
|
file_counts = poll_exit_response.file_counts
|
wandb/sdk/wandb_settings.py
CHANGED
@@ -45,6 +45,7 @@ from wandb.proto import wandb_settings_pb2
|
|
45
45
|
from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
|
46
46
|
from wandb.sdk.lib import filesystem
|
47
47
|
from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
|
48
|
+
from wandb.sdk.lib.run_moment import RunMoment
|
48
49
|
from wandb.sdk.wandb_setup import _EarlyLogger
|
49
50
|
|
50
51
|
from .lib import apikey
|
@@ -160,6 +161,14 @@ def _get_program() -> Optional[str]:
|
|
160
161
|
return None
|
161
162
|
|
162
163
|
|
164
|
+
def _runmoment_preprocessor(val: Any) -> Optional[RunMoment]:
|
165
|
+
if isinstance(val, RunMoment) or val is None:
|
166
|
+
return val
|
167
|
+
elif isinstance(val, str):
|
168
|
+
return RunMoment.from_uri(val)
|
169
|
+
raise UsageError(f"Could not parse value {val} as a RunMoment.")
|
170
|
+
|
171
|
+
|
163
172
|
def _get_program_relpath(
|
164
173
|
program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
|
165
174
|
) -> Optional[str]:
|
@@ -291,13 +300,14 @@ class SettingsData:
|
|
291
300
|
_aws_lambda: bool
|
292
301
|
_async_upload_concurrency_limit: int
|
293
302
|
_cli_only_mode: bool # Avoid running any code specific for runs
|
303
|
+
_code_path_local: str
|
294
304
|
_colab: bool
|
295
305
|
# _config_dict: Config
|
296
306
|
_cuda: str
|
297
307
|
_disable_meta: bool # Do not collect system metadata
|
298
308
|
_disable_service: (
|
299
|
-
bool
|
300
|
-
)
|
309
|
+
bool # Disable wandb-service, spin up internal process the old way
|
310
|
+
)
|
301
311
|
_disable_setproctitle: bool # Do not use setproctitle on internal process
|
302
312
|
_disable_stats: bool # Do not collect system metrics
|
303
313
|
_disable_viewer: bool # Prevent early viewer query
|
@@ -354,20 +364,18 @@ class SettingsData:
|
|
354
364
|
_stats_sample_rate_seconds: float
|
355
365
|
_stats_samples_to_average: int
|
356
366
|
_stats_join_assets: (
|
357
|
-
bool
|
358
|
-
)
|
367
|
+
bool # join metrics from different assets before sending to backend
|
368
|
+
)
|
359
369
|
_stats_neuron_monitor_config_path: (
|
360
|
-
str
|
361
|
-
)
|
370
|
+
str # path to place config file for neuron-monitor (AWS Trainium)
|
371
|
+
)
|
362
372
|
_stats_open_metrics_endpoints: Mapping[str, str] # open metrics endpoint names/urls
|
363
373
|
# open metrics filters in one of the two formats:
|
364
374
|
# - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
|
365
375
|
# - ("metric regex pattern 1", "metric regex pattern 2", ...)
|
366
376
|
_stats_open_metrics_filters: Union[Sequence[str], Mapping[str, Mapping[str, str]]]
|
367
377
|
_stats_disk_paths: Sequence[str] # paths to monitor disk usage
|
368
|
-
_stats_buffer_size:
|
369
|
-
int
|
370
|
-
) # number of consolidated samples to buffer before flushing, available in run obj
|
378
|
+
_stats_buffer_size: int # number of consolidated samples to buffer before flushing, available in run obj
|
371
379
|
_tmp_code_dir: str
|
372
380
|
_tracelog: str
|
373
381
|
_unsaved_keys: Sequence[str]
|
@@ -392,6 +400,7 @@ class SettingsData:
|
|
392
400
|
entity: str
|
393
401
|
files_dir: str
|
394
402
|
force: bool
|
403
|
+
fork_from: Optional[RunMoment]
|
395
404
|
git_commit: str
|
396
405
|
git_remote: str
|
397
406
|
git_remote_url: str
|
@@ -619,6 +628,10 @@ class Settings(SettingsData):
|
|
619
628
|
"hook": lambda _: is_aws_lambda(),
|
620
629
|
"auto_hook": True,
|
621
630
|
},
|
631
|
+
_code_path_local={
|
632
|
+
"hook": lambda _: _get_program_relpath(self.program),
|
633
|
+
"auto_hook": True,
|
634
|
+
},
|
622
635
|
_colab={
|
623
636
|
"hook": lambda _: "google.colab" in sys.modules,
|
624
637
|
"auto_hook": True,
|
@@ -800,6 +813,10 @@ class Settings(SettingsData):
|
|
800
813
|
),
|
801
814
|
},
|
802
815
|
force={"preprocessor": _str_as_bool},
|
816
|
+
fork_from={
|
817
|
+
"value": None,
|
818
|
+
"preprocessor": _runmoment_preprocessor,
|
819
|
+
},
|
803
820
|
git_remote={"value": "origin"},
|
804
821
|
heartbeat_seconds={"value": 30},
|
805
822
|
ignore_globs={
|
@@ -1570,6 +1587,14 @@ class Settings(SettingsData):
|
|
1570
1587
|
for key, value in v.items():
|
1571
1588
|
# we only support dicts with string values for now
|
1572
1589
|
mapping.value[key] = value
|
1590
|
+
elif isinstance(v, RunMoment):
|
1591
|
+
getattr(settings, k).CopyFrom(
|
1592
|
+
wandb_settings_pb2.RunMoment(
|
1593
|
+
run=v.run,
|
1594
|
+
value=v.value,
|
1595
|
+
metric=v.metric,
|
1596
|
+
)
|
1597
|
+
)
|
1573
1598
|
elif v is None:
|
1574
1599
|
# None is the default value for all settings, so we don't need to set it,
|
1575
1600
|
# i.e. None means that the value was not set.
|
@@ -1892,16 +1917,33 @@ class Settings(SettingsData):
|
|
1892
1917
|
f.write(json.dumps({"run_id": self.run_id}))
|
1893
1918
|
|
1894
1919
|
def _apply_login(
|
1895
|
-
self,
|
1920
|
+
self,
|
1921
|
+
login_settings: Dict[str, Any],
|
1922
|
+
_logger: Optional[_EarlyLogger] = None,
|
1896
1923
|
) -> None:
|
1897
|
-
|
1924
|
+
key_map = {
|
1925
|
+
"key": "api_key",
|
1926
|
+
"host": "base_url",
|
1927
|
+
"timeout": "login_timeout",
|
1928
|
+
}
|
1929
|
+
|
1930
|
+
# Rename keys and keep only the non-None values.
|
1931
|
+
#
|
1932
|
+
# The input keys are parameters to wandb.login(), but we use different
|
1933
|
+
# names for some of them in Settings.
|
1898
1934
|
login_settings = {
|
1899
|
-
|
1935
|
+
key_map.get(key, key): value
|
1936
|
+
for key, value in login_settings.items()
|
1937
|
+
if value is not None
|
1900
1938
|
}
|
1901
|
-
|
1902
|
-
|
1903
|
-
|
1904
|
-
|
1939
|
+
|
1940
|
+
if _logger:
|
1941
|
+
_logger.info(f"Applying login settings: {_redact_dict(login_settings)}")
|
1942
|
+
|
1943
|
+
self.update(
|
1944
|
+
login_settings,
|
1945
|
+
source=Source.LOGIN,
|
1946
|
+
)
|
1905
1947
|
|
1906
1948
|
def _apply_run_start(self, run_start_settings: Dict[str, Any]) -> None:
|
1907
1949
|
# This dictionary maps from the "run message dict" to relevant fields in settings
|