wandb 0.16.3__py3-none-any.whl → 0.16.5__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. wandb/__init__.py +2 -2
  2. wandb/agents/pyagent.py +1 -1
  3. wandb/apis/importers/__init__.py +1 -4
  4. wandb/apis/importers/internals/internal.py +386 -0
  5. wandb/apis/importers/internals/protocols.py +125 -0
  6. wandb/apis/importers/internals/util.py +78 -0
  7. wandb/apis/importers/mlflow.py +125 -88
  8. wandb/apis/importers/validation.py +108 -0
  9. wandb/apis/importers/wandb.py +1604 -0
  10. wandb/apis/public/api.py +7 -10
  11. wandb/apis/public/artifacts.py +38 -0
  12. wandb/apis/public/files.py +11 -2
  13. wandb/apis/reports/v2/__init__.py +0 -19
  14. wandb/apis/reports/v2/expr_parsing.py +0 -1
  15. wandb/apis/reports/v2/interface.py +15 -18
  16. wandb/apis/reports/v2/internal.py +12 -45
  17. wandb/cli/cli.py +52 -55
  18. wandb/integration/gym/__init__.py +2 -1
  19. wandb/integration/keras/callbacks/model_checkpoint.py +1 -1
  20. wandb/integration/keras/keras.py +6 -4
  21. wandb/integration/kfp/kfp_patch.py +2 -2
  22. wandb/integration/openai/fine_tuning.py +1 -2
  23. wandb/integration/ultralytics/callback.py +0 -1
  24. wandb/proto/v3/wandb_internal_pb2.py +332 -312
  25. wandb/proto/v3/wandb_settings_pb2.py +13 -3
  26. wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
  27. wandb/proto/v4/wandb_internal_pb2.py +316 -312
  28. wandb/proto/v4/wandb_settings_pb2.py +5 -3
  29. wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
  30. wandb/sdk/artifacts/artifact.py +75 -31
  31. wandb/sdk/artifacts/artifact_manifest.py +5 -2
  32. wandb/sdk/artifacts/artifact_manifest_entry.py +6 -1
  33. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +8 -2
  34. wandb/sdk/artifacts/artifact_saver.py +19 -47
  35. wandb/sdk/artifacts/storage_handler.py +2 -1
  36. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +22 -9
  37. wandb/sdk/artifacts/storage_policy.py +4 -1
  38. wandb/sdk/data_types/base_types/wb_value.py +1 -1
  39. wandb/sdk/data_types/image.py +2 -2
  40. wandb/sdk/interface/interface.py +49 -13
  41. wandb/sdk/interface/interface_shared.py +17 -11
  42. wandb/sdk/internal/file_stream.py +20 -1
  43. wandb/sdk/internal/handler.py +1 -4
  44. wandb/sdk/internal/internal_api.py +3 -1
  45. wandb/sdk/internal/job_builder.py +49 -19
  46. wandb/sdk/internal/profiler.py +1 -1
  47. wandb/sdk/internal/sender.py +96 -124
  48. wandb/sdk/internal/sender_config.py +197 -0
  49. wandb/sdk/internal/settings_static.py +9 -0
  50. wandb/sdk/internal/system/system_info.py +5 -3
  51. wandb/sdk/internal/update.py +1 -1
  52. wandb/sdk/launch/_launch.py +3 -3
  53. wandb/sdk/launch/_launch_add.py +28 -29
  54. wandb/sdk/launch/_project_spec.py +148 -136
  55. wandb/sdk/launch/agent/agent.py +3 -7
  56. wandb/sdk/launch/agent/config.py +0 -27
  57. wandb/sdk/launch/builder/build.py +54 -28
  58. wandb/sdk/launch/builder/docker_builder.py +4 -15
  59. wandb/sdk/launch/builder/kaniko_builder.py +72 -45
  60. wandb/sdk/launch/create_job.py +6 -40
  61. wandb/sdk/launch/loader.py +10 -0
  62. wandb/sdk/launch/registry/anon.py +29 -0
  63. wandb/sdk/launch/registry/local_registry.py +4 -1
  64. wandb/sdk/launch/runner/kubernetes_runner.py +20 -2
  65. wandb/sdk/launch/runner/local_container.py +15 -10
  66. wandb/sdk/launch/runner/sagemaker_runner.py +1 -1
  67. wandb/sdk/launch/sweeps/scheduler.py +11 -3
  68. wandb/sdk/launch/utils.py +14 -0
  69. wandb/sdk/lib/__init__.py +2 -5
  70. wandb/sdk/lib/_settings_toposort_generated.py +4 -1
  71. wandb/sdk/lib/apikey.py +0 -5
  72. wandb/sdk/lib/config_util.py +0 -31
  73. wandb/sdk/lib/filesystem.py +11 -1
  74. wandb/sdk/lib/run_moment.py +72 -0
  75. wandb/sdk/service/service.py +7 -2
  76. wandb/sdk/service/streams.py +1 -6
  77. wandb/sdk/verify/verify.py +2 -1
  78. wandb/sdk/wandb_init.py +12 -1
  79. wandb/sdk/wandb_login.py +43 -26
  80. wandb/sdk/wandb_run.py +164 -110
  81. wandb/sdk/wandb_settings.py +58 -16
  82. wandb/testing/relay.py +5 -6
  83. wandb/util.py +50 -7
  84. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/METADATA +8 -1
  85. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/RECORD +89 -82
  86. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/WHEEL +1 -1
  87. wandb/apis/importers/base.py +0 -400
  88. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/LICENSE +0 -0
  89. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/entry_points.txt +0 -0
  90. {wandb-0.16.3.dist-info → wandb-0.16.5.dist-info}/top_level.txt +0 -0
wandb/sdk/wandb_run.py CHANGED
@@ -6,6 +6,7 @@ import json
6
6
  import logging
7
7
  import numbers
8
8
  import os
9
+ import pathlib
9
10
  import re
10
11
  import sys
11
12
  import threading
@@ -42,7 +43,6 @@ from wandb.apis import internal, public
42
43
  from wandb.apis.internal import Api
43
44
  from wandb.apis.public import Api as PublicApi
44
45
  from wandb.proto.wandb_internal_pb2 import (
45
- JobInfoResponse,
46
46
  MetricRecord,
47
47
  PollExitResponse,
48
48
  Result,
@@ -531,7 +531,6 @@ class Run:
531
531
  _check_version: Optional["CheckVersionResponse"]
532
532
  _sampled_history: Optional["SampledHistoryResponse"]
533
533
  _final_summary: Optional["GetSummaryResponse"]
534
- _job_info: Optional["JobInfoResponse"]
535
534
  _poll_exit_handle: Optional[MailboxHandle]
536
535
  _poll_exit_response: Optional[PollExitResponse]
537
536
  _server_info_response: Optional[ServerInfoResponse]
@@ -642,7 +641,6 @@ class Run:
642
641
  self._server_info_response = None
643
642
  self._internal_messages_response = None
644
643
  self._poll_exit_handle = None
645
- self._job_info = None
646
644
 
647
645
  # Initialize telemetry object
648
646
  self._telemetry_obj = telemetry.TelemetryRecord()
@@ -673,6 +671,12 @@ class Run:
673
671
  os.path.join("code", self._settings.program_relpath)
674
672
  )
675
673
 
674
+ if self._settings.fork_from is not None:
675
+ config[wandb_key]["branch_point"] = {
676
+ "run_id": self._settings.fork_from.run,
677
+ "step": self._settings.fork_from.value,
678
+ }
679
+
676
680
  self._config._update(config, ignore_locked=True)
677
681
 
678
682
  if sweep_config:
@@ -734,7 +738,7 @@ class Run:
734
738
  and self._settings.launch_config_path
735
739
  and os.path.exists(self._settings.launch_config_path)
736
740
  ):
737
- self._save(self._settings.launch_config_path)
741
+ self.save(self._settings.launch_config_path)
738
742
  with open(self._settings.launch_config_path) as fp:
739
743
  launch_config = json.loads(fp.read())
740
744
  if launch_config.get("overrides", {}).get("artifacts") is not None:
@@ -1385,6 +1389,8 @@ class Run:
1385
1389
 
1386
1390
  @_run_decorator._noop_on_finish()
1387
1391
  def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
1392
+ with telemetry.context(run=self) as tel:
1393
+ tel.feature.set_summary = True
1388
1394
  if self._backend and self._backend.interface:
1389
1395
  self._backend.interface.publish_summary(summary_record)
1390
1396
 
@@ -1811,6 +1817,10 @@ class Run:
1811
1817
  ValueError: if invalid data is passed
1812
1818
 
1813
1819
  """
1820
+ if step is not None:
1821
+ with telemetry.context(run=self) as tel:
1822
+ tel.feature.set_step_log = True
1823
+
1814
1824
  if sync is not None:
1815
1825
  deprecate.deprecate(
1816
1826
  field_name=deprecate.Deprecated.run__log_sync,
@@ -1831,20 +1841,53 @@ class Run:
1831
1841
  @_run_decorator._attach
1832
1842
  def save(
1833
1843
  self,
1834
- glob_str: Optional[str] = None,
1835
- base_path: Optional[str] = None,
1844
+ glob_str: Optional[Union[str, os.PathLike]] = None,
1845
+ base_path: Optional[Union[str, os.PathLike]] = None,
1836
1846
  policy: "PolicyName" = "live",
1837
1847
  ) -> Union[bool, List[str]]:
1838
- """Ensure all files matching `glob_str` are synced to wandb with the policy specified.
1848
+ """Sync one or more files to W&B.
1849
+
1850
+ Relative paths are relative to the current working directory.
1851
+
1852
+ A Unix glob, such as "myfiles/*", is expanded at the time `save` is
1853
+ called regardless of the `policy`. In particular, new files are not
1854
+ picked up automatically.
1855
+
1856
+ A `base_path` may be provided to control the directory structure of
1857
+ uploaded files. It should be a prefix of `glob_str`, and the direcotry
1858
+ structure beneath it is preserved. It's best understood through
1859
+ examples:
1860
+
1861
+ ```
1862
+ wandb.save("these/are/myfiles/*")
1863
+ # => Saves files in a "these/are/myfiles/" folder in the run.
1864
+
1865
+ wandb.save("these/are/myfiles/*", base_path="these")
1866
+ # => Saves files in an "are/myfiles/" folder in the run.
1867
+
1868
+ wandb.save("/User/username/Documents/run123/*.txt")
1869
+ # => Saves files in a "run123/" folder in the run.
1870
+
1871
+ wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
1872
+ # => Saves files in a "username/Documents/run123/" folder in the run.
1873
+
1874
+ wandb.save("files/*/saveme.txt")
1875
+ # => Saves each "saveme.txt" file in an appropriate subdirectory
1876
+ # of "files/".
1877
+ ```
1839
1878
 
1840
1879
  Arguments:
1841
- glob_str: (string) a relative or absolute path to a unix glob or regular
1842
- path. If this isn't specified the method is a noop.
1843
- base_path: (string) the base path to run the glob relative to
1844
- policy: (string) one of `live`, `now`, or `end`
1845
- - live: upload the file as it changes, overwriting the previous version
1846
- - now: upload the file once now
1847
- - end: only upload file when the run ends
1880
+ glob_str: A relative or absolute path or Unix glob.
1881
+ base_path: A path to use to infer a directory structure; see examples.
1882
+ policy: One of `live`, `now`, or `end`.
1883
+ * live: upload the file as it changes, overwriting the previous version
1884
+ * now: upload the file once now
1885
+ * end: upload file when the run ends
1886
+
1887
+ Returns:
1888
+ Paths to the symlinks created for the matched files.
1889
+
1890
+ For historical reasons, this may return a boolean in legacy code.
1848
1891
  """
1849
1892
  if glob_str is None:
1850
1893
  # noop for historical reasons, run.save() may be called in legacy code
@@ -1857,77 +1900,116 @@ class Run:
1857
1900
  )
1858
1901
  return True
1859
1902
 
1860
- return self._save(glob_str, base_path, policy)
1903
+ if isinstance(glob_str, bytes):
1904
+ # Preserved for backward compatibility: allow bytes inputs.
1905
+ glob_str = glob_str.decode("utf-8")
1906
+ if isinstance(glob_str, str) and (
1907
+ glob_str.startswith("gs://") or glob_str.startswith("s3://")
1908
+ ):
1909
+ # Provide a better error message for a common misuse.
1910
+ wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
1911
+ return []
1912
+ glob_path = pathlib.Path(glob_str)
1913
+
1914
+ if base_path is not None:
1915
+ base_path = pathlib.Path(base_path)
1916
+ elif not glob_path.is_absolute():
1917
+ base_path = pathlib.Path(".")
1918
+ else:
1919
+ # Absolute glob paths with no base path get special handling.
1920
+ wandb.termwarn(
1921
+ "Saving files without folders. If you want to preserve "
1922
+ "subdirectories pass base_path to wandb.save, i.e. "
1923
+ 'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
1924
+ repeat=False,
1925
+ )
1926
+ base_path = glob_path.resolve().parent.parent
1861
1927
 
1862
- def _save(
1863
- self,
1864
- glob_str: Optional[str] = None,
1865
- base_path: Optional[str] = None,
1866
- policy: "PolicyName" = "live",
1867
- ) -> Union[bool, List[str]]:
1868
1928
  if policy not in ("live", "end", "now"):
1869
1929
  raise ValueError(
1870
- 'Only "live" "end" and "now" policies are currently supported.'
1930
+ 'Only "live", "end" and "now" policies are currently supported.'
1871
1931
  )
1872
- if isinstance(glob_str, bytes):
1873
- glob_str = glob_str.decode("utf-8")
1874
- if not isinstance(glob_str, str):
1875
- raise ValueError("Must call wandb.save(glob_str) with glob_str a str")
1876
1932
 
1877
- if base_path is None:
1878
- if os.path.isabs(glob_str):
1879
- base_path = os.path.dirname(glob_str)
1880
- wandb.termwarn(
1881
- "Saving files without folders. If you want to preserve "
1882
- "sub directories pass base_path to wandb.save, i.e. "
1883
- 'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
1884
- repeat=False,
1885
- )
1886
- else:
1887
- base_path = "."
1888
- wandb_glob_str = GlobStr(os.path.relpath(glob_str, base_path))
1889
- if ".." + os.sep in wandb_glob_str:
1890
- raise ValueError("globs can't walk above base_path")
1933
+ resolved_glob_path = glob_path.resolve()
1934
+ resolved_base_path = base_path.resolve()
1935
+
1936
+ return self._save(
1937
+ resolved_glob_path,
1938
+ resolved_base_path,
1939
+ policy,
1940
+ )
1941
+
1942
+ def _save(
1943
+ self,
1944
+ glob_path: pathlib.Path,
1945
+ base_path: pathlib.Path,
1946
+ policy: "PolicyName",
1947
+ ) -> List[str]:
1948
+ # Can't use is_relative_to() because that's added in Python 3.9,
1949
+ # but we support down to Python 3.7.
1950
+ if not str(glob_path).startswith(str(base_path)):
1951
+ raise ValueError("Glob may not walk above the base path")
1952
+
1953
+ if glob_path == base_path:
1954
+ raise ValueError("Glob cannot be the same as the base path")
1955
+
1956
+ relative_glob = glob_path.relative_to(base_path)
1957
+ if relative_glob.parts[0] == "*":
1958
+ raise ValueError("Glob may not start with '*' relative to the base path")
1959
+ relative_glob_str = GlobStr(str(relative_glob))
1891
1960
 
1892
1961
  with telemetry.context(run=self) as tel:
1893
1962
  tel.feature.save = True
1894
1963
 
1895
- if glob_str.startswith("gs://") or glob_str.startswith("s3://"):
1896
- wandb.termlog(
1897
- "%s is a cloud storage url, can't save file to wandb." % glob_str
1898
- )
1899
- return []
1900
- files = glob.glob(os.path.join(self._settings.files_dir, wandb_glob_str))
1901
- warn = False
1902
- if len(files) == 0 and "*" in wandb_glob_str:
1903
- warn = True
1904
- for path in glob.glob(glob_str):
1905
- file_name = os.path.relpath(path, base_path)
1906
- abs_path = os.path.abspath(path)
1907
- wandb_path = os.path.join(self._settings.files_dir, file_name)
1908
- filesystem.mkdir_exists_ok(os.path.dirname(wandb_path))
1909
- # We overwrite symlinks because namespaces can change in Tensorboard
1910
- if os.path.islink(wandb_path) and abs_path != os.readlink(wandb_path):
1911
- os.remove(wandb_path)
1912
- os.symlink(abs_path, wandb_path)
1913
- elif not os.path.exists(wandb_path):
1914
- os.symlink(abs_path, wandb_path)
1915
- files.append(wandb_path)
1916
- if warn:
1917
- file_str = "%i file" % len(files)
1918
- if len(files) > 1:
1964
+ # Paths to the symlinks created for the globbed files.
1965
+ wandb_files = [
1966
+ str(path)
1967
+ for path in pathlib.Path(
1968
+ self._settings.files_dir,
1969
+ ).glob(relative_glob_str)
1970
+ ]
1971
+
1972
+ had_symlinked_files = len(wandb_files) > 0
1973
+ is_star_glob = "*" in relative_glob_str
1974
+
1975
+ # The base_path may itself be a glob, so we can't do
1976
+ # base_path.glob(relative_glob_str)
1977
+ for path_str in glob.glob(str(base_path / relative_glob_str)):
1978
+ path = pathlib.Path(path_str).absolute()
1979
+
1980
+ # We can't use relative_to() because base_path may be a glob.
1981
+ saved_path = pathlib.Path(*path.parts[len(base_path.parts) :])
1982
+
1983
+ wandb_path = pathlib.Path(self._settings.files_dir, saved_path)
1984
+
1985
+ wandb_files.append(str(wandb_path))
1986
+ wandb_path.parent.mkdir(parents=True, exist_ok=True)
1987
+
1988
+ # Delete the symlink if it exists.
1989
+ try:
1990
+ wandb_path.unlink()
1991
+ except FileNotFoundError:
1992
+ # In Python 3.8, we would pass missing_ok=True, but as of now
1993
+ # we support down to Python 3.7.
1994
+ pass
1995
+
1996
+ wandb_path.symlink_to(path)
1997
+
1998
+ # Inform users that new files aren't detected automatically.
1999
+ if not had_symlinked_files and is_star_glob:
2000
+ file_str = f"{len(wandb_files)} file"
2001
+ if len(wandb_files) > 1:
1919
2002
  file_str += "s"
1920
2003
  wandb.termwarn(
1921
- (
1922
- "Symlinked %s into the W&B run directory, "
1923
- "call wandb.save again to sync new files."
1924
- )
1925
- % file_str
2004
+ f"Symlinked {file_str} into the W&B run directory, "
2005
+ "call wandb.save again to sync new files."
1926
2006
  )
1927
- files_dict: FilesDict = dict(files=[(wandb_glob_str, policy)])
2007
+
2008
+ files_dict: FilesDict = {"files": [(relative_glob_str, policy)]}
1928
2009
  if self._backend and self._backend.interface:
1929
2010
  self._backend.interface.publish_files(files_dict)
1930
- return files
2011
+
2012
+ return wandb_files
1931
2013
 
1932
2014
  @_run_decorator._attach
1933
2015
  def restore(
@@ -2286,14 +2368,12 @@ class Run:
2286
2368
 
2287
2369
  if self._settings._save_requirements:
2288
2370
  if self._backend and self._backend.interface:
2289
- import pkg_resources
2371
+ from wandb.util import working_set
2290
2372
 
2291
2373
  logger.debug(
2292
2374
  "Saving list of pip packages installed into the current environment"
2293
2375
  )
2294
- self._backend.interface.publish_python_packages(
2295
- pkg_resources.working_set
2296
- )
2376
+ self._backend.interface.publish_python_packages(working_set())
2297
2377
 
2298
2378
  if self._backend and self._backend.interface and not self._settings._offline:
2299
2379
  self._run_status_checker = RunStatusChecker(
@@ -2353,11 +2433,9 @@ class Run:
2353
2433
  os.remove(self._settings.resume_fname)
2354
2434
 
2355
2435
  def _make_job_source_reqs(self) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]:
2356
- import pkg_resources
2436
+ from wandb.util import working_set
2357
2437
 
2358
- installed_packages_list = sorted(
2359
- f"{d.key}=={d.version}" for d in iter(pkg_resources.working_set)
2360
- )
2438
+ installed_packages_list = sorted(f"{d.key}=={d.version}" for d in working_set())
2361
2439
  input_types = TypeRegistry.type_of(self.config.as_dict()).to_json()
2362
2440
  output_types = TypeRegistry.type_of(self.summary._as_dict()).to_json()
2363
2441
 
@@ -2428,8 +2506,6 @@ class Run:
2428
2506
  else:
2429
2507
  return artifact
2430
2508
 
2431
- # Add a recurring callback (probe) to poll the backend process
2432
- # for its status using the "poll_exit" message.
2433
2509
  def _on_probe_exit(self, probe_handle: MailboxProbe) -> None:
2434
2510
  handle = probe_handle.get_mailbox_handle()
2435
2511
  if handle:
@@ -2441,8 +2517,6 @@ class Run:
2441
2517
  handle = self._backend.interface.deliver_poll_exit()
2442
2518
  probe_handle.set_mailbox_handle(handle)
2443
2519
 
2444
- # Handles the progress message from the backend process and prints
2445
- # the current status to the terminal footer
2446
2520
  def _on_progress_exit(self, progress_handle: MailboxProgress) -> None:
2447
2521
  probe_handles = progress_handle.get_probe_handles()
2448
2522
  assert probe_handles and len(probe_handles) == 1
@@ -2493,7 +2567,6 @@ class Run:
2493
2567
  sampled_history_handle = (
2494
2568
  self._backend.interface.deliver_request_sampled_history()
2495
2569
  )
2496
- job_info_handle = self._backend.interface.deliver_request_job_info()
2497
2570
 
2498
2571
  result = server_info_handle.wait(timeout=-1)
2499
2572
  assert result
@@ -2507,10 +2580,6 @@ class Run:
2507
2580
  assert result
2508
2581
  self._final_summary = result.response.get_summary_response
2509
2582
 
2510
- result = job_info_handle.wait(timeout=-1)
2511
- assert result
2512
- self._job_info = result.response.job_info_response
2513
-
2514
2583
  if self._backend:
2515
2584
  self._backend.cleanup()
2516
2585
 
@@ -2533,7 +2602,6 @@ class Run:
2533
2602
  server_info_response=self._server_info_response,
2534
2603
  check_version_response=self._check_version,
2535
2604
  internal_messages_response=self._internal_messages_response,
2536
- job_info=self._job_info,
2537
2605
  reporter=self._reporter,
2538
2606
  quiet=self._quiet,
2539
2607
  settings=self._settings,
@@ -2730,9 +2798,6 @@ class Run:
2730
2798
  if self._backend and self._backend.interface:
2731
2799
  if artifact.is_draft() and not artifact._is_draft_save_started():
2732
2800
  artifact = self._log_artifact(artifact)
2733
- # artifact logging is async, wait until the artifact is committed
2734
- # before trying to link it
2735
- artifact.wait()
2736
2801
  if not self._settings._offline:
2737
2802
  self._backend.interface.publish_link_artifact(
2738
2803
  self,
@@ -2769,7 +2834,6 @@ class Run:
2769
2834
  can be in the following forms:
2770
2835
  - name:version
2771
2836
  - name:alias
2772
- - digest
2773
2837
  You can also pass an Artifact object created by calling `wandb.Artifact`
2774
2838
  type: (str, optional) The type of artifact to use.
2775
2839
  aliases: (list, optional) Aliases to apply to this artifact
@@ -3034,7 +3098,7 @@ class Run:
3034
3098
  self._assert_can_log_artifact(artifact)
3035
3099
  if self._backend and self._backend.interface:
3036
3100
  if not self._settings._offline:
3037
- handle = self._backend.interface.deliver_artifact(
3101
+ future = self._backend.interface.communicate_artifact(
3038
3102
  self,
3039
3103
  artifact,
3040
3104
  aliases,
@@ -3043,9 +3107,7 @@ class Run:
3043
3107
  is_user_created=is_user_created,
3044
3108
  use_after_commit=use_after_commit,
3045
3109
  )
3046
- handle.add_probe(self._on_probe_exit)
3047
- handle.add_progress(self._on_progress_exit)
3048
- artifact._set_save_handle(handle, self._public_api().client)
3110
+ artifact._set_save_future(future, self._public_api().client)
3049
3111
  else:
3050
3112
  self._backend.interface.publish_artifact(
3051
3113
  self,
@@ -3199,7 +3261,6 @@ class Run:
3199
3261
  can be in the following forms:
3200
3262
  - model_artifact_name:version
3201
3263
  - model_artifact_name:alias
3202
- - model_artifact_name:digest.
3203
3264
 
3204
3265
  Examples:
3205
3266
  ```python
@@ -3524,7 +3585,7 @@ class Run:
3524
3585
  if settings._offline or settings.silent:
3525
3586
  return
3526
3587
 
3527
- run_url = settings.run_url
3588
+ workspace_url = f"{settings.run_url}/workspace"
3528
3589
  project_url = settings.project_url
3529
3590
  sweep_url = settings.sweep_url
3530
3591
 
@@ -3535,7 +3596,7 @@ class Run:
3535
3596
 
3536
3597
  if printer._html:
3537
3598
  if not wandb.jupyter.maybe_display():
3538
- run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
3599
+ run_line = f"<strong>{printer.link(workspace_url, run_name)}</strong>"
3539
3600
  project_line, sweep_line = "", ""
3540
3601
 
3541
3602
  # TODO(settings): make settings the source of truth
@@ -3567,7 +3628,7 @@ class Run:
3567
3628
  f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
3568
3629
  )
3569
3630
  printer.display(
3570
- f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
3631
+ f'{printer.emoji("rocket")} View run at {printer.link(workspace_url)}',
3571
3632
  )
3572
3633
 
3573
3634
  # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
@@ -3591,7 +3652,6 @@ class Run:
3591
3652
  server_info_response: Optional[ServerInfoResponse] = None,
3592
3653
  check_version_response: Optional["CheckVersionResponse"] = None,
3593
3654
  internal_messages_response: Optional["InternalMessagesResponse"] = None,
3594
- job_info: Optional["JobInfoResponse"] = None,
3595
3655
  reporter: Optional[Reporter] = None,
3596
3656
  quiet: Optional[bool] = None,
3597
3657
  *,
@@ -3608,7 +3668,6 @@ class Run:
3608
3668
 
3609
3669
  Run._footer_sync_info(
3610
3670
  poll_exit_response=poll_exit_response,
3611
- job_info=job_info,
3612
3671
  quiet=quiet,
3613
3672
  settings=settings,
3614
3673
  printer=printer,
@@ -3793,7 +3852,6 @@ class Run:
3793
3852
  @staticmethod
3794
3853
  def _footer_sync_info(
3795
3854
  poll_exit_response: Optional[PollExitResponse] = None,
3796
- job_info: Optional["JobInfoResponse"] = None,
3797
3855
  quiet: Optional[bool] = None,
3798
3856
  *,
3799
3857
  settings: "Settings",
@@ -3813,14 +3871,10 @@ class Run:
3813
3871
  else:
3814
3872
  info = []
3815
3873
  if settings.run_name and settings.run_url:
3874
+ run_workspace = f"{settings.run_url}/workspace"
3816
3875
  info = [
3817
- f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
3876
+ f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(run_workspace)}"
3818
3877
  ]
3819
- if job_info and job_info.version and job_info.sequenceId:
3820
- link = f"{settings.project_url}/jobs/{job_info.sequenceId}/version_details/{job_info.version}"
3821
- info.append(
3822
- f"{printer.emoji('lightning')} View job at {printer.link(link)}",
3823
- )
3824
3878
  if poll_exit_response and poll_exit_response.file_counts:
3825
3879
  logger.info("logging synced files")
3826
3880
  file_counts = poll_exit_response.file_counts
@@ -45,6 +45,7 @@ from wandb.proto import wandb_settings_pb2
45
45
  from wandb.sdk.internal.system.env_probe_helpers import is_aws_lambda
46
46
  from wandb.sdk.lib import filesystem
47
47
  from wandb.sdk.lib._settings_toposort_generated import SETTINGS_TOPOLOGICALLY_SORTED
48
+ from wandb.sdk.lib.run_moment import RunMoment
48
49
  from wandb.sdk.wandb_setup import _EarlyLogger
49
50
 
50
51
  from .lib import apikey
@@ -160,6 +161,14 @@ def _get_program() -> Optional[str]:
160
161
  return None
161
162
 
162
163
 
164
+ def _runmoment_preprocessor(val: Any) -> Optional[RunMoment]:
165
+ if isinstance(val, RunMoment) or val is None:
166
+ return val
167
+ elif isinstance(val, str):
168
+ return RunMoment.from_uri(val)
169
+ raise UsageError(f"Could not parse value {val} as a RunMoment.")
170
+
171
+
163
172
  def _get_program_relpath(
164
173
  program: str, root: Optional[str] = None, _logger: Optional[_EarlyLogger] = None
165
174
  ) -> Optional[str]:
@@ -291,13 +300,14 @@ class SettingsData:
291
300
  _aws_lambda: bool
292
301
  _async_upload_concurrency_limit: int
293
302
  _cli_only_mode: bool # Avoid running any code specific for runs
303
+ _code_path_local: str
294
304
  _colab: bool
295
305
  # _config_dict: Config
296
306
  _cuda: str
297
307
  _disable_meta: bool # Do not collect system metadata
298
308
  _disable_service: (
299
- bool
300
- ) # Disable wandb-service, spin up internal process the old way
309
+ bool # Disable wandb-service, spin up internal process the old way
310
+ )
301
311
  _disable_setproctitle: bool # Do not use setproctitle on internal process
302
312
  _disable_stats: bool # Do not collect system metrics
303
313
  _disable_viewer: bool # Prevent early viewer query
@@ -354,20 +364,18 @@ class SettingsData:
354
364
  _stats_sample_rate_seconds: float
355
365
  _stats_samples_to_average: int
356
366
  _stats_join_assets: (
357
- bool
358
- ) # join metrics from different assets before sending to backend
367
+ bool # join metrics from different assets before sending to backend
368
+ )
359
369
  _stats_neuron_monitor_config_path: (
360
- str
361
- ) # path to place config file for neuron-monitor (AWS Trainium)
370
+ str # path to place config file for neuron-monitor (AWS Trainium)
371
+ )
362
372
  _stats_open_metrics_endpoints: Mapping[str, str] # open metrics endpoint names/urls
363
373
  # open metrics filters in one of the two formats:
364
374
  # - {"metric regex pattern, including endpoint name as prefix": {"label": "label value regex pattern"}}
365
375
  # - ("metric regex pattern 1", "metric regex pattern 2", ...)
366
376
  _stats_open_metrics_filters: Union[Sequence[str], Mapping[str, Mapping[str, str]]]
367
377
  _stats_disk_paths: Sequence[str] # paths to monitor disk usage
368
- _stats_buffer_size: (
369
- int
370
- ) # number of consolidated samples to buffer before flushing, available in run obj
378
+ _stats_buffer_size: int # number of consolidated samples to buffer before flushing, available in run obj
371
379
  _tmp_code_dir: str
372
380
  _tracelog: str
373
381
  _unsaved_keys: Sequence[str]
@@ -392,6 +400,7 @@ class SettingsData:
392
400
  entity: str
393
401
  files_dir: str
394
402
  force: bool
403
+ fork_from: Optional[RunMoment]
395
404
  git_commit: str
396
405
  git_remote: str
397
406
  git_remote_url: str
@@ -619,6 +628,10 @@ class Settings(SettingsData):
619
628
  "hook": lambda _: is_aws_lambda(),
620
629
  "auto_hook": True,
621
630
  },
631
+ _code_path_local={
632
+ "hook": lambda _: _get_program_relpath(self.program),
633
+ "auto_hook": True,
634
+ },
622
635
  _colab={
623
636
  "hook": lambda _: "google.colab" in sys.modules,
624
637
  "auto_hook": True,
@@ -800,6 +813,10 @@ class Settings(SettingsData):
800
813
  ),
801
814
  },
802
815
  force={"preprocessor": _str_as_bool},
816
+ fork_from={
817
+ "value": None,
818
+ "preprocessor": _runmoment_preprocessor,
819
+ },
803
820
  git_remote={"value": "origin"},
804
821
  heartbeat_seconds={"value": 30},
805
822
  ignore_globs={
@@ -1570,6 +1587,14 @@ class Settings(SettingsData):
1570
1587
  for key, value in v.items():
1571
1588
  # we only support dicts with string values for now
1572
1589
  mapping.value[key] = value
1590
+ elif isinstance(v, RunMoment):
1591
+ getattr(settings, k).CopyFrom(
1592
+ wandb_settings_pb2.RunMoment(
1593
+ run=v.run,
1594
+ value=v.value,
1595
+ metric=v.metric,
1596
+ )
1597
+ )
1573
1598
  elif v is None:
1574
1599
  # None is the default value for all settings, so we don't need to set it,
1575
1600
  # i.e. None means that the value was not set.
@@ -1892,16 +1917,33 @@ class Settings(SettingsData):
1892
1917
  f.write(json.dumps({"run_id": self.run_id}))
1893
1918
 
1894
1919
  def _apply_login(
1895
- self, login_settings: Dict[str, Any], _logger: Optional[_EarlyLogger] = None
1920
+ self,
1921
+ login_settings: Dict[str, Any],
1922
+ _logger: Optional[_EarlyLogger] = None,
1896
1923
  ) -> None:
1897
- param_map = dict(key="api_key", host="base_url", timeout="login_timeout")
1924
+ key_map = {
1925
+ "key": "api_key",
1926
+ "host": "base_url",
1927
+ "timeout": "login_timeout",
1928
+ }
1929
+
1930
+ # Rename keys and keep only the non-None values.
1931
+ #
1932
+ # The input keys are parameters to wandb.login(), but we use different
1933
+ # names for some of them in Settings.
1898
1934
  login_settings = {
1899
- param_map.get(k, k): v for k, v in login_settings.items() if v is not None
1935
+ key_map.get(key, key): value
1936
+ for key, value in login_settings.items()
1937
+ if value is not None
1900
1938
  }
1901
- if login_settings:
1902
- if _logger:
1903
- _logger.info(f"Applying login settings: {_redact_dict(login_settings)}")
1904
- self.update(login_settings, source=Source.LOGIN)
1939
+
1940
+ if _logger:
1941
+ _logger.info(f"Applying login settings: {_redact_dict(login_settings)}")
1942
+
1943
+ self.update(
1944
+ login_settings,
1945
+ source=Source.LOGIN,
1946
+ )
1905
1947
 
1906
1948
  def _apply_run_start(self, run_start_settings: Dict[str, Any]) -> None:
1907
1949
  # This dictionary maps from the "run message dict" to relevant fields in settings