wandb 0.16.4__py3-none-any.whl → 0.16.5__py3-none-any.whl

wandb/sdk/wandb_init.py

@@ -7,6 +7,7 @@ your evaluation script, and each step would be tracked as a run in W&B.
 For more on using `wandb.init()`, including code snippets, check out our
 [guide and FAQs](https://docs.wandb.ai/guides/track/launch).
 """
+
 import copy
 import json
 import logging
@@ -828,7 +829,7 @@ class _WandbInit:
             and self.settings.launch_config_path
             and os.path.exists(self.settings.launch_config_path)
         ):
-            run._save(self.settings.launch_config_path)
+            run.save(self.settings.launch_config_path)
         # put artifacts in run config here
         # since doing so earlier will cause an error
         # as the run is not upserted
@@ -961,6 +962,7 @@ def init(
     monitor_gym: Optional[bool] = None,
     save_code: Optional[bool] = None,
     id: Optional[str] = None,
+    fork_from: Optional[str] = None,
     settings: Union[Settings, Dict[str, Any], None] = None,
 ) -> Union[Run, RunDisabled, None]:
     r"""Start a new run to track and log to W&B.
@@ -1122,6 +1124,10 @@ def init(
             for saving hyperparameters to compare across runs. The ID cannot
             contain the following special characters: `/\#?%:`.
             See [our guide to resuming runs](https://docs.wandb.com/guides/runs/resuming).
+        fork_from: (str, optional) A string with the format <run_id>?_step=<step> describing
+            a moment in a previous run to fork a new run from. Creates a new run that picks up
+            logging history from the specified run at the specified moment. The target run must
+            be in the current project.
 
     Examples:
     ### Set where the run is logged
@@ -1167,6 +1173,11 @@ def init(
     error_seen = None
     except_exit = None
     run: Optional[Union[Run, RunDisabled]] = None
+
+    # convert fork_from into a version that can be passed to settings
+    if fork_from is not None and resume is not None:
+        raise ValueError("Cannot specify both `fork_from` and `resume`")
+
     try:
         wi = _WandbInit()
         wi.setup(kwargs)

wandb/sdk/wandb_login.py

@@ -22,7 +22,7 @@ from wandb.old.settings import Settings as OldSettings
 from ..apis import InternalApi
 from .internal.internal_api import Api
 from .lib import apikey
-from .wandb_settings import Settings, Source
+from .wandb_settings import Settings
 
 
 def _handle_host_wandb_setting(host: Optional[str], cloud: bool = False) -> None:
@@ -80,11 +80,17 @@ def login(
     _handle_host_wandb_setting(host)
     if wandb.setup()._settings._noop:
         return True
-    kwargs = dict(locals())
-    _verify = kwargs.pop("verify", False)
-    configured = _login(**kwargs)
 
-    if _verify:
+    configured = _login(
+        anonymous=anonymous,
+        key=key,
+        relogin=relogin,
+        host=host,
+        force=force,
+        timeout=timeout,
+    )
+
+    if verify:
         from . import wandb_setup
 
         singleton = wandb_setup._WandbSetup._instance
@@ -115,22 +121,32 @@ class _WandbLogin:
         self._key = None
         self._relogin = None
 
-    def setup(self, kwargs):
-        self.kwargs = kwargs
+    def setup(
+        self,
+        *,
+        anonymous: Optional[Literal["must", "allow", "never"]] = None,
+        key: Optional[str] = None,
+        relogin: Optional[bool] = None,
+        host: Optional[str] = None,
+        force: Optional[bool] = None,
+        timeout: Optional[int] = None,
+    ):
+        self._relogin = relogin
 
         # built up login settings
         login_settings: Settings = wandb.Settings()
-        settings_param = kwargs.pop("_settings", None)
-        # note that this case does not come up anywhere except for the tests
-        if settings_param is not None:
-            if isinstance(settings_param, Settings):
-                login_settings._apply_settings(settings_param)
-            elif isinstance(settings_param, dict):
-                login_settings.update(settings_param, source=Source.LOGIN)
-        _logger = wandb.setup()._get_logger()
-        # Do not save relogin into settings as we just want to relogin once
-        self._relogin = kwargs.pop("relogin", None)
-        login_settings._apply_login(kwargs, _logger=_logger)
+        logger = wandb.setup()._get_logger()
+
+        login_settings._apply_login(
+            {
+                "anonymous": anonymous,
+                "key": key,
+                "host": host,
+                "force": force,
+                "timeout": timeout,
+            },
+            _logger=logger,
+        )
 
         # make sure they are applied globally
         self._wl = wandb.setup(settings=login_settings)
@@ -259,6 +275,7 @@ class _WandbLogin:
 
 
 def _login(
+    *,
     anonymous: Optional[Literal["must", "allow", "never"]] = None,
     key: Optional[str] = None,
     relogin: Optional[bool] = None,
@@ -270,9 +287,6 @@ def _login(
     _disable_warning: Optional[bool] = None,
     _entity: Optional[str] = None,
 ):
-    kwargs = dict(locals())
-    _disable_warning = kwargs.pop("_disable_warning", None)
-
     if wandb.run is not None:
         if not _disable_warning:
             wandb.termwarn("Calling wandb.login() after wandb.init() has no effect.")
@@ -280,20 +294,24 @@ def _login(
 
     wlogin = _WandbLogin()
 
-    _backend = kwargs.pop("_backend", None)
     if _backend:
         wlogin.set_backend(_backend)
 
-    _silent = kwargs.pop("_silent", None)
     if _silent:
         wlogin.set_silent(_silent)
 
-    _entity = kwargs.pop("_entity", None)
     if _entity:
         wlogin.set_entity(_entity)
 
     # configure login object
-    wlogin.setup(kwargs)
+    wlogin.setup(
+        anonymous=anonymous,
+        key=key,
+        relogin=relogin,
+        host=host,
+        force=force,
+        timeout=timeout,
+    )
 
     if wlogin._settings._offline:
         return False
@@ -306,7 +324,6 @@ def _login(
     # perform a login
     logged_in = wlogin.login()
 
-    key = kwargs.get("key")
     if key:
         wlogin.configure_api_key(key)
 

wandb/sdk/wandb_run.py

@@ -6,6 +6,7 @@ import json
 import logging
 import numbers
 import os
+import pathlib
 import re
 import sys
 import threading
@@ -42,7 +43,6 @@ from wandb.apis import internal, public
 from wandb.apis.internal import Api
 from wandb.apis.public import Api as PublicApi
 from wandb.proto.wandb_internal_pb2 import (
-    JobInfoResponse,
     MetricRecord,
     PollExitResponse,
     Result,
@@ -531,7 +531,6 @@ class Run:
     _check_version: Optional["CheckVersionResponse"]
     _sampled_history: Optional["SampledHistoryResponse"]
     _final_summary: Optional["GetSummaryResponse"]
-    _job_info: Optional["JobInfoResponse"]
     _poll_exit_handle: Optional[MailboxHandle]
     _poll_exit_response: Optional[PollExitResponse]
     _server_info_response: Optional[ServerInfoResponse]
@@ -642,7 +641,6 @@ class Run:
         self._server_info_response = None
         self._internal_messages_response = None
         self._poll_exit_handle = None
-        self._job_info = None
 
         # Initialize telemetry object
         self._telemetry_obj = telemetry.TelemetryRecord()
@@ -673,6 +671,12 @@ class Run:
                 os.path.join("code", self._settings.program_relpath)
             )
 
+        if self._settings.fork_from is not None:
+            config[wandb_key]["branch_point"] = {
+                "run_id": self._settings.fork_from.run,
+                "step": self._settings.fork_from.value,
+            }
+
         self._config._update(config, ignore_locked=True)
 
         if sweep_config:
@@ -734,7 +738,7 @@ class Run:
             and self._settings.launch_config_path
             and os.path.exists(self._settings.launch_config_path)
         ):
-            self._save(self._settings.launch_config_path)
+            self.save(self._settings.launch_config_path)
             with open(self._settings.launch_config_path) as fp:
                 launch_config = json.loads(fp.read())
             if launch_config.get("overrides", {}).get("artifacts") is not None:
@@ -1385,6 +1389,8 @@ class Run:
 
     @_run_decorator._noop_on_finish()
     def _summary_update_callback(self, summary_record: SummaryRecord) -> None:
+        with telemetry.context(run=self) as tel:
+            tel.feature.set_summary = True
         if self._backend and self._backend.interface:
             self._backend.interface.publish_summary(summary_record)
 
@@ -1811,6 +1817,10 @@ class Run:
             ValueError: if invalid data is passed
 
         """
+        if step is not None:
+            with telemetry.context(run=self) as tel:
+                tel.feature.set_step_log = True
+
         if sync is not None:
             deprecate.deprecate(
                 field_name=deprecate.Deprecated.run__log_sync,
@@ -1831,20 +1841,53 @@ class Run:
     @_run_decorator._attach
     def save(
         self,
-        glob_str: Optional[str] = None,
-        base_path: Optional[str] = None,
+        glob_str: Optional[Union[str, os.PathLike]] = None,
+        base_path: Optional[Union[str, os.PathLike]] = None,
         policy: "PolicyName" = "live",
     ) -> Union[bool, List[str]]:
-        """Ensure all files matching `glob_str` are synced to wandb with the policy specified.
+        """Sync one or more files to W&B.
+
+        Relative paths are relative to the current working directory.
+
+        A Unix glob, such as "myfiles/*", is expanded at the time `save` is
+        called regardless of the `policy`. In particular, new files are not
+        picked up automatically.
+
+        A `base_path` may be provided to control the directory structure of
+        uploaded files. It should be a prefix of `glob_str`, and the direcotry
+        structure beneath it is preserved. It's best understood through
+        examples:
+
+        ```
+        wandb.save("these/are/myfiles/*")
+        # => Saves files in a "these/are/myfiles/" folder in the run.
+
+        wandb.save("these/are/myfiles/*", base_path="these")
+        # => Saves files in an "are/myfiles/" folder in the run.
+
+        wandb.save("/User/username/Documents/run123/*.txt")
+        # => Saves files in a "run123/" folder in the run.
+
+        wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
+        # => Saves files in a "username/Documents/run123/" folder in the run.
+
+        wandb.save("files/*/saveme.txt")
+        # => Saves each "saveme.txt" file in an appropriate subdirectory
+        # of "files/".
+        ```
 
         Arguments:
-            glob_str: (string) a relative or absolute path to a unix glob or regular
-                path.  If this isn't specified the method is a noop.
-            base_path: (string) the base path to run the glob relative to
-            policy: (string) one of `live`, `now`, or `end`
-                - live: upload the file as it changes, overwriting the previous version
-                - now: upload the file once now
-                - end: only upload file when the run ends
+            glob_str: A relative or absolute path or Unix glob.
+            base_path: A path to use to infer a directory structure; see examples.
+            policy: One of `live`, `now`, or `end`.
+                * live: upload the file as it changes, overwriting the previous version
+                * now: upload the file once now
+                * end: upload file when the run ends
+
+        Returns:
+            Paths to the symlinks created for the matched files.
+
+            For historical reasons, this may return a boolean in legacy code.
         """
         if glob_str is None:
             # noop for historical reasons, run.save() may be called in legacy code
@@ -1857,77 +1900,116 @@ class Run:
             )
             return True
 
-        return self._save(glob_str, base_path, policy)
+        if isinstance(glob_str, bytes):
+            # Preserved for backward compatibility: allow bytes inputs.
+            glob_str = glob_str.decode("utf-8")
+        if isinstance(glob_str, str) and (
+            glob_str.startswith("gs://") or glob_str.startswith("s3://")
+        ):
+            # Provide a better error message for a common misuse.
+            wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
+            return []
+        glob_path = pathlib.Path(glob_str)
+
+        if base_path is not None:
+            base_path = pathlib.Path(base_path)
+        elif not glob_path.is_absolute():
+            base_path = pathlib.Path(".")
+        else:
+            # Absolute glob paths with no base path get special handling.
+            wandb.termwarn(
+                "Saving files without folders. If you want to preserve "
+                "subdirectories pass base_path to wandb.save, i.e. "
+                'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
+                repeat=False,
+            )
+            base_path = glob_path.resolve().parent.parent
 
-    def _save(
-        self,
-        glob_str: Optional[str] = None,
-        base_path: Optional[str] = None,
-        policy: "PolicyName" = "live",
-    ) -> Union[bool, List[str]]:
         if policy not in ("live", "end", "now"):
             raise ValueError(
-                'Only "live" "end" and "now" policies are currently supported.'
+                'Only "live", "end" and "now" policies are currently supported.'
             )
-        if isinstance(glob_str, bytes):
-            glob_str = glob_str.decode("utf-8")
-        if not isinstance(glob_str, str):
-            raise ValueError("Must call wandb.save(glob_str) with glob_str a str")
 
-        if base_path is None:
-            if os.path.isabs(glob_str):
-                base_path = os.path.dirname(glob_str)
-                wandb.termwarn(
-                    "Saving files without folders. If you want to preserve "
-                    "sub directories pass base_path to wandb.save, i.e. "
-                    'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
-                    repeat=False,
-                )
-            else:
-                base_path = "."
-        wandb_glob_str = GlobStr(os.path.relpath(glob_str, base_path))
-        if ".." + os.sep in wandb_glob_str:
-            raise ValueError("globs can't walk above base_path")
+        resolved_glob_path = glob_path.resolve()
+        resolved_base_path = base_path.resolve()
+
+        return self._save(
+            resolved_glob_path,
+            resolved_base_path,
+            policy,
+        )
+
+    def _save(
+        self,
+        glob_path: pathlib.Path,
+        base_path: pathlib.Path,
+        policy: "PolicyName",
+    ) -> List[str]:
+        # Can't use is_relative_to() because that's added in Python 3.9,
+        # but we support down to Python 3.7.
+        if not str(glob_path).startswith(str(base_path)):
+            raise ValueError("Glob may not walk above the base path")
+
+        if glob_path == base_path:
+            raise ValueError("Glob cannot be the same as the base path")
+
+        relative_glob = glob_path.relative_to(base_path)
+        if relative_glob.parts[0] == "*":
+            raise ValueError("Glob may not start with '*' relative to the base path")
+        relative_glob_str = GlobStr(str(relative_glob))
 
         with telemetry.context(run=self) as tel:
             tel.feature.save = True
 
-        if glob_str.startswith("gs://") or glob_str.startswith("s3://"):
-            wandb.termlog(
-                "%s is a cloud storage url, can't save file to wandb." % glob_str
-            )
-            return []
-        files = glob.glob(os.path.join(self._settings.files_dir, wandb_glob_str))
-        warn = False
-        if len(files) == 0 and "*" in wandb_glob_str:
-            warn = True
-        for path in glob.glob(glob_str):
-            file_name = os.path.relpath(path, base_path)
-            abs_path = os.path.abspath(path)
-            wandb_path = os.path.join(self._settings.files_dir, file_name)
-            filesystem.mkdir_exists_ok(os.path.dirname(wandb_path))
-            # We overwrite symlinks because namespaces can change in Tensorboard
-            if os.path.islink(wandb_path) and abs_path != os.readlink(wandb_path):
-                os.remove(wandb_path)
-                os.symlink(abs_path, wandb_path)
-            elif not os.path.exists(wandb_path):
-                os.symlink(abs_path, wandb_path)
-            files.append(wandb_path)
-        if warn:
-            file_str = "%i file" % len(files)
-            if len(files) > 1:
+        # Paths to the symlinks created for the globbed files.
+        wandb_files = [
+            str(path)
+            for path in pathlib.Path(
+                self._settings.files_dir,
+            ).glob(relative_glob_str)
+        ]
+
+        had_symlinked_files = len(wandb_files) > 0
+        is_star_glob = "*" in relative_glob_str
+
+        # The base_path may itself be a glob, so we can't do
+        #     base_path.glob(relative_glob_str)
+        for path_str in glob.glob(str(base_path / relative_glob_str)):
+            path = pathlib.Path(path_str).absolute()
+
+            # We can't use relative_to() because base_path may be a glob.
+            saved_path = pathlib.Path(*path.parts[len(base_path.parts) :])
+
+            wandb_path = pathlib.Path(self._settings.files_dir, saved_path)
+
+            wandb_files.append(str(wandb_path))
+            wandb_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Delete the symlink if it exists.
+            try:
+                wandb_path.unlink()
+            except FileNotFoundError:
+                # In Python 3.8, we would pass missing_ok=True, but as of now
+                # we support down to Python 3.7.
+                pass
+
+            wandb_path.symlink_to(path)
+
+        # Inform users that new files aren't detected automatically.
+        if not had_symlinked_files and is_star_glob:
+            file_str = f"{len(wandb_files)} file"
+            if len(wandb_files) > 1:
                 file_str += "s"
             wandb.termwarn(
-                (
-                    "Symlinked %s into the W&B run directory, "
-                    "call wandb.save again to sync new files."
-                )
-                % file_str
+                f"Symlinked {file_str} into the W&B run directory, "
+                "call wandb.save again to sync new files."
             )
-        files_dict: FilesDict = dict(files=[(wandb_glob_str, policy)])
+
+        files_dict: FilesDict = {"files": [(relative_glob_str, policy)]}
         if self._backend and self._backend.interface:
             self._backend.interface.publish_files(files_dict)
-        return files
+
+        return wandb_files
 
     @_run_decorator._attach
     def restore(
@@ -2485,7 +2567,6 @@ class Run:
         sampled_history_handle = (
             self._backend.interface.deliver_request_sampled_history()
         )
-        job_info_handle = self._backend.interface.deliver_request_job_info()
 
         result = server_info_handle.wait(timeout=-1)
         assert result
@@ -2499,10 +2580,6 @@ class Run:
         assert result
         self._final_summary = result.response.get_summary_response
 
-        result = job_info_handle.wait(timeout=-1)
-        assert result
-        self._job_info = result.response.job_info_response
-
         if self._backend:
             self._backend.cleanup()
 
@@ -2525,7 +2602,6 @@ class Run:
             server_info_response=self._server_info_response,
             check_version_response=self._check_version,
             internal_messages_response=self._internal_messages_response,
-            job_info=self._job_info,
             reporter=self._reporter,
             quiet=self._quiet,
             settings=self._settings,
@@ -3509,7 +3585,7 @@ class Run:
         if settings._offline or settings.silent:
             return
 
-        run_url = settings.run_url
+        workspace_url = f"{settings.run_url}/workspace"
         project_url = settings.project_url
         sweep_url = settings.sweep_url
 
@@ -3520,7 +3596,7 @@ class Run:
 
         if printer._html:
             if not wandb.jupyter.maybe_display():
-                run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
+                run_line = f"<strong>{printer.link(workspace_url, run_name)}</strong>"
                 project_line, sweep_line = "", ""
 
                 # TODO(settings): make settings the source of truth
@@ -3552,7 +3628,7 @@ class Run:
                     f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
                 )
         printer.display(
-            f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
+            f'{printer.emoji("rocket")} View run at {printer.link(workspace_url)}',
         )
 
         # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
@@ -3576,7 +3652,6 @@ class Run:
         server_info_response: Optional[ServerInfoResponse] = None,
         check_version_response: Optional["CheckVersionResponse"] = None,
         internal_messages_response: Optional["InternalMessagesResponse"] = None,
-        job_info: Optional["JobInfoResponse"] = None,
         reporter: Optional[Reporter] = None,
         quiet: Optional[bool] = None,
         *,
@@ -3593,7 +3668,6 @@ class Run:
 
         Run._footer_sync_info(
             poll_exit_response=poll_exit_response,
-            job_info=job_info,
             quiet=quiet,
             settings=settings,
             printer=printer,
@@ -3778,7 +3852,6 @@ class Run:
     @staticmethod
     def _footer_sync_info(
         poll_exit_response: Optional[PollExitResponse] = None,
-        job_info: Optional["JobInfoResponse"] = None,
         quiet: Optional[bool] = None,
         *,
         settings: "Settings",
@@ -3798,14 +3871,10 @@ class Run:
         else:
             info = []
             if settings.run_name and settings.run_url:
+                run_workspace = f"{settings.run_url}/workspace"
                 info = [
-                    f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
+                    f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(run_workspace)}"
                 ]
-            if job_info and job_info.version and job_info.sequenceId:
-                link = f"{settings.project_url}/jobs/{job_info.sequenceId}/version_details/{job_info.version}"
-                info.append(
-                    f"{printer.emoji('lightning')} View job at {printer.link(link)}",
-                )
             if poll_exit_response and poll_exit_response.file_counts:
                 logger.info("logging synced files")
                 file_counts = poll_exit_response.file_counts