wandb 0.16.5__py3-none-any.whl → 0.17.0__py3-none-any.whl

wandb/sdk/wandb_init.py

@@ -15,7 +15,6 @@ import os
 import platform
 import sys
 import tempfile
-import traceback
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
 
 import wandb
@@ -195,12 +194,6 @@ class _WandbInit:
         # Start with settings from wandb library singleton
         settings: Settings = self._wl.settings.copy()
 
-        # when using launch, we don't want to reuse the same run id from the singleton
-        # since users might launch multiple runs in the same process
-        # TODO(kdg): allow users to control this via launch settings
-        if settings.launch and singleton is not None:
-            settings.update({"run_id": None}, source=Source.INIT)
-
         settings_param = kwargs.pop("settings", None)
         if settings_param is not None and isinstance(settings_param, (Settings, dict)):
             settings.update(settings_param, source=Source.INIT)
@@ -561,7 +554,7 @@ class _WandbInit:
         percent_done = handle.percent_done
         self.printer.progress_update(line, percent_done=percent_done)
 
-    def init(self) -> Union[Run, RunDisabled, None]:  # noqa: C901
+    def init(self) -> Union[Run, RunDisabled]:  # noqa: C901
         if logger is None:
             raise RuntimeError("Logger not initialized")
         logger.info("calling init triggers")
@@ -660,9 +653,6 @@ class _WandbInit:
             if self.settings.launch:
                 tel.feature.launch = True
 
-            if self.settings._async_upload_concurrency_limit:
-                tel.feature.async_uploads = True
-
             for module_name in telemetry.list_telemetry_imports(only_imported=True):
                 setattr(tel.imports_init, module_name, True)
 
@@ -849,13 +839,6 @@ class _WandbInit:
         return run
 
 
-def getcaller() -> None:
-    if not logger:
-        return None
-    src, line, func, stack = logger.findCaller(stack_info=True)
-    print("Problem at:", src, line, func)
-
-
 def _attach(
     attach_id: Optional[str] = None,
     run_id: Optional[str] = None,
@@ -964,7 +947,7 @@ def init(
     id: Optional[str] = None,
     fork_from: Optional[str] = None,
     settings: Union[Settings, Dict[str, Any], None] = None,
-) -> Union[Run, RunDisabled, None]:
+) -> Union[Run, RunDisabled]:
     r"""Start a new run to track and log to W&B.
 
     In an ML training pipeline, you could add `wandb.init()`
@@ -1124,10 +1107,10 @@ def init(
             for saving hyperparameters to compare across runs. The ID cannot
             contain the following special characters: `/\#?%:`.
             See [our guide to resuming runs](https://docs.wandb.com/guides/runs/resuming).
-        fork_from: (str, optional) A string with the format <run_id>?_step=<step> describing
+        fork_from: (str, optional) A string with the format {run_id}?_step={step} describing
             a moment in a previous run to fork a new run from. Creates a new run that picks up
             logging history from the specified run at the specified moment. The target run must
-            be in the current project.
+            be in the current project. Example: `fork_from="my-run-id?_step=1234"`.
 
     Examples:
     ### Set where the run is logged
@@ -1170,9 +1153,6 @@ def init(
     wandb._assert_is_user_process()
 
     kwargs = dict(locals())
-    error_seen = None
-    except_exit = None
-    run: Optional[Union[Run, RunDisabled]] = None
 
     # convert fork_from into a version that can be passed to settings
     if fork_from is not None and resume is not None:
@@ -1181,46 +1161,19 @@ def init(
     try:
         wi = _WandbInit()
         wi.setup(kwargs)
-        assert wi.settings
-        except_exit = wi.settings._except_exit
-        try:
-            run = wi.init()
-            except_exit = wi.settings._except_exit
-        except (KeyboardInterrupt, Exception) as e:
-            if not isinstance(e, KeyboardInterrupt):
-                wandb._sentry.exception(e)
-            if not (
-                wandb.wandb_agent._is_running() and isinstance(e, KeyboardInterrupt)
-            ):
-                getcaller()
-            assert logger
-            if wi.settings.problem == "fatal":
-                raise
-            if wi.settings.problem == "warn":
-                pass
-            # TODO(jhr): figure out how to make this RunDummy
-            run = None
-    except Error as e:
-        if logger is not None:
-            logger.exception(str(e))
-        raise e
+        return wi.init()
+
     except KeyboardInterrupt as e:
-        assert logger
-        logger.warning("interrupted", exc_info=e)
-        raise e
+        if logger is not None:
+            logger.warning("interrupted", exc_info=e)
+
+        raise
+
     except Exception as e:
-        error_seen = e
-        traceback.print_exc()
-        assert logger
-        logger.error("error", exc_info=e)
+        if logger is not None:
+            logger.exception("error in wandb.init()", exc_info=e)
+
         # Need to build delay into this sentry capture because our exit hooks
         # mess with sentry's ability to send out errors before the program ends.
-        wandb._sentry.exception(e)
-        # reraise(*sys.exc_info())
-    finally:
-        if error_seen:
-            if except_exit:
-                wandb.termerror("Abnormal program exit")
-                os._exit(1)
-            raise Error("An unexpected error occurred") from error_seen
-    return run
+        wandb._sentry.reraise(e)
+        raise AssertionError()  # unreachable

wandb/sdk/wandb_manager.py

@@ -5,7 +5,7 @@ Create a manager channel.
 
 import atexit
 import os
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Callable, Optional
 
 import psutil
 
@@ -205,7 +205,7 @@ class _Manager:
         svc_iface = self._get_service_interface()
         svc_iface._svc_inform_start(settings=settings, run_id=run_id)
 
-    def _inform_attach(self, attach_id: str) -> Optional[Dict[str, Any]]:
+    def _inform_attach(self, attach_id: str) -> Optional["wandb_settings_pb2.Settings"]:
         svc_iface = self._get_service_interface()
         try:
             response = svc_iface._svc_inform_attach(attach_id=attach_id)

wandb/sdk/wandb_require.py

@@ -9,9 +9,11 @@ Example:
     wandb.require("incremental-artifacts@beta")
 """
 
+import os
 from typing import Optional, Sequence, Union
 
 import wandb
+from wandb.env import _REQUIRE_CORE
 from wandb.errors import UnsupportedError
 from wandb.sdk import wandb_run
 from wandb.sdk.lib.wburls import wburls
@@ -38,6 +40,9 @@ class _Requires:
     def require_service(self) -> None:
         self._require_service()
 
+    def require_core(self) -> None:
+        os.environ[_REQUIRE_CORE] = "true"
+
     def apply(self) -> None:
         """Call require_* method for supported features."""
         last_message: str = ""

wandb/sdk/wandb_run.py

@@ -207,6 +207,15 @@ class RunStatusChecker:
         self._network_status_thread.start()
         self._internal_messages_thread.start()
 
+    @staticmethod
+    def _abandon_status_check(
+        lock: threading.Lock,
+        handle: Optional[MailboxHandle],
+    ):
+        with lock:
+            if handle:
+                handle.abandon()
+
     def _loop_check_status(
         self,
         *,
@@ -265,13 +274,19 @@ class RunStatusChecker:
                         )
                     )
 
-        self._loop_check_status(
-            lock=self._network_status_lock,
-            set_handle=lambda x: setattr(self, "_network_status_handle", x),
-            timeout=self._retry_polling_interval,
-            request=self._interface.deliver_network_status,
-            process=_process_network_status,
-        )
+        try:
+            self._loop_check_status(
+                lock=self._network_status_lock,
+                set_handle=lambda x: setattr(self, "_network_status_handle", x),
+                timeout=self._retry_polling_interval,
+                request=self._interface.deliver_network_status,
+                process=_process_network_status,
+            )
+        except BrokenPipeError:
+            self._abandon_status_check(
+                self._network_status_lock,
+                self._network_status_handle,
+            )
 
     def check_stop_status(self) -> None:
         def _process_stop_status(result: Result) -> None:
@@ -283,13 +298,19 @@ class RunStatusChecker:
                     thread.interrupt_main()
                     return
 
-        self._loop_check_status(
-            lock=self._stop_status_lock,
-            set_handle=lambda x: setattr(self, "_stop_status_handle", x),
-            timeout=self._stop_polling_interval,
-            request=self._interface.deliver_stop_status,
-            process=_process_stop_status,
-        )
+        try:
+            self._loop_check_status(
+                lock=self._stop_status_lock,
+                set_handle=lambda x: setattr(self, "_stop_status_handle", x),
+                timeout=self._stop_polling_interval,
+                request=self._interface.deliver_stop_status,
+                process=_process_stop_status,
+            )
+        except BrokenPipeError:
+            self._abandon_status_check(
+                self._stop_status_lock,
+                self._stop_status_handle,
+            )
 
     def check_internal_messages(self) -> None:
         def _process_internal_messages(result: Result) -> None:
@@ -297,25 +318,34 @@ class RunStatusChecker:
             for msg in internal_messages.messages.warning:
                 wandb.termwarn(msg)
 
-        self._loop_check_status(
-            lock=self._internal_messages_lock,
-            set_handle=lambda x: setattr(self, "_internal_messages_handle", x),
-            timeout=1,
-            request=self._interface.deliver_internal_messages,
-            process=_process_internal_messages,
-        )
+            try:
+                self._loop_check_status(
+                    lock=self._internal_messages_lock,
+                    set_handle=lambda x: setattr(self, "_internal_messages_handle", x),
+                    timeout=1,
+                    request=self._interface.deliver_internal_messages,
+                    process=_process_internal_messages,
+                )
+            except BrokenPipeError:
+                self._abandon_status_check(
+                    self._internal_messages_lock,
+                    self._internal_messages_handle,
+                )
 
     def stop(self) -> None:
         self._join_event.set()
-        with self._stop_status_lock:
-            if self._stop_status_handle:
-                self._stop_status_handle.abandon()
-        with self._network_status_lock:
-            if self._network_status_handle:
-                self._network_status_handle.abandon()
-        with self._internal_messages_lock:
-            if self._internal_messages_handle:
-                self._internal_messages_handle.abandon()
+        self._abandon_status_check(
+            self._stop_status_lock,
+            self._stop_status_handle,
+        )
+        self._abandon_status_check(
+            self._network_status_lock,
+            self._network_status_handle,
+        )
+        self._abandon_status_check(
+            self._internal_messages_lock,
+            self._internal_messages_handle,
+        )
 
     def join(self) -> None:
         self.stop()
@@ -327,8 +357,7 @@ class RunStatusChecker:
 class _run_decorator:  # noqa: N801
     _is_attaching: str = ""
 
-    class Dummy:
-        ...
+    class Dummy: ...
 
     @classmethod
     def _attach(cls, func: Callable) -> Callable:
@@ -1401,6 +1430,10 @@ class Run:
         # self._printer.display(line)
 
     def _summary_get_current_summary_callback(self) -> Dict[str, Any]:
+        if self._is_finished:
+            # TODO: WB-18420: fetch summary from backend and stage it before run is finished
+            wandb.termwarn("Summary data not available in finished run")
+            return {}
         if not self._backend or not self._backend.interface:
             return {}
         handle = self._backend.interface.deliver_get_summary()
@@ -1793,7 +1826,7 @@ class Run:
             import wandb
 
             run = wandb.init()
-            run.log({"pr": wandb.plots.precision_recall(y_test, y_probas, labels)})
+            run.log({"pr": wandb.plot.pr_curve(y_test, y_probas, labels)})
             ```
 
             ### 3D Object
@@ -1854,7 +1887,7 @@ class Run:
         picked up automatically.
 
         A `base_path` may be provided to control the directory structure of
-        uploaded files. It should be a prefix of `glob_str`, and the direcotry
+        uploaded files. It should be a prefix of `glob_str`, and the directory
         structure beneath it is preserved. It's best understood through
         examples:
 
@@ -1866,16 +1899,19 @@ class Run:
         # => Saves files in an "are/myfiles/" folder in the run.
 
         wandb.save("/User/username/Documents/run123/*.txt")
-        # => Saves files in a "run123/" folder in the run.
+        # => Saves files in a "run123/" folder in the run. See note below.
 
         wandb.save("/User/username/Documents/run123/*.txt", base_path="/User")
         # => Saves files in a "username/Documents/run123/" folder in the run.
 
         wandb.save("files/*/saveme.txt")
         # => Saves each "saveme.txt" file in an appropriate subdirectory
-        # of "files/".
+        #    of "files/".
         ```
 
+        Note: when given an absolute path or glob and no `base_path`, one
+        directory level is preserved as in the example above.
+
         Arguments:
             glob_str: A relative or absolute path or Unix glob.
             base_path: A path to use to infer a directory structure; see examples.
@@ -1909,7 +1945,11 @@ class Run:
             # Provide a better error message for a common misuse.
             wandb.termlog(f"{glob_str} is a cloud storage url, can't save file to W&B.")
             return []
-        glob_path = pathlib.Path(glob_str)
+        # NOTE: We use PurePath instead of Path because WindowsPath doesn't
+        # like asterisks and errors out in resolve(). It also makes logical
+        # sense: globs aren't real paths, they're just path-like strings.
+        glob_path = pathlib.PurePath(glob_str)
+        resolved_glob_path = pathlib.PurePath(os.path.abspath(glob_path))
 
         if base_path is not None:
             base_path = pathlib.Path(base_path)
@@ -1923,15 +1963,14 @@ class Run:
                 'wandb.save("/mnt/folder/file.h5", base_path="/mnt")',
                 repeat=False,
             )
-            base_path = glob_path.resolve().parent.parent
+            base_path = resolved_glob_path.parent.parent
 
         if policy not in ("live", "end", "now"):
             raise ValueError(
                 'Only "live", "end" and "now" policies are currently supported.'
             )
 
-        resolved_glob_path = glob_path.resolve()
-        resolved_base_path = base_path.resolve()
+        resolved_base_path = pathlib.PurePath(os.path.abspath(base_path))
 
         return self._save(
             resolved_glob_path,
@@ -1941,8 +1980,8 @@ class Run:
 
     def _save(
         self,
-        glob_path: pathlib.Path,
-        base_path: pathlib.Path,
+        glob_path: pathlib.PurePath,
+        base_path: pathlib.PurePath,
         policy: "PolicyName",
     ) -> List[str]:
         # Can't use is_relative_to() because that's added in Python 3.9,
@@ -1961,55 +2000,67 @@ class Run:
         with telemetry.context(run=self) as tel:
             tel.feature.save = True
 
-        # Paths to the symlinks created for the globbed files.
-        wandb_files = [
-            str(path)
-            for path in pathlib.Path(
+        # Files in the files directory matched by the glob, including old and
+        # new ones.
+        globbed_files = set(
+            pathlib.Path(
                 self._settings.files_dir,
             ).glob(relative_glob_str)
-        ]
+        )
 
-        had_symlinked_files = len(wandb_files) > 0
+        had_symlinked_files = len(globbed_files) > 0
         is_star_glob = "*" in relative_glob_str
 
         # The base_path may itself be a glob, so we can't do
         #     base_path.glob(relative_glob_str)
         for path_str in glob.glob(str(base_path / relative_glob_str)):
-            path = pathlib.Path(path_str).absolute()
+            source_path = pathlib.Path(path_str).absolute()
 
             # We can't use relative_to() because base_path may be a glob.
-            saved_path = pathlib.Path(*path.parts[len(base_path.parts) :])
+            relative_path = pathlib.Path(*source_path.parts[len(base_path.parts) :])
 
-            wandb_path = pathlib.Path(self._settings.files_dir, saved_path)
+            target_path = pathlib.Path(self._settings.files_dir, relative_path)
+            globbed_files.add(target_path)
 
-            wandb_files.append(str(wandb_path))
-            wandb_path.parent.mkdir(parents=True, exist_ok=True)
+            # If the file is already where it needs to be, don't create a symlink.
+            if source_path.resolve() == target_path.resolve():
+                continue
+
+            target_path.parent.mkdir(parents=True, exist_ok=True)
 
             # Delete the symlink if it exists.
             try:
-                wandb_path.unlink()
+                target_path.unlink()
             except FileNotFoundError:
                 # In Python 3.8, we would pass missing_ok=True, but as of now
                 # we support down to Python 3.7.
                 pass
 
-            wandb_path.symlink_to(path)
+            target_path.symlink_to(source_path)
 
         # Inform users that new files aren't detected automatically.
         if not had_symlinked_files and is_star_glob:
-            file_str = f"{len(wandb_files)} file"
-            if len(wandb_files) > 1:
+            file_str = f"{len(globbed_files)} file"
+            if len(globbed_files) > 1:
                 file_str += "s"
             wandb.termwarn(
                 f"Symlinked {file_str} into the W&B run directory, "
                 "call wandb.save again to sync new files."
             )
 
-        files_dict: FilesDict = {"files": [(relative_glob_str, policy)]}
+        files_dict: FilesDict = {
+            "files": [
+                (
+                    GlobStr(str(f.relative_to(self._settings.files_dir))),
+                    policy,
+                )
+                for f in globbed_files
+            ]
+        }
         if self._backend and self._backend.interface:
             self._backend.interface.publish_files(files_dict)
 
-        return wandb_files
+        return [str(f) for f in globbed_files]
 
     @_run_decorator._attach
     def restore(
@@ -2341,16 +2392,17 @@ class Run:
         if self._settings._offline:
             return
         if self._backend and self._backend.interface:
-            logger.info("communicating current version")
-            version_handle = self._backend.interface.deliver_check_version(
-                current_version=wandb.__version__
-            )
-            version_result = version_handle.wait(timeout=30)
-            if not version_result:
-                version_handle.abandon()
-                return
-            self._check_version = version_result.response.check_version_response
-            logger.info(f"got version response {self._check_version}")
+            if not self._settings._disable_update_check:
+                logger.info("communicating current version")
+                version_handle = self._backend.interface.deliver_check_version(
+                    current_version=wandb.__version__
+                )
+                version_result = version_handle.wait(timeout=30)
+                if not version_result:
+                    version_handle.abandon()
+                else:
+                    self._check_version = version_result.response.check_version_response
+                    logger.info("got version response %s", self._check_version)
 
     def _on_start(self) -> None:
         # would like to move _set_global to _on_ready to unify _on_start and _on_attach
@@ -2425,6 +2477,8 @@ class Run:
         self._telemetry_obj_active = True
         self._telemetry_flush()
 
+        self._detect_and_apply_job_inputs()
+
         # object is about to be returned to the user, don't let them modify it
         self._freeze()
 
@@ -2432,6 +2486,12 @@ class Run:
             if os.path.exists(self._settings.resume_fname):
                 os.remove(self._settings.resume_fname)
 
+    def _detect_and_apply_job_inputs(self) -> None:
+        """If the user has staged launch inputs, apply them to the run."""
+        from wandb.sdk.launch.inputs.internal import StagedLaunchInputs
+
+        StagedLaunchInputs().apply(self)
+
     def _make_job_source_reqs(self) -> Tuple[List[str], Dict[str, Any], Dict[str, Any]]:
         from wandb.util import working_set
 
@@ -2690,12 +2750,23 @@ class Run:
                 if i not in valid:
                     raise wandb.Error(f"Unhandled define_metric() arg: summary op: {i}")
                 summary_ops.append(i)
+            with telemetry.context(run=self) as tel:
+                tel.feature.metric_summary = True
         goal_cleaned: Optional[str] = None
         if goal is not None:
             goal_cleaned = goal[:3].lower()
             valid_goal = {"min", "max"}
             if goal_cleaned not in valid_goal:
                 raise wandb.Error(f"Unhandled define_metric() arg: goal: {goal}")
+            with telemetry.context(run=self) as tel:
+                tel.feature.metric_goal = True
+        if hidden:
+            with telemetry.context(run=self) as tel:
+                tel.feature.metric_hidden = True
+        if step_sync:
+            with telemetry.context(run=self) as tel:
+                tel.feature.metric_step_sync = True
+
         m = wandb_metric.Metric(
             name=name,
             step_metric=step_metric,
@@ -3158,13 +3229,13 @@ class Run:
             )
         if entity and artifact._source_entity and entity != artifact._source_entity:
             raise ValueError(
-                f"Artifact {artifact.name} is owned by entity '{entity}'; it can't be "
-                f"moved to '{artifact._source_entity}'"
+                f"Artifact {artifact.name} is owned by entity "
+                f"'{artifact._source_entity}'; it can't be moved to '{entity}'"
             )
         if project and artifact._source_project and project != artifact._source_project:
             raise ValueError(
-                f"Artifact {artifact.name} exists in project '{project}'; it can't be "
-                f"moved to '{artifact._source_project}'"
+                f"Artifact {artifact.name} exists in project "
+                f"'{artifact._source_project}'; it can't be moved to '{project}'"
             )
 
     def _prepare_artifact(
@@ -3291,8 +3362,8 @@ class Run:
             path: (str) path to downloaded model artifact file(s).
         """
         artifact = self.use_artifact(artifact_or_name=name)
-        assert "model" in str(
-            artifact.type.lower()
+        assert (
+            "model" in str(artifact.type.lower())
         ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
         path = artifact.download()
 
@@ -3384,8 +3455,8 @@ class Run:
         public_api = self._public_api()
         try:
             artifact = public_api.artifact(name=f"{name}:latest")
-            assert "model" in str(
-                artifact.type.lower()
+            assert (
+                "model" in str(artifact.type.lower())
             ), "You can only use this method for 'model' artifacts. For an artifact to be a 'model' artifact, its type property must contain the substring 'model'."
             artifact = self._log_artifact(
                 artifact_or_path=path, name=name, type=artifact.type
@@ -3585,7 +3656,7 @@ class Run:
         if settings._offline or settings.silent:
             return
 
-        workspace_url = f"{settings.run_url}/workspace"
+        run_url = settings.run_url
         project_url = settings.project_url
         sweep_url = settings.sweep_url
 
@@ -3596,7 +3667,7 @@ class Run:
 
         if printer._html:
             if not wandb.jupyter.maybe_display():
-                run_line = f"<strong>{printer.link(workspace_url, run_name)}</strong>"
+                run_line = f"<strong>{printer.link(run_url, run_name)}</strong>"
                 project_line, sweep_line = "", ""
 
                 # TODO(settings): make settings the source of truth
@@ -3628,7 +3699,7 @@ class Run:
                     f'{printer.emoji("broom")} View sweep at {printer.link(sweep_url)}'
                 )
         printer.display(
-            f'{printer.emoji("rocket")} View run at {printer.link(workspace_url)}',
+            f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
         )
 
         # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
@@ -3643,7 +3714,7 @@ class Run:
     # FOOTER
     # ------------------------------------------------------------------------------
     # Note: All the footer methods are static methods since we want to share the printing logic
-    # with the service execution path that doesn't have acess to the run instance
+    # with the service execution path that doesn't have access to the run instance
     @staticmethod
     def _footer(
         sampled_history: Optional["SampledHistoryResponse"] = None,
@@ -3871,10 +3942,13 @@ class Run:
         else:
             info = []
             if settings.run_name and settings.run_url:
-                run_workspace = f"{settings.run_url}/workspace"
-                info = [
-                    f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(run_workspace)}"
-                ]
+                info.append(
+                    f"{printer.emoji('rocket')} View run {printer.name(settings.run_name)} at: {printer.link(settings.run_url)}"
+                )
+            if settings.project_url:
+                info.append(
+                    f"{printer.emoji('star')} View project at: {printer.link(settings.project_url)}"
+                )
             if poll_exit_response and poll_exit_response.file_counts:
                 logger.info("logging synced files")
                 file_counts = poll_exit_response.file_counts
@@ -3943,11 +4017,11 @@ class Run:
 
         # Render summary if available
         if summary:
-            final_summary = {
-                item.key: json.loads(item.value_json)
-                for item in summary.item
-                if not item.key.startswith("_")
-            }
+            final_summary = {}
+            for item in summary.item:
+                if item.key.startswith("_") or len(item.nested_key) > 0:
+                    continue
+                final_summary[item.key] = json.loads(item.value_json)
 
             logger.info("rendering summary")
             summary_rows = []