wandb 0.19.1rc1__py3-none-macosx_11_0_arm64.whl → 0.19.2__py3-none-macosx_11_0_arm64.whl

wandb/sdk/service/server_sock.py

@@ -4,6 +4,7 @@ import threading
 import time
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
 
+import wandb
 from wandb.proto import wandb_server_pb2 as spb
 from wandb.sdk.internal.settings_static import SettingsStatic
 
@@ -225,7 +226,7 @@ class DebugThread(threading.Thread):
         while True:
             time.sleep(30)
             for thread in threading.enumerate():
-                print(f"DEBUG: {thread.name}")
+                wandb.termwarn(f"DEBUG: {thread.name}")
 
 
 class SocketServer:
@@ -266,7 +267,7 @@ class SocketServer:
                 # socket.shutdown() is a more heavy handed approach to interrupting socket.accept()
                 # in the future we might want to consider a more graceful shutdown which would involve setting
                 # a threading Event and then initiating one last connection just to close down the thread
-                # The advantage of the heavy handed approach is that it doesnt depend on the threads functioning
+                # The advantage of the heavy handed approach is that it does not depend on the threads functioning
                 # properly, that is, if something has gone wrong, we probably want to use this hammer to shut things down
                 self._sock.shutdown(socket.SHUT_RDWR)
             except OSError:

wandb/sdk/service/service.py

@@ -160,7 +160,7 @@ class _Service:
                     service_args.append("--no-observability")
 
                 if core_debug(default="False"):
-                    service_args.append("--debug")
+                    service_args.extend(["--log-level", "-4"])
 
                 exec_cmd_list = []
                 termlog(

wandb/sdk/service/streams.py

@@ -6,13 +6,15 @@ StreamAction: Lightweight record for stream ops for thread safety
 StreamMux: Container for dictionary of stream threads per runid
 """
 
+from __future__ import annotations
+
 import functools
 import multiprocessing
 import queue
 import threading
 import time
 from threading import Event
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable
 
 import psutil
 
@@ -38,7 +40,7 @@ from ..interface.interface_relay import InterfaceRelay
 class StreamThread(threading.Thread):
     """Class to running internal process as a thread."""
 
-    def __init__(self, target: Callable, kwargs: Dict[str, Any]) -> None:
+    def __init__(self, target: Callable, kwargs: dict[str, Any]) -> None:
         threading.Thread.__init__(self)
         self.name = "StreamThr"
         self._target = target
@@ -51,9 +53,9 @@ class StreamThread(threading.Thread):
 
 
 class StreamRecord:
-    _record_q: "queue.Queue[pb.Record]"
-    _result_q: "queue.Queue[pb.Result]"
-    _relay_q: "queue.Queue[pb.Result]"
+    _record_q: queue.Queue[pb.Record]
+    _result_q: queue.Queue[pb.Result]
+    _relay_q: queue.Queue[pb.Result]
     _iface: InterfaceRelay
     _thread: StreamThread
     _settings: SettingsStatic
@@ -111,7 +113,7 @@ class StreamAction:
     _processed: Event
     _data: Any
 
-    def __init__(self, action: str, stream_id: str, data: Optional[Any] = None):
+    def __init__(self, action: str, stream_id: str, data: Any | None = None):
         self._action = action
         self._stream_id = stream_id
         self._data = data
@@ -133,12 +135,12 @@ class StreamAction:
 
 class StreamMux:
     _streams_lock: threading.Lock
-    _streams: Dict[str, StreamRecord]
-    _port: Optional[int]
-    _pid: Optional[int]
-    _action_q: "queue.Queue[StreamAction]"
+    _streams: dict[str, StreamRecord]
+    _port: int | None
+    _pid: int | None
+    _action_q: queue.Queue[StreamAction]
     _stopped: Event
-    _pid_checked_ts: Optional[float]
+    _pid_checked_ts: float | None
     _mailbox: Mailbox
 
     def __init__(self) -> None:
@@ -152,7 +154,7 @@ class StreamMux:
         self._mailbox = Mailbox()
         self._mailbox.enable_keepalive()
 
-    def _get_stopped_event(self) -> "Event":
+    def _get_stopped_event(self) -> Event:
         # TODO: clean this up, there should be a better way to abstract this
         return self._stopped
 
@@ -192,7 +194,7 @@ class StreamMux:
         self._action_q.put(action)
         action.wait_handled()
 
-    def stream_names(self) -> List[str]:
+    def stream_names(self) -> list[str]:
         with self._streams_lock:
             names = list(self._streams.keys())
             return names
@@ -273,7 +275,7 @@ class StreamMux:
         if self._check_orphaned():
             self._stopped.set()
 
-        poll_exit_responses: List[pb.PollExitResponse] = []
+        poll_exit_responses: list[pb.PollExitResponse] = []
         for probe_handle in probe_handles:
             result = probe_handle.get_probe_result()
             if result:
@@ -281,7 +283,7 @@ class StreamMux:
 
         progress_printer.update(poll_exit_responses)
 
-    def _finish_all(self, streams: Dict[str, StreamRecord], exit_code: int) -> None:
+    def _finish_all(self, streams: dict[str, StreamRecord], exit_code: int) -> None:
         if not streams:
             return
 
@@ -292,8 +294,8 @@ class StreamMux:
         exit_handles = []
 
         # only finish started streams, non started streams failed early
-        started_streams: Dict[str, StreamRecord] = {}
-        not_started_streams: Dict[str, StreamRecord] = {}
+        started_streams: dict[str, StreamRecord] = {}
+        not_started_streams: dict[str, StreamRecord] = {}
         for stream_id, stream in streams.items():
             d = started_streams if stream._started else not_started_streams
             d[stream_id] = stream

wandb/sdk/verify/verify.py

@@ -37,17 +37,17 @@ def print_results(
     else:
         color = "red"
     if isinstance(failed_test_or_tests, str):
-        print(RED_X)
-        print(click.style(failed_test_or_tests, fg=color, bold=True))
+        print(RED_X)  # noqa: T201
+        print(click.style(failed_test_or_tests, fg=color, bold=True))  # noqa: T201
     elif isinstance(failed_test_or_tests, list) and len(failed_test_or_tests) > 0:
-        print(RED_X)
-        print(
+        print(RED_X)  # noqa: T201
+        print(  # noqa: T201
             "\n".join(
                 [click.style(f, fg=color, bold=True) for f in failed_test_or_tests]
             )
         )
     else:
-        print(CHECKMARK)
+        print(CHECKMARK)  # noqa: T201
 
 
 def check_host(host: str) -> bool:
@@ -58,7 +58,7 @@ def check_host(host: str) -> bool:
 
 
 def check_logged_in(api: Api, host: str) -> bool:
-    print("Checking if logged in".ljust(72, "."), end="")
+    print("Checking if logged in".ljust(72, "."), end="")  # noqa: T201
     login_doc_url = "https://docs.wandb.ai/ref/cli/wandb-login"
     fail_string = None
     if api.api_key is None:
@@ -83,7 +83,7 @@ def check_logged_in(api: Api, host: str) -> bool:
 
 def check_secure_requests(url: str, test_url_string: str, failure_output: str) -> None:
     # check if request is over https
-    print(test_url_string.ljust(72, "."), end="")
+    print(test_url_string.ljust(72, "."), end="")  # noqa: T201
     fail_string = None
     if not url.startswith("https"):
         fail_string = failure_output
@@ -91,7 +91,7 @@ def check_secure_requests(url: str, test_url_string: str, failure_output: str) -
 
 
 def check_cors_configuration(url: str, origin: str) -> None:
-    print("Checking CORs configuration of the bucket".ljust(72, "."), end="")
+    print("Checking CORs configuration of the bucket".ljust(72, "."), end="")  # noqa: T201
     fail_string = None
     res_get = requests.options(
         url, headers={"Origin": origin, "Access-Control-Request-Method": "GET"}
@@ -107,7 +107,7 @@ def check_cors_configuration(url: str, origin: str) -> None:
 
 
 def check_run(api: Api) -> bool:
-    print(
+    print(  # noqa: T201
         "Checking logged metrics, saving and downloading a file".ljust(72, "."), end=""
     )
     failed_test_strings = []
@@ -308,7 +308,7 @@ def log_use_download_artifact(
 
 
 def check_artifacts() -> bool:
-    print("Checking artifact save and download workflows".ljust(72, "."), end="")
+    print("Checking artifact save and download workflows".ljust(72, "."), end="")  # noqa: T201
     failed_test_strings: List[str] = []
 
     # test checksum
@@ -365,7 +365,7 @@ def check_artifacts() -> bool:
 
 def check_graphql_put(api: Api, host: str) -> Tuple[bool, Optional[str]]:
     # check graphql endpoint using an upload
-    print("Checking signed URL upload".ljust(72, "."), end="")
+    print("Checking signed URL upload".ljust(72, "."), end="")  # noqa: T201
     failed_test_strings = []
     gql_fp = "gql_test_file.txt"
     f = open(gql_fp, "w")
@@ -410,7 +410,7 @@ def check_graphql_put(api: Api, host: str) -> Tuple[bool, Optional[str]]:
 
 
 def check_large_post() -> bool:
-    print(
+    print(  # noqa: T201
         "Checking ability to send large payloads through proxy".ljust(72, "."), end=""
     )
     descy = "a" * int(10**7)
@@ -461,7 +461,7 @@ def check_large_post() -> bool:
 
 
 def check_wandb_version(api: Api) -> None:
-    print("Checking wandb package version is up to date".ljust(72, "."), end="")
+    print("Checking wandb package version is up to date".ljust(72, "."), end="")  # noqa: T201
     _, server_info = api.viewer_server_info()
     fail_string = None
     warning = False

wandb/sdk/wandb_init.py

@@ -48,10 +48,10 @@ from .wandb_settings import Settings
 if TYPE_CHECKING:
     from wandb.proto import wandb_internal_pb2 as pb
 
-logger: logging.Logger | None = None  # logger configured during wandb.init()
+logger: wandb_setup.Logger | None = None  # logger configured during wandb.init()
 
 
-def _set_logger(log_object: logging.Logger) -> None:
+def _set_logger(log_object: wandb_setup.Logger | None) -> None:
     """Configure module logger."""
     global logger
     logger = log_object
@@ -125,7 +125,7 @@ class _WandbInit:
         self.backend: Backend | None = None
 
         self._teardown_hooks: list[TeardownHook] = []
-        self._wl: wandb_setup._WandbSetup | None = None
+        self._wl = wandb.setup()
         self.notebook: wandb.jupyter.Notebook | None = None  # type: ignore
         self.printer = printer.new_printer()
 
@@ -139,7 +139,7 @@ class _WandbInit:
         Any settings from environment variables set after the singleton is initialized
         (via login/setup/etc.) will be ignored.
         """
-        singleton = wandb_setup._WandbSetup._instance
+        singleton = wandb_setup.singleton()
         if singleton is None:
             return
 
@@ -167,6 +167,41 @@ class _WandbInit:
             )
             self.printer.display(line, level="warn")
 
+    def clear_run_path_if_sweep_or_launch(
+        self,
+        init_settings: Settings,
+    ) -> None:
+        """Clear project/entity/run_id keys if in a Sweep or a Launch context.
+
+        Args:
+            init_settings: Settings specified in the call to `wandb.init()`.
+        """
+        when_doing_thing = ""
+
+        if self._wl.settings.sweep_id:
+            when_doing_thing = "when running a sweep"
+        elif self._wl.settings.launch:
+            when_doing_thing = "when running from a wandb launch context"
+
+        if not when_doing_thing:
+            return
+
+        def warn(key: str, value: str) -> None:
+            self.printer.display(
+                f"Ignoring {key} {value!r} {when_doing_thing}.",
+                level="warn",
+            )
+
+        if init_settings.project is not None:
+            warn("project", init_settings.project)
+            init_settings.project = None
+        if init_settings.entity is not None:
+            warn("entity", init_settings.entity)
+            init_settings.entity = None
+        if init_settings.run_id is not None:
+            warn("run_id", init_settings.run_id)
+            init_settings.run_id = None
+
     def setup(  # noqa: C901
         self,
         init_settings: Settings,
@@ -182,50 +217,23 @@ class _WandbInit:
         """
         self.warn_env_vars_change_after_setup()
 
-        # mode="disabled" is a special case where we don't want to start wandb-core
-        setup_settings_dict: dict[str, Any] = {}
-        if init_settings.mode == "disabled":
-            setup_settings_dict["mode"] = init_settings.mode
-        # TODO: x_disable_service is deprecated, remove this once officially deprecated
-        if init_settings.x_disable_service:
-            setup_settings_dict["x_disable_service"] = init_settings.x_disable_service
-        setup_settings = (
-            wandb.Settings(**setup_settings_dict) if setup_settings_dict else None
-        )
-
-        self._wl = wandb_setup.setup(settings=setup_settings)
-
-        assert self._wl is not None
         _set_logger(self._wl._get_logger())
+        assert logger
 
-        # Start with settings from wandb library singleton
-        settings = self._wl.settings.copy()
+        self.clear_run_path_if_sweep_or_launch(init_settings)
 
-        # handle custom sweep- and launch-related logic for init settings
-        if settings.sweep_id:
-            init_settings.sweep_id = settings.sweep_id
-            init_settings.handle_sweep_logic()
-        if settings.launch:
-            init_settings.launch = settings.launch
-            init_settings.handle_launch_logic()
+        # Inherit global settings.
+        settings = self._wl.settings.model_copy()
 
-        # Apply settings from wandb.init() call
+        # Apply settings from wandb.init() call.
         settings.update_from_settings(init_settings)
 
-        sagemaker_config: dict = (
-            dict() if settings.sagemaker_disable else sagemaker.parse_sm_config()
-        )
-        if sagemaker_config:
-            sagemaker_api_key = sagemaker_config.get("wandb_api_key", None)
-            sagemaker_run, sagemaker_env = sagemaker.parse_sm_resources()
-            if sagemaker_env:
-                if sagemaker_api_key:
-                    sagemaker_env["WANDB_API_KEY"] = sagemaker_api_key
-                settings.update_from_env_vars(sagemaker_env)
-                wandb.setup(settings=settings)
-            settings.update_from_dict(sagemaker_run)
-            with telemetry.context(obj=self._init_telemetry_obj) as tel:
-                tel.feature.sagemaker = True
+        # Infer the run ID from SageMaker.
+        if not settings.sagemaker_disable and sagemaker.is_using_sagemaker():
+            if sagemaker.set_run_id(settings):
+                logger.info("set run ID and group based on SageMaker")
+                with telemetry.context(obj=self._init_telemetry_obj) as tel:
+                    tel.feature.sagemaker = True
 
         with telemetry.context(obj=self._init_telemetry_obj) as tel:
             if config is not None:
@@ -252,23 +260,25 @@ class _WandbInit:
             exclude=config_exclude_keys,
         )
 
-        # merge config with sweep or sagemaker (or config file)
-        self.sweep_config = dict()
-        sweep_config = self._wl._sweep_config or dict()
+        # Construct the run's config.
         self.config = dict()
         self.init_artifact_config: dict[str, Any] = dict()
-        for config_data in (
-            sagemaker_config,
-            self._wl._config,
-            config,
-        ):
-            if not config_data:
-                continue
-            # split out artifacts, since when inserted into
-            # config they will trigger use_artifact
-            # but the run is not yet upserted
-            self._split_artifacts_from_config(config_data, self.config)  # type: ignore
 
+        if not settings.sagemaker_disable and sagemaker.is_using_sagemaker():
+            sagemaker_config = sagemaker.parse_sm_config()
+            self._split_artifacts_from_config(sagemaker_config, self.config)
+
+            with telemetry.context(obj=self._init_telemetry_obj) as tel:
+                tel.feature.sagemaker = True
+
+        if self._wl._config:
+            self._split_artifacts_from_config(self._wl._config, self.config)
+
+        if config and isinstance(config, dict):
+            self._split_artifacts_from_config(config, self.config)
+
+        self.sweep_config = dict()
+        sweep_config = self._wl._sweep_config or dict()
         if sweep_config:
             self._split_artifacts_from_config(sweep_config, self.sweep_config)
 
@@ -295,16 +305,14 @@ class _WandbInit:
             )
 
         # apply updated global state after login was handled
-        wl = wandb.setup()
-        assert wl is not None
         login_settings = {
             k: v
             for k, v in {
-                "anonymous": wl.settings.anonymous,
-                "api_key": wl.settings.api_key,
-                "base_url": wl.settings.base_url,
-                "force": wl.settings.force,
-                "login_timeout": wl.settings.login_timeout,
+                "anonymous": self._wl.settings.anonymous,
+                "api_key": self._wl.settings.api_key,
+                "base_url": self._wl.settings.base_url,
+                "force": self._wl.settings.force,
+                "login_timeout": self._wl.settings.login_timeout,
             }.items()
             if v is not None
         }
@@ -328,7 +336,7 @@ class _WandbInit:
 
         # TODO: remove this once we refactor the client. This is a temporary
         # fix to make sure that we use the same project name for wandb-core.
-        # The reason this is not going throught the settings object is to
+        # The reason this is not going through the settings object is to
         # avoid failure cases in other parts of the code that will be
         # removed with the switch to wandb-core.
         if settings.project is None:
@@ -364,7 +372,7 @@ class _WandbInit:
             else:
                 config_target.setdefault(k, v)
 
-    def _enable_logging(self, log_fname: str, run_id: str | None = None) -> None:
+    def _enable_logging(self, log_fname: str) -> None:
         """Enable logging to the global debug log.
 
         This adds a run_id to the log, in case of multiple processes on the same machine.
@@ -373,26 +381,13 @@ class _WandbInit:
         handler = logging.FileHandler(log_fname)
         handler.setLevel(logging.INFO)
 
-        class WBFilter(logging.Filter):
-            def filter(self, record: logging.LogRecord) -> bool:
-                record.run_id = run_id
-                return True
-
-        if run_id:
-            formatter = logging.Formatter(
-                "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
-                "[%(run_id)s:%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
-            )
-        else:
-            formatter = logging.Formatter(
-                "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
-                "[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
-            )
+        formatter = logging.Formatter(
+            "%(asctime)s %(levelname)-7s %(threadName)-10s:%(process)d "
+            "[%(filename)s:%(funcName)s():%(lineno)s] %(message)s"
+        )
 
         handler.setFormatter(formatter)
-        if run_id:
-            handler.addFilter(WBFilter())
-        assert logger is not None
+        assert isinstance(logger, logging.Logger)
         logger.propagate = False
         logger.addHandler(handler)
         # TODO: make me configurable
@@ -407,7 +402,7 @@ class _WandbInit:
     def _safe_symlink(
         self, base: str, target: str, name: str, delete: bool = False
     ) -> None:
-        # TODO(jhr): do this with relpaths, but i cant figure it out on no sleep
+        # TODO(jhr): do this with relpaths, but i can't figure it out on no sleep
         if not hasattr(os, "symlink"):
             return
 
@@ -552,11 +547,13 @@ class _WandbInit:
                 entity="dummy",
             )
         )
-        # config and summary objects
+        # config, summary, and metadata objects
         drun._config = wandb.sdk.wandb_config.Config()
         drun._config.update(self.sweep_config)
         drun._config.update(self.config)
         drun.summary = SummaryDisabled()  # type: ignore
+        drun._Run__metadata = wandb.sdk.wandb_metadata.Metadata()
+
         # methods
         drun.log = lambda data, *_, **__: drun.summary.update(data)  # type: ignore
         drun.finish = lambda *_, **__: module.unset_globals()  # type: ignore
@@ -666,26 +663,25 @@ class _WandbInit:
                 )
 
             latest_run = self._wl._global_run_stack[-1]
-            logger.info(f"found existing run on stack: {latest_run._run_id}")
+            logger.info(f"found existing run on stack: {latest_run.id}")
             latest_run.finish()
-        elif isinstance(wandb.run, Run):
-            service = self._wl.service
-            # We shouldn't return a stale global run if we are in a new pid
-            if not service or os.getpid() == wandb.run._init_pid:
-                logger.info("wandb.init() called when a run is still active")
-                with telemetry.context() as tel:
-                    tel.feature.init_return_run = True
-                return wandb.run
+        elif wandb.run is not None and os.getpid() == wandb.run._init_pid:
+            logger.info("wandb.init() called when a run is still active")
+            with telemetry.context() as tel:
+                tel.feature.init_return_run = True
+            return wandb.run
 
         logger.info("starting backend")
 
-        service = self._wl.service
-        if service:
+        if not self.settings.x_disable_service:
+            service = self._wl.ensure_service()
             logger.info("sending inform_init request")
             service.inform_init(
                 settings=self.settings.to_proto(),
                 run_id=self.settings.run_id,  # type: ignore
             )
+        else:
+            service = None
 
         mailbox = Mailbox()
         backend = Backend(
@@ -869,8 +865,6 @@ class _WandbInit:
                 tel.feature.resumed = run_result.run.resumed
         run._set_run_obj(run_result.run)
 
-        run._on_init()
-
         logger.info("starting run threads in backend")
         # initiate run (stats and metadata probing)
 
@@ -943,16 +937,13 @@ def _attach(
         )
     wandb._assert_is_user_process()  # type: ignore
 
-    _wl = wandb_setup._setup()
-    assert _wl
+    _wl = wandb.setup()
 
     _set_logger(_wl._get_logger())
     if logger is None:
         raise UsageError("logger is not initialized")
 
-    service = _wl.service
-    if not service:
-        raise UsageError(f"Unable to attach to run {attach_id} (no service process)")
+    service = _wl.ensure_service()
 
     try:
         attach_settings = service.inform_attach(attach_id=attach_id)
@@ -1081,7 +1072,7 @@ def init(  # noqa: C901
         entity: The username or team name under which the runs will be logged.
             The entity must already exist, so ensure you’ve created your account
             or team in the UI before starting to log runs. If not specified, the
-            run will default your defualt entity. To change the default entity,
+            run will default your default entity. To change the default entity,
             go to [your settings](https://wandb.ai/settings) and update the
             "Default location to create new projects" under "Default team".
         project: The name of the project under which this run will be logged.

wandb/sdk/wandb_login.py

@@ -88,7 +88,7 @@ def login(
     if verify:
         from . import wandb_setup
 
-        singleton = wandb_setup._WandbSetup._instance
+        singleton = wandb_setup.singleton()
         assert singleton is not None
         viewer = singleton._server._viewer
         if not viewer: