wandb 0.19.1rc1__py3-none-any.whl → 0.19.3__py3-none-any.whl

wandb/sdk/wandb_run.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import _thread as thread
 import atexit
 import functools
 import glob
@@ -30,13 +29,13 @@ import wandb.util
 from wandb import trigger
 from wandb._globals import _datatypes_set_callback
 from wandb.apis import internal, public
-from wandb.apis.internal import Api
 from wandb.apis.public import Api as PublicApi
 from wandb.errors import CommError, UnsupportedError, UsageError
 from wandb.errors.links import url_registry
 from wandb.integration.torch import wandb_torch
 from wandb.plot import CustomChart, Visualize
 from wandb.proto.wandb_internal_pb2 import (
+    MetadataRequest,
     MetricRecord,
     PollExitResponse,
     Result,
@@ -73,6 +72,7 @@ from .lib import (
     deprecate,
     filenames,
     filesystem,
+    interrupt,
     ipython,
     module,
     printer,
@@ -84,6 +84,7 @@ from .lib import (
 from .lib.exit_hooks import ExitHooks
 from .lib.mailbox import MailboxError, MailboxHandle, MailboxProbe, MailboxProgress
 from .wandb_alerts import AlertLevel
+from .wandb_metadata import Metadata
 from .wandb_settings import Settings
 from .wandb_setup import _WandbSetup
 
@@ -287,7 +288,7 @@ class RunStatusChecker:
                 # TODO(frz): This check is required
                 # until WB-3606 is resolved on server side.
                 if not wandb.agents.pyagent.is_running():  # type: ignore
-                    thread.interrupt_main()
+                    interrupt.interrupt_main()
                     return
 
         try:
@@ -595,15 +596,17 @@ class Run:
         self._config._set_artifact_callback(self._config_artifact_callback)
         self._config._set_settings(self._settings)
 
-        # todo: perhaps this should be a property that is a noop on a finished run
+        # TODO: perhaps this should be a property that is a noop on a finished run
         self.summary = wandb_summary.Summary(
             self._summary_get_current_summary_callback,
         )
         self.summary._set_update_callback(self._summary_update_callback)
 
+        self.__metadata: Metadata | None = None
+
         self._step = 0
         self._starting_step = 0
-        # todo: eventually would be nice to make this configurable using self._settings._start_time
+        # TODO: eventually would be nice to make this configurable using self._settings._start_time
         #  need to test (jhr): if you set start time to 2 days ago and run a test for 15 minutes,
         #  does the total time get calculated right (not as 2 days and 15 minutes)?
         self._start_time = time.time()
@@ -1359,32 +1362,68 @@ class Run:
         files: FilesDict = dict(files=[(GlobStr(glob.escape(fname)), "now")])
         self._backend.interface.publish_files(files)
 
-    def _serialize_custom_charts(self, data: dict[str, Any]) -> dict[str, Any]:
-        if not data:
-            return data
+    def _pop_all_charts(
+        self,
+        data: dict[str, Any],
+        key_prefix: str | None = None,
+    ) -> dict[str, Any]:
+        """Pops all charts from a dictionary including nested charts.
 
-        chart_keys = set()
+        This function will return a mapping of the charts and a dot-separated
+        key for each chart. Indicating the path to the chart in the data dictionary.
+        """
+        keys_to_remove = set()
+        charts: dict[str, Any] = {}
         for k, v in data.items():
+            key = f"{key_prefix}.{k}" if key_prefix else k
             if isinstance(v, Visualize):
-                data[k] = v.table
-                v.set_key(k)
-                self._config_callback(
-                    val=v.spec.config_value,
-                    key=v.spec.config_key,
-                )
+                keys_to_remove.add(k)
+                charts[key] = v
             elif isinstance(v, CustomChart):
-                chart_keys.add(k)
-                v.set_key(k)
-                self._config_callback(
-                    key=v.spec.config_key,
-                    val=v.spec.config_value,
-                )
+                keys_to_remove.add(k)
+                charts[key] = v
+            elif isinstance(v, dict):
+                nested_charts = self._pop_all_charts(v, key)
+                charts.update(nested_charts)
+
+        for k in keys_to_remove:
+            data.pop(k)
+
+        return charts
+
+    def _serialize_custom_charts(
+        self,
+        data: dict[str, Any],
+    ) -> dict[str, Any]:
+        """Process and replace chart objects with their underlying table values.
+
+        This processes the chart objects passed to `run.log()`, replacing their entries
+        in the given dictionary (which is saved to the run's history) and adding them
+        to the run's config.
+
+        Args:
+            data: Dictionary containing data that may include plot objects
+                Plot objects can be nested in dictionaries, which will be processed recursively.
+
+        Returns:
+            The processed dictionary with custom charts transformed into tables.
+        """
+        if not data:
+            return data
+
+        charts = self._pop_all_charts(data)
+        for k, v in charts.items():
+            v.set_key(k)
+            self._config_callback(
+                val=v.spec.config_value,
+                key=v.spec.config_key,
+            )
 
-        for k in chart_keys:
-            # remove the chart key from the row
-            v = data.pop(k)
             if isinstance(v, CustomChart):
                 data[v.spec.table_key] = v.table
+            elif isinstance(v, Visualize):
+                data[k] = v.table
+
         return data
 
     def _partial_history_callback(
@@ -1774,7 +1813,10 @@ class Run:
             examples = []
             for i in range(3):
                 pixels = np.random.randint(
-                    low=0, high=256, size=(100, 100, 3), dtype=np.uint8
+                    low=0,
+                    high=256,
+                    size=(100, 100, 3),
+                    dtype=np.uint8,
                 )
                 pil_image = PILImage.fromarray(pixels, mode="RGB")
                 image = wandb.Image(pil_image, caption=f"random field {i}")
@@ -1791,7 +1833,10 @@ class Run:
             run = wandb.init()
             # axes are (time, channel, height, width)
             frames = np.random.randint(
-                low=0, high=256, size=(10, 3, 100, 100), dtype=np.uint8
+                low=0,
+                high=256,
+                size=(10, 3, 100, 100),
+                dtype=np.uint8,
             )
             run.log({"video": wandb.Video(frames, fps=4)})
             ```
@@ -2133,8 +2178,9 @@ class Run:
             # Inform the service that we're done sending messages for this run.
             #
             # TODO: Why not do this in _atexit_cleanup()?
-            service = self._wl and self._wl.service
-            if service and self._settings.run_id:
+            if self._settings.run_id:
+                assert self._wl
+                service = self._wl.assert_service()
                 service.inform_finish(run_id=self._settings.run_id)
 
         finally:
@@ -2313,7 +2359,7 @@ class Run:
             self._err_redir = err_redir
             logger.info("Redirects installed.")
         except Exception as e:
-            print(e)
+            wandb.termwarn(f"Failed to redirect: {e}")
             logger.error("Failed to redirect.", exc_info=e)
         return
 
@@ -2374,8 +2420,7 @@ class Run:
         logger.info("atexit reg")
         self._hooks = ExitHooks()
 
-        service = self._wl and self._wl.service
-        if not service:
+        if self.settings.x_disable_service:
             self._hooks.hook()
             # NB: manager will perform atexit hook like behavior for outstanding runs
             atexit.register(lambda: self._atexit_cleanup())
@@ -2388,10 +2433,6 @@ class Run:
             self._output_writer.close()
             self._output_writer = None
 
-    def _on_init(self) -> None:
-        if self._settings._offline:
-            return
-
     def _on_start(self) -> None:
         # would like to move _set_global to _on_ready to unify _on_start and _on_attach
         # (we want to do the set globals after attach)
@@ -3230,8 +3271,7 @@ class Run:
         is_user_created: bool = False,
         use_after_commit: bool = False,
     ) -> Artifact:
-        api = internal.Api()
-        if api.settings().get("anonymous") == "true":
+        if self._settings.anonymous in ["allow", "must"]:
             wandb.termwarn(
                 "Artifacts logged anonymously cannot be claimed and expire after 7 days."
             )
@@ -3665,6 +3705,62 @@ class Run:
                 logger.error("Error getting system metrics: %s", e)
         return {}
 
+    @property
+    @_run_decorator._attach
+    @_run_decorator._noop_on_finish()
+    def _metadata(self) -> Metadata | None:
+        """The metadata associated with this run.
+
+        NOTE: Automatically collected metadata can be overridden by the user.
+        """
+        if not self._backend or not self._backend.interface:
+            return self.__metadata
+
+        # Initialize the metadata object if it doesn't exist.
+        if self.__metadata is None:
+            self.__metadata = Metadata()
+            self.__metadata._set_callback(self._metadata_callback)
+
+        handle = self._backend.interface.deliver_get_system_metadata()
+        result = handle.wait(timeout=1)
+
+        if not result:
+            logger.error("Error getting run metadata: no result")
+            return None
+
+        try:
+            response = result.response.get_system_metadata_response
+
+            # Temporarily disable the callback to prevent triggering
+            # an update call to wandb-core with the callback.
+            with self.__metadata.disable_callback():
+                # Values stored in the metadata object take precedence.
+                self.__metadata.update_from_proto(response.metadata, skip_existing=True)
+
+            return self.__metadata
+        except Exception as e:
+            logger.error("Error getting run metadata: %s", e)
+
+        return None
+
+    @_run_decorator._noop_on_finish()
+    @_run_decorator._attach
+    def _metadata_callback(
+        self,
+        metadata: MetadataRequest,
+    ) -> None:
+        """Callback to publish Metadata to wandb-core upon user updates."""
+        # ignore updates if the attached to another run
+        if self._is_attached:
+            wandb.termwarn(
+                "Metadata updates are ignored when attached to another run.",
+                repeat=False,
+            )
+            return
+
+        if self._backend and self._backend.interface:
+            self._backend.interface.publish_metadata(metadata)
+
     # ------------------------------------------------------------------------------
     # HEADER
     # ------------------------------------------------------------------------------
@@ -3772,8 +3868,7 @@ class Run:
             f'{printer.emoji("rocket")} View run at {printer.link(run_url)}',
         )
 
-        # TODO(settings) use `wandb_settings` (if self.settings.anonymous == "true":)
-        if run_name and Api().api.settings().get("anonymous") == "true":
+        if run_name and settings.anonymous in ["allow", "must"]:
             printer.display(
                 (
                     "Do NOT share these links with anyone."

wandb/sdk/wandb_settings.py

@@ -34,14 +34,12 @@ from pydantic_core import SchemaValidator, core_schema
 
 import wandb
 from wandb import env, termwarn, util
-from wandb.apis.internal import Api
 from wandb.errors import UsageError
 from wandb.proto import wandb_settings_pb2
 
-from .lib import apikey, credentials, filesystem, ipython
+from .lib import apikey, credentials, ipython
 from .lib.gitlib import GitRepo
 from .lib.run_moment import RunMoment
-from .lib.runid import generate_id
 
 
 def _path_convert(*args: str) -> str:
@@ -305,7 +303,7 @@ class Settings(BaseModel, validate_assignment=True):
     x_save_requirements: bool = True
     x_service_transport: str | None = None
     x_service_wait: float = 30.0
-    x_show_operation_stats: bool = False
+    x_show_operation_stats: bool = True
     # The start time of the run in seconds since the Unix epoch.
     x_start_time: float | None = None
     # PID of the process that started the wandb-core process to collect system stats for.
@@ -349,7 +347,7 @@ class Settings(BaseModel, validate_assignment=True):
     @model_validator(mode="before")
     @classmethod
     def catch_private_settings(cls, values):
-        """Check if a private field is provided and assign to the corrsponding public one.
+        """Check if a private field is provided and assign to the corresponding public one.
 
         This is a compatibility layer to handle previous versions of the settings.
         """
@@ -961,7 +959,7 @@ class Settings(BaseModel, validate_assignment=True):
     # The Settings class does not track the source of the settings,
     # so it is up to the developer to ensure that the settings are applied
     # in the correct order. Most of the updates are done in
-    # wandb/sdk/wandb_setup.py::_WandbSetup__WandbSetup._settings_setup.
+    # wandb/sdk/wandb_setup.py::_WandbSetup._settings_setup.
 
     def update_from_system_config_file(self):
         """Update settings from the system config file."""
@@ -1063,7 +1061,8 @@ class Settings(BaseModel, validate_assignment=True):
         )
         self.x_executable = _executable
 
-        self.docker = env.get_docker(util.image_id_from_k8s())
+        if self.docker is None:
+            self.docker = env.get_docker(util.image_id_from_k8s())
 
         # proceed if not in CLI mode
         if self.x_cli_only_mode:
@@ -1153,61 +1152,6 @@ class Settings(BaseModel, validate_assignment=True):
 
         return settings_proto
 
-    def handle_resume_logic(self):
-        """Handle logic for resuming runs."""
-        # handle auto resume logic
-        if self.resume == "auto":
-            if os.path.exists(self.resume_fname):
-                with open(self.resume_fname) as f:
-                    resume_run_id = json.load(f)["run_id"]
-                if self.run_id is None:
-                    self.run_id = resume_run_id
-                elif self.run_id != resume_run_id:
-                    wandb.termwarn(
-                        "Tried to auto resume run with "
-                        f"id {resume_run_id} but id {self.run_id} is set.",
-                    )
-        if self.run_id is None:
-            self.run_id = generate_id()
-
-        # persist run_id in case of failure
-        if self.resume == "auto" and self.resume_fname is not None:
-            filesystem.mkdir_exists_ok(self.wandb_dir)
-            with open(self.resume_fname, "w") as f:
-                f.write(json.dumps({"run_id": self.run_id}))
-
-    def handle_sweep_logic(self):
-        """Update settings based on sweep context.
-
-        When running a sweep, the project, entity, and run_id are handled externally,
-        and should be ignored if they are set.
-        """
-        if self.sweep_id is None:
-            return
-
-        for key in ("project", "entity", "run_id"):
-            value = getattr(self, key)
-            if value is not None:
-                wandb.termwarn(f"Ignoring {key} {value!r} when running a sweep.")
-                setattr(self, key, None)
-
-    def handle_launch_logic(self):
-        """Update settings based on launch context.
-
-        When running in a launch context, the project, entity, and run_id are handled
-        externally, and should be ignored if they are set.
-        """
-        if not self.launch:
-            return
-
-        for key in ("project", "entity", "run_id"):
-            value = getattr(self, key)
-            if value is not None:
-                wandb.termwarn(
-                    f"Ignoring {key} {value!r} when running from wandb launch context."
-                )
-                setattr(self, key, None)
-
     @staticmethod
     def validate_url(url: str) -> None:
         """Validate a URL string."""
@@ -1293,7 +1237,7 @@ class Settings(BaseModel, validate_assignment=True):
     def _get_url_query_string(self) -> str:
         """Construct the query string for project, run, and sweep URLs."""
         # TODO: remove dependency on Api()
-        if Api().settings().get("anonymous") != "true":
+        if self.anonymous not in ["allow", "must"]:
             return ""
 
         api_key = apikey.api_key(settings=self)