ob-metaflow-extensions 1.1.99__tar.gz → 1.1.100rc0__tar.gz

{ob-metaflow-extensions-1.1.99 → ob-metaflow-extensions-1.1.100rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.99
+Version: 1.1.100rc0
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.99 → ob-metaflow-extensions-1.1.100rc0}/metaflow_extensions/outerbounds/profilers/gpu.py

@@ -315,7 +315,11 @@ def _update_charts(results, md_dict):
 
 # This code is adapted from: https://github.com/outerbounds/monitorbench
 class GPUProfiler:
-    def __init__(self, interval=1, monitor_batch_duration=200):
+    def __init__(
+        self, interval=1, monitor_batch_duration=200, artifact_name="gpu_profile_data"
+    ):
+        self.artifact_name = artifact_name
+        self._card_setup_finished = False
         self.driver_ver, self.cuda_ver, self.error = self._read_versions()
         (
             self.interconnect_data,
@@ -389,16 +393,35 @@ class GPUProfiler:
                 print("GPU Profiler readings are none", file=sys.stderr)
                 time.sleep(self._interval)
                 continue
+            if not self._card_setup_finished:
+                self._setup_card()
+                time.sleep(self._interval)
+                continue
+
             _update_utilization(readings, self._card_comps["max_utilization"])
             _update_charts(readings, self._card_comps["charts"])
             current.card["gpu_profile"].refresh()
             time.sleep(self._interval)
 
-    def _setup_card(self, artifact_name):
+    def _setup_card(self):
         from metaflow import current
 
         results = self._make_reading()
+        if "profile" not in results:
+            return
+
         els = current.card["gpu_profile"]
+        els.clear()
+
+        current.card["gpu_profile"].append(
+            Markdown("# GPU profile for `%s`" % current.pathspec)
+        )
+        current.card["gpu_profile"].append(
+            Markdown(
+                "_Started at: %s_"
+                % datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S %z")
+            )
+        )
 
         def _drivers():
             els.append(Markdown("## Drivers"))
@@ -444,7 +467,7 @@ class GPUProfiler:
                 Table(data=_rows, headers=["Device ID", "Max GPU %", "Max memory"])
             )
             els.append(
-                Markdown(f"Detailed data saved in an artifact `{artifact_name}`")
+                Markdown(f"Detailed data saved in an artifact `{self.artifact_name}`")
             )
             return rows
 
@@ -479,6 +502,7 @@ class GPUProfiler:
         _interconnect()
         self._card_comps["max_utilization"] = _utilization()
         self._card_comps["charts"] = _plots()
+        self._card_setup_finished = True
 
     def _read_versions(self):
         def parse(r, s):
@@ -572,7 +596,9 @@ class gpu_profile:
     def __call__(self, f):
         @wraps(f)
         def func(s):
-            prof = GPUProfiler(interval=self.interval)
+            prof = GPUProfiler(
+                interval=self.interval, artifact_name=self.artifact_prefix + "data"
+            )
             if self.include_artifacts:
                 setattr(s, self.artifact_prefix + "num_gpus", len(prof.devices))
 
@@ -585,7 +611,7 @@ class gpu_profile:
                     % datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S %z")
                 )
             )
-            prof._setup_card(self.artifact_prefix + "data")
+            prof._setup_card()
             current.card["gpu_profile"].refresh()
             update_thread = threading.Thread(target=prof._update_card, daemon=True)
             update_thread.start()

{ob-metaflow-extensions-1.1.99 → ob-metaflow-extensions-1.1.100rc0}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.99
+Version: 1.1.100rc0
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.99 → ob-metaflow-extensions-1.1.100rc0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.1.99"
+version = "1.1.100rc0"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()