ob-metaflow-extensions 1.1.98__py2.py3-none-any.whl → 1.1.100rc0__py2.py3-none-any.whl

metaflow_extensions/outerbounds/profilers/gpu.py

@@ -315,7 +315,11 @@ def _update_charts(results, md_dict):
 
 # This code is adapted from: https://github.com/outerbounds/monitorbench
 class GPUProfiler:
-    def __init__(self, interval=1, monitor_batch_duration=200):
+    def __init__(
+        self, interval=1, monitor_batch_duration=200, artifact_name="gpu_profile_data"
+    ):
+        self.artifact_name = artifact_name
+        self._card_setup_finished = False
         self.driver_ver, self.cuda_ver, self.error = self._read_versions()
         (
             self.interconnect_data,
@@ -389,16 +393,35 @@ class GPUProfiler:
                 print("GPU Profiler readings are none", file=sys.stderr)
                 time.sleep(self._interval)
                 continue
+            if not self._card_setup_finished:
+                self._setup_card()
+                time.sleep(self._interval)
+                continue
+
             _update_utilization(readings, self._card_comps["max_utilization"])
             _update_charts(readings, self._card_comps["charts"])
             current.card["gpu_profile"].refresh()
             time.sleep(self._interval)
 
-    def _setup_card(self, artifact_name):
+    def _setup_card(self):
         from metaflow import current
 
         results = self._make_reading()
+        if "profile" not in results:
+            return
+
         els = current.card["gpu_profile"]
+        els.clear()
+
+        current.card["gpu_profile"].append(
+            Markdown("# GPU profile for `%s`" % current.pathspec)
+        )
+        current.card["gpu_profile"].append(
+            Markdown(
+                "_Started at: %s_"
+                % datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S %z")
+            )
+        )
 
         def _drivers():
             els.append(Markdown("## Drivers"))
@@ -444,7 +467,7 @@ class GPUProfiler:
                 Table(data=_rows, headers=["Device ID", "Max GPU %", "Max memory"])
             )
             els.append(
-                Markdown(f"Detailed data saved in an artifact `{artifact_name}`")
+                Markdown(f"Detailed data saved in an artifact `{self.artifact_name}`")
             )
             return rows
 
@@ -479,6 +502,7 @@ class GPUProfiler:
         _interconnect()
         self._card_comps["max_utilization"] = _utilization()
         self._card_comps["charts"] = _plots()
+        self._card_setup_finished = True
 
     def _read_versions(self):
         def parse(r, s):
@@ -572,7 +596,9 @@ class gpu_profile:
     def __call__(self, f):
         @wraps(f)
         def func(s):
-            prof = GPUProfiler(interval=self.interval)
+            prof = GPUProfiler(
+                interval=self.interval, artifact_name=self.artifact_prefix + "data"
+            )
             if self.include_artifacts:
                 setattr(s, self.artifact_prefix + "num_gpus", len(prof.devices))
 
@@ -585,7 +611,7 @@ class gpu_profile:
                     % datetime.now().astimezone().strftime("%Y-%m-%dT%H:%M:%S %z")
                 )
             )
-            prof._setup_card(self.artifact_prefix + "data")
+            prof._setup_card()
             current.card["gpu_profile"].refresh()
             update_thread = threading.Thread(target=prof._update_card, daemon=True)
             update_thread.start()

{ob_metaflow_extensions-1.1.98.dist-info → ob_metaflow_extensions-1.1.100rc0.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.98
+Version: 1.1.100rc0
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial
 Description-Content-Type: text/markdown
 Requires-Dist: boto3
 Requires-Dist: kubernetes
-Requires-Dist: ob-metaflow (==2.12.25.1)
+Requires-Dist: ob-metaflow (==2.12.25.2)
 
 # Outerbounds platform package
 

{ob_metaflow_extensions-1.1.98.dist-info → ob_metaflow_extensions-1.1.100rc0.dist-info}/RECORD

@@ -28,13 +28,13 @@ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=d_5UhXqZ
 metaflow_extensions/outerbounds/plugins/snowpark/snowpark_service_spec.py,sha256=AI_kcm1hZV3JRxJkookcH6twiGnAYjk9Dx-MeoYz60Y,8511
 metaflow_extensions/outerbounds/plugins/tensorboard/__init__.py,sha256=9lUM4Cqi5RjrHBRfG6AQMRz8-R96eZC8Ih0KD2lv22Y,1858
 metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
-metaflow_extensions/outerbounds/profilers/gpu.py,sha256=a5YZAepujuP0uDqG9UpXBlZS3wjUt4Yv8CjybXqeT2c,24342
+metaflow_extensions/outerbounds/profilers/gpu.py,sha256=JYu8tQl0p5CIq1xSVUHjK3ldeqoLaZzqLBu86OKr5rg,25104
 metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
 metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=Zq3OuL1bOod8KJra-Zk8B3gNhSHoWEGteM9T7g0pp6E,1881
 metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
 metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
-ob_metaflow_extensions-1.1.98.dist-info/METADATA,sha256=4Qixz_AtC57tc30Z4u2iDj4s7K0oY0zJK7qVaznZ8J0,520
-ob_metaflow_extensions-1.1.98.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.98.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.98.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.100rc0.dist-info/METADATA,sha256=y93pSmIa-j7KdnaJMxeq9Fie5dWrjFqxncdS9zfVtww,524
+ob_metaflow_extensions-1.1.100rc0.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.100rc0.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.100rc0.dist-info/RECORD,,