ob-metaflow-extensions 1.1.100rc0__tar.gz → 1.1.102__tar.gz

{ob-metaflow-extensions-1.1.100rc0 → ob-metaflow-extensions-1.1.102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.100rc0
+Version: 1.1.102
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.100rc0 → ob-metaflow-extensions-1.1.102}/metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py

@@ -319,15 +319,17 @@ class DockerEnvironment(MetaflowEnvironment):
         config.append("--disable=F0401")
         return config
 
+    def get_package_commands(self, codepackage_url, datastore_type):
+        # we must set the skip install flag at this stage in order to skip package downloads,
+        # doing so in bootstrap_commands is too late in the lifecycle.
+        return [
+            "export METAFLOW_SKIP_INSTALL_DEPENDENCIES=$FASTBAKERY_IMAGE",
+        ] + super().get_package_commands(codepackage_url, datastore_type)
+
     def bootstrap_commands(self, step_name, datastore_type):
         if step_name in self.skipped_steps:
             return self.delegate.bootstrap_commands(step_name, datastore_type)
-        # Bootstrap conda and execution environment for step
-        # we set the environment flag for skipping bootstrap dependencies, as these are
-        # provided in all baked images.
-        return [
-            "export METAFLOW_SKIP_INSTALL_DEPENDENCIES=$FASTBAKERY_IMAGE",
-        ] + super().bootstrap_commands(step_name, datastore_type)
+        return super().bootstrap_commands(step_name, datastore_type)
 
 
 def get_fastbakery_metafile_path(local_root, flow_name):

{ob-metaflow-extensions-1.1.100rc0 → ob-metaflow-extensions-1.1.102}/metaflow_extensions/outerbounds/profilers/gpu.py

@@ -13,6 +13,7 @@ import json
 import sys
 from tempfile import TemporaryDirectory
 from subprocess import check_output, Popen
+import subprocess
 from datetime import datetime, timedelta
 from functools import wraps
 from collections import namedtuple
@@ -318,8 +319,20 @@ class GPUProfiler:
     def __init__(
         self, interval=1, monitor_batch_duration=200, artifact_name="gpu_profile_data"
     ):
+        self._interval = interval
+        self._monitor_batch_duration = monitor_batch_duration
         self.artifact_name = artifact_name
         self._card_setup_finished = False
+        self._card_comps = {
+            "max_utilization": {},
+            "charts": {},
+            "reading_duration": {},
+            "error_component": None,
+        }
+        self._monitor_started = False
+        self._start_monitor()
+
+    def _start_monitor(self):
         self.driver_ver, self.cuda_ver, self.error = self._read_versions()
         (
             self.interconnect_data,
@@ -331,16 +344,14 @@ class GPUProfiler:
         else:
             self.devices = self._read_devices()
             self._monitor = GPUMonitor(
-                interval=interval, duration=monitor_batch_duration
+                interval=self._interval, duration=self._monitor_batch_duration
             )
             self._monitor_thread = threading.Thread(
                 target=self._monitor._monitor_update_thread, daemon=True
             )
+            self.error = None
             self._monitor_thread.start()
-            self._interval = interval
-
-        self._card_comps = {"max_utilization": {}, "charts": {}, "reading_duration": {}}
-        self._card_created = False
+            self._monitor_started = True
 
     def finish(self):
         ret = {
@@ -388,11 +399,20 @@ class GPUProfiler:
             return
 
         while True:
+            # There is a possibility that the `monitor` thread is not started yet
+            # because it somehow crashed at the very start.
+            if not self._monitor_started:
+                self._start_monitor()
+                time.sleep(self._interval)
+                continue
+
             readings = self._make_reading()
+
             if readings is None:
                 print("GPU Profiler readings are none", file=sys.stderr)
                 time.sleep(self._interval)
                 continue
+
             if not self._card_setup_finished:
                 self._setup_card()
                 time.sleep(self._interval)
@@ -408,9 +428,20 @@ class GPUProfiler:
 
         results = self._make_reading()
         if "profile" not in results:
+            if self._card_comps["error_component"] is None:
+                self._card_comps["error_component"] = Markdown(
+                    "## GPU profile failed: %s" % results["error"]
+                )
+                current.card["gpu_profile"].append(self._card_comps["error_component"])
+            else:
+                self._card_comps["error_component"].update(
+                    Markdown("## GPU profile failed: %s" % results["error"])
+                )
+                current.card["gpu_profile"].refresh()
             return
 
         els = current.card["gpu_profile"]
+        self._card_comps["error_component"] = None
         els.clear()
 
         current.card["gpu_profile"].append(
@@ -509,14 +540,29 @@ class GPUProfiler:
             return r.search(s).group(1).strip().decode("utf-8")
 
         try:
-            out = check_output(["nvidia-smi"])
+            result = subprocess.run(
+                ["nvidia-smi"],
+                check=True,  # This will raise a CalledProcessError if the command fails
+                stdout=subprocess.PIPE,  # Capture stdout
+                stderr=subprocess.PIPE,  # Capture stderr
+            )
+            # Access the standard output
+            out = result.stdout
             return parse(DRIVER_VER, out), parse(CUDA_VER, out), None
         except FileNotFoundError:
             return None, None, "nvidia-smi not found"
         except AttributeError:
             return None, None, "nvidia-smi output is unexpected"
-        except:
-            return None, None, "nvidia-smi error"
+        except subprocess.CalledProcessError as e:
+            _error_message = "nvidia-smi error (CalledProcessError calling nvidia-smi)"
+            if e.stderr is not None:
+                _error_message = (
+                    "nvidia-smi error (CalledProcessError stderr) \n %s \n %s"
+                    % (e.stderr.decode("utf-8"), e.stdout.decode("utf-8"))
+                )
+            return None, None, _error_message
+        except Exception as e:
+            return None, None, "nvidia-smi error (unknown error) \n%s" % str(e)
 
     def _read_devices(self):
         out = check_output(

{ob-metaflow-extensions-1.1.100rc0 → ob-metaflow-extensions-1.1.102}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.100rc0
+Version: 1.1.102
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

ob-metaflow-extensions-1.1.102/ob_metaflow_extensions.egg-info/requires.txt

@@ -0,0 +1,3 @@
+boto3
+kubernetes
+ob-metaflow==2.12.27.1

{ob-metaflow-extensions-1.1.100rc0 → ob-metaflow-extensions-1.1.102}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.1.100rc0"
+version = "1.1.102"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
@@ -18,6 +18,6 @@ setup(
     install_requires=[
         "boto3",
         "kubernetes",
-        "ob-metaflow == 2.12.25.2",
+        "ob-metaflow == 2.12.27.1",
     ],
 )

ob-metaflow-extensions-1.1.100rc0/ob_metaflow_extensions.egg-info/requires.txt

@@ -1,3 +0,0 @@
-boto3
-kubernetes
-ob-metaflow==2.12.25.2