ob-metaflow 2.10.7.4__py2.py3-none-any.whl → 2.10.9.2__py2.py3-none-any.whl

metaflow/plugins/cards/card_creator.py

@@ -0,0 +1,228 @@
+import time
+import subprocess
+import tempfile
+import json
+import sys
+import os
+from metaflow import current
+
+ASYNC_TIMEOUT = 30
+
+
+class CardProcessManager:
+    """
+    This class is responsible for managing the card creation processes.
+
+    """
+
+    async_card_processes = {
+        # "carduuid": {
+        #     "proc": subprocess.Popen,
+        #     "started": time.time()
+        # }
+    }
+
+    @classmethod
+    def _register_card_process(cls, carduuid, proc):
+        cls.async_card_processes[carduuid] = {
+            "proc": proc,
+            "started": time.time(),
+        }
+
+    @classmethod
+    def _get_card_process(cls, carduuid):
+        proc_dict = cls.async_card_processes.get(carduuid, None)
+        if proc_dict is not None:
+            return proc_dict["proc"], proc_dict["started"]
+        return None, None
+
+    @classmethod
+    def _remove_card_process(cls, carduuid):
+        if carduuid in cls.async_card_processes:
+            cls.async_card_processes[carduuid]["proc"].kill()
+            del cls.async_card_processes[carduuid]
+
+
+class CardCreator:
+    def __init__(self, top_level_options):
+        self._top_level_options = top_level_options
+
+    def create(
+        self,
+        card_uuid=None,
+        user_set_card_id=None,
+        runtime_card=False,
+        decorator_attributes=None,
+        card_options=None,
+        logger=None,
+        mode="render",
+        final=False,
+        sync=False,
+    ):
+        # Setting `final` will affect the Reload token set during the card refresh
+        # data creation along with synchronous execution of subprocess.
+        # Setting `sync` will only cause synchronous execution of subprocess.
+        if mode != "render" and not runtime_card:
+            # silently ignore runtime updates for cards that don't support them
+            return
+        elif mode == "refresh":
+            # don't serialize components, which can be a somewhat expensive operation,
+            # if we are just updating data
+            component_strings = []
+        else:
+            component_strings = current.card._serialize_components(card_uuid)
+        data = current.card._get_latest_data(card_uuid, final=final, mode=mode)
+        runspec = "/".join([current.run_id, current.step_name, current.task_id])
+        self._run_cards_subprocess(
+            card_uuid,
+            user_set_card_id,
+            mode,
+            runspec,
+            decorator_attributes,
+            card_options,
+            component_strings,
+            logger,
+            data,
+            final=final,
+            sync=sync,
+        )
+
+    def _run_cards_subprocess(
+        self,
+        card_uuid,
+        user_set_card_id,
+        mode,
+        runspec,
+        decorator_attributes,
+        card_options,
+        component_strings,
+        logger,
+        data=None,
+        final=False,
+        sync=False,
+    ):
+        components_file = data_file = None
+        wait = final or sync
+
+        if len(component_strings) > 0:
+            # note that we can't delete temporary files here when calling the subprocess
+            # async due to a race condition. The subprocess must delete them
+            components_file = tempfile.NamedTemporaryFile(
+                "w", suffix=".json", delete=False
+            )
+            json.dump(component_strings, components_file)
+            components_file.seek(0)
+        if data is not None:
+            data_file = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False)
+            json.dump(data, data_file)
+            data_file.seek(0)
+
+        executable = sys.executable
+        cmd = [
+            executable,
+            sys.argv[0],
+        ]
+        cmd += self._top_level_options + [
+            "card",
+            "create",
+            runspec,
+            "--delete-input-files",
+            "--card-uuid",
+            card_uuid,
+            "--mode",
+            mode,
+            "--type",
+            decorator_attributes["type"],
+            # Add the options relating to card arguments.
+            # todo : add scope as a CLI arg for the create method.
+        ]
+        if card_options is not None and len(card_options) > 0:
+            cmd += ["--options", json.dumps(card_options)]
+        # set the id argument.
+
+        if decorator_attributes["timeout"] is not None:
+            cmd += ["--timeout", str(decorator_attributes["timeout"])]
+
+        if user_set_card_id is not None:
+            cmd += ["--id", str(user_set_card_id)]
+
+        if decorator_attributes["save_errors"]:
+            cmd += ["--render-error-card"]
+
+        if components_file is not None:
+            cmd += ["--component-file", components_file.name]
+
+        if data_file is not None:
+            cmd += ["--data-file", data_file.name]
+
+        response, fail = self._run_command(
+            cmd,
+            card_uuid,
+            os.environ,
+            timeout=decorator_attributes["timeout"],
+            wait=wait,
+        )
+        if fail:
+            resp = "" if response is None else response.decode("utf-8")
+            logger(
+                "Card render failed with error : \n\n %s" % resp,
+                timestamp=False,
+                bad=True,
+            )
+
+    def _wait_for_async_processes_to_finish(self, card_uuid, async_timeout):
+        _async_proc, _async_started = CardProcessManager._get_card_process(card_uuid)
+        while _async_proc is not None and _async_proc.poll() is None:
+            if time.time() - _async_started > async_timeout:
+                # This means the process has crossed the timeout and we need to kill it.
+                CardProcessManager._remove_card_process(card_uuid)
+                break
+
+    def _run_command(self, cmd, card_uuid, env, wait=True, timeout=None):
+        fail = False
+        timeout_args = {}
+        async_timeout = ASYNC_TIMEOUT
+        if timeout is not None:
+            async_timeout = int(timeout) + 10
+            timeout_args = dict(timeout=int(timeout) + 10)
+
+        if wait:
+            self._wait_for_async_processes_to_finish(card_uuid, async_timeout)
+            try:
+                rep = subprocess.check_output(
+                    cmd, env=env, stderr=subprocess.STDOUT, **timeout_args
+                )
+            except subprocess.CalledProcessError as e:
+                rep = e.output
+                fail = True
+            except subprocess.TimeoutExpired as e:
+                rep = e.output
+                fail = True
+            return rep, fail
+        else:
+            _async_proc, _async_started = CardProcessManager._get_card_process(
+                card_uuid
+            )
+            if _async_proc and _async_proc.poll() is None:
+                if time.time() - _async_started > async_timeout:
+                    CardProcessManager._remove_card_process(card_uuid)
+                    # Since we have removed the card process, we are free to run a new one
+                    # This will also ensure that when a old process is removed a new one is replaced.
+                    return self._run_command(
+                        cmd, card_uuid, env, wait=wait, timeout=timeout
+                    )
+                else:
+                    # silently refuse to run an async process if a previous one is still running
+                    # and timeout hasn't been reached
+                    return "".encode(), False
+            else:
+                CardProcessManager._register_card_process(
+                    card_uuid,
+                    subprocess.Popen(
+                        cmd,
+                        env=env,
+                        stderr=subprocess.DEVNULL,
+                        stdout=subprocess.DEVNULL,
+                    ),
+                )
+                return "".encode(), False

metaflow/plugins/cards/card_datastore.py

@@ -3,9 +3,9 @@
 """
 
 from collections import namedtuple
-from hashlib import sha1
 from io import BytesIO
 import os
+import json
 import shutil
 
 from metaflow.plugins.datastores.local_storage import LocalStorage
@@ -28,6 +28,16 @@ NUM_SHORT_HASH_CHARS = 5
 CardInfo = namedtuple("CardInfo", ["type", "hash", "id", "filename"])
 
 
+class CardNameSuffix:
+    DATA = "data.json"
+    CARD = "html"
+
+
+class CardPathSuffix:
+    DATA = "runtime"
+    CARD = "cards"
+
+
 def path_spec_resolver(pathspec):
     splits = pathspec.split("/")
     splits.extend([None] * (4 - len(splits)))
@@ -85,18 +95,22 @@ class CardDatastore(object):
         self._run_id = run_id
         self._step_name = step_name
         self._pathspec = pathspec
-        self._temp_card_save_path = self._get_write_path(base_pth=TEMP_DIR_NAME)
+        self._temp_card_save_path = self._get_card_write_path(base_pth=TEMP_DIR_NAME)
 
     @classmethod
-    def get_card_location(cls, base_path, card_name, card_html, card_id=None):
-        chash = sha1(bytes(card_html, "utf-8")).hexdigest()
+    def get_card_location(
+        cls, base_path, card_name, uuid, card_id=None, suffix=CardNameSuffix.CARD
+    ):
+        chash = uuid
         if card_id is None:
-            card_file_name = "%s-%s.html" % (card_name, chash)
+            card_file_name = "%s-%s.%s" % (card_name, chash, suffix)
         else:
-            card_file_name = "%s-%s-%s.html" % (card_name, card_id, chash)
+            card_file_name = "%s-%s-%s.%s" % (card_name, card_id, chash, suffix)
         return os.path.join(base_path, card_file_name)
 
-    def _make_path(self, base_pth, pathspec=None, with_steps=False):
+    def _make_path(
+        self, base_pth, pathspec=None, with_steps=False, suffix=CardPathSuffix.CARD
+    ):
         sysroot = base_pth
         if pathspec is not None:
             # since most cards are at a task level there will always be 4 non-none values returned
@@ -121,7 +135,7 @@ class CardDatastore(object):
                 step_name,
                 "tasks",
                 task_id,
-                "cards",
+                suffix,
             ]
         else:
             pth_arr = [
@@ -131,20 +145,49 @@ class CardDatastore(object):
                 run_id,
                 "tasks",
                 task_id,
-                "cards",
+                suffix,
             ]
         if sysroot == "" or sysroot is None:
             pth_arr.pop(0)
         return os.path.join(*pth_arr)
 
-    def _get_write_path(self, base_pth=""):
-        return self._make_path(base_pth, pathspec=self._pathspec, with_steps=True)
+    def _get_data_read_path(self, base_pth=""):
+        return self._make_path(
+            base_pth=base_pth,
+            pathspec=self._pathspec,
+            with_steps=True,
+            suffix=CardPathSuffix.DATA,
+        )
+
+    def _get_data_write_path(self, base_pth=""):
+        return self._make_path(
+            base_pth=base_pth,
+            pathspec=self._pathspec,
+            with_steps=True,
+            suffix=CardPathSuffix.DATA,
+        )
+
+    def _get_card_write_path(
+        self,
+        base_pth="",
+    ):
+        return self._make_path(
+            base_pth,
+            pathspec=self._pathspec,
+            with_steps=True,
+            suffix=CardPathSuffix.CARD,
+        )
 
-    def _get_read_path(self, base_pth="", with_steps=False):
-        return self._make_path(base_pth, pathspec=self._pathspec, with_steps=with_steps)
+    def _get_card_read_path(self, base_pth="", with_steps=False):
+        return self._make_path(
+            base_pth,
+            pathspec=self._pathspec,
+            with_steps=with_steps,
+            suffix=CardPathSuffix.CARD,
+        )
 
     @staticmethod
-    def card_info_from_path(path):
+    def info_from_path(path, suffix=CardNameSuffix.CARD):
         """
         Args:
             path (str): The path to the card
@@ -160,8 +203,8 @@ class CardDatastore(object):
 
         if len(file_split) not in [2, 3]:
             raise Exception(
-                "Invalid card file name %s. Card file names should be of form TYPE-HASH.html or TYPE-ID-HASH.html"
-                % card_file_name
+                "Invalid file name %s. Card/Data file names should be of form TYPE-HASH.%s or TYPE-ID-HASH.%s"
+                % (card_file_name, suffix, suffix)
             )
         card_type, card_hash, card_id = None, None, None
 
@@ -170,10 +213,23 @@ class CardDatastore(object):
         else:
             card_type, card_id, card_hash = file_split
 
-        card_hash = card_hash.split(".html")[0]
+        card_hash = card_hash.split("." + suffix)[0]
         return CardInfo(card_type, card_hash, card_id, card_file_name)
 
-    def save_card(self, card_type, card_html, card_id=None, overwrite=True):
+    def save_data(self, uuid, card_type, json_data, card_id=None):
+        card_file_name = card_type
+        loc = self.get_card_location(
+            self._get_data_write_path(),
+            card_file_name,
+            uuid,
+            card_id=card_id,
+            suffix=CardNameSuffix.DATA,
+        )
+        self._backend.save_bytes(
+            [(loc, BytesIO(json.dumps(json_data).encode("utf-8")))], overwrite=True
+        )
+
+    def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
         card_file_name = card_type
         # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
         # Check file blame to understand the age of this temporary workaround.
@@ -193,7 +249,11 @@ class CardDatastore(object):
         # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
         # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
         card_path_with_steps = self.get_card_location(
-            self._get_write_path(), card_file_name, card_html, card_id=card_id
+            self._get_card_write_path(),
+            card_file_name,
+            uuid,
+            card_id=card_id,
+            suffix=CardNameSuffix.CARD,
         )
         if SKIP_CARD_DUALWRITE:
             self._backend.save_bytes(
@@ -202,28 +262,29 @@ class CardDatastore(object):
             )
         else:
             card_path_without_steps = self.get_card_location(
-                self._get_read_path(with_steps=False),
+                self._get_card_read_path(with_steps=False),
                 card_file_name,
-                card_html,
+                uuid,
                 card_id=card_id,
+                suffix=CardNameSuffix.CARD,
             )
             for cp in [card_path_with_steps, card_path_without_steps]:
                 self._backend.save_bytes(
                     [(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite
                 )
 
-        return self.card_info_from_path(card_path_with_steps)
+        return self.info_from_path(card_path_with_steps, suffix=CardNameSuffix.CARD)
 
     def _list_card_paths(self, card_type=None, card_hash=None, card_id=None):
         # Check for new cards first
         card_paths = []
         card_paths_with_steps = self._backend.list_content(
-            [self._get_read_path(with_steps=True)]
+            [self._get_card_read_path(with_steps=True)]
         )
 
         if len(card_paths_with_steps) == 0:
             card_paths_without_steps = self._backend.list_content(
-                [self._get_read_path(with_steps=False)]
+                [self._get_card_read_path(with_steps=False)]
             )
             if len(card_paths_without_steps) == 0:
                 # If there are no files found on the Path then raise an error of
@@ -240,7 +301,7 @@ class CardDatastore(object):
         cards_found = []
         for task_card_path in card_paths:
             card_path = task_card_path.path
-            card_info = self.card_info_from_path(card_path)
+            card_info = self.info_from_path(card_path, suffix=CardNameSuffix.CARD)
             if card_type is not None and card_info.type != card_type:
                 continue
             elif card_hash is not None:
@@ -254,11 +315,32 @@ class CardDatastore(object):
 
         return cards_found
 
+    def _list_card_data(self, card_type=None, card_hash=None, card_id=None):
+        card_data_paths = self._backend.list_content([self._get_data_read_path()])
+        data_found = []
+
+        for data_path in card_data_paths:
+            _pth = data_path.path
+            card_info = self.info_from_path(_pth, suffix=CardNameSuffix.DATA)
+            if card_type is not None and card_info.type != card_type:
+                continue
+            elif card_hash is not None:
+                if not card_info.hash.startswith(card_hash):
+                    continue
+            elif card_id is not None and card_info.id != card_id:
+                continue
+            if data_path.is_file:
+                data_found.append(_pth)
+
+        return data_found
+
     def create_full_path(self, card_path):
         return os.path.join(self._backend.datastore_root, card_path)
 
     def get_card_names(self, card_paths):
-        return [self.card_info_from_path(path) for path in card_paths]
+        return [
+            self.info_from_path(path, suffix=CardNameSuffix.CARD) for path in card_paths
+        ]
 
     def get_card_html(self, path):
         with self._backend.load_bytes([path]) as get_results:
@@ -267,6 +349,13 @@ class CardDatastore(object):
                     with open(path, "r") as f:
                         return f.read()
 
+    def get_card_data(self, path):
+        with self._backend.load_bytes([path]) as get_results:
+            for _, path, _ in get_results:
+                if path is not None:
+                    with open(path, "r") as f:
+                        return json.loads(f.read())
+
     def cache_locally(self, path, save_path=None):
         """
         Saves the data present in the `path` the `metaflow_card_cache` directory or to the `save_path`.
@@ -292,6 +381,15 @@ class CardDatastore(object):
                     shutil.copy(path, main_path)
                     return main_path
 
+    def extract_data_paths(self, card_type=None, card_hash=None, card_id=None):
+        return self._list_card_data(
+            # card_hash is the unique identifier to the card.
+            # Its no longer the actual hash!
+            card_type=card_type,
+            card_hash=card_hash,
+            card_id=card_id,
+        )
+
     def extract_card_paths(self, card_type=None, card_hash=None, card_id=None):
         return self._list_card_paths(
             card_type=card_type, card_hash=card_hash, card_id=card_id