ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/plugins/cards/card_cli.py

@@ -1,5 +1,6 @@
 from metaflow.client import Task
-from metaflow import JSONType, namespace
+from metaflow.parameters import JSONTypeClass
+from metaflow import namespace
 from metaflow.util import resolve_identity
 from metaflow.exception import (
     CommandException,
@@ -29,7 +30,7 @@ from .exception import (
 )
 import traceback
 from collections import namedtuple
-
+from .metadata import _save_metadata
 from .card_resolver import resolve_paths_from_task, resumed_info
 
 id_func = id
@@ -388,6 +389,22 @@ def card_read_options_and_arguments(func):
     return wrapper
 
 
+def _extract_reload_token(data, task, mf_card):
+    if "render_seq" not in data:
+        return "never"
+
+    if data["render_seq"] == "final":
+        # final data update should always trigger a card reload to show
+        # the final card, hence a different token for the final update
+        return "final"
+    elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ALWAYS:
+        return "render-seq-%s" % data["render_seq"]
+    elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_NEVER:
+        return "never"
+    elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ONCHANGE:
+        return mf_card.reload_content_token(task, data)
+
+
 def update_card(mf_card, mode, task, data, timeout_value=None):
     """
     This method will be responsible for creating a card/data-update based on the `mode`.
@@ -432,31 +449,21 @@ def update_card(mf_card, mode, task, data, timeout_value=None):
             - `timeout_stack_trace` : stack trace of the function if it timed out.
     """
 
-    def _reload_token():
-        if "render_seq" not in data:
-            return "never"
-
-        if data["render_seq"] == "final":
-            # final data update should always trigger a card reload to show
-            # the final card, hence a different token for the final update
-            return "final"
-        elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ALWAYS:
-            return "render-seq-%s" % data["render_seq"]
-        elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_NEVER:
-            return "never"
-        elif mf_card.RELOAD_POLICY == mf_card.RELOAD_POLICY_ONCHANGE:
-            return mf_card.reload_content_token(task, data)
-
     def _add_token_html(html):
         if html is None:
             return None
-        return html.replace(mf_card.RELOAD_POLICY_TOKEN, _reload_token())
+        return html.replace(
+            mf_card.RELOAD_POLICY_TOKEN,
+            _extract_reload_token(data=data, task=task, mf_card=mf_card),
+        )
 
     def _add_token_json(json_msg):
         if json_msg is None:
             return None
         return {
-            "reload_token": _reload_token(),
+            "reload_token": _extract_reload_token(
+                data=data, task=task, mf_card=mf_card
+            ),
             "data": json_msg,
             "created_on": time.time(),
         }
@@ -545,7 +552,7 @@ def update_card(mf_card, mode, task, data, timeout_value=None):
     "--options",
     default=None,
     show_default=True,
-    type=JSONType,
+    type=JSONTypeClass(),
     help="arguments of the card being created.",
 )
 @click.option(
@@ -606,6 +613,14 @@ def update_card(mf_card, mode, task, data, timeout_value=None):
     hidden=True,
     help="Delete data-file and component-file after reading. (internal)",
 )
+@click.option(
+    "--save-metadata",
+    default=None,
+    show_default=True,
+    type=JSONTypeClass(),
+    hidden=True,
+    help="JSON string containing metadata to be saved. (internal)",
+)
 @click.pass_context
 def create(
     ctx,
@@ -620,6 +635,7 @@ def create(
     card_uuid=None,
     delete_input_files=None,
     id=None,
+    save_metadata=None,
 ):
     card_id = id
     rendered_info = None  # Variable holding all the information which will be rendered
@@ -684,10 +700,15 @@ def create(
         try:
             if options is not None:
                 mf_card = filtered_card(
-                    options=options, components=component_arr, graph=graph_dict
+                    options=options,
+                    components=component_arr,
+                    graph=graph_dict,
+                    flow=ctx.obj.flow,
                 )
             else:
-                mf_card = filtered_card(components=component_arr, graph=graph_dict)
+                mf_card = filtered_card(
+                    components=component_arr, graph=graph_dict, flow=ctx.obj.flow
+                )
         except TypeError as e:
             if render_error_card:
                 mf_card = None
@@ -740,8 +761,16 @@ def create(
     #   3. `refresh` is implemented but it raises an exception. (We do nothing. Don't store anything.)
     #   4. `refresh` is implemented but it times out. (We do nothing. Don't store anything.)
 
+    def _render_error_card(stack_trace):
+        _card = error_card()
+        token = _extract_reload_token(data, task, _card)
+        return _card.render(
+            task,
+            stack_trace=stack_trace,
+        ).replace(_card.RELOAD_POLICY_TOKEN, token)
+
     if error_stack_trace is not None and mode != "refresh":
-        rendered_content = error_card().render(task, stack_trace=error_stack_trace)
+        rendered_content = _render_error_card(error_stack_trace)
     elif (
         rendered_info.is_implemented
         and rendered_info.timed_out
@@ -753,18 +782,15 @@ def create(
             "To increase the timeout duration for card rendering, please set the `timeout` parameter in the @card decorator. "
             "\nStack Trace : \n%s"
         ) % (timeout, rendered_info.timeout_stack_trace)
-        rendered_content = error_card().render(
-            task,
-            stack_trace=timeout_stack_trace,
-        )
+        rendered_content = _render_error_card(timeout_stack_trace)
     elif (
         rendered_info.is_implemented
         and rendered_info.data is None
         and render_error_card
         and mode != "refresh"
     ):
-        rendered_content = error_card().render(
-            task, stack_trace="No information rendered from card of type %s" % type
+        rendered_content = _render_error_card(
+            "No information rendered from card of type %s" % type
         )
     elif (
         not rendered_info.is_implemented
@@ -775,7 +801,7 @@ def create(
             "Card of type %s is a runtime time card with no `render_runtime` implemented. "
             "Please implement `render_runtime` method to allow rendering this card at runtime."
         ) % type
-        rendered_content = error_card().render(task, stack_trace=message)
+        rendered_content = _render_error_card(message)
 
     # todo : should we save native type for error card or error type ?
     if type is not None and re.match(CARD_ID_PATTERN, type) is not None:
@@ -807,6 +833,16 @@ def create(
                 % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]),
                 fg="green",
             )
+            if save_metadata:
+                _save_metadata(
+                    ctx.obj.metadata,
+                    task.parent.parent.id,
+                    task.parent.id,
+                    task.id,
+                    task.current_attempt,
+                    card_uuid,
+                    save_metadata,
+                )
 
 
 @card.command()

metaflow/plugins/cards/card_creator.py

@@ -5,6 +5,8 @@ import json
 import sys
 import os
 from metaflow import current
+from typing import Callable, Tuple, Dict
+
 
 ASYNC_TIMEOUT = 30
 
@@ -44,8 +46,18 @@ class CardProcessManager:
 
 
 class CardCreator:
-    def __init__(self, top_level_options):
+    def __init__(
+        self,
+        top_level_options,
+        should_save_metadata_lambda: Callable[[str], Tuple[bool, Dict]],
+    ):
+        # should_save_metadata_lambda is a lambda that provides a flag to indicate if
+        # card metadata should be written to the metadata store.
+        # It gets called only once when the card is created inside the subprocess.
+        # The intent is that this is a stateful lambda that will ensure that we only end
+        # up writing to the metadata store once.
         self._top_level_options = top_level_options
+        self._should_save_metadata = should_save_metadata_lambda
 
     def create(
         self,
@@ -62,6 +74,8 @@ class CardCreator:
         # Setting `final` will affect the Reload token set during the card refresh
         # data creation along with synchronous execution of subprocess.
         # Setting `sync` will only cause synchronous execution of subprocess.
+        save_metadata = False
+        metadata_dict = {}
         if mode != "render" and not runtime_card:
             # silently ignore runtime updates for cards that don't support them
             return
@@ -71,6 +85,8 @@ class CardCreator:
             component_strings = []
         else:
             component_strings = current.card._serialize_components(card_uuid)
+            # Since the mode is a render, we can check if we need to write to the metadata store.
+            save_metadata, metadata_dict = self._should_save_metadata(card_uuid)
         data = current.card._get_latest_data(card_uuid, final=final, mode=mode)
         runspec = "/".join([current.run_id, current.step_name, current.task_id])
         self._run_cards_subprocess(
@@ -85,6 +101,8 @@ class CardCreator:
             data,
             final=final,
             sync=sync,
+            save_metadata=save_metadata,
+            metadata_dict=metadata_dict,
         )
 
     def _run_cards_subprocess(
@@ -100,6 +118,8 @@ class CardCreator:
         data=None,
         final=False,
         sync=False,
+        save_metadata=False,
+        metadata_dict=None,
     ):
         components_file = data_file = None
         wait = final or sync
@@ -122,6 +142,7 @@ class CardCreator:
             executable,
             sys.argv[0],
         ]
+
         cmd += self._top_level_options + [
             "card",
             "create",
@@ -155,6 +176,9 @@ class CardCreator:
         if data_file is not None:
             cmd += ["--data-file", data_file.name]
 
+        if save_metadata:
+            cmd += ["--save-metadata", json.dumps(metadata_dict)]
+
         response, fail = self._run_command(
             cmd,
             card_uuid,

metaflow/plugins/cards/card_datastore.py

@@ -1,7 +1,3 @@
-""" 
-
-"""
-
 from collections import namedtuple
 from io import BytesIO
 import os
@@ -13,10 +9,10 @@ from metaflow.metaflow_config import (
     CARD_S3ROOT,
     CARD_LOCALROOT,
     DATASTORE_LOCAL_DIR,
+    DATASTORE_SPIN_LOCAL_DIR,
     CARD_SUFFIX,
     CARD_AZUREROOT,
     CARD_GSROOT,
-    SKIP_CARD_DUALWRITE,
 )
 import metaflow.metaflow_config as metaflow_config
 
@@ -63,25 +59,28 @@ class CardDatastore(object):
             return CARD_AZUREROOT
         elif storage_type == "gs":
             return CARD_GSROOT
-        elif storage_type == "local":
+        elif storage_type == "local" or storage_type == "spin":
             # Borrowing some of the logic from LocalStorage.get_storage_root
             result = CARD_LOCALROOT
+            local_dir = (
+                DATASTORE_SPIN_LOCAL_DIR
+                if storage_type == "spin"
+                else DATASTORE_LOCAL_DIR
+            )
             if result is None:
                 current_path = os.getcwd()
-                check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR, CARD_SUFFIX)
+                check_dir = os.path.join(current_path, local_dir)
                 check_dir = os.path.realpath(check_dir)
                 orig_path = check_dir
                 while not os.path.isdir(check_dir):
                     new_path = os.path.dirname(current_path)
                     if new_path == current_path:
-                        break  # We are no longer making upward progress
+                        # No longer making upward progress so we
+                        # return the top level path
+                        return os.path.join(orig_path, CARD_SUFFIX)
                     current_path = new_path
-                    check_dir = os.path.join(
-                        current_path, DATASTORE_LOCAL_DIR, CARD_SUFFIX
-                    )
-                result = orig_path
-
-            return result
+                    check_dir = os.path.join(current_path, local_dir)
+                return os.path.join(check_dir, CARD_SUFFIX)
         else:
             # Let's make it obvious we need to update this block for each new datastore backend...
             raise NotImplementedError(
@@ -231,23 +230,6 @@ class CardDatastore(object):
 
     def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
         card_file_name = card_type
-        # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
-        # Check file blame to understand the age of this temporary workaround.
-
-        # This function will end up saving cards at two locations.
-        # Thereby doubling the number of cards. (Which is a temporary fix)
-        # Why do this ? :
-        # When cards were introduced there was an assumption made about task-ids being unique.
-        # This assumption was incorrect.
-        # Only the pathspec needs to be unique but there is no such guarantees about task-ids.
-        # When task-ids are non-unique, card read would result in finding incorrect cards.
-        # This happens because cards were stored based on task-ids.
-        # If we immediately switch from storing based on task-ids to a step-name abstraction folder,
-        # then card reading will crash for many users.
-        # It would especially happen for users who are accessing cards created by a newer
-        # MF client from an older version of MF client.
-        # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
-        # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
         card_path_with_steps = self.get_card_location(
             self._get_card_write_path(),
             card_file_name,
@@ -255,24 +237,10 @@ class CardDatastore(object):
             card_id=card_id,
             suffix=CardNameSuffix.CARD,
         )
-        if SKIP_CARD_DUALWRITE:
-            self._backend.save_bytes(
-                [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
-                overwrite=overwrite,
-            )
-        else:
-            card_path_without_steps = self.get_card_location(
-                self._get_card_read_path(with_steps=False),
-                card_file_name,
-                uuid,
-                card_id=card_id,
-                suffix=CardNameSuffix.CARD,
-            )
-            for cp in [card_path_with_steps, card_path_without_steps]:
-                self._backend.save_bytes(
-                    [(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite
-                )
-
+        self._backend.save_bytes(
+            [(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
+            overwrite=overwrite,
+        )
         return self.info_from_path(card_path_with_steps, suffix=CardNameSuffix.CARD)
 
     def _list_card_paths(self, card_type=None, card_hash=None, card_id=None):
@@ -283,6 +251,10 @@ class CardDatastore(object):
         )
 
         if len(card_paths_with_steps) == 0:
+            # The listing logic is reading the cards with steps and without steps
+            # because earlier versions of clients (ones that wrote cards before June 2022),
+            # would have written cards without steps. So as a fallback we will try to check for the
+            # cards without steps.
             card_paths_without_steps = self._backend.list_content(
                 [self._get_card_read_path(with_steps=False)]
             )

metaflow/plugins/cards/card_decorator.py

@@ -1,13 +1,18 @@
+import json
+import os
+import re
+import tempfile
+from typing import Tuple, Dict
+
 from metaflow.decorators import StepDecorator
+from metaflow.metadata_provider import MetaDatum
 from metaflow.metaflow_current import current
+from metaflow.user_configs.config_options import ConfigInput
+from metaflow.user_configs.config_parameters import dump_config_values
 from metaflow.util import to_unicode
+
 from .component_serializer import CardComponentCollector, get_card_class
 from .card_creator import CardCreator
-
-
-# from metaflow import get_metadata
-import re
-
 from .exception import CARD_ID_PATTERN, TYPE_CHECK_REGEX
 
 ASYNC_TIMEOUT = 30
@@ -19,6 +24,24 @@ def warning_message(message, logger=None, ts=False):
         logger(msg, timestamp=ts, bad=True)
 
 
+class MetadataStateManager(object):
+    def __init__(self, info_func):
+        self._info_func = info_func
+        self._metadata_registered = {}
+
+    def register_metadata(self, card_uuid) -> Tuple[bool, Dict]:
+        info = self._info_func()
+        # Check that metadata was not written yet. We only want to write once.
+        if (
+            info is None
+            or info.get(card_uuid) is None
+            or self._metadata_registered.get(card_uuid)
+        ):
+            return False, {}
+        self._metadata_registered[card_uuid] = True
+        return True, info.get(card_uuid)
+
+
 class CardDecorator(StepDecorator):
     """
     Creates a human-readable report, a Metaflow Card, after this step completes.
@@ -52,11 +75,14 @@ class CardDecorator(StepDecorator):
             The or one of the cards attached to this step.
     """
 
+    _GLOBAL_CARD_INFO = {}
+
     name = "card"
     defaults = {
         "type": "default",
         "options": {},
         "scope": "task",
+        "rank": None,  # Can be one of "high", "medium", "low". Can help derive ordering on the UI.
         "timeout": 45,
         "id": None,
         "save_errors": True,
@@ -73,6 +99,12 @@ class CardDecorator(StepDecorator):
 
     card_creator = None
 
+    _config_values = None
+
+    _config_file_name = None
+
+    task_finished_decos = 0
+
     def __init__(self, *args, **kwargs):
         super(CardDecorator, self).__init__(*args, **kwargs)
         self._task_datastore = None
@@ -82,6 +114,7 @@ class CardDecorator(StepDecorator):
         self._is_editable = False
         self._card_uuid = None
         self._user_set_card_id = None
+        self._metadata_registered = False
 
     @classmethod
     def _set_card_creator(cls, card_creator):
@@ -103,20 +136,61 @@ class CardDecorator(StepDecorator):
     def _increment_step_counter(cls):
         cls.step_counter += 1
 
+    @classmethod
+    def _increment_completed_counter(cls):
+        cls.task_finished_decos += 1
+
+    @classmethod
+    def _set_config_values(cls, config_values):
+        cls._config_values = config_values
+
+    @classmethod
+    def _set_config_file_name(cls, flow):
+        # Only create a config file from the very first card decorator.
+        if cls._config_values and not cls._config_file_name:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", delete=False
+            ) as config_file:
+                config_value = dump_config_values(flow)
+                json.dump(config_value, config_file)
+                cls._config_file_name = config_file.name
+
+    @classmethod
+    def _register_card_info(cls, **kwargs):
+        if not kwargs.get("card_uuid"):
+            raise ValueError("card_uuid is required")
+        cls._GLOBAL_CARD_INFO[kwargs["card_uuid"]] = kwargs
+
+    @classmethod
+    def all_cards_info(cls):
+        return cls._GLOBAL_CARD_INFO.copy()
+
     def step_init(
         self, flow, graph, step_name, decorators, environment, flow_datastore, logger
     ):
         self._flow_datastore = flow_datastore
         self._environment = environment
         self._logger = logger
+
         self.card_options = None
 
+        # We check for configuration options. We do this here before they are
+        # converted to properties.
+        self._set_config_values(
+            [
+                (config.name, ConfigInput.make_key_name(config.name))
+                for _, config in flow._get_parameters()
+                if config.IS_CONFIG_PARAMETER
+            ]
+        )
+
         self.card_options = self.attributes["options"]
 
         evt_name = "step-init"
         # `'%s-%s'%(evt_name,step_name)` ensures that we capture this once per @card per @step.
         # Since there can be many steps checking if event is registered for `evt_name` will only make it check it once for all steps.
         # Hence, we have `_is_event_registered('%s-%s'%(evt_name,step_name))`
+        self._is_runtime_card = False
         evt = "%s-%s" % (evt_name, step_name)
         if not self._is_event_registered(evt):
             # We set the total count of decorators so that we can use it for
@@ -145,6 +219,19 @@ class CardDecorator(StepDecorator):
         self._task_datastore = task_datastore
         self._metadata = metadata
 
+        # If we have configs, we need to dump them to a file so we can re-use them
+        # when calling the card creation subprocess.
+        # Since a step can contain multiple card decorators, and all the card creation processes
+        # will reference the same config file (because of how the CardCreator is created (only single class instance)),
+        # we need to ensure that a single config file is being referenced for all card create commands.
+        # This config file will be removed when the last card decorator has finished creating its card.
+        self._set_config_file_name(flow)
+        # The MetadataStateManager is used to track the state of the metadata registration.
+        # It is there to ensure that we only register metadata for the card once. This is so that we
+        # avoid any un-necessary metadata writes because the create command can be called multiple times during the
+        # card creation process.
+        self._metadata_state_manager = MetadataStateManager(self.all_cards_info)
+
         card_type = self.attributes["type"]
         card_class = get_card_class(card_type)
 
@@ -178,7 +265,12 @@ class CardDecorator(StepDecorator):
         # we need to ensure that `current.card` has `CardComponentCollector` instantiated only once.
         if not self._is_event_registered("pre-step"):
             self._register_event("pre-step")
-            self._set_card_creator(CardCreator(self._create_top_level_args()))
+            self._set_card_creator(
+                CardCreator(
+                    self._create_top_level_args(flow),
+                    self._metadata_state_manager.register_metadata,
+                )
+            )
 
             current._update_env(
                 {"card": CardComponentCollector(self._logger, self.card_creator)}
@@ -201,6 +293,18 @@ class CardDecorator(StepDecorator):
         )
         self._card_uuid = card_metadata["uuid"]
 
+        self._register_card_info(
+            card_uuid=self._card_uuid,
+            rank=self.attributes["rank"],
+            type=self.attributes["type"],
+            options=self.card_options,
+            is_editable=self._is_editable,
+            is_runtime_card=self._is_runtime_card,
+            refresh_interval=self.attributes["refresh_interval"],
+            customize=customize,
+            id=self._user_set_card_id,
+        )
+
         # This means that we are calling `task_pre_step` on the last card decorator.
         # We can now `finalize` method in the CardComponentCollector object.
         # This will set up the `current.card` object for usage inside `@step` code.
@@ -222,6 +326,8 @@ class CardDecorator(StepDecorator):
             self.card_creator.create(mode="render", final=True, **create_options)
             self.card_creator.create(mode="refresh", final=True, **create_options)
 
+        self._cleanup(step_name)
+
     @staticmethod
     def _options(mapping):
         for k, v in mapping.items():
@@ -231,9 +337,13 @@ class CardDecorator(StepDecorator):
                 for value in v:
                     yield "--%s" % k
                     if not isinstance(value, bool):
-                        yield to_unicode(value)
+                        if isinstance(value, tuple):
+                            for val in value:
+                                yield to_unicode(val)
+                        else:
+                            yield to_unicode(value)
 
-    def _create_top_level_args(self):
+    def _create_top_level_args(self, flow):
         top_level_options = {
             "quiet": True,
             "metadata": self._metadata.TYPE,
@@ -246,4 +356,18 @@ class CardDecorator(StepDecorator):
             # We don't provide --with as all execution is taking place in
             # the context of the main process
         }
+        if self._config_values:
+            top_level_options["config-value"] = self._config_values
+            top_level_options["local-config-file"] = self._config_file_name
+
         return list(self._options(top_level_options))
+
+    def _cleanup(self, step_name):
+        self._increment_completed_counter()
+        if self.task_finished_decos == self.total_decos_on_step[step_name]:
+            # Unlink the config file if it exists
+            if self._config_file_name:
+                try:
+                    os.unlink(self._config_file_name)
+                except Exception as e:
+                    pass