ob-metaflow 2.13.2.1__py2.py3-none-any.whl → 2.13.6.1__py2.py3-none-any.whl

metaflow/cards.py

@@ -8,6 +8,7 @@ from metaflow.plugins.cards.card_modules.components import (
     Markdown,
     VegaChart,
     ProgressBar,
+    PythonCode,
 )
 from metaflow.plugins.cards.card_modules.basic import (
     DefaultCard,

metaflow/cli.py

@@ -504,6 +504,11 @@ def start(
     # *after* the run decospecs so that they don't take precedence. In other
     # words, for the same decorator, we want `myflow.py run --with foo` to
     # take precedence over any other `foo` decospec
+
+    # Note that top-level decospecs are used primarily with non run/resume
+    # options as well as with the airflow/argo/sfn integrations which pass
+    # all the decospecs (the ones from top-level but also the ones from the
+    # run/resume level) through the tl decospecs.
     ctx.obj.tl_decospecs = list(decospecs or [])
 
     # initialize current and parameter context for deploy-time parameters

metaflow/cli_components/run_cmds.py

@@ -39,6 +39,8 @@ def before_run(obj, tags, decospecs):
         + list(obj.environment.decospecs() or [])
     )
     if all_decospecs:
+        # These decospecs are the ones from run/resume PLUS the ones from the
+        # environment (for example the @conda)
         decorators._attach_decorators(obj.flow, all_decospecs)
         decorators._init(obj.flow)
         # Regenerate graph if we attached more decorators

metaflow/cli_components/step_cmd.py

@@ -77,14 +77,6 @@ from ..util import decompress_list
     default=None,
     help="Run id of the origin flow, if this task is part of a flow being resumed.",
 )
-@click.option(
-    "--with",
-    "decospecs",
-    multiple=True,
-    help="Add a decorator to this task. You can specify this "
-    "option multiple times to attach multiple decorators "
-    "to this task.",
-)
 @click.option(
     "--ubf-context",
     default="none",
@@ -112,7 +104,6 @@ def step(
     max_user_code_retries=None,
     clone_only=None,
     clone_run_id=None,
-    decospecs=None,
     ubf_context="none",
     num_parallel=None,
 ):
@@ -136,10 +127,6 @@ def step(
         raise CommandException("Function *%s* is not a step." % step_name)
     echo("Executing a step, *%s*" % step_name, fg="magenta", bold=False)
 
-    if decospecs:
-        decorators._attach_decorators_to_step(func, decospecs)
-        decorators._init(ctx.obj.flow)
-
     step_kwargs = ctx.params
     # Remove argument `step_name` from `step_kwargs`.
     step_kwargs.pop("step_name", None)

metaflow/decorators.py

@@ -150,8 +150,8 @@ class Decorator(object):
             return
 
         # Note that by design, later values override previous ones.
-        self.attributes = unpack_delayed_evaluator(self.attributes)
-        self._user_defined_attributes.update(self.attributes.keys())
+        self.attributes, new_user_attributes = unpack_delayed_evaluator(self.attributes)
+        self._user_defined_attributes.update(new_user_attributes)
         self.attributes = resolve_delayed_evaluator(self.attributes)
 
         self._ran_init = True

metaflow/parameters.py

@@ -367,7 +367,9 @@ class Parameter(object):
         )
 
         # Resolve any value from configurations
-        self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
+        self.kwargs, _ = unpack_delayed_evaluator(
+            self.kwargs, ignore_errors=ignore_errors
+        )
         # Do it one item at a time so errors are ignored at that level (as opposed to
         # at the entire kwargs level)
         self.kwargs = {

metaflow/plugins/__init__.py

@@ -16,6 +16,7 @@ CLIS_DESC = [
     ("argo-workflows", ".argo.argo_workflows_cli.cli"),
     ("card", ".cards.card_cli.cli"),
     ("tag", ".tag_cli.cli"),
+    ("spot-metadata", ".kubernetes.spot_metadata_cli.cli"),
     ("logs", ".logs_cli.cli"),
 ]
 
@@ -104,6 +105,10 @@ SIDECARS_DESC = [
         "save_logs_periodically",
         "..mflog.save_logs_periodically.SaveLogsPeriodicallySidecar",
     ),
+    (
+        "spot_termination_monitor",
+        ".kubernetes.spot_monitor_sidecar.SpotTerminationMonitorSidecar",
+    ),
     ("heartbeat", "metaflow.metadata_provider.heartbeat.MetadataHeartBeat"),
 ]
 

metaflow/plugins/argo/argo_workflows.py

@@ -625,6 +625,16 @@ class ArgoWorkflows(object):
             for event in trigger_on_finish_deco.triggers:
                 # Actual filters are deduced here since we don't have access to
                 # the current object in the @trigger_on_finish decorator.
+                project_name = event.get("project") or current.get("project_name")
+                branch_name = event.get("branch") or current.get("branch_name")
+                # validate that we have complete project info for an event name
+                if project_name or branch_name:
+                    if not (project_name and branch_name):
+                        # if one of the two is missing, we would end up listening to an event that will never be broadcast.
+                        raise ArgoWorkflowsException(
+                            "Incomplete project info. Please specify both 'project' and 'project_branch' or use the @project decorator"
+                        )
+
                 triggers.append(
                     {
                         # Make sure this remains consistent with the event name format
@@ -633,18 +643,16 @@ class ArgoWorkflows(object):
                         % ".".join(
                             v
                             for v in [
-                                event.get("project") or current.get("project_name"),
-                                event.get("branch") or current.get("branch_name"),
+                                project_name,
+                                branch_name,
                                 event["flow"],
                             ]
                             if v
                         ),
                         "filters": {
                             "auto-generated-by-metaflow": True,
-                            "project_name": event.get("project")
-                            or current.get("project_name"),
-                            "branch_name": event.get("branch")
-                            or current.get("branch_name"),
+                            "project_name": project_name,
+                            "branch_name": branch_name,
                             # TODO: Add a time filters to guard against cached events
                         },
                         "type": "run",
@@ -842,8 +850,15 @@ class ArgoWorkflows(object):
                     Metadata()
                     .labels(self._base_labels)
                     .label("app.kubernetes.io/name", "metaflow-task")
-                    .annotations(annotations)
-                    .annotations(self._base_annotations)
+                    .annotations(
+                        {
+                            **annotations,
+                            **self._base_annotations,
+                            **{
+                                "metaflow/run_id": "argo-{{workflow.name}}"
+                            },  # we want pods of the workflow to have the run_id as an annotation as well
+                        }
+                    )
                 )
                 # Set the entrypoint to flow name
                 .entrypoint(self.flow.name)
@@ -1706,6 +1721,7 @@ class ArgoWorkflows(object):
                     },
                     **{
                         # Some optional values for bookkeeping
+                        "METAFLOW_FLOW_FILENAME": os.path.basename(sys.argv[0]),
                         "METAFLOW_FLOW_NAME": self.flow.name,
                         "METAFLOW_STEP_NAME": node.name,
                         "METAFLOW_RUN_ID": run_id,

metaflow/plugins/cards/card_cli.py

@@ -691,10 +691,15 @@ def create(
         try:
             if options is not None:
                 mf_card = filtered_card(
-                    options=options, components=component_arr, graph=graph_dict
+                    options=options,
+                    components=component_arr,
+                    graph=graph_dict,
+                    flow=ctx.obj.flow,
                 )
             else:
-                mf_card = filtered_card(components=component_arr, graph=graph_dict)
+                mf_card = filtered_card(
+                    components=component_arr, graph=graph_dict, flow=ctx.obj.flow
+                )
         except TypeError as e:
             if render_error_card:
                 mf_card = None

metaflow/plugins/cards/card_decorator.py

@@ -76,6 +76,12 @@ class CardDecorator(StepDecorator):
 
     card_creator = None
 
+    _config_values = None
+
+    _config_file_name = None
+
+    task_finished_decos = 0
+
     def __init__(self, *args, **kwargs):
         super(CardDecorator, self).__init__(*args, **kwargs)
         self._task_datastore = None
@@ -106,6 +112,25 @@ class CardDecorator(StepDecorator):
     def _increment_step_counter(cls):
         cls.step_counter += 1
 
+    @classmethod
+    def _increment_completed_counter(cls):
+        cls.task_finished_decos += 1
+
+    @classmethod
+    def _set_config_values(cls, config_values):
+        cls._config_values = config_values
+
+    @classmethod
+    def _set_config_file_name(cls, flow):
+        # Only create a config file from the very first card decorator.
+        if cls._config_values and not cls._config_file_name:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", delete=False
+            ) as config_file:
+                config_value = dump_config_values(flow)
+                json.dump(config_value, config_file)
+                cls._config_file_name = config_file.name
+
     def step_init(
         self, flow, graph, step_name, decorators, environment, flow_datastore, logger
     ):
@@ -116,11 +141,13 @@ class CardDecorator(StepDecorator):
 
         # We check for configuration options. We do this here before they are
         # converted to properties.
-        self._config_values = [
-            (config.name, ConfigInput.make_key_name(config.name))
-            for _, config in flow._get_parameters()
-            if config.IS_CONFIG_PARAMETER
-        ]
+        self._set_config_values(
+            [
+                (config.name, ConfigInput.make_key_name(config.name))
+                for _, config in flow._get_parameters()
+                if config.IS_CONFIG_PARAMETER
+            ]
+        )
 
         self.card_options = self.attributes["options"]
 
@@ -159,15 +186,11 @@ class CardDecorator(StepDecorator):
 
         # If we have configs, we need to dump them to a file so we can re-use them
         # when calling the card creation subprocess.
-        if self._config_values:
-            with tempfile.NamedTemporaryFile(
-                mode="w", encoding="utf-8", delete=False
-            ) as config_file:
-                config_value = dump_config_values(flow)
-                json.dump(config_value, config_file)
-                self._config_file_name = config_file.name
-        else:
-            self._config_file_name = None
+        # Since a step can contain multiple card decorators, and all the card creation processes
+        # will reference the same config file (because of how the CardCreator is created (only single class instance)),
+        # we need to ensure that a single config file is being referenced for all card create commands.
+        # This config file will be removed when the last card decorator has finished creating its card.
+        self._set_config_file_name(flow)
 
         card_type = self.attributes["type"]
         card_class = get_card_class(card_type)
@@ -246,12 +269,7 @@ class CardDecorator(StepDecorator):
             self.card_creator.create(mode="render", final=True, **create_options)
             self.card_creator.create(mode="refresh", final=True, **create_options)
 
-        # Unlink the config file if it exists
-        if self._config_file_name:
-            try:
-                os.unlink(self._config_file_name)
-            except Exception as e:
-                pass
+        self._cleanup(step_name)
 
     @staticmethod
     def _options(mapping):
@@ -286,3 +304,18 @@ class CardDecorator(StepDecorator):
             top_level_options["local-config-file"] = self._config_file_name
 
         return list(self._options(top_level_options))
+
+    def task_exception(
+        self, exception, step_name, flow, graph, retry_count, max_user_code_retries
+    ):
+        self._cleanup(step_name)
+
+    def _cleanup(self, step_name):
+        self._increment_completed_counter()
+        if self.task_finished_decos == self.total_decos_on_step[step_name]:
+            # Unlink the config file if it exists
+            if self._config_file_name:
+                try:
+                    os.unlink(self._config_file_name)
+                except Exception as e:
+                    pass

metaflow/plugins/cards/card_modules/basic.py

@@ -1,9 +1,10 @@
 import base64
 import json
 import os
-from .card import MetaflowCard, MetaflowCardComponent
+from .card import MetaflowCard, MetaflowCardComponent, with_default_component_id
 from .convert_to_native_type import TaskToDict
 import uuid
+import inspect
 
 ABS_DIR_PATH = os.path.dirname(os.path.abspath(__file__))
 RENDER_TEMPLATE_PATH = os.path.join(ABS_DIR_PATH, "base.html")
@@ -236,9 +237,28 @@ class LogComponent(DefaultComponent):
         super().__init__(title=None, subtitle=None)
         self._data = data
 
+    @with_default_component_id
     def render(self):
         datadict = super().render()
         datadict["data"] = self._data
+        if self.component_id is not None:
+            datadict["id"] = self.component_id
+        return datadict
+
+
+class PythonCodeComponent(DefaultComponent):
+
+    type = "pythonCode"
+
+    def __init__(self, data=None):
+        super().__init__(title=None, subtitle=None)
+        self._data = data
+
+    def render(self):
+        datadict = super().render()
+        datadict["data"] = self._data
+        if self.component_id is not None:
+            datadict["id"] = self.component_id
         return datadict
 
 
@@ -343,6 +363,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         graph=None,
         components=[],
         runtime=False,
+        flow=None,
     ):
         self._task = task
         self._only_repr = only_repr
@@ -352,6 +373,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         self.final_component = None
         self.page_component = None
         self.runtime = runtime
+        self.flow = flow
 
     def render(self):
         """
@@ -475,6 +497,16 @@ class TaskInfoComponent(MetaflowCardComponent):
             contents=[param_component],
         ).render()
 
+        step_func = getattr(self.flow, self._task.parent.id)
+        code_table = SectionComponent(
+            title="Task Code",
+            contents=[
+                TableComponent(
+                    data=[[PythonCodeComponent(inspect.getsource(step_func)).render()]]
+                )
+            ],
+        ).render()
+
         # Don't include parameter ids + "name" in the task artifacts
         artifactlist = [
             task_data_dict["data"][k]
@@ -500,6 +532,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         page_contents.extend(
             [
                 metadata_table,
+                code_table,
                 parameter_table,
                 artifact_section,
             ]
@@ -546,7 +579,7 @@ class ErrorCard(MetaflowCard):
 
     RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ONCHANGE
 
-    def __init__(self, options={}, components=[], graph=None):
+    def __init__(self, options={}, components=[], graph=None, **kwargs):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._components = components
@@ -602,9 +635,17 @@ class DefaultCardJSON(MetaflowCard):
 
     type = "default_json"
 
-    def __init__(self, options=dict(only_repr=True), components=[], graph=None):
+    def __init__(
+        self,
+        options=dict(only_repr=True),
+        components=[],
+        graph=None,
+        flow=None,
+        **kwargs
+    ):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
+        self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
         self._components = components
@@ -615,6 +656,7 @@ class DefaultCardJSON(MetaflowCard):
             only_repr=self._only_repr,
             graph=self._graph,
             components=self._components,
+            flow=self._flow,
         ).render()
         return json.dumps(final_component_dict)
 
@@ -629,9 +671,17 @@ class DefaultCard(MetaflowCard):
 
     type = "default"
 
-    def __init__(self, options=dict(only_repr=True), components=[], graph=None):
+    def __init__(
+        self,
+        options=dict(only_repr=True),
+        components=[],
+        graph=None,
+        flow=None,
+        **kwargs
+    ):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
+        self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
         self._components = components
@@ -646,6 +696,7 @@ class DefaultCard(MetaflowCard):
             graph=self._graph,
             components=self._components,
             runtime=runtime,
+            flow=self._flow,
         ).render()
         pt = self._get_mustache()
         data_dict = dict(
@@ -688,7 +739,7 @@ class BlankCard(MetaflowCard):
 
     type = "blank"
 
-    def __init__(self, options=dict(title=""), components=[], graph=None):
+    def __init__(self, options=dict(title=""), components=[], graph=None, **kwargs):
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._title = ""
         if "title" in options:

metaflow/plugins/cards/card_modules/card.py

@@ -1,4 +1,5 @@
 from typing import TYPE_CHECKING
+import uuid
 
 if TYPE_CHECKING:
     import metaflow
@@ -66,7 +67,7 @@ class MetaflowCard(object):
     # FIXME document runtime_data
     runtime_data = None
 
-    def __init__(self, options={}, components=[], graph=None):
+    def __init__(self, options={}, components=[], graph=None, flow=None):
         pass
 
     def _get_mustache(self):
@@ -140,3 +141,17 @@ class MetaflowCardComponent(object):
         `render` returns a string or dictionary. This class can be called on the client side to dynamically add components to the `MetaflowCard`
         """
         raise NotImplementedError()
+
+
+def create_component_id(component):
+    uuid_bit = "".join(uuid.uuid4().hex.split("-"))[:6]
+    return type(component).__name__.lower() + "_" + uuid_bit
+
+
+def with_default_component_id(func):
+    def ret_func(self, *args, **kwargs):
+        if self.component_id is None:
+            self.component_id = create_component_id(self)
+        return func(self, *args, **kwargs)
+
+    return ret_func

metaflow/plugins/cards/card_modules/components.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Optional, Union
+from typing import Any, List, Optional, Union, Callable
 from .basic import (
     LogComponent,
     ErrorComponent,
@@ -7,25 +7,13 @@ from .basic import (
     ImageComponent,
     SectionComponent,
     MarkdownComponent,
+    PythonCodeComponent,
 )
-from .card import MetaflowCardComponent
+from .card import MetaflowCardComponent, with_default_component_id
 from .convert_to_native_type import TaskToDict, _full_classname
 from .renderer_tools import render_safely
 import uuid
-
-
-def create_component_id(component):
-    uuid_bit = "".join(uuid.uuid4().hex.split("-"))[:6]
-    return type(component).__name__.lower() + "_" + uuid_bit
-
-
-def with_default_component_id(func):
-    def ret_func(self, *args, **kwargs):
-        if self.component_id is None:
-            self.component_id = create_component_id(self)
-        return func(self, *args, **kwargs)
-
-    return ret_func
+import inspect
 
 
 def _warning_with_component(component, msg):
@@ -823,3 +811,63 @@ class VegaChart(UserComponent):
         if self._chart_inside_table and "autosize" not in self._spec:
             data["spec"]["autosize"] = "fit-x"
         return data
+
+
+class PythonCode(UserComponent):
+    """
+    A component to display Python code with syntax highlighting.
+
+    Example:
+    ```python
+    @card
+    @step
+    def my_step(self):
+        # Using code_func
+        def my_function():
+            x = 1
+            y = 2
+            return x + y
+        current.card.append(
+            PythonCode(my_function)
+        )
+
+        # Using code_string
+        code = '''
+        def another_function():
+            return "Hello World"
+        '''
+        current.card.append(
+            PythonCode(code_string=code)
+        )
+    ```
+
+    Parameters
+    ----------
+    code_func : Callable[..., Any], optional, default None
+        The function whose source code should be displayed.
+    code_string : str, optional, default None
+        A string containing Python code to display.
+        Either code_func or code_string must be provided.
+    """
+
+    def __init__(
+        self,
+        code_func: Optional[Callable[..., Any]] = None,
+        code_string: Optional[str] = None,
+    ):
+        if code_func is not None:
+            self._code_string = inspect.getsource(code_func)
+        else:
+            self._code_string = code_string
+
+    @with_default_component_id
+    @render_safely
+    def render(self):
+        if self._code_string is None:
+            return ErrorComponent(
+                "`PythonCode` component requires a `code_func` or `code_string` argument. ",
+                "None provided for both",
+            ).render()
+        _code_component = PythonCodeComponent(self._code_string)
+        _code_component.component_id = self.component_id
+        return _code_component.render()