metaflow 2.13.1__py2.py3-none-any.whl → 2.13.3__py2.py3-none-any.whl

metaflow/cards.py

@@ -8,6 +8,7 @@ from metaflow.plugins.cards.card_modules.components import (
     Markdown,
     VegaChart,
     ProgressBar,
+    PythonCode,
 )
 from metaflow.plugins.cards.card_modules.basic import (
     DefaultCard,

metaflow/extension_support/plugins.py

@@ -197,6 +197,7 @@ _plugin_categories = {
     "cli": lambda x: (
         list(x.commands)[0] if len(x.commands) == 1 else "too many commands"
     ),
+    "runner_cli": lambda x: x.name,
 }
 
 

metaflow/parameters.py

@@ -369,7 +369,7 @@ class Parameter(object):
         # Resolve any value from configurations
         self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
         # Do it one item at a time so errors are ignored at that level (as opposed to
-        # at the entire kwargs leve)
+        # at the entire kwargs level)
         self.kwargs = {
             k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
             for k, v in self.kwargs.items()

metaflow/plugins/__init__.py

@@ -19,6 +19,11 @@ CLIS_DESC = [
     ("logs", ".logs_cli.cli"),
 ]
 
+# Add additional commands to the runner here
+# These will be accessed using Runner().<command>()
+RUNNER_CLIS_DESC = []
+
+
 from .test_unbounded_foreach_decorator import InternalTestUnboundedForeachInput
 
 # Add new step decorators here
@@ -168,6 +173,14 @@ def get_plugin_cli_path():
     return resolve_plugins("cli", path_only=True)
 
 
+def get_runner_cli():
+    return resolve_plugins("runner_cli")
+
+
+def get_runner_cli_path():
+    return resolve_plugins("runner_cli", path_only=True)
+
+
 STEP_DECORATORS = resolve_plugins("step_decorator")
 FLOW_DECORATORS = resolve_plugins("flow_decorator")
 ENVIRONMENTS = resolve_plugins("environment")

metaflow/plugins/argo/argo_workflows.py

@@ -743,6 +743,17 @@ class ArgoWorkflows(object):
                     )
                 }
             )
+        try:
+            # Build the DAG based on the DAGNodes given by the FlowGraph for the found FlowSpec class.
+            _steps_info, graph_structure = self.graph.output_steps()
+            graph_info = {
+                # for the time being, we only need the graph_structure. Being mindful of annotation size limits we do not include anything extra.
+                "graph_structure": graph_structure
+            }
+        except Exception:
+            graph_info = None
+
+        dag_annotation = {"metaflow/dag": json.dumps(graph_info)}
 
         return (
             WorkflowTemplate()
@@ -758,6 +769,7 @@ class ArgoWorkflows(object):
                 .annotations(self._base_annotations)
                 .labels(self._base_labels)
                 .label("app.kubernetes.io/name", "metaflow-flow")
+                .annotations(dag_annotation)
             )
             .spec(
                 WorkflowSpec()

metaflow/plugins/cards/card_cli.py

@@ -691,10 +691,15 @@ def create(
         try:
             if options is not None:
                 mf_card = filtered_card(
-                    options=options, components=component_arr, graph=graph_dict
+                    options=options,
+                    components=component_arr,
+                    graph=graph_dict,
+                    flow=ctx.obj.flow,
                 )
             else:
-                mf_card = filtered_card(components=component_arr, graph=graph_dict)
+                mf_card = filtered_card(
+                    components=component_arr, graph=graph_dict, flow=ctx.obj.flow
+                )
         except TypeError as e:
             if render_error_card:
                 mf_card = None

metaflow/plugins/cards/card_modules/basic.py

@@ -1,9 +1,10 @@
 import base64
 import json
 import os
-from .card import MetaflowCard, MetaflowCardComponent
+from .card import MetaflowCard, MetaflowCardComponent, with_default_component_id
 from .convert_to_native_type import TaskToDict
 import uuid
+import inspect
 
 ABS_DIR_PATH = os.path.dirname(os.path.abspath(__file__))
 RENDER_TEMPLATE_PATH = os.path.join(ABS_DIR_PATH, "base.html")
@@ -236,9 +237,28 @@ class LogComponent(DefaultComponent):
         super().__init__(title=None, subtitle=None)
         self._data = data
 
+    @with_default_component_id
     def render(self):
         datadict = super().render()
         datadict["data"] = self._data
+        if self.component_id is not None:
+            datadict["id"] = self.component_id
+        return datadict
+
+
+class PythonCodeComponent(DefaultComponent):
+
+    type = "pythonCode"
+
+    def __init__(self, data=None):
+        super().__init__(title=None, subtitle=None)
+        self._data = data
+
+    def render(self):
+        datadict = super().render()
+        datadict["data"] = self._data
+        if self.component_id is not None:
+            datadict["id"] = self.component_id
         return datadict
 
 
@@ -343,6 +363,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         graph=None,
         components=[],
         runtime=False,
+        flow=None,
     ):
         self._task = task
         self._only_repr = only_repr
@@ -352,6 +373,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         self.final_component = None
         self.page_component = None
         self.runtime = runtime
+        self.flow = flow
 
     def render(self):
         """
@@ -475,6 +497,16 @@ class TaskInfoComponent(MetaflowCardComponent):
             contents=[param_component],
         ).render()
 
+        step_func = getattr(self.flow, self._task.parent.id)
+        code_table = SectionComponent(
+            title="Task Code",
+            contents=[
+                TableComponent(
+                    data=[[PythonCodeComponent(inspect.getsource(step_func)).render()]]
+                )
+            ],
+        ).render()
+
         # Don't include parameter ids + "name" in the task artifacts
         artifactlist = [
             task_data_dict["data"][k]
@@ -500,6 +532,7 @@ class TaskInfoComponent(MetaflowCardComponent):
         page_contents.extend(
             [
                 metadata_table,
+                code_table,
                 parameter_table,
                 artifact_section,
             ]
@@ -546,7 +579,7 @@ class ErrorCard(MetaflowCard):
 
     RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ONCHANGE
 
-    def __init__(self, options={}, components=[], graph=None):
+    def __init__(self, options={}, components=[], graph=None, **kwargs):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._components = components
@@ -602,9 +635,17 @@ class DefaultCardJSON(MetaflowCard):
 
     type = "default_json"
 
-    def __init__(self, options=dict(only_repr=True), components=[], graph=None):
+    def __init__(
+        self,
+        options=dict(only_repr=True),
+        components=[],
+        graph=None,
+        flow=None,
+        **kwargs
+    ):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
+        self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
         self._components = components
@@ -615,6 +656,7 @@ class DefaultCardJSON(MetaflowCard):
             only_repr=self._only_repr,
             graph=self._graph,
             components=self._components,
+            flow=self._flow,
         ).render()
         return json.dumps(final_component_dict)
 
@@ -629,9 +671,17 @@ class DefaultCard(MetaflowCard):
 
     type = "default"
 
-    def __init__(self, options=dict(only_repr=True), components=[], graph=None):
+    def __init__(
+        self,
+        options=dict(only_repr=True),
+        components=[],
+        graph=None,
+        flow=None,
+        **kwargs
+    ):
         self._only_repr = True
         self._graph = None if graph is None else transform_flow_graph(graph)
+        self._flow = flow
         if "only_repr" in options:
             self._only_repr = options["only_repr"]
         self._components = components
@@ -646,6 +696,7 @@ class DefaultCard(MetaflowCard):
             graph=self._graph,
             components=self._components,
             runtime=runtime,
+            flow=self._flow,
         ).render()
         pt = self._get_mustache()
         data_dict = dict(
@@ -688,7 +739,7 @@ class BlankCard(MetaflowCard):
 
     type = "blank"
 
-    def __init__(self, options=dict(title=""), components=[], graph=None):
+    def __init__(self, options=dict(title=""), components=[], graph=None, **kwargs):
         self._graph = None if graph is None else transform_flow_graph(graph)
         self._title = ""
         if "title" in options:

metaflow/plugins/cards/card_modules/card.py

@@ -1,4 +1,5 @@
 from typing import TYPE_CHECKING
+import uuid
 
 if TYPE_CHECKING:
     import metaflow
@@ -66,7 +67,7 @@ class MetaflowCard(object):
     # FIXME document runtime_data
     runtime_data = None
 
-    def __init__(self, options={}, components=[], graph=None):
+    def __init__(self, options={}, components=[], graph=None, flow=None):
         pass
 
     def _get_mustache(self):
@@ -140,3 +141,17 @@ class MetaflowCardComponent(object):
         `render` returns a string or dictionary. This class can be called on the client side to dynamically add components to the `MetaflowCard`
         """
         raise NotImplementedError()
+
+
+def create_component_id(component):
+    uuid_bit = "".join(uuid.uuid4().hex.split("-"))[:6]
+    return type(component).__name__.lower() + "_" + uuid_bit
+
+
+def with_default_component_id(func):
+    def ret_func(self, *args, **kwargs):
+        if self.component_id is None:
+            self.component_id = create_component_id(self)
+        return func(self, *args, **kwargs)
+
+    return ret_func

metaflow/plugins/cards/card_modules/components.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Optional, Union
+from typing import Any, List, Optional, Union, Callable
 from .basic import (
     LogComponent,
     ErrorComponent,
@@ -7,25 +7,13 @@ from .basic import (
     ImageComponent,
     SectionComponent,
     MarkdownComponent,
+    PythonCodeComponent,
 )
-from .card import MetaflowCardComponent
+from .card import MetaflowCardComponent, with_default_component_id
 from .convert_to_native_type import TaskToDict, _full_classname
 from .renderer_tools import render_safely
 import uuid
-
-
-def create_component_id(component):
-    uuid_bit = "".join(uuid.uuid4().hex.split("-"))[:6]
-    return type(component).__name__.lower() + "_" + uuid_bit
-
-
-def with_default_component_id(func):
-    def ret_func(self, *args, **kwargs):
-        if self.component_id is None:
-            self.component_id = create_component_id(self)
-        return func(self, *args, **kwargs)
-
-    return ret_func
+import inspect
 
 
 def _warning_with_component(component, msg):
@@ -823,3 +811,63 @@ class VegaChart(UserComponent):
         if self._chart_inside_table and "autosize" not in self._spec:
             data["spec"]["autosize"] = "fit-x"
         return data
+
+
+class PythonCode(UserComponent):
+    """
+    A component to display Python code with syntax highlighting.
+
+    Example:
+    ```python
+    @card
+    @step
+    def my_step(self):
+        # Using code_func
+        def my_function():
+            x = 1
+            y = 2
+            return x + y
+        current.card.append(
+            PythonCode(my_function)
+        )
+
+        # Using code_string
+        code = '''
+        def another_function():
+            return "Hello World"
+        '''
+        current.card.append(
+            PythonCode(code_string=code)
+        )
+    ```
+
+    Parameters
+    ----------
+    code_func : Callable[..., Any], optional, default None
+        The function whose source code should be displayed.
+    code_string : str, optional, default None
+        A string containing Python code to display.
+        Either code_func or code_string must be provided.
+    """
+
+    def __init__(
+        self,
+        code_func: Optional[Callable[..., Any]] = None,
+        code_string: Optional[str] = None,
+    ):
+        if code_func is not None:
+            self._code_string = inspect.getsource(code_func)
+        else:
+            self._code_string = code_string
+
+    @with_default_component_id
+    @render_safely
+    def render(self):
+        if self._code_string is None:
+            return ErrorComponent(
+                "`PythonCode` component requires a `code_func` or `code_string` argument. ",
+                "None provided for both",
+            ).render()
+        _code_component = PythonCodeComponent(self._code_string)
+        _code_component.component_id = self.component_id
+        return _code_component.render()