PyPI - ob-metaflow - Versions diffs - 2.10.7.4__py2.py3-none-any.whl → 2.10.9.1__py2.py3-none-any.whl - Mend

ob-metaflow 2.10.7.4py2.py3-none-any.whl → 2.10.9.1py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of ob-metaflow might be problematic. Click here for more details.

Files changed (52) hide show

metaflow/cards.py +2 -0
metaflow/decorators.py +1 -1
metaflow/metaflow_config.py +2 -0
metaflow/plugins/__init__.py +4 -0
metaflow/plugins/airflow/airflow_cli.py +1 -1
metaflow/plugins/argo/argo_workflows_cli.py +1 -1
metaflow/plugins/aws/aws_utils.py +1 -1
metaflow/plugins/aws/batch/batch.py +4 -0
metaflow/plugins/aws/batch/batch_cli.py +3 -0
metaflow/plugins/aws/batch/batch_client.py +40 -11
metaflow/plugins/aws/batch/batch_decorator.py +1 -0
metaflow/plugins/aws/step_functions/step_functions.py +1 -0
metaflow/plugins/aws/step_functions/step_functions_cli.py +1 -1
metaflow/plugins/azure/azure_exceptions.py +1 -1
metaflow/plugins/cards/card_cli.py +413 -28
metaflow/plugins/cards/card_client.py +16 -7
metaflow/plugins/cards/card_creator.py +228 -0
metaflow/plugins/cards/card_datastore.py +124 -26
metaflow/plugins/cards/card_decorator.py +40 -86
metaflow/plugins/cards/card_modules/base.html +12 -0
metaflow/plugins/cards/card_modules/basic.py +74 -8
metaflow/plugins/cards/card_modules/bundle.css +1 -170
metaflow/plugins/cards/card_modules/card.py +65 -0
metaflow/plugins/cards/card_modules/components.py +446 -81
metaflow/plugins/cards/card_modules/convert_to_native_type.py +9 -3
metaflow/plugins/cards/card_modules/main.js +250 -21
metaflow/plugins/cards/card_modules/test_cards.py +117 -0
metaflow/plugins/cards/card_resolver.py +0 -2
metaflow/plugins/cards/card_server.py +361 -0
metaflow/plugins/cards/component_serializer.py +506 -42
metaflow/plugins/cards/exception.py +20 -1
metaflow/plugins/datastores/azure_storage.py +1 -2
metaflow/plugins/datastores/gs_storage.py +1 -2
metaflow/plugins/datastores/s3_storage.py +2 -1
metaflow/plugins/datatools/s3/s3.py +24 -11
metaflow/plugins/env_escape/client.py +2 -12
metaflow/plugins/env_escape/client_modules.py +18 -14
metaflow/plugins/env_escape/server.py +18 -11
metaflow/plugins/env_escape/utils.py +12 -0
metaflow/plugins/gcp/gs_exceptions.py +1 -1
metaflow/plugins/gcp/gs_utils.py +1 -1
metaflow/plugins/pypi/conda_environment.py +5 -6
metaflow/plugins/pypi/pip.py +2 -2
metaflow/plugins/pypi/utils.py +15 -0
metaflow/task.py +1 -0
metaflow/version.py +1 -1
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/METADATA +1 -1
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/RECORD +52 -50
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/LICENSE +0 -0
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/WHEEL +0 -0
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/entry_points.txt +0 -0
{ob_metaflow-2.10.7.4.dist-info → ob_metaflow-2.10.9.1.dist-info}/top_level.txt +0 -0

metaflow/plugins/cards/card_modules/test_cards.py CHANGED Viewed

@@ -1,13 +1,20 @@
+import json
 from .card import MetaflowCard, MetaflowCardComponent
+from .renderer_tools import render_safely
 class TestStringComponent(MetaflowCardComponent):
+    REALTIME_UPDATABLE = True
     def __init__(self, text):
         self._text = text
     def render(self):
         return str(self._text)
+    def update(self, text):
+        self._text = text
 class TestPathSpecCard(MetaflowCard):
     type = "test_pathspec_card"
@@ -98,3 +105,113 @@ class TestTimeoutCard(MetaflowCard):
         time.sleep(self._timeout)
         return "%s" % task.pathspec
+REFRESHABLE_HTML_TEMPLATE = """
+<html>
+<script>
+var METAFLOW_RELOAD_TOKEN = "[METAFLOW_RELOAD_TOKEN]"
+window.metaflow_card_update = function(data) {
+    document.querySelector("h1").innerHTML = JSON.stringify(data);
+}
+</script>
+<h1>[PATHSPEC]</h1>
+<h1>[REPLACE_CONTENT_HERE]</h1>
+</html>
+"""
+class TestJSONComponent(MetaflowCardComponent):
+    REALTIME_UPDATABLE = True
+    def __init__(self, data):
+        self._data = data
+    @render_safely
+    def render(self):
+        return self._data
+    def update(self, data):
+        self._data = data
+class TestRefreshCard(MetaflowCard):
+    """
+    This card takes no components and helps test the `current.card.refresh(data)` interface.
+    """
+    HTML_TEMPLATE = REFRESHABLE_HTML_TEMPLATE
+    RUNTIME_UPDATABLE = True
+    ALLOW_USER_COMPONENTS = True
+    # Not implementing Reload Policy here since the reload Policy is set to always
+    RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ALWAYS
+    type = "test_refresh_card"
+    def render(self, task) -> str:
+        return self._render_func(task, self.runtime_data)
+    def _render_func(self, task, data):
+        return self.HTML_TEMPLATE.replace(
+            "[REPLACE_CONTENT_HERE]", json.dumps(data["user"])
+        ).replace("[PATHSPEC]", task.pathspec)
+    def render_runtime(self, task, data):
+        return self._render_func(task, data)
+    def refresh(self, task, data):
+        return data
+import hashlib
+def _component_values_to_hash(components):
+    comma_str = ",".join(["".join(x) for v in components.values() for x in v])
+    return hashlib.sha256(comma_str.encode("utf-8")).hexdigest()
+class TestRefreshComponentCard(MetaflowCard):
+    """
+    This card takes components and helps test the `current.card.components["A"].update()`
+    interface
+    """
+    HTML_TEMPLATE = REFRESHABLE_HTML_TEMPLATE
+    RUNTIME_UPDATABLE = True
+    ALLOW_USER_COMPONENTS = True
+    # Not implementing Reload Policy here since the reload Policy is set to always
+    RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ONCHANGE
+    type = "test_component_refresh_card"
+    def __init__(self, options={}, components=[], graph=None):
+        self._components = components
+    def render(self, task) -> str:
+        # Calling `render`/`render_runtime` wont require the `data` object
+        return self.HTML_TEMPLATE.replace(
+            "[REPLACE_CONTENT_HERE]", json.dumps(self._components)
+        ).replace("[PATHSPEC]", task.pathspec)
+    def render_runtime(self, task, data):
+        return self.render(task)
+    def refresh(self, task, data):
+        # Govers the information passed in the data update
+        return data["components"]
+    def reload_content_token(self, task, data):
+        if task.finished:
+            return "final"
+        return "runtime-%s" % _component_values_to_hash(data["components"])

metaflow/plugins/cards/card_resolver.py CHANGED Viewed

@@ -1,5 +1,3 @@
-from collections import namedtuple
 from .card_datastore import CardDatastore

metaflow/plugins/cards/card_server.py ADDED Viewed

@@ -0,0 +1,361 @@
+import os
+import json
+from http.server import BaseHTTPRequestHandler
+from threading import Thread
+from multiprocessing import Pipe
+from multiprocessing.connection import Connection
+from urllib.parse import urlparse
+import time
+try:
+    from http.server import ThreadingHTTPServer
+except ImportError:
+    from socketserver import ThreadingMixIn
+    from http.server import HTTPServer
+    class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+        daemon_threads = True
+from .card_client import CardContainer
+from .exception import CardNotPresentException
+from .card_resolver import resolve_paths_from_task
+from metaflow.metaflow_config import DATASTORE_LOCAL_DIR
+from metaflow import namespace
+from metaflow.exception import (
+    CommandException,
+    MetaflowNotFound,
+    MetaflowNamespaceMismatch,
+)
+VIEWER_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "card_viewer", "viewer.html"
+)
+CARD_VIEWER_HTML = open(VIEWER_PATH).read()
+TASK_CACHE = {}
+_ClickLogger = None
+class RunWatcher(Thread):
+    """
+    A thread that watches for new runs and sends the run_id to the
+    card server when a new run is detected. It observes the `latest_run`
+    file in the `.metaflow/<flowname>` directory.
+    """
+    def __init__(self, flow_name, connection: Connection):
+        super().__init__()
+        self._watch_file = os.path.join(
+            os.getcwd(), DATASTORE_LOCAL_DIR, flow_name, "latest_run"
+        )
+        self._current_run_id = self.get_run_id()
+        self.daemon = True
+        self._connection = connection
+    def get_run_id(self):
+        if not os.path.exists(self._watch_file):
+            return None
+        with open(self._watch_file, "r") as f:
+            return f.read().strip()
+    def watch(self):
+        while True:
+            run_id = self.get_run_id()
+            if run_id != self._current_run_id:
+                self._current_run_id = run_id
+                self._connection.send(run_id)
+            time.sleep(2)
+    def run(self):
+        self.watch()
+class CardServerOptions:
+    def __init__(
+        self,
+        flow_name,
+        run_object,
+        only_running,
+        follow_resumed,
+        flow_datastore,
+        follow_new_runs,
+        max_cards=20,
+        poll_interval=5,
+    ):
+        from metaflow import Run
+        self.RunClass = Run
+        self.run_object = run_object
+        self.flow_name = flow_name
+        self.only_running = only_running
+        self.follow_resumed = follow_resumed
+        self.flow_datastore = flow_datastore
+        self.max_cards = max_cards
+        self.follow_new_runs = follow_new_runs
+        self.poll_interval = poll_interval
+        self._parent_conn, self._child_conn = Pipe()
+    def refresh_run(self):
+        if not self.follow_new_runs:
+            return False
+        if not self.parent_conn.poll():
+            return False
+        run_id = self.parent_conn.recv()
+        if run_id is None:
+            return False
+        namespace(None)
+        try:
+            self.run_object = self.RunClass(f"{self.flow_name}/{run_id}")
+            return True
+        except MetaflowNotFound:
+            return False
+    @property
+    def parent_conn(self):
+        return self._parent_conn
+    @property
+    def child_conn(self):
+        return self._child_conn
+def cards_for_task(
+    flow_datastore, task_pathspec, card_type=None, card_hash=None, card_id=None
+):
+    try:
+        paths, card_ds = resolve_paths_from_task(
+            flow_datastore,
+            task_pathspec,
+            type=card_type,
+            hash=card_hash,
+            card_id=card_id,
+        )
+    except CardNotPresentException:
+        return None
+    for card in CardContainer(paths, card_ds, origin_pathspec=None):
+        yield card
+def cards_for_run(
+    flow_datastore,
+    run_object,
+    only_running,
+    card_type=None,
+    card_hash=None,
+    card_id=None,
+    max_cards=20,
+):
+    curr_idx = 0
+    for step in run_object.steps():
+        for task in step.tasks():
+            if only_running and task.finished:
+                continue
+            card_generator = cards_for_task(
+                flow_datastore,
+                task.pathspec,
+                card_type=card_type,
+                card_hash=card_hash,
+                card_id=card_id,
+            )
+            if card_generator is None:
+                continue
+            for card in card_generator:
+                curr_idx += 1
+                if curr_idx >= max_cards:
+                    raise StopIteration
+                yield task.pathspec, card
+class CardViewerRoutes(BaseHTTPRequestHandler):
+    card_options: CardServerOptions = None
+    run_watcher: RunWatcher = None
+    def do_GET(self):
+        try:
+            _, path = self.path.split("/", 1)
+            try:
+                prefix, suffix = path.split("/", 1)
+            except:
+                prefix = path
+                suffix = None
+        except:
+            prefix = None
+        if prefix in self.ROUTES:
+            self.ROUTES[prefix](self, suffix)
+        else:
+            self._response(open(VIEWER_PATH).read().encode("utf-8"))
+    def get_runinfo(self, suffix):
+        run_id_changed = self.card_options.refresh_run()
+        if run_id_changed:
+            self.log_message(
+                "RunID changed in the background to %s"
+                % self.card_options.run_object.pathspec
+            )
+            _ClickLogger(
+                "RunID changed in the background to %s"
+                % self.card_options.run_object.pathspec,
+                fg="blue",
+            )
+        if self.card_options.run_object is None:
+            self._response(
+                {"status": "No Run Found", "flow": self.card_options.flow_name},
+                code=404,
+                is_json=True,
+            )
+            return
+        task_card_generator = cards_for_run(
+            self.card_options.flow_datastore,
+            self.card_options.run_object,
+            self.card_options.only_running,
+            max_cards=self.card_options.max_cards,
+        )
+        flow_name = self.card_options.run_object.parent.id
+        run_id = self.card_options.run_object.id
+        cards = []
+        for pathspec, card in task_card_generator:
+            step, task = pathspec.split("/")[-2:]
+            _task = self.card_options.run_object[step][task]
+            task_finished = True if _task.finished else False
+            cards.append(
+                dict(
+                    task=pathspec,
+                    label="%s/%s %s" % (step, task, card.hash),
+                    card_object=dict(
+                        hash=card.hash,
+                        type=card.type,
+                        path=card.path,
+                        id=card.id,
+                    ),
+                    finished=task_finished,
+                    card="%s/%s" % (pathspec, card.hash),
+                )
+            )
+        resp = {
+            "status": "ok",
+            "flow": flow_name,
+            "run_id": run_id,
+            "cards": cards,
+            "poll_interval": self.card_options.poll_interval,
+        }
+        self._response(resp, is_json=True)
+    def get_card(self, suffix):
+        _suffix = urlparse(self.path).path
+        _, flow, run_id, step, task_id, card_hash = _suffix.strip("/").split("/")
+        pathspec = "/".join([flow, run_id, step, task_id])
+        cards = list(
+            cards_for_task(
+                self.card_options.flow_datastore, pathspec, card_hash=card_hash
+            )
+        )
+        if len(cards) == 0:
+            self._response({"status": "Card Not Found"}, code=404)
+            return
+        selected_card = cards[0]
+        self._response(selected_card.get().encode("utf-8"))
+    def get_data(self, suffix):
+        _suffix = urlparse(self.path).path
+        _, flow, run_id, step, task_id, card_hash = _suffix.strip("/").split("/")
+        pathspec = "/".join([flow, run_id, step, task_id])
+        cards = list(
+            cards_for_task(
+                self.card_options.flow_datastore, pathspec, card_hash=card_hash
+            )
+        )
+        if len(cards) == 0:
+            self._response(
+                {
+                    "status": "Card Not Found",
+                },
+                is_json=True,
+                code=404,
+            )
+            return
+        status = "ok"
+        try:
+            task_object = self.card_options.run_object[step][task_id]
+        except KeyError:
+            return self._response(
+                {"status": "Task Not Found", "is_complete": False},
+                is_json=True,
+                code=404,
+            )
+        is_complete = task_object.finished
+        selected_card = cards[0]
+        card_data = selected_card.get_data()
+        if card_data is not None:
+            self.log_message(
+                "Task Success: %s, Task Finished: %s"
+                % (task_object.successful, is_complete)
+            )
+            if not task_object.successful and is_complete:
+                status = "Task Failed"
+            self._response(
+                {"status": status, "payload": card_data, "is_complete": is_complete},
+                is_json=True,
+            )
+        else:
+            self._response(
+                {"status": "ok", "is_complete": is_complete},
+                is_json=True,
+                code=404,
+            )
+    def _response(self, body, is_json=False, code=200):
+        self.send_response(code)
+        mime = "application/json" if is_json else "text/html"
+        self.send_header("Content-type", mime)
+        self.end_headers()
+        if is_json:
+            self.wfile.write(json.dumps(body).encode("utf-8"))
+        else:
+            self.wfile.write(body)
+    ROUTES = {"runinfo": get_runinfo, "card": get_card, "data": get_data}
+def _is_debug_mode():
+    debug_flag = os.environ.get("METAFLOW_DEBUG_CARD_SERVER")
+    if debug_flag is None:
+        return False
+    return debug_flag.lower() in ["true", "1"]
+def create_card_server(card_options: CardServerOptions, port, ctx_obj):
+    CardViewerRoutes.card_options = card_options
+    global _ClickLogger
+    _ClickLogger = ctx_obj.echo
+    if card_options.follow_new_runs:
+        CardViewerRoutes.run_watcher = RunWatcher(
+            card_options.flow_name, card_options.child_conn
+        )
+        CardViewerRoutes.run_watcher.start()
+    server_addr = ("", port)
+    ctx_obj.echo(
+        "Starting card server on port %d " % (port),
+        fg="green",
+        bold=True,
+    )
+    # Disable logging if not in debug mode
+    if not _is_debug_mode():
+        CardViewerRoutes.log_request = lambda *args, **kwargs: None
+        CardViewerRoutes.log_message = lambda *args, **kwargs: None
+    server = ThreadingHTTPServer(server_addr, CardViewerRoutes)
+    server.serve_forever()

ob-metaflow 2.10.7.4__py2.py3-none-any.whl → 2.10.9.1__py2.py3-none-any.whl

Potentially problematic release.

ob-metaflow 2.10.7.4py2.py3-none-any.whl → 2.10.9.1py2.py3-none-any.whl