metaflow 2.10.9__tar.gz → 2.10.11__tar.gz

{metaflow-2.10.9 → metaflow-2.10.11}/MANIFEST.in

@@ -1,4 +1,5 @@
 include LICENSE
 include metaflow/plugins/cards/card_modules/*.html
 include metaflow/plugins/cards/card_modules/*.js
-include metaflow/plugins/cards/card_modules/*.css
+include metaflow/plugins/cards/card_modules/*.css
+include metaflow/plugins/cards/card_viewer/*.html

{metaflow-2.10.9/metaflow.egg-info → metaflow-2.10.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.10.9
+Version: 2.10.11
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org

{metaflow-2.10.9 → metaflow-2.10.11}/metaflow/plugins/cards/card_modules/basic.py

@@ -313,7 +313,14 @@ class MarkdownComponent(DefaultComponent):
 
     def render(self):
         datadict = super().render()
-        datadict["source"] = self._text
+        _text = self._text
+        # When we have a markdown text that doesn't start with a `#`,
+        # we need to add a newline to make sure that the markdown
+        # is rendered correctly. Otherwise `svelte-markdown` will render
+        # the empty `<p>` tags during re-renders on card data updates.
+        if self._text is not None and not self._text.startswith("#"):
+            _text = "\n" + _text
+        datadict["source"] = _text
         if self.component_id is not None:
             datadict["id"] = self.component_id
         return datadict

{metaflow-2.10.9 → metaflow-2.10.11}/metaflow/plugins/cards/card_modules/components.py

@@ -712,8 +712,16 @@ class ProgressBar(UserComponent):
 
     Parameters
     ----------
-    text : str
-        Text formatted in Markdown.
+    max : int
+        The maximum value of the progress bar.
+    label : str, optional
+        Optional label for the progress bar.
+    value : int, optional
+        Optional initial value of the progress bar.
+    unit : str, optional
+        Optional unit for the progress bar.
+    metadata : str, optional
+        Optional additional information to show on the progress bar.
     """
 
     type = "progressBar"
@@ -762,31 +770,41 @@ class VegaChart(UserComponent):
 
     REALTIME_UPDATABLE = True
 
-    def __init__(self, spec: dict, show_controls=False):
+    def __init__(self, spec: dict, show_controls: bool = False):
         self._spec = spec
         self._show_controls = show_controls
         self._chart_inside_table = False
 
     def update(self, spec=None):
-        if spec is not None:
-            self._spec = spec
+        """
+        Update the chart.
 
-    @classmethod
-    def from_altair_chart(cls, altair_chart):
-        from metaflow.plugins.cards.card_modules.convert_to_native_type import (
-            _full_classname,
-        )
+        Parameters
+        ----------
+        spec : dict or altair.Chart
+            The updated chart spec or an altair Chart Object.
+        """
+        _spec = spec
+        if self._object_is_altair_chart(spec):
+            _spec = spec.to_dict()
+        if _spec is not None:
+            self._spec = _spec
 
+    @staticmethod
+    def _object_is_altair_chart(altair_chart):
         # This will feel slightly hacky but I am unable to find a natural way of determining the class
-        # name of the Altair chart. The only way I can think of is to use the full class name and then
+        # name of the Altair chart. The only way simple way is to extract the full class name and then
         # match with heuristics
-
         fulclsname = _full_classname(altair_chart)
         if not all([x in fulclsname for x in ["altair", "vegalite", "Chart"]]):
-            raise ValueError(fulclsname + " is not an altair chart")
+            return False
+        return True
 
+    @classmethod
+    def from_altair_chart(cls, altair_chart):
+        if not cls._object_is_altair_chart(altair_chart):
+            raise ValueError(_full_classname(altair_chart) + " is not an altair chart")
         altair_chart_dict = altair_chart.to_dict()
-
         cht = cls(spec=altair_chart_dict)
         return cht
 

metaflow-2.10.11/metaflow/plugins/cards/card_viewer/viewer.html

@@ -0,0 +1,344 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Metaflow Run</title>
+    <style>
+      @import url("https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap");
+
+      body {
+        font-size: 14px;
+        font-family: "Roboto", -apple-system, BlinkMacSystemFont, "Segoe UI",
+          "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans",
+          "Helvetica Neue", sans-serif;
+        padding: 0;
+        margin: 0;
+        display: flex;
+        flex-direction: column;
+        height: 100vh;
+        width: 100%;
+        overflow: hidden;
+        color: #31302f;
+      }
+
+      header {
+        height: 65px;
+        flex: 0 1 auto;
+        flex-grow: none;
+        background: #f9f9f9;
+        display: flex;
+        justify-content: space-evenly;
+      }
+
+      h1,
+      h2,
+      h3,
+      h4 {
+        font-size: inherit;
+        font-weight: 500;
+        margin: 0;
+        padding: 0;
+      }
+
+      header > * {
+        padding: 20px 25px;
+        flex: 1;
+        display: flex;
+        align-items: center;
+      }
+
+      header > div:nth-child(2) {
+        text-align: center;
+        flex: 1;
+        justify-content: center;
+      }
+
+      header > div:nth-child(3) {
+        text-align: right;
+        justify-content: flex-end;
+        font-weight: 400;
+      }
+
+      .taskStatus {
+        color: #6a6867;
+      }
+
+      .taskStatus.error {
+        color: #e67058;
+      }
+
+      .content {
+        flex: 1 1 auto;
+        overflow-y: scroll;
+      }
+
+      iframe {
+        width: 100%;
+        height: calc(100vh - 65px);
+        outline: none;
+        border: 0;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+      }
+
+      header select {
+        width: 100%;
+        border: 0;
+        outline: none;
+        font-size: inherit;
+        font-family: inherit;
+        background: transparent;
+        padding: 0 1rem;
+      }
+
+      .selectorContainer {
+        border: 0;
+        outline: none;
+        font-size: inherit;
+        font-family: inherit;
+        border-radius: 8px;
+        background: #ebeaea;
+        padding: 0 1rem;
+        margin: 1rem;
+      }
+
+      .logoContainer {
+        display: none !important;
+      }
+
+      .errorContainer svg {
+        position: relative;
+        top: 3px;
+      }
+    </style>
+  </head>
+  <body>
+    <header>
+      <h1 id="header-title">Metaflow Run</h1>
+      <div class="selectorContainer">
+        <select id="selector">
+          <option value="" disabled selected>
+            Select a card
+          </option>
+        </select>
+      </div>
+      <div>
+        <div id="task-status" class="taskStatus">Initializing</div>
+        <div id="task-action" class="" style="display: none">Pause</div>
+      </div>
+    </header>
+    <div class="content">
+      <iframe id="card-frame"></iframe>
+    </div>
+
+    <script type="application/javascript">
+      (() => {
+        // setup constants
+        let DATA_UPDATE_POLL_INTERVAL = 5000;
+        let RUN_UPDATE_POLL_INTERVAL = 10000;
+
+        // setup state
+        let currentCardsLength = 0;
+        let currentRunId = "";
+        let selectedCard = "";
+
+        // setup selector
+        const selector = document.getElementById("selector");
+        selector?.addEventListener("change", (event) => {
+          selectedCard = event.target.value;
+          handleLoadCard(event.target.name, event.target.value);
+        });
+
+        /**
+         * Handle any changes to the status of the page
+         */
+        function handleStatus(status, isComplete = false) {
+          const statusDiv = document.getElementById("task-status");
+
+          // change the text of the status depending if its in progress or not
+          if (statusDiv) {
+            // complete
+            if (isComplete && status === "ok") {
+              statusDiv.innerText = "Task Is Complete";
+              // running
+            } else if (!isComplete && status === "ok") {
+              statusDiv.innerText = "Task Is Running";
+              // error
+            } else {
+              statusDiv.innerHTML = `<div class="errorContainer"><svg width="17" height="17" viewBox="0 0 17 17" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clip-path="url(#clip0_164_815)"> <circle cx="8.5" cy="8.5" r="8" fill="#E67058"/> <path d="M11.6475 6.0575C11.8422 5.86282 11.8422 5.54718 11.6475 5.3525C11.4528 5.15782 11.1372 5.15782 10.9425 5.3525L8.5 7.795L6.0575 5.3525C5.86282 5.15782 5.54718 5.15782 5.3525 5.3525C5.15782 5.54718 5.15782 5.86282 5.3525 6.0575L7.795 8.5L5.3525 10.9425C5.15782 11.1372 5.15782 11.4528 5.3525 11.6475C5.54718 11.8422 5.86282 11.8422 6.0575 11.6475L8.5 9.205L10.9425 11.6475C11.1372 11.8422 11.4528 11.8422 11.6475 11.6475C11.8422 11.4528 11.8422 11.1372 11.6475 10.9425L9.205 8.5L11.6475 6.0575Z" fill="white"/> </g><defs><clipPath id="clip0_164_815"><rect width="16" height="16" fill="white" transform="translate(0.5 0.5)"/></clipPath></defs></svg>
+       <span>${status}</span></div>
+      `;
+            }
+          }
+        }
+
+        /**
+         * Handle any changes to the cards
+         */
+        function onCardsLoaded(runInfo) {
+          const { cards, flow, run_id } = runInfo;
+          if (!Array.isArray(cards) || !flow || !run_id) {
+            console.log("Data was not in the correct format");
+            return;
+          }
+          // console.log("Running ON CARDS LOADED", cards, flow, run_id)
+
+          // reset
+          currentCardsLength = cards.length;
+          selector.innerHTML = "";
+          // if the task has finished, change to the next card
+          changeToNewCardIfTaskHasFinished(runInfo);
+
+          // create select options for each card
+          cards.reverse().forEach((card) => {
+            const option = document.createElement("option");
+            option.value = card.card;
+            option.innerText = card.label.split(" ")?.[0] || card.label;
+            option.selected = card.card === selectedCard;
+            selector.appendChild(option);
+          });
+
+          // get card-title element and change it to name
+          const title = document.getElementById("header-title");
+          title.innerText = `${flow}/${run_id}`;
+        }
+
+        /**
+         * This function will determine if changes need to be made to the
+         * card and if so, will we do it by push or by reload
+         */
+
+        async function watchForCardChanges(iframe) {
+          // replace the /card/ url with the data url
+          const dataUri = iframe.src.replace(/.*\/card\//, "/data/");
+          let resp;
+          try {
+            const req = await fetch(dataUri);
+            resp = await req.json();
+            // setup UI to show complete and status
+            handleStatus(resp.status, resp?.is_complete);
+
+            // grab update fn from iframe
+            const update = iframe.contentWindow?.metaflow_card_update;
+            // make sure we're able to update the card
+            if (resp.status !== "ok" || !update || !resp.payload?.data) {
+              console.log(
+                "Data-updates not taking place because either : status is not `ok` or update function is not available or data is not available",
+                resp,
+                update
+              );
+            } else {
+              // make appropriate updates by pushing if the reload token matches
+              update &&
+              resp.payload?.reload_token ===
+                iframe.contentWindow.METAFLOW_RELOAD_TOKEN
+                ? update(resp.payload.data)
+                : iframe.contentWindow.location.reload();
+            }
+            // If there is no Update function than we don't need to wire the
+            // periodic updates
+            if (!update) {
+              console.log(
+                "No metaflow_card_update function available. So not running any periodic updates"
+              );
+              return;
+            }
+          } catch (error) {
+            console.error(error);
+          }
+          // if this update is not the last one, run it again!
+          if (!(resp.payload?.reload_token === "final")) {
+            setTimeout(
+              () => watchForCardChanges(iframe),
+              DATA_UPDATE_POLL_INTERVAL
+            );
+          }
+        }
+
+        /**
+         * Setup a new card and start the watcher
+         */
+        function handleLoadCard(name, value) {
+          document.getElementsByTagName("title").innerHTML = name;
+          const iframe = document.getElementById("card-frame");
+          iframe.src = `/card/${value}?embed=true`;
+          iframe.onload = () => {
+            watchForCardChanges(iframe);
+          };
+        }
+
+        function changeToNewCardIfTaskHasFinished(runInfo) {
+          // search for the selectedCard in the runInfo
+          const card = runInfo.cards.find((card) => card.card === selectedCard);
+          // if the card object contains `finished` === True, then change to the next card that has not finished
+          if (card?.finished) {
+            // find the index of the current card
+            const currentCardIndex = runInfo.cards.findIndex(
+              (card) => card.card === selectedCard
+            );
+            // find the next card that has not finished
+            const nextCard = runInfo.cards.find((card) => !card.finished);
+            // if there is a next card, change to it
+
+            console.log("nextCard", nextCard);
+            if (nextCard) {
+              selectedCard = nextCard.card;
+              handleLoadCard(nextCard.label, nextCard.card);
+            }
+          }
+        }
+
+        /**
+         * Main loop function
+         */
+        async function main() {
+          try {
+            // get info on the entire run
+            const runInfoRequest = await fetch("/runinfo");
+            const runInfo = await runInfoRequest.json();
+
+            // reset the state if the run_id changes or on first time
+            if (currentRunId !== runInfo.run_id) {
+              currentRunId = runInfo.run_id;
+              currentCardsLength = 0;
+              selectedCard = "";
+              console.log("runInfo", runInfo);
+            }
+            if (runInfo.poll_interval * 1000 !== DATA_UPDATE_POLL_INTERVAL) {
+              DATA_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
+            }
+            if (runInfo.poll_interval * 1000 !== RUN_UPDATE_POLL_INTERVAL) {
+              RUN_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
+            }
+            if (runInfo?.status === "ok") {
+              // only make changes if the cards length has changed
+              if (runInfo?.cards.length !== currentCardsLength) {
+                onCardsLoaded(runInfo);
+              }
+              // if there is no selected card, select the first one
+              if (!selectedCard) {
+                selectedCard = runInfo.cards[0].card;
+                handleLoadCard(runInfo.cards[0].label, runInfo.cards[0].card);
+              }
+            } else {
+              handleStatus(runInfo?.status, false);
+            }
+          } catch (error) {
+            console.error(error);
+          }
+          // always run the main loop, because a user can start a new run,
+          // or restart the web server, and we want the page to continue working
+          setTimeout(main, RUN_UPDATE_POLL_INTERVAL);
+        }
+
+        // start the main loop
+        window.onload = () => {
+          main();
+        };
+      })();
+    </script>
+  </body>
+</html>

{metaflow-2.10.9 → metaflow-2.10.11}/metaflow/plugins/pypi/conda_environment.py

@@ -1,3 +1,4 @@
+import copy
 import errno
 import fcntl
 import functools
@@ -10,17 +11,17 @@ from concurrent.futures import ThreadPoolExecutor
 from hashlib import sha256
 from io import BufferedIOBase
 from itertools import chain
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlparse
 
 import requests
 
-from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.exception import MetaflowException
+from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.metaflow_environment import MetaflowEnvironment
 from metaflow.metaflow_profile import profile
 
 from . import MAGIC_FILE, _datastore_packageroot
-from .utils import conda_platform, generate_cache_path, parse_filename_from_url
+from .utils import conda_platform
 
 
 class CondaEnvironmentException(MetaflowException):
@@ -104,10 +105,24 @@ class CondaEnvironment(MetaflowEnvironment):
             )
 
         def cache(storage, results, type_):
+            def _path(url, local_path):
+                # Special handling for VCS packages
+                if url.startswith("git+"):
+                    base, _ = os.path.split(urlparse(url).path)
+                    _, file = os.path.split(local_path)
+                    prefix = url.split("@")[-1]
+                    return urlparse(url).netloc + os.path.join(
+                        unquote(base), prefix, file
+                    )
+                else:
+                    return urlparse(url).netloc + urlparse(url).path
+
             local_packages = {
                 url: {
                     # Path to package in datastore.
-                    "path": generate_cache_path(url, local_path),
+                    "path": _path(
+                        url, local_path
+                    ),  # urlparse(url).netloc + urlparse(url).path,
                     # Path to package on local disk.
                     "local_path": local_path,
                 }
@@ -116,22 +131,26 @@ class CondaEnvironment(MetaflowEnvironment):
             }
             dirty = set()
             # Prune list of packages to cache.
+
+            _meta = copy.deepcopy(local_packages)
             for id_, packages, _, _ in results:
                 for package in packages:
                     if package.get("path"):
                         # Cache only those packages that manifest is unaware of
                         local_packages.pop(package["url"], None)
                     else:
-                        package["path"] = generate_cache_path(
-                            package["url"], parse_filename_from_url(package["url"])
-                        )
+                        package["path"] = _meta[package["url"]]["path"]
                         dirty.add(id_)
 
             list_of_path_and_filehandle = [
                 (
                     package["path"],
                     # Lazily fetch package from the interweb if needed.
-                    LazyOpen(package["local_path"], "rb", url),
+                    LazyOpen(
+                        package["local_path"],
+                        "rb",
+                        url,
+                    ),
                 )
                 for url, package in local_packages.items()
             ]
@@ -393,6 +412,10 @@ class LazyOpen(BufferedIOBase):
             if self.filename and os.path.exists(self.filename):
                 self._file = open(self.filename, self.mode)
             elif self.url:
+                if self.url.startswith("git+"):
+                    raise ValueError(
+                        "LazyOpen doesn't support VCS url %s yet!" % self.url
+                    )
                 self._buffer = self._download_to_buffer()
                 self._file = io.BytesIO(self._buffer)
             else: