metaflow 2.10.9__py2.py3-none-any.whl → 2.10.11__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -313,7 +313,14 @@ class MarkdownComponent(DefaultComponent):
313
313
 
314
314
  def render(self):
315
315
  datadict = super().render()
316
- datadict["source"] = self._text
316
+ _text = self._text
317
+ # When we have a markdown text that doesn't start with a `#`,
318
+ # we need to add a newline to make sure that the markdown
319
+ # is rendered correctly. Otherwise `svelte-markdown` will render
320
+ # the empty `<p>` tags during re-renders on card data updates.
321
+ if self._text is not None and not self._text.startswith("#"):
322
+ _text = "\n" + _text
323
+ datadict["source"] = _text
317
324
  if self.component_id is not None:
318
325
  datadict["id"] = self.component_id
319
326
  return datadict
@@ -712,8 +712,16 @@ class ProgressBar(UserComponent):
712
712
 
713
713
  Parameters
714
714
  ----------
715
- text : str
716
- Text formatted in Markdown.
715
+ max : int
716
+ The maximum value of the progress bar.
717
+ label : str, optional
718
+ Optional label for the progress bar.
719
+ value : int, optional
720
+ Optional initial value of the progress bar.
721
+ unit : str, optional
722
+ Optional unit for the progress bar.
723
+ metadata : str, optional
724
+ Optional additional information to show on the progress bar.
717
725
  """
718
726
 
719
727
  type = "progressBar"
@@ -762,31 +770,41 @@ class VegaChart(UserComponent):
762
770
 
763
771
  REALTIME_UPDATABLE = True
764
772
 
765
- def __init__(self, spec: dict, show_controls=False):
773
+ def __init__(self, spec: dict, show_controls: bool = False):
766
774
  self._spec = spec
767
775
  self._show_controls = show_controls
768
776
  self._chart_inside_table = False
769
777
 
770
778
  def update(self, spec=None):
771
- if spec is not None:
772
- self._spec = spec
779
+ """
780
+ Update the chart.
773
781
 
774
- @classmethod
775
- def from_altair_chart(cls, altair_chart):
776
- from metaflow.plugins.cards.card_modules.convert_to_native_type import (
777
- _full_classname,
778
- )
782
+ Parameters
783
+ ----------
784
+ spec : dict or altair.Chart
785
+ The updated chart spec or an altair Chart Object.
786
+ """
787
+ _spec = spec
788
+ if self._object_is_altair_chart(spec):
789
+ _spec = spec.to_dict()
790
+ if _spec is not None:
791
+ self._spec = _spec
779
792
 
793
+ @staticmethod
794
+ def _object_is_altair_chart(altair_chart):
780
795
  # This will feel slightly hacky but I am unable to find a natural way of determining the class
781
- # name of the Altair chart. The only way I can think of is to use the full class name and then
796
+ # name of the Altair chart. The only way simple way is to extract the full class name and then
782
797
  # match with heuristics
783
-
784
798
  fulclsname = _full_classname(altair_chart)
785
799
  if not all([x in fulclsname for x in ["altair", "vegalite", "Chart"]]):
786
- raise ValueError(fulclsname + " is not an altair chart")
800
+ return False
801
+ return True
787
802
 
803
+ @classmethod
804
+ def from_altair_chart(cls, altair_chart):
805
+ if not cls._object_is_altair_chart(altair_chart):
806
+ raise ValueError(_full_classname(altair_chart) + " is not an altair chart")
788
807
  altair_chart_dict = altair_chart.to_dict()
789
-
790
808
  cht = cls(spec=altair_chart_dict)
791
809
  return cht
792
810
 
@@ -0,0 +1,344 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>Metaflow Run</title>
7
+ <style>
8
+ @import url("https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap");
9
+
10
+ body {
11
+ font-size: 14px;
12
+ font-family: "Roboto", -apple-system, BlinkMacSystemFont, "Segoe UI",
13
+ "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans",
14
+ "Helvetica Neue", sans-serif;
15
+ padding: 0;
16
+ margin: 0;
17
+ display: flex;
18
+ flex-direction: column;
19
+ height: 100vh;
20
+ width: 100%;
21
+ overflow: hidden;
22
+ color: #31302f;
23
+ }
24
+
25
+ header {
26
+ height: 65px;
27
+ flex: 0 1 auto;
28
+ flex-grow: none;
29
+ background: #f9f9f9;
30
+ display: flex;
31
+ justify-content: space-evenly;
32
+ }
33
+
34
+ h1,
35
+ h2,
36
+ h3,
37
+ h4 {
38
+ font-size: inherit;
39
+ font-weight: 500;
40
+ margin: 0;
41
+ padding: 0;
42
+ }
43
+
44
+ header > * {
45
+ padding: 20px 25px;
46
+ flex: 1;
47
+ display: flex;
48
+ align-items: center;
49
+ }
50
+
51
+ header > div:nth-child(2) {
52
+ text-align: center;
53
+ flex: 1;
54
+ justify-content: center;
55
+ }
56
+
57
+ header > div:nth-child(3) {
58
+ text-align: right;
59
+ justify-content: flex-end;
60
+ font-weight: 400;
61
+ }
62
+
63
+ .taskStatus {
64
+ color: #6a6867;
65
+ }
66
+
67
+ .taskStatus.error {
68
+ color: #e67058;
69
+ }
70
+
71
+ .content {
72
+ flex: 1 1 auto;
73
+ overflow-y: scroll;
74
+ }
75
+
76
+ iframe {
77
+ width: 100%;
78
+ height: calc(100vh - 65px);
79
+ outline: none;
80
+ border: 0;
81
+ display: flex;
82
+ justify-content: center;
83
+ align-items: center;
84
+ }
85
+
86
+ header select {
87
+ width: 100%;
88
+ border: 0;
89
+ outline: none;
90
+ font-size: inherit;
91
+ font-family: inherit;
92
+ background: transparent;
93
+ padding: 0 1rem;
94
+ }
95
+
96
+ .selectorContainer {
97
+ border: 0;
98
+ outline: none;
99
+ font-size: inherit;
100
+ font-family: inherit;
101
+ border-radius: 8px;
102
+ background: #ebeaea;
103
+ padding: 0 1rem;
104
+ margin: 1rem;
105
+ }
106
+
107
+ .logoContainer {
108
+ display: none !important;
109
+ }
110
+
111
+ .errorContainer svg {
112
+ position: relative;
113
+ top: 3px;
114
+ }
115
+ </style>
116
+ </head>
117
+ <body>
118
+ <header>
119
+ <h1 id="header-title">Metaflow Run</h1>
120
+ <div class="selectorContainer">
121
+ <select id="selector">
122
+ <option value="" disabled selected>
123
+ Select a card
124
+ </option>
125
+ </select>
126
+ </div>
127
+ <div>
128
+ <div id="task-status" class="taskStatus">Initializing</div>
129
+ <div id="task-action" class="" style="display: none">Pause</div>
130
+ </div>
131
+ </header>
132
+ <div class="content">
133
+ <iframe id="card-frame"></iframe>
134
+ </div>
135
+
136
+ <script type="application/javascript">
137
+ (() => {
138
+ // setup constants
139
+ let DATA_UPDATE_POLL_INTERVAL = 5000;
140
+ let RUN_UPDATE_POLL_INTERVAL = 10000;
141
+
142
+ // setup state
143
+ let currentCardsLength = 0;
144
+ let currentRunId = "";
145
+ let selectedCard = "";
146
+
147
+ // setup selector
148
+ const selector = document.getElementById("selector");
149
+ selector?.addEventListener("change", (event) => {
150
+ selectedCard = event.target.value;
151
+ handleLoadCard(event.target.name, event.target.value);
152
+ });
153
+
154
+ /**
155
+ * Handle any changes to the status of the page
156
+ */
157
+ function handleStatus(status, isComplete = false) {
158
+ const statusDiv = document.getElementById("task-status");
159
+
160
+ // change the text of the status depending if its in progress or not
161
+ if (statusDiv) {
162
+ // complete
163
+ if (isComplete && status === "ok") {
164
+ statusDiv.innerText = "Task Is Complete";
165
+ // running
166
+ } else if (!isComplete && status === "ok") {
167
+ statusDiv.innerText = "Task Is Running";
168
+ // error
169
+ } else {
170
+ statusDiv.innerHTML = `<div class="errorContainer"><svg width="17" height="17" viewBox="0 0 17 17" fill="none" xmlns="http://www.w3.org/2000/svg">
171
+ <g clip-path="url(#clip0_164_815)"> <circle cx="8.5" cy="8.5" r="8" fill="#E67058"/> <path d="M11.6475 6.0575C11.8422 5.86282 11.8422 5.54718 11.6475 5.3525C11.4528 5.15782 11.1372 5.15782 10.9425 5.3525L8.5 7.795L6.0575 5.3525C5.86282 5.15782 5.54718 5.15782 5.3525 5.3525C5.15782 5.54718 5.15782 5.86282 5.3525 6.0575L7.795 8.5L5.3525 10.9425C5.15782 11.1372 5.15782 11.4528 5.3525 11.6475C5.54718 11.8422 5.86282 11.8422 6.0575 11.6475L8.5 9.205L10.9425 11.6475C11.1372 11.8422 11.4528 11.8422 11.6475 11.6475C11.8422 11.4528 11.8422 11.1372 11.6475 10.9425L9.205 8.5L11.6475 6.0575Z" fill="white"/> </g><defs><clipPath id="clip0_164_815"><rect width="16" height="16" fill="white" transform="translate(0.5 0.5)"/></clipPath></defs></svg>
172
+ <span>${status}</span></div>
173
+ `;
174
+ }
175
+ }
176
+ }
177
+
178
+ /**
179
+ * Handle any changes to the cards
180
+ */
181
+ function onCardsLoaded(runInfo) {
182
+ const { cards, flow, run_id } = runInfo;
183
+ if (!Array.isArray(cards) || !flow || !run_id) {
184
+ console.log("Data was not in the correct format");
185
+ return;
186
+ }
187
+ // console.log("Running ON CARDS LOADED", cards, flow, run_id)
188
+
189
+ // reset
190
+ currentCardsLength = cards.length;
191
+ selector.innerHTML = "";
192
+ // if the task has finished, change to the next card
193
+ changeToNewCardIfTaskHasFinished(runInfo);
194
+
195
+ // create select options for each card
196
+ cards.reverse().forEach((card) => {
197
+ const option = document.createElement("option");
198
+ option.value = card.card;
199
+ option.innerText = card.label.split(" ")?.[0] || card.label;
200
+ option.selected = card.card === selectedCard;
201
+ selector.appendChild(option);
202
+ });
203
+
204
+ // get card-title element and change it to name
205
+ const title = document.getElementById("header-title");
206
+ title.innerText = `${flow}/${run_id}`;
207
+ }
208
+
209
+ /**
210
+ * This function will determine if changes need to be made to the
211
+ * card and if so, will we do it by push or by reload
212
+ */
213
+
214
+ async function watchForCardChanges(iframe) {
215
+ // replace the /card/ url with the data url
216
+ const dataUri = iframe.src.replace(/.*\/card\//, "/data/");
217
+ let resp;
218
+ try {
219
+ const req = await fetch(dataUri);
220
+ resp = await req.json();
221
+ // setup UI to show complete and status
222
+ handleStatus(resp.status, resp?.is_complete);
223
+
224
+ // grab update fn from iframe
225
+ const update = iframe.contentWindow?.metaflow_card_update;
226
+ // make sure we're able to update the card
227
+ if (resp.status !== "ok" || !update || !resp.payload?.data) {
228
+ console.log(
229
+ "Data-updates not taking place because either : status is not `ok` or update function is not available or data is not available",
230
+ resp,
231
+ update
232
+ );
233
+ } else {
234
+ // make appropriate updates by pushing if the reload token matches
235
+ update &&
236
+ resp.payload?.reload_token ===
237
+ iframe.contentWindow.METAFLOW_RELOAD_TOKEN
238
+ ? update(resp.payload.data)
239
+ : iframe.contentWindow.location.reload();
240
+ }
241
+ // If there is no Update function than we don't need to wire the
242
+ // periodic updates
243
+ if (!update) {
244
+ console.log(
245
+ "No metaflow_card_update function available. So not running any periodic updates"
246
+ );
247
+ return;
248
+ }
249
+ } catch (error) {
250
+ console.error(error);
251
+ }
252
+ // if this update is not the last one, run it again!
253
+ if (!(resp.payload?.reload_token === "final")) {
254
+ setTimeout(
255
+ () => watchForCardChanges(iframe),
256
+ DATA_UPDATE_POLL_INTERVAL
257
+ );
258
+ }
259
+ }
260
+
261
+ /**
262
+ * Setup a new card and start the watcher
263
+ */
264
+ function handleLoadCard(name, value) {
265
+ document.getElementsByTagName("title").innerHTML = name;
266
+ const iframe = document.getElementById("card-frame");
267
+ iframe.src = `/card/${value}?embed=true`;
268
+ iframe.onload = () => {
269
+ watchForCardChanges(iframe);
270
+ };
271
+ }
272
+
273
+ function changeToNewCardIfTaskHasFinished(runInfo) {
274
+ // search for the selectedCard in the runInfo
275
+ const card = runInfo.cards.find((card) => card.card === selectedCard);
276
+ // if the card object contains `finished` === True, then change to the next card that has not finished
277
+ if (card?.finished) {
278
+ // find the index of the current card
279
+ const currentCardIndex = runInfo.cards.findIndex(
280
+ (card) => card.card === selectedCard
281
+ );
282
+ // find the next card that has not finished
283
+ const nextCard = runInfo.cards.find((card) => !card.finished);
284
+ // if there is a next card, change to it
285
+
286
+ console.log("nextCard", nextCard);
287
+ if (nextCard) {
288
+ selectedCard = nextCard.card;
289
+ handleLoadCard(nextCard.label, nextCard.card);
290
+ }
291
+ }
292
+ }
293
+
294
+ /**
295
+ * Main loop function
296
+ */
297
+ async function main() {
298
+ try {
299
+ // get info on the entire run
300
+ const runInfoRequest = await fetch("/runinfo");
301
+ const runInfo = await runInfoRequest.json();
302
+
303
+ // reset the state if the run_id changes or on first time
304
+ if (currentRunId !== runInfo.run_id) {
305
+ currentRunId = runInfo.run_id;
306
+ currentCardsLength = 0;
307
+ selectedCard = "";
308
+ console.log("runInfo", runInfo);
309
+ }
310
+ if (runInfo.poll_interval * 1000 !== DATA_UPDATE_POLL_INTERVAL) {
311
+ DATA_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
312
+ }
313
+ if (runInfo.poll_interval * 1000 !== RUN_UPDATE_POLL_INTERVAL) {
314
+ RUN_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
315
+ }
316
+ if (runInfo?.status === "ok") {
317
+ // only make changes if the cards length has changed
318
+ if (runInfo?.cards.length !== currentCardsLength) {
319
+ onCardsLoaded(runInfo);
320
+ }
321
+ // if there is no selected card, select the first one
322
+ if (!selectedCard) {
323
+ selectedCard = runInfo.cards[0].card;
324
+ handleLoadCard(runInfo.cards[0].label, runInfo.cards[0].card);
325
+ }
326
+ } else {
327
+ handleStatus(runInfo?.status, false);
328
+ }
329
+ } catch (error) {
330
+ console.error(error);
331
+ }
332
+ // always run the main loop, because a user can start a new run,
333
+ // or restart the web server, and we want the page to continue working
334
+ setTimeout(main, RUN_UPDATE_POLL_INTERVAL);
335
+ }
336
+
337
+ // start the main loop
338
+ window.onload = () => {
339
+ main();
340
+ };
341
+ })();
342
+ </script>
343
+ </body>
344
+ </html>
@@ -1,3 +1,4 @@
1
+ import copy
1
2
  import errno
2
3
  import fcntl
3
4
  import functools
@@ -10,17 +11,17 @@ from concurrent.futures import ThreadPoolExecutor
10
11
  from hashlib import sha256
11
12
  from io import BufferedIOBase
12
13
  from itertools import chain
13
- from urllib.parse import urlparse
14
+ from urllib.parse import unquote, urlparse
14
15
 
15
16
  import requests
16
17
 
17
- from metaflow.metaflow_config import get_pinned_conda_libs
18
18
  from metaflow.exception import MetaflowException
19
+ from metaflow.metaflow_config import get_pinned_conda_libs
19
20
  from metaflow.metaflow_environment import MetaflowEnvironment
20
21
  from metaflow.metaflow_profile import profile
21
22
 
22
23
  from . import MAGIC_FILE, _datastore_packageroot
23
- from .utils import conda_platform, generate_cache_path, parse_filename_from_url
24
+ from .utils import conda_platform
24
25
 
25
26
 
26
27
  class CondaEnvironmentException(MetaflowException):
@@ -104,10 +105,24 @@ class CondaEnvironment(MetaflowEnvironment):
104
105
  )
105
106
 
106
107
  def cache(storage, results, type_):
108
+ def _path(url, local_path):
109
+ # Special handling for VCS packages
110
+ if url.startswith("git+"):
111
+ base, _ = os.path.split(urlparse(url).path)
112
+ _, file = os.path.split(local_path)
113
+ prefix = url.split("@")[-1]
114
+ return urlparse(url).netloc + os.path.join(
115
+ unquote(base), prefix, file
116
+ )
117
+ else:
118
+ return urlparse(url).netloc + urlparse(url).path
119
+
107
120
  local_packages = {
108
121
  url: {
109
122
  # Path to package in datastore.
110
- "path": generate_cache_path(url, local_path),
123
+ "path": _path(
124
+ url, local_path
125
+ ), # urlparse(url).netloc + urlparse(url).path,
111
126
  # Path to package on local disk.
112
127
  "local_path": local_path,
113
128
  }
@@ -116,22 +131,26 @@ class CondaEnvironment(MetaflowEnvironment):
116
131
  }
117
132
  dirty = set()
118
133
  # Prune list of packages to cache.
134
+
135
+ _meta = copy.deepcopy(local_packages)
119
136
  for id_, packages, _, _ in results:
120
137
  for package in packages:
121
138
  if package.get("path"):
122
139
  # Cache only those packages that manifest is unaware of
123
140
  local_packages.pop(package["url"], None)
124
141
  else:
125
- package["path"] = generate_cache_path(
126
- package["url"], parse_filename_from_url(package["url"])
127
- )
142
+ package["path"] = _meta[package["url"]]["path"]
128
143
  dirty.add(id_)
129
144
 
130
145
  list_of_path_and_filehandle = [
131
146
  (
132
147
  package["path"],
133
148
  # Lazily fetch package from the interweb if needed.
134
- LazyOpen(package["local_path"], "rb", url),
149
+ LazyOpen(
150
+ package["local_path"],
151
+ "rb",
152
+ url,
153
+ ),
135
154
  )
136
155
  for url, package in local_packages.items()
137
156
  ]
@@ -393,6 +412,10 @@ class LazyOpen(BufferedIOBase):
393
412
  if self.filename and os.path.exists(self.filename):
394
413
  self._file = open(self.filename, self.mode)
395
414
  elif self.url:
415
+ if self.url.startswith("git+"):
416
+ raise ValueError(
417
+ "LazyOpen doesn't support VCS url %s yet!" % self.url
418
+ )
396
419
  self._buffer = self._download_to_buffer()
397
420
  self._file = io.BytesIO(self._buffer)
398
421
  else:
@@ -1,15 +1,18 @@
1
1
  import json
2
2
  import os
3
3
  import re
4
+ import shutil
4
5
  import subprocess
5
6
  import tempfile
7
+ from concurrent.futures import ThreadPoolExecutor
6
8
  from itertools import chain, product
9
+ from urllib.parse import unquote
7
10
 
8
11
  from metaflow.exception import MetaflowException
9
12
  from metaflow.util import which
10
13
 
11
14
  from .micromamba import Micromamba
12
- from .utils import parse_filename_from_url, pip_tags
15
+ from .utils import pip_tags, wheel_tags
13
16
 
14
17
 
15
18
  class PipException(MetaflowException):
@@ -26,7 +29,7 @@ METADATA_FILE = "{prefix}/.pip/metadata"
26
29
  INSTALLATION_MARKER = "{prefix}/.pip/id"
27
30
 
28
31
  # TODO:
29
- # 1. Support git repositories, local dirs, non-wheel like packages
32
+ # 1. Support local dirs, non-wheel like packages
30
33
  # 2. Support protected indices
31
34
 
32
35
 
@@ -72,15 +75,32 @@ class Pip(object):
72
75
  # *(chain.from_iterable(product(["--implementations"], set(implementations)))),
73
76
  ]
74
77
  for package, version in packages.items():
75
- if version.startswith(("<", ">", "!", "~")):
78
+ if version.startswith(("<", ">", "!", "~", "@")):
79
+ cmd.append(f"{package}{version}")
80
+ elif not version:
76
81
  cmd.append(f"{package}{version}")
77
82
  else:
78
83
  cmd.append(f"{package}=={version}")
79
84
  self._call(prefix, cmd)
85
+
86
+ def _format(dl_info):
87
+ res = {k: v for k, v in dl_info.items() if k in ["url"]}
88
+ # If source url is not a wheel, we need to build the target.
89
+ res["require_build"] = not res["url"].endswith(".whl")
90
+
91
+ # Reconstruct the VCS url and pin to current commit_id
92
+ # so using @branch as a version acts as expected.
93
+ vcs_info = dl_info.get("vcs_info")
94
+ if vcs_info:
95
+ res["url"] = "{vcs}+{url}@{commit_id}".format(**vcs_info, **res)
96
+ # used to deduplicate the storage location in case wheel does not
97
+ # build with enough unique identifiers.
98
+ res["hash"] = vcs_info["commit_id"]
99
+ return res
100
+
80
101
  with open(report, mode="r", encoding="utf-8") as f:
81
102
  return [
82
- {k: v for k, v in item["download_info"].items() if k in ["url"]}
83
- for item in json.load(f)["install"]
103
+ _format(item["download_info"]) for item in json.load(f)["install"]
84
104
  ]
85
105
 
86
106
  def download(self, id_, packages, python, platform):
@@ -89,21 +109,78 @@ class Pip(object):
89
109
  # download packages only if they haven't ever been downloaded before
90
110
  if os.path.isfile(metadata_file):
91
111
  return
112
+
92
113
  metadata = {}
114
+ custom_index_url, extra_index_urls = self.indices(prefix)
115
+
116
+ # build wheels if needed
117
+ with ThreadPoolExecutor() as executor:
118
+
119
+ def _build(key, package):
120
+ dest = "{prefix}/.pip/built_wheels/{key}".format(prefix=prefix, key=key)
121
+ cmd = [
122
+ "wheel",
123
+ "--no-deps",
124
+ "--progress-bar=off",
125
+ "--wheel-dir=%s" % dest,
126
+ "--quiet",
127
+ *(["--index-url", custom_index_url] if custom_index_url else []),
128
+ *(
129
+ chain.from_iterable(
130
+ product(["--extra-index-url"], set(extra_index_urls))
131
+ )
132
+ ),
133
+ package["url"],
134
+ ]
135
+ self._call(prefix, cmd)
136
+ return package, dest
137
+
138
+ results = list(
139
+ executor.map(
140
+ lambda x: _build(*x),
141
+ enumerate(
142
+ package for package in packages if package["require_build"]
143
+ ),
144
+ )
145
+ )
146
+
147
+ for package, path in results:
148
+ (wheel,) = [
149
+ f
150
+ for f in os.listdir(path)
151
+ if os.path.isfile(os.path.join(path, f)) and f.endswith(".whl")
152
+ ]
153
+ if (
154
+ len(set(pip_tags(python, platform)).intersection(wheel_tags(wheel)))
155
+ == 0
156
+ ):
157
+ raise PipException(
158
+ "The built wheel %s is not supported for %s with Python %s"
159
+ % (wheel, platform, python)
160
+ )
161
+ target = "{prefix}/.pip/wheels/{hash}/{wheel}".format(
162
+ prefix=prefix,
163
+ wheel=wheel,
164
+ hash=package["hash"],
165
+ )
166
+ os.makedirs(os.path.dirname(target), exist_ok=True)
167
+ shutil.move(os.path.join(path, wheel), target)
168
+ metadata["{url}".format(**package)] = target
169
+
93
170
  implementations, platforms, abis = zip(
94
171
  *[
95
172
  (tag.interpreter, tag.platform, tag.abi)
96
173
  for tag in pip_tags(python, platform)
97
174
  ]
98
175
  )
99
- custom_index_url, extra_index_urls = self.indices(prefix)
176
+
100
177
  cmd = [
101
178
  "download",
102
179
  "--no-deps",
103
180
  "--no-index",
104
181
  "--progress-bar=off",
105
182
  # if packages are present in Pip cache, this will be a local copy
106
- "--dest=%s/.pip/wheels" % prefix,
183
+ "--dest={prefix}/.pip/wheels".format(prefix=prefix),
107
184
  "--quiet",
108
185
  *(["--index-url", custom_index_url] if custom_index_url else []),
109
186
  *(
@@ -115,10 +192,11 @@ class Pip(object):
115
192
  *(chain.from_iterable(product(["--platform"], set(platforms)))),
116
193
  # *(chain.from_iterable(product(["--implementations"], set(implementations)))),
117
194
  ]
195
+ packages = [package for package in packages if not package["require_build"]]
118
196
  for package in packages:
119
197
  cmd.append("{url}".format(**package))
120
198
  metadata["{url}".format(**package)] = "{prefix}/.pip/wheels/{wheel}".format(
121
- prefix=prefix, wheel=parse_filename_from_url(package["url"])
199
+ prefix=prefix, wheel=unquote(package["url"].split("/")[-1])
122
200
  )
123
201
  self._call(prefix, cmd)
124
202
  # write the url to wheel mappings in a magic location
@@ -128,6 +206,7 @@ class Pip(object):
128
206
  def create(self, id_, packages, python, platform):
129
207
  prefix = self.micromamba.path_to_environment(id_)
130
208
  installation_marker = INSTALLATION_MARKER.format(prefix=prefix)
209
+ metadata = self.metadata(id_, packages, python, platform)
131
210
  # install packages only if they haven't been installed before
132
211
  if os.path.isfile(installation_marker):
133
212
  return
@@ -143,7 +222,7 @@ class Pip(object):
143
222
  "--quiet",
144
223
  ]
145
224
  for package in packages:
146
- cmd.append("{url}".format(**package))
225
+ cmd.append(metadata[package["url"]])
147
226
  self._call(prefix, cmd)
148
227
  with open(installation_marker, "w") as file:
149
228
  file.write(json.dumps({"id": id_}))
@@ -10,12 +10,17 @@ if sys.version_info < (3, 6):
10
10
  )
11
11
 
12
12
  tags = Tags()
13
+ parse_wheel_filename = lambda: (_ for _ in ()).throw(
14
+ RuntimeError("packaging.utils is not avaliable for Python < 3.6")
15
+ )
13
16
  else:
14
17
  from metaflow._vendor.packaging import tags
18
+ from metaflow._vendor.packaging.utils import parse_wheel_filename
15
19
 
16
- from metaflow.exception import MetaflowException
17
20
  from urllib.parse import unquote, urlparse
18
21
 
22
+ from metaflow.exception import MetaflowException
23
+
19
24
 
20
25
  def conda_platform():
21
26
  # Returns the conda platform for the Python interpreter
@@ -34,6 +39,11 @@ def conda_platform():
34
39
  return "osx-64"
35
40
 
36
41
 
42
+ def wheel_tags(wheel):
43
+ _, _, _, tags = parse_wheel_filename(wheel)
44
+ return list(tags)
45
+
46
+
37
47
  def pip_tags(python_version, mamba_platform):
38
48
  # Returns a list of pip tags containing (implementation, platforms, abis) tuples
39
49
  # assuming a CPython implementation for Python interpreter.
@@ -81,10 +91,3 @@ def parse_filename_from_url(url):
81
91
  # Separate method as it might require additional checks for the parsing.
82
92
  filename = url.split("/")[-1]
83
93
  return unquote(filename)
84
-
85
-
86
- def generate_cache_path(url, local_path):
87
- base, _ = os.path.split(urlparse(url).path)
88
- _, localfile = os.path.split(local_path)
89
- unquoted_base = unquote(base)
90
- return urlparse(url).netloc + os.path.join(unquoted_base, localfile)
metaflow/version.py CHANGED
@@ -1 +1 @@
1
- metaflow_version = "2.10.9"
1
+ metaflow_version = "2.10.11"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: metaflow
3
- Version: 2.10.9
3
+ Version: 2.10.11
4
4
  Summary: Metaflow: More Data Science, Less Engineering
5
5
  Author: Metaflow Developers
6
6
  Author-email: help@metaflow.org
@@ -33,7 +33,7 @@ metaflow/task.py,sha256=yGNU3T3giKiG--vE0DUj_K-8jur2TclCS45XjPVLcq4,25314
33
33
  metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
34
34
  metaflow/util.py,sha256=jbMJ17rK-dFTBCjimWqxkfcr3v__bHa3tZtX0g8iS2c,13257
35
35
  metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
36
- metaflow/version.py,sha256=UIROGFLe3vYAfuZS6rvIiGAC4PDT7qabUNUVREI5e7U,28
36
+ metaflow/version.py,sha256=NpJM4rx0ubVOsZrPH8YGJtEzrQhzHj-x614Y8e5rl6c,29
37
37
  metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
38
38
  metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
39
39
  metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -187,10 +187,10 @@ metaflow/plugins/cards/component_serializer.py,sha256=Row7c_8_euJcF_I1lWHdgMRj7d
187
187
  metaflow/plugins/cards/exception.py,sha256=2UqlNb-Kxpg6cuLu2sBEIPTIElwlVBsSpeCgDYKTxWg,5222
188
188
  metaflow/plugins/cards/card_modules/__init__.py,sha256=WI2IAsFiKGyqPrHtO9S9-MbyVtUTgWJNL4xjJaBErRo,3437
189
189
  metaflow/plugins/cards/card_modules/base.html,sha256=Y208ZKIZqEWWUcoBFTLTdWKAG0C8xH5lmyCRSjaN2FY,21004
190
- metaflow/plugins/cards/card_modules/basic.py,sha256=CieAZJTyN2bJfC59bM5JKTLhdN0uRvgMV9evB75usv8,23011
190
+ metaflow/plugins/cards/card_modules/basic.py,sha256=h8v-3S_katr6YOteNVXuFuFY_9a2SeMjLT_TTrcQbPM,23416
191
191
  metaflow/plugins/cards/card_modules/bundle.css,sha256=ms2wOKftlPM_i6bC_4BkrmqCOj8mYw9OFvRCJF9FSV4,11981
192
192
  metaflow/plugins/cards/card_modules/card.py,sha256=8pyBDEd-on7tayTfGxAQcAiqLlp6rSQDVKYvp87SglM,4209
193
- metaflow/plugins/cards/card_modules/components.py,sha256=ekIUQUdrOJbF6Ng2TU8_NzSRSGxOieFw2GqlGYTmSOc,24631
193
+ metaflow/plugins/cards/card_modules/components.py,sha256=Yvq9lqR_5EFVPM0ruzYTft635_dGIbkJhgxrQlPz1eg,25321
194
194
  metaflow/plugins/cards/card_modules/convert_to_native_type.py,sha256=CX4Z9Xdu6ZRELOyzILCNTGOz0kNWecEUet4ddgd2JVI,15782
195
195
  metaflow/plugins/cards/card_modules/main.js,sha256=VxKo5ZcINVzp71vnu79XqzeRLN6yoPmdISb0jzWiHFw,1027769
196
196
  metaflow/plugins/cards/card_modules/renderer_tools.py,sha256=V3H92FvIl2DtzVsU7SQuQd5_MFwvG6GSAmpqqrqeIQM,1641
@@ -200,6 +200,7 @@ metaflow/plugins/cards/card_modules/chevron/main.py,sha256=FKpDW4PS2dYshgkm6yIxr
200
200
  metaflow/plugins/cards/card_modules/chevron/metadata.py,sha256=gQDpB9KezW9Mcv-n9K2Pt1akjiPGG1zYB5YPdQ0FlNc,19
201
201
  metaflow/plugins/cards/card_modules/chevron/renderer.py,sha256=dm5ufR9-qnx94D9BtV7q_64oqYUpwdCRgBTcjBRkZyM,13938
202
202
  metaflow/plugins/cards/card_modules/chevron/tokenizer.py,sha256=lQU9OELUE9a5Xu4snGhEV_YTT34iyUHVBuM1YO016Sw,7400
203
+ metaflow/plugins/cards/card_viewer/viewer.html,sha256=qZJGzhZhQ1gugsknRP7zkAPPfUAtvemK1UKqXoGff5M,11593
203
204
  metaflow/plugins/datastores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
204
205
  metaflow/plugins/datastores/azure_storage.py,sha256=V15_BZNZIp9_2rqzEjuOSxUnPzrNZTd5f9hsJ9SEFLc,16234
205
206
  metaflow/plugins/datastores/gs_storage.py,sha256=ED1xfXNDiPkqma5PEui8HCv0b4ImHK_rlUMQJD9UNes,9622
@@ -252,12 +253,12 @@ metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j
252
253
  metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
253
254
  metaflow/plugins/pypi/bootstrap.py,sha256=PBIStzaEuhAl0xBd0Ap6oUaWMDwvz5bZEWo6h2SFru0,5073
254
255
  metaflow/plugins/pypi/conda_decorator.py,sha256=RHFzVJ5mdd4WeXC5cKe92xHgK775g8m-vY4REKJu4YI,13525
255
- metaflow/plugins/pypi/conda_environment.py,sha256=Q9DxinHbD8L-UPKxcVjrcwyUM7VdMXYZVLJ97-_mLUg,17050
256
+ metaflow/plugins/pypi/conda_environment.py,sha256=FeHwE9xkI3nYv_OZ_iFzMI2Z2s5BG7llFSi-ndPeId8,17867
256
257
  metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
257
- metaflow/plugins/pypi/pip.py,sha256=YIscbcqpItljJegi1I_cTQts63-NElJD_JBKEGyUFoU,8535
258
+ metaflow/plugins/pypi/pip.py,sha256=paL-hbj5j-vcYOeSm04OiBix-TpoKMF7bRfGVdXfGZs,11685
258
259
  metaflow/plugins/pypi/pypi_decorator.py,sha256=6-8KzJqduV5JBnvJwZTl-ZC3lNUZF1BxAYiO1npttiQ,5553
259
260
  metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
260
- metaflow/plugins/pypi/utils.py,sha256=0k0ZT2TFrGMn-3jHHqqOMfdJ2JTEPJ5d-c76qApco5E,2776
261
+ metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
261
262
  metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
262
263
  metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
263
264
  metaflow/plugins/secrets/secrets_decorator.py,sha256=Y41XCOEGt7asxx6FEOBdKMqYFlU_jcA5u7erHN8DqJM,10514
@@ -293,9 +294,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
293
294
  metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
294
295
  metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
295
296
  metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
296
- metaflow-2.10.9.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
297
- metaflow-2.10.9.dist-info/METADATA,sha256=dplMzgJC4vMwKXvcMEQLUvCfvVPoX4owSm8YF4i9PkQ,5826
298
- metaflow-2.10.9.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
299
- metaflow-2.10.9.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
300
- metaflow-2.10.9.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
301
- metaflow-2.10.9.dist-info/RECORD,,
297
+ metaflow-2.10.11.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
298
+ metaflow-2.10.11.dist-info/METADATA,sha256=Nzv4M2QBD8aZpENBzwYCadi_bRFT2imd3cKRth40mIY,5827
299
+ metaflow-2.10.11.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
300
+ metaflow-2.10.11.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
301
+ metaflow-2.10.11.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
302
+ metaflow-2.10.11.dist-info/RECORD,,