metaflow 2.10.9__py2.py3-none-any.whl → 2.10.11__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/plugins/cards/card_modules/basic.py +8 -1
- metaflow/plugins/cards/card_modules/components.py +32 -14
- metaflow/plugins/cards/card_viewer/viewer.html +344 -0
- metaflow/plugins/pypi/conda_environment.py +31 -8
- metaflow/plugins/pypi/pip.py +88 -9
- metaflow/plugins/pypi/utils.py +11 -8
- metaflow/version.py +1 -1
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/METADATA +1 -1
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/RECORD +13 -12
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/LICENSE +0 -0
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/WHEEL +0 -0
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/entry_points.txt +0 -0
- {metaflow-2.10.9.dist-info → metaflow-2.10.11.dist-info}/top_level.txt +0 -0
@@ -313,7 +313,14 @@ class MarkdownComponent(DefaultComponent):
|
|
313
313
|
|
314
314
|
def render(self):
|
315
315
|
datadict = super().render()
|
316
|
-
|
316
|
+
_text = self._text
|
317
|
+
# When we have a markdown text that doesn't start with a `#`,
|
318
|
+
# we need to add a newline to make sure that the markdown
|
319
|
+
# is rendered correctly. Otherwise `svelte-markdown` will render
|
320
|
+
# the empty `<p>` tags during re-renders on card data updates.
|
321
|
+
if self._text is not None and not self._text.startswith("#"):
|
322
|
+
_text = "\n" + _text
|
323
|
+
datadict["source"] = _text
|
317
324
|
if self.component_id is not None:
|
318
325
|
datadict["id"] = self.component_id
|
319
326
|
return datadict
|
@@ -712,8 +712,16 @@ class ProgressBar(UserComponent):
|
|
712
712
|
|
713
713
|
Parameters
|
714
714
|
----------
|
715
|
-
|
716
|
-
|
715
|
+
max : int
|
716
|
+
The maximum value of the progress bar.
|
717
|
+
label : str, optional
|
718
|
+
Optional label for the progress bar.
|
719
|
+
value : int, optional
|
720
|
+
Optional initial value of the progress bar.
|
721
|
+
unit : str, optional
|
722
|
+
Optional unit for the progress bar.
|
723
|
+
metadata : str, optional
|
724
|
+
Optional additional information to show on the progress bar.
|
717
725
|
"""
|
718
726
|
|
719
727
|
type = "progressBar"
|
@@ -762,31 +770,41 @@ class VegaChart(UserComponent):
|
|
762
770
|
|
763
771
|
REALTIME_UPDATABLE = True
|
764
772
|
|
765
|
-
def __init__(self, spec: dict, show_controls=False):
|
773
|
+
def __init__(self, spec: dict, show_controls: bool = False):
|
766
774
|
self._spec = spec
|
767
775
|
self._show_controls = show_controls
|
768
776
|
self._chart_inside_table = False
|
769
777
|
|
770
778
|
def update(self, spec=None):
|
771
|
-
|
772
|
-
|
779
|
+
"""
|
780
|
+
Update the chart.
|
773
781
|
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
782
|
+
Parameters
|
783
|
+
----------
|
784
|
+
spec : dict or altair.Chart
|
785
|
+
The updated chart spec or an altair Chart Object.
|
786
|
+
"""
|
787
|
+
_spec = spec
|
788
|
+
if self._object_is_altair_chart(spec):
|
789
|
+
_spec = spec.to_dict()
|
790
|
+
if _spec is not None:
|
791
|
+
self._spec = _spec
|
779
792
|
|
793
|
+
@staticmethod
|
794
|
+
def _object_is_altair_chart(altair_chart):
|
780
795
|
# This will feel slightly hacky but I am unable to find a natural way of determining the class
|
781
|
-
# name of the Altair chart. The only way
|
796
|
+
# name of the Altair chart. The only way simple way is to extract the full class name and then
|
782
797
|
# match with heuristics
|
783
|
-
|
784
798
|
fulclsname = _full_classname(altair_chart)
|
785
799
|
if not all([x in fulclsname for x in ["altair", "vegalite", "Chart"]]):
|
786
|
-
|
800
|
+
return False
|
801
|
+
return True
|
787
802
|
|
803
|
+
@classmethod
|
804
|
+
def from_altair_chart(cls, altair_chart):
|
805
|
+
if not cls._object_is_altair_chart(altair_chart):
|
806
|
+
raise ValueError(_full_classname(altair_chart) + " is not an altair chart")
|
788
807
|
altair_chart_dict = altair_chart.to_dict()
|
789
|
-
|
790
808
|
cht = cls(spec=altair_chart_dict)
|
791
809
|
return cht
|
792
810
|
|
@@ -0,0 +1,344 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
<html lang="en">
|
3
|
+
<head>
|
4
|
+
<meta charset="UTF-8" />
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
6
|
+
<title>Metaflow Run</title>
|
7
|
+
<style>
|
8
|
+
@import url("https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap");
|
9
|
+
|
10
|
+
body {
|
11
|
+
font-size: 14px;
|
12
|
+
font-family: "Roboto", -apple-system, BlinkMacSystemFont, "Segoe UI",
|
13
|
+
"Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans",
|
14
|
+
"Helvetica Neue", sans-serif;
|
15
|
+
padding: 0;
|
16
|
+
margin: 0;
|
17
|
+
display: flex;
|
18
|
+
flex-direction: column;
|
19
|
+
height: 100vh;
|
20
|
+
width: 100%;
|
21
|
+
overflow: hidden;
|
22
|
+
color: #31302f;
|
23
|
+
}
|
24
|
+
|
25
|
+
header {
|
26
|
+
height: 65px;
|
27
|
+
flex: 0 1 auto;
|
28
|
+
flex-grow: none;
|
29
|
+
background: #f9f9f9;
|
30
|
+
display: flex;
|
31
|
+
justify-content: space-evenly;
|
32
|
+
}
|
33
|
+
|
34
|
+
h1,
|
35
|
+
h2,
|
36
|
+
h3,
|
37
|
+
h4 {
|
38
|
+
font-size: inherit;
|
39
|
+
font-weight: 500;
|
40
|
+
margin: 0;
|
41
|
+
padding: 0;
|
42
|
+
}
|
43
|
+
|
44
|
+
header > * {
|
45
|
+
padding: 20px 25px;
|
46
|
+
flex: 1;
|
47
|
+
display: flex;
|
48
|
+
align-items: center;
|
49
|
+
}
|
50
|
+
|
51
|
+
header > div:nth-child(2) {
|
52
|
+
text-align: center;
|
53
|
+
flex: 1;
|
54
|
+
justify-content: center;
|
55
|
+
}
|
56
|
+
|
57
|
+
header > div:nth-child(3) {
|
58
|
+
text-align: right;
|
59
|
+
justify-content: flex-end;
|
60
|
+
font-weight: 400;
|
61
|
+
}
|
62
|
+
|
63
|
+
.taskStatus {
|
64
|
+
color: #6a6867;
|
65
|
+
}
|
66
|
+
|
67
|
+
.taskStatus.error {
|
68
|
+
color: #e67058;
|
69
|
+
}
|
70
|
+
|
71
|
+
.content {
|
72
|
+
flex: 1 1 auto;
|
73
|
+
overflow-y: scroll;
|
74
|
+
}
|
75
|
+
|
76
|
+
iframe {
|
77
|
+
width: 100%;
|
78
|
+
height: calc(100vh - 65px);
|
79
|
+
outline: none;
|
80
|
+
border: 0;
|
81
|
+
display: flex;
|
82
|
+
justify-content: center;
|
83
|
+
align-items: center;
|
84
|
+
}
|
85
|
+
|
86
|
+
header select {
|
87
|
+
width: 100%;
|
88
|
+
border: 0;
|
89
|
+
outline: none;
|
90
|
+
font-size: inherit;
|
91
|
+
font-family: inherit;
|
92
|
+
background: transparent;
|
93
|
+
padding: 0 1rem;
|
94
|
+
}
|
95
|
+
|
96
|
+
.selectorContainer {
|
97
|
+
border: 0;
|
98
|
+
outline: none;
|
99
|
+
font-size: inherit;
|
100
|
+
font-family: inherit;
|
101
|
+
border-radius: 8px;
|
102
|
+
background: #ebeaea;
|
103
|
+
padding: 0 1rem;
|
104
|
+
margin: 1rem;
|
105
|
+
}
|
106
|
+
|
107
|
+
.logoContainer {
|
108
|
+
display: none !important;
|
109
|
+
}
|
110
|
+
|
111
|
+
.errorContainer svg {
|
112
|
+
position: relative;
|
113
|
+
top: 3px;
|
114
|
+
}
|
115
|
+
</style>
|
116
|
+
</head>
|
117
|
+
<body>
|
118
|
+
<header>
|
119
|
+
<h1 id="header-title">Metaflow Run</h1>
|
120
|
+
<div class="selectorContainer">
|
121
|
+
<select id="selector">
|
122
|
+
<option value="" disabled selected>
|
123
|
+
Select a card
|
124
|
+
</option>
|
125
|
+
</select>
|
126
|
+
</div>
|
127
|
+
<div>
|
128
|
+
<div id="task-status" class="taskStatus">Initializing</div>
|
129
|
+
<div id="task-action" class="" style="display: none">Pause</div>
|
130
|
+
</div>
|
131
|
+
</header>
|
132
|
+
<div class="content">
|
133
|
+
<iframe id="card-frame"></iframe>
|
134
|
+
</div>
|
135
|
+
|
136
|
+
<script type="application/javascript">
|
137
|
+
(() => {
|
138
|
+
// setup constants
|
139
|
+
let DATA_UPDATE_POLL_INTERVAL = 5000;
|
140
|
+
let RUN_UPDATE_POLL_INTERVAL = 10000;
|
141
|
+
|
142
|
+
// setup state
|
143
|
+
let currentCardsLength = 0;
|
144
|
+
let currentRunId = "";
|
145
|
+
let selectedCard = "";
|
146
|
+
|
147
|
+
// setup selector
|
148
|
+
const selector = document.getElementById("selector");
|
149
|
+
selector?.addEventListener("change", (event) => {
|
150
|
+
selectedCard = event.target.value;
|
151
|
+
handleLoadCard(event.target.name, event.target.value);
|
152
|
+
});
|
153
|
+
|
154
|
+
/**
|
155
|
+
* Handle any changes to the status of the page
|
156
|
+
*/
|
157
|
+
function handleStatus(status, isComplete = false) {
|
158
|
+
const statusDiv = document.getElementById("task-status");
|
159
|
+
|
160
|
+
// change the text of the status depending if its in progress or not
|
161
|
+
if (statusDiv) {
|
162
|
+
// complete
|
163
|
+
if (isComplete && status === "ok") {
|
164
|
+
statusDiv.innerText = "Task Is Complete";
|
165
|
+
// running
|
166
|
+
} else if (!isComplete && status === "ok") {
|
167
|
+
statusDiv.innerText = "Task Is Running";
|
168
|
+
// error
|
169
|
+
} else {
|
170
|
+
statusDiv.innerHTML = `<div class="errorContainer"><svg width="17" height="17" viewBox="0 0 17 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
171
|
+
<g clip-path="url(#clip0_164_815)"> <circle cx="8.5" cy="8.5" r="8" fill="#E67058"/> <path d="M11.6475 6.0575C11.8422 5.86282 11.8422 5.54718 11.6475 5.3525C11.4528 5.15782 11.1372 5.15782 10.9425 5.3525L8.5 7.795L6.0575 5.3525C5.86282 5.15782 5.54718 5.15782 5.3525 5.3525C5.15782 5.54718 5.15782 5.86282 5.3525 6.0575L7.795 8.5L5.3525 10.9425C5.15782 11.1372 5.15782 11.4528 5.3525 11.6475C5.54718 11.8422 5.86282 11.8422 6.0575 11.6475L8.5 9.205L10.9425 11.6475C11.1372 11.8422 11.4528 11.8422 11.6475 11.6475C11.8422 11.4528 11.8422 11.1372 11.6475 10.9425L9.205 8.5L11.6475 6.0575Z" fill="white"/> </g><defs><clipPath id="clip0_164_815"><rect width="16" height="16" fill="white" transform="translate(0.5 0.5)"/></clipPath></defs></svg>
|
172
|
+
<span>${status}</span></div>
|
173
|
+
`;
|
174
|
+
}
|
175
|
+
}
|
176
|
+
}
|
177
|
+
|
178
|
+
/**
|
179
|
+
* Handle any changes to the cards
|
180
|
+
*/
|
181
|
+
function onCardsLoaded(runInfo) {
|
182
|
+
const { cards, flow, run_id } = runInfo;
|
183
|
+
if (!Array.isArray(cards) || !flow || !run_id) {
|
184
|
+
console.log("Data was not in the correct format");
|
185
|
+
return;
|
186
|
+
}
|
187
|
+
// console.log("Running ON CARDS LOADED", cards, flow, run_id)
|
188
|
+
|
189
|
+
// reset
|
190
|
+
currentCardsLength = cards.length;
|
191
|
+
selector.innerHTML = "";
|
192
|
+
// if the task has finished, change to the next card
|
193
|
+
changeToNewCardIfTaskHasFinished(runInfo);
|
194
|
+
|
195
|
+
// create select options for each card
|
196
|
+
cards.reverse().forEach((card) => {
|
197
|
+
const option = document.createElement("option");
|
198
|
+
option.value = card.card;
|
199
|
+
option.innerText = card.label.split(" ")?.[0] || card.label;
|
200
|
+
option.selected = card.card === selectedCard;
|
201
|
+
selector.appendChild(option);
|
202
|
+
});
|
203
|
+
|
204
|
+
// get card-title element and change it to name
|
205
|
+
const title = document.getElementById("header-title");
|
206
|
+
title.innerText = `${flow}/${run_id}`;
|
207
|
+
}
|
208
|
+
|
209
|
+
/**
|
210
|
+
* This function will determine if changes need to be made to the
|
211
|
+
* card and if so, will we do it by push or by reload
|
212
|
+
*/
|
213
|
+
|
214
|
+
async function watchForCardChanges(iframe) {
|
215
|
+
// replace the /card/ url with the data url
|
216
|
+
const dataUri = iframe.src.replace(/.*\/card\//, "/data/");
|
217
|
+
let resp;
|
218
|
+
try {
|
219
|
+
const req = await fetch(dataUri);
|
220
|
+
resp = await req.json();
|
221
|
+
// setup UI to show complete and status
|
222
|
+
handleStatus(resp.status, resp?.is_complete);
|
223
|
+
|
224
|
+
// grab update fn from iframe
|
225
|
+
const update = iframe.contentWindow?.metaflow_card_update;
|
226
|
+
// make sure we're able to update the card
|
227
|
+
if (resp.status !== "ok" || !update || !resp.payload?.data) {
|
228
|
+
console.log(
|
229
|
+
"Data-updates not taking place because either : status is not `ok` or update function is not available or data is not available",
|
230
|
+
resp,
|
231
|
+
update
|
232
|
+
);
|
233
|
+
} else {
|
234
|
+
// make appropriate updates by pushing if the reload token matches
|
235
|
+
update &&
|
236
|
+
resp.payload?.reload_token ===
|
237
|
+
iframe.contentWindow.METAFLOW_RELOAD_TOKEN
|
238
|
+
? update(resp.payload.data)
|
239
|
+
: iframe.contentWindow.location.reload();
|
240
|
+
}
|
241
|
+
// If there is no Update function than we don't need to wire the
|
242
|
+
// periodic updates
|
243
|
+
if (!update) {
|
244
|
+
console.log(
|
245
|
+
"No metaflow_card_update function available. So not running any periodic updates"
|
246
|
+
);
|
247
|
+
return;
|
248
|
+
}
|
249
|
+
} catch (error) {
|
250
|
+
console.error(error);
|
251
|
+
}
|
252
|
+
// if this update is not the last one, run it again!
|
253
|
+
if (!(resp.payload?.reload_token === "final")) {
|
254
|
+
setTimeout(
|
255
|
+
() => watchForCardChanges(iframe),
|
256
|
+
DATA_UPDATE_POLL_INTERVAL
|
257
|
+
);
|
258
|
+
}
|
259
|
+
}
|
260
|
+
|
261
|
+
/**
|
262
|
+
* Setup a new card and start the watcher
|
263
|
+
*/
|
264
|
+
function handleLoadCard(name, value) {
|
265
|
+
document.getElementsByTagName("title").innerHTML = name;
|
266
|
+
const iframe = document.getElementById("card-frame");
|
267
|
+
iframe.src = `/card/${value}?embed=true`;
|
268
|
+
iframe.onload = () => {
|
269
|
+
watchForCardChanges(iframe);
|
270
|
+
};
|
271
|
+
}
|
272
|
+
|
273
|
+
function changeToNewCardIfTaskHasFinished(runInfo) {
|
274
|
+
// search for the selectedCard in the runInfo
|
275
|
+
const card = runInfo.cards.find((card) => card.card === selectedCard);
|
276
|
+
// if the card object contains `finished` === True, then change to the next card that has not finished
|
277
|
+
if (card?.finished) {
|
278
|
+
// find the index of the current card
|
279
|
+
const currentCardIndex = runInfo.cards.findIndex(
|
280
|
+
(card) => card.card === selectedCard
|
281
|
+
);
|
282
|
+
// find the next card that has not finished
|
283
|
+
const nextCard = runInfo.cards.find((card) => !card.finished);
|
284
|
+
// if there is a next card, change to it
|
285
|
+
|
286
|
+
console.log("nextCard", nextCard);
|
287
|
+
if (nextCard) {
|
288
|
+
selectedCard = nextCard.card;
|
289
|
+
handleLoadCard(nextCard.label, nextCard.card);
|
290
|
+
}
|
291
|
+
}
|
292
|
+
}
|
293
|
+
|
294
|
+
/**
|
295
|
+
* Main loop function
|
296
|
+
*/
|
297
|
+
async function main() {
|
298
|
+
try {
|
299
|
+
// get info on the entire run
|
300
|
+
const runInfoRequest = await fetch("/runinfo");
|
301
|
+
const runInfo = await runInfoRequest.json();
|
302
|
+
|
303
|
+
// reset the state if the run_id changes or on first time
|
304
|
+
if (currentRunId !== runInfo.run_id) {
|
305
|
+
currentRunId = runInfo.run_id;
|
306
|
+
currentCardsLength = 0;
|
307
|
+
selectedCard = "";
|
308
|
+
console.log("runInfo", runInfo);
|
309
|
+
}
|
310
|
+
if (runInfo.poll_interval * 1000 !== DATA_UPDATE_POLL_INTERVAL) {
|
311
|
+
DATA_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
|
312
|
+
}
|
313
|
+
if (runInfo.poll_interval * 1000 !== RUN_UPDATE_POLL_INTERVAL) {
|
314
|
+
RUN_UPDATE_POLL_INTERVAL = runInfo.poll_interval * 1000;
|
315
|
+
}
|
316
|
+
if (runInfo?.status === "ok") {
|
317
|
+
// only make changes if the cards length has changed
|
318
|
+
if (runInfo?.cards.length !== currentCardsLength) {
|
319
|
+
onCardsLoaded(runInfo);
|
320
|
+
}
|
321
|
+
// if there is no selected card, select the first one
|
322
|
+
if (!selectedCard) {
|
323
|
+
selectedCard = runInfo.cards[0].card;
|
324
|
+
handleLoadCard(runInfo.cards[0].label, runInfo.cards[0].card);
|
325
|
+
}
|
326
|
+
} else {
|
327
|
+
handleStatus(runInfo?.status, false);
|
328
|
+
}
|
329
|
+
} catch (error) {
|
330
|
+
console.error(error);
|
331
|
+
}
|
332
|
+
// always run the main loop, because a user can start a new run,
|
333
|
+
// or restart the web server, and we want the page to continue working
|
334
|
+
setTimeout(main, RUN_UPDATE_POLL_INTERVAL);
|
335
|
+
}
|
336
|
+
|
337
|
+
// start the main loop
|
338
|
+
window.onload = () => {
|
339
|
+
main();
|
340
|
+
};
|
341
|
+
})();
|
342
|
+
</script>
|
343
|
+
</body>
|
344
|
+
</html>
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import copy
|
1
2
|
import errno
|
2
3
|
import fcntl
|
3
4
|
import functools
|
@@ -10,17 +11,17 @@ from concurrent.futures import ThreadPoolExecutor
|
|
10
11
|
from hashlib import sha256
|
11
12
|
from io import BufferedIOBase
|
12
13
|
from itertools import chain
|
13
|
-
from urllib.parse import urlparse
|
14
|
+
from urllib.parse import unquote, urlparse
|
14
15
|
|
15
16
|
import requests
|
16
17
|
|
17
|
-
from metaflow.metaflow_config import get_pinned_conda_libs
|
18
18
|
from metaflow.exception import MetaflowException
|
19
|
+
from metaflow.metaflow_config import get_pinned_conda_libs
|
19
20
|
from metaflow.metaflow_environment import MetaflowEnvironment
|
20
21
|
from metaflow.metaflow_profile import profile
|
21
22
|
|
22
23
|
from . import MAGIC_FILE, _datastore_packageroot
|
23
|
-
from .utils import conda_platform
|
24
|
+
from .utils import conda_platform
|
24
25
|
|
25
26
|
|
26
27
|
class CondaEnvironmentException(MetaflowException):
|
@@ -104,10 +105,24 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
104
105
|
)
|
105
106
|
|
106
107
|
def cache(storage, results, type_):
|
108
|
+
def _path(url, local_path):
|
109
|
+
# Special handling for VCS packages
|
110
|
+
if url.startswith("git+"):
|
111
|
+
base, _ = os.path.split(urlparse(url).path)
|
112
|
+
_, file = os.path.split(local_path)
|
113
|
+
prefix = url.split("@")[-1]
|
114
|
+
return urlparse(url).netloc + os.path.join(
|
115
|
+
unquote(base), prefix, file
|
116
|
+
)
|
117
|
+
else:
|
118
|
+
return urlparse(url).netloc + urlparse(url).path
|
119
|
+
|
107
120
|
local_packages = {
|
108
121
|
url: {
|
109
122
|
# Path to package in datastore.
|
110
|
-
"path":
|
123
|
+
"path": _path(
|
124
|
+
url, local_path
|
125
|
+
), # urlparse(url).netloc + urlparse(url).path,
|
111
126
|
# Path to package on local disk.
|
112
127
|
"local_path": local_path,
|
113
128
|
}
|
@@ -116,22 +131,26 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
116
131
|
}
|
117
132
|
dirty = set()
|
118
133
|
# Prune list of packages to cache.
|
134
|
+
|
135
|
+
_meta = copy.deepcopy(local_packages)
|
119
136
|
for id_, packages, _, _ in results:
|
120
137
|
for package in packages:
|
121
138
|
if package.get("path"):
|
122
139
|
# Cache only those packages that manifest is unaware of
|
123
140
|
local_packages.pop(package["url"], None)
|
124
141
|
else:
|
125
|
-
package["path"] =
|
126
|
-
package["url"], parse_filename_from_url(package["url"])
|
127
|
-
)
|
142
|
+
package["path"] = _meta[package["url"]]["path"]
|
128
143
|
dirty.add(id_)
|
129
144
|
|
130
145
|
list_of_path_and_filehandle = [
|
131
146
|
(
|
132
147
|
package["path"],
|
133
148
|
# Lazily fetch package from the interweb if needed.
|
134
|
-
LazyOpen(
|
149
|
+
LazyOpen(
|
150
|
+
package["local_path"],
|
151
|
+
"rb",
|
152
|
+
url,
|
153
|
+
),
|
135
154
|
)
|
136
155
|
for url, package in local_packages.items()
|
137
156
|
]
|
@@ -393,6 +412,10 @@ class LazyOpen(BufferedIOBase):
|
|
393
412
|
if self.filename and os.path.exists(self.filename):
|
394
413
|
self._file = open(self.filename, self.mode)
|
395
414
|
elif self.url:
|
415
|
+
if self.url.startswith("git+"):
|
416
|
+
raise ValueError(
|
417
|
+
"LazyOpen doesn't support VCS url %s yet!" % self.url
|
418
|
+
)
|
396
419
|
self._buffer = self._download_to_buffer()
|
397
420
|
self._file = io.BytesIO(self._buffer)
|
398
421
|
else:
|
metaflow/plugins/pypi/pip.py
CHANGED
@@ -1,15 +1,18 @@
|
|
1
1
|
import json
|
2
2
|
import os
|
3
3
|
import re
|
4
|
+
import shutil
|
4
5
|
import subprocess
|
5
6
|
import tempfile
|
7
|
+
from concurrent.futures import ThreadPoolExecutor
|
6
8
|
from itertools import chain, product
|
9
|
+
from urllib.parse import unquote
|
7
10
|
|
8
11
|
from metaflow.exception import MetaflowException
|
9
12
|
from metaflow.util import which
|
10
13
|
|
11
14
|
from .micromamba import Micromamba
|
12
|
-
from .utils import
|
15
|
+
from .utils import pip_tags, wheel_tags
|
13
16
|
|
14
17
|
|
15
18
|
class PipException(MetaflowException):
|
@@ -26,7 +29,7 @@ METADATA_FILE = "{prefix}/.pip/metadata"
|
|
26
29
|
INSTALLATION_MARKER = "{prefix}/.pip/id"
|
27
30
|
|
28
31
|
# TODO:
|
29
|
-
# 1. Support
|
32
|
+
# 1. Support local dirs, non-wheel like packages
|
30
33
|
# 2. Support protected indices
|
31
34
|
|
32
35
|
|
@@ -72,15 +75,32 @@ class Pip(object):
|
|
72
75
|
# *(chain.from_iterable(product(["--implementations"], set(implementations)))),
|
73
76
|
]
|
74
77
|
for package, version in packages.items():
|
75
|
-
if version.startswith(("<", ">", "!", "~")):
|
78
|
+
if version.startswith(("<", ">", "!", "~", "@")):
|
79
|
+
cmd.append(f"{package}{version}")
|
80
|
+
elif not version:
|
76
81
|
cmd.append(f"{package}{version}")
|
77
82
|
else:
|
78
83
|
cmd.append(f"{package}=={version}")
|
79
84
|
self._call(prefix, cmd)
|
85
|
+
|
86
|
+
def _format(dl_info):
|
87
|
+
res = {k: v for k, v in dl_info.items() if k in ["url"]}
|
88
|
+
# If source url is not a wheel, we need to build the target.
|
89
|
+
res["require_build"] = not res["url"].endswith(".whl")
|
90
|
+
|
91
|
+
# Reconstruct the VCS url and pin to current commit_id
|
92
|
+
# so using @branch as a version acts as expected.
|
93
|
+
vcs_info = dl_info.get("vcs_info")
|
94
|
+
if vcs_info:
|
95
|
+
res["url"] = "{vcs}+{url}@{commit_id}".format(**vcs_info, **res)
|
96
|
+
# used to deduplicate the storage location in case wheel does not
|
97
|
+
# build with enough unique identifiers.
|
98
|
+
res["hash"] = vcs_info["commit_id"]
|
99
|
+
return res
|
100
|
+
|
80
101
|
with open(report, mode="r", encoding="utf-8") as f:
|
81
102
|
return [
|
82
|
-
|
83
|
-
for item in json.load(f)["install"]
|
103
|
+
_format(item["download_info"]) for item in json.load(f)["install"]
|
84
104
|
]
|
85
105
|
|
86
106
|
def download(self, id_, packages, python, platform):
|
@@ -89,21 +109,78 @@ class Pip(object):
|
|
89
109
|
# download packages only if they haven't ever been downloaded before
|
90
110
|
if os.path.isfile(metadata_file):
|
91
111
|
return
|
112
|
+
|
92
113
|
metadata = {}
|
114
|
+
custom_index_url, extra_index_urls = self.indices(prefix)
|
115
|
+
|
116
|
+
# build wheels if needed
|
117
|
+
with ThreadPoolExecutor() as executor:
|
118
|
+
|
119
|
+
def _build(key, package):
|
120
|
+
dest = "{prefix}/.pip/built_wheels/{key}".format(prefix=prefix, key=key)
|
121
|
+
cmd = [
|
122
|
+
"wheel",
|
123
|
+
"--no-deps",
|
124
|
+
"--progress-bar=off",
|
125
|
+
"--wheel-dir=%s" % dest,
|
126
|
+
"--quiet",
|
127
|
+
*(["--index-url", custom_index_url] if custom_index_url else []),
|
128
|
+
*(
|
129
|
+
chain.from_iterable(
|
130
|
+
product(["--extra-index-url"], set(extra_index_urls))
|
131
|
+
)
|
132
|
+
),
|
133
|
+
package["url"],
|
134
|
+
]
|
135
|
+
self._call(prefix, cmd)
|
136
|
+
return package, dest
|
137
|
+
|
138
|
+
results = list(
|
139
|
+
executor.map(
|
140
|
+
lambda x: _build(*x),
|
141
|
+
enumerate(
|
142
|
+
package for package in packages if package["require_build"]
|
143
|
+
),
|
144
|
+
)
|
145
|
+
)
|
146
|
+
|
147
|
+
for package, path in results:
|
148
|
+
(wheel,) = [
|
149
|
+
f
|
150
|
+
for f in os.listdir(path)
|
151
|
+
if os.path.isfile(os.path.join(path, f)) and f.endswith(".whl")
|
152
|
+
]
|
153
|
+
if (
|
154
|
+
len(set(pip_tags(python, platform)).intersection(wheel_tags(wheel)))
|
155
|
+
== 0
|
156
|
+
):
|
157
|
+
raise PipException(
|
158
|
+
"The built wheel %s is not supported for %s with Python %s"
|
159
|
+
% (wheel, platform, python)
|
160
|
+
)
|
161
|
+
target = "{prefix}/.pip/wheels/{hash}/{wheel}".format(
|
162
|
+
prefix=prefix,
|
163
|
+
wheel=wheel,
|
164
|
+
hash=package["hash"],
|
165
|
+
)
|
166
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
167
|
+
shutil.move(os.path.join(path, wheel), target)
|
168
|
+
metadata["{url}".format(**package)] = target
|
169
|
+
|
93
170
|
implementations, platforms, abis = zip(
|
94
171
|
*[
|
95
172
|
(tag.interpreter, tag.platform, tag.abi)
|
96
173
|
for tag in pip_tags(python, platform)
|
97
174
|
]
|
98
175
|
)
|
99
|
-
|
176
|
+
|
100
177
|
cmd = [
|
101
178
|
"download",
|
102
179
|
"--no-deps",
|
103
180
|
"--no-index",
|
104
181
|
"--progress-bar=off",
|
105
182
|
# if packages are present in Pip cache, this will be a local copy
|
106
|
-
"--dest
|
183
|
+
"--dest={prefix}/.pip/wheels".format(prefix=prefix),
|
107
184
|
"--quiet",
|
108
185
|
*(["--index-url", custom_index_url] if custom_index_url else []),
|
109
186
|
*(
|
@@ -115,10 +192,11 @@ class Pip(object):
|
|
115
192
|
*(chain.from_iterable(product(["--platform"], set(platforms)))),
|
116
193
|
# *(chain.from_iterable(product(["--implementations"], set(implementations)))),
|
117
194
|
]
|
195
|
+
packages = [package for package in packages if not package["require_build"]]
|
118
196
|
for package in packages:
|
119
197
|
cmd.append("{url}".format(**package))
|
120
198
|
metadata["{url}".format(**package)] = "{prefix}/.pip/wheels/{wheel}".format(
|
121
|
-
prefix=prefix, wheel=
|
199
|
+
prefix=prefix, wheel=unquote(package["url"].split("/")[-1])
|
122
200
|
)
|
123
201
|
self._call(prefix, cmd)
|
124
202
|
# write the url to wheel mappings in a magic location
|
@@ -128,6 +206,7 @@ class Pip(object):
|
|
128
206
|
def create(self, id_, packages, python, platform):
|
129
207
|
prefix = self.micromamba.path_to_environment(id_)
|
130
208
|
installation_marker = INSTALLATION_MARKER.format(prefix=prefix)
|
209
|
+
metadata = self.metadata(id_, packages, python, platform)
|
131
210
|
# install packages only if they haven't been installed before
|
132
211
|
if os.path.isfile(installation_marker):
|
133
212
|
return
|
@@ -143,7 +222,7 @@ class Pip(object):
|
|
143
222
|
"--quiet",
|
144
223
|
]
|
145
224
|
for package in packages:
|
146
|
-
cmd.append("
|
225
|
+
cmd.append(metadata[package["url"]])
|
147
226
|
self._call(prefix, cmd)
|
148
227
|
with open(installation_marker, "w") as file:
|
149
228
|
file.write(json.dumps({"id": id_}))
|
metaflow/plugins/pypi/utils.py
CHANGED
@@ -10,12 +10,17 @@ if sys.version_info < (3, 6):
|
|
10
10
|
)
|
11
11
|
|
12
12
|
tags = Tags()
|
13
|
+
parse_wheel_filename = lambda: (_ for _ in ()).throw(
|
14
|
+
RuntimeError("packaging.utils is not avaliable for Python < 3.6")
|
15
|
+
)
|
13
16
|
else:
|
14
17
|
from metaflow._vendor.packaging import tags
|
18
|
+
from metaflow._vendor.packaging.utils import parse_wheel_filename
|
15
19
|
|
16
|
-
from metaflow.exception import MetaflowException
|
17
20
|
from urllib.parse import unquote, urlparse
|
18
21
|
|
22
|
+
from metaflow.exception import MetaflowException
|
23
|
+
|
19
24
|
|
20
25
|
def conda_platform():
|
21
26
|
# Returns the conda platform for the Python interpreter
|
@@ -34,6 +39,11 @@ def conda_platform():
|
|
34
39
|
return "osx-64"
|
35
40
|
|
36
41
|
|
42
|
+
def wheel_tags(wheel):
|
43
|
+
_, _, _, tags = parse_wheel_filename(wheel)
|
44
|
+
return list(tags)
|
45
|
+
|
46
|
+
|
37
47
|
def pip_tags(python_version, mamba_platform):
|
38
48
|
# Returns a list of pip tags containing (implementation, platforms, abis) tuples
|
39
49
|
# assuming a CPython implementation for Python interpreter.
|
@@ -81,10 +91,3 @@ def parse_filename_from_url(url):
|
|
81
91
|
# Separate method as it might require additional checks for the parsing.
|
82
92
|
filename = url.split("/")[-1]
|
83
93
|
return unquote(filename)
|
84
|
-
|
85
|
-
|
86
|
-
def generate_cache_path(url, local_path):
|
87
|
-
base, _ = os.path.split(urlparse(url).path)
|
88
|
-
_, localfile = os.path.split(local_path)
|
89
|
-
unquoted_base = unquote(base)
|
90
|
-
return urlparse(url).netloc + os.path.join(unquoted_base, localfile)
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.10.
|
1
|
+
metaflow_version = "2.10.11"
|
@@ -33,7 +33,7 @@ metaflow/task.py,sha256=yGNU3T3giKiG--vE0DUj_K-8jur2TclCS45XjPVLcq4,25314
|
|
33
33
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
34
34
|
metaflow/util.py,sha256=jbMJ17rK-dFTBCjimWqxkfcr3v__bHa3tZtX0g8iS2c,13257
|
35
35
|
metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
|
36
|
-
metaflow/version.py,sha256=
|
36
|
+
metaflow/version.py,sha256=NpJM4rx0ubVOsZrPH8YGJtEzrQhzHj-x614Y8e5rl6c,29
|
37
37
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
38
38
|
metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
|
39
39
|
metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
|
@@ -187,10 +187,10 @@ metaflow/plugins/cards/component_serializer.py,sha256=Row7c_8_euJcF_I1lWHdgMRj7d
|
|
187
187
|
metaflow/plugins/cards/exception.py,sha256=2UqlNb-Kxpg6cuLu2sBEIPTIElwlVBsSpeCgDYKTxWg,5222
|
188
188
|
metaflow/plugins/cards/card_modules/__init__.py,sha256=WI2IAsFiKGyqPrHtO9S9-MbyVtUTgWJNL4xjJaBErRo,3437
|
189
189
|
metaflow/plugins/cards/card_modules/base.html,sha256=Y208ZKIZqEWWUcoBFTLTdWKAG0C8xH5lmyCRSjaN2FY,21004
|
190
|
-
metaflow/plugins/cards/card_modules/basic.py,sha256=
|
190
|
+
metaflow/plugins/cards/card_modules/basic.py,sha256=h8v-3S_katr6YOteNVXuFuFY_9a2SeMjLT_TTrcQbPM,23416
|
191
191
|
metaflow/plugins/cards/card_modules/bundle.css,sha256=ms2wOKftlPM_i6bC_4BkrmqCOj8mYw9OFvRCJF9FSV4,11981
|
192
192
|
metaflow/plugins/cards/card_modules/card.py,sha256=8pyBDEd-on7tayTfGxAQcAiqLlp6rSQDVKYvp87SglM,4209
|
193
|
-
metaflow/plugins/cards/card_modules/components.py,sha256=
|
193
|
+
metaflow/plugins/cards/card_modules/components.py,sha256=Yvq9lqR_5EFVPM0ruzYTft635_dGIbkJhgxrQlPz1eg,25321
|
194
194
|
metaflow/plugins/cards/card_modules/convert_to_native_type.py,sha256=CX4Z9Xdu6ZRELOyzILCNTGOz0kNWecEUet4ddgd2JVI,15782
|
195
195
|
metaflow/plugins/cards/card_modules/main.js,sha256=VxKo5ZcINVzp71vnu79XqzeRLN6yoPmdISb0jzWiHFw,1027769
|
196
196
|
metaflow/plugins/cards/card_modules/renderer_tools.py,sha256=V3H92FvIl2DtzVsU7SQuQd5_MFwvG6GSAmpqqrqeIQM,1641
|
@@ -200,6 +200,7 @@ metaflow/plugins/cards/card_modules/chevron/main.py,sha256=FKpDW4PS2dYshgkm6yIxr
|
|
200
200
|
metaflow/plugins/cards/card_modules/chevron/metadata.py,sha256=gQDpB9KezW9Mcv-n9K2Pt1akjiPGG1zYB5YPdQ0FlNc,19
|
201
201
|
metaflow/plugins/cards/card_modules/chevron/renderer.py,sha256=dm5ufR9-qnx94D9BtV7q_64oqYUpwdCRgBTcjBRkZyM,13938
|
202
202
|
metaflow/plugins/cards/card_modules/chevron/tokenizer.py,sha256=lQU9OELUE9a5Xu4snGhEV_YTT34iyUHVBuM1YO016Sw,7400
|
203
|
+
metaflow/plugins/cards/card_viewer/viewer.html,sha256=qZJGzhZhQ1gugsknRP7zkAPPfUAtvemK1UKqXoGff5M,11593
|
203
204
|
metaflow/plugins/datastores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
204
205
|
metaflow/plugins/datastores/azure_storage.py,sha256=V15_BZNZIp9_2rqzEjuOSxUnPzrNZTd5f9hsJ9SEFLc,16234
|
205
206
|
metaflow/plugins/datastores/gs_storage.py,sha256=ED1xfXNDiPkqma5PEui8HCv0b4ImHK_rlUMQJD9UNes,9622
|
@@ -252,12 +253,12 @@ metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j
|
|
252
253
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
253
254
|
metaflow/plugins/pypi/bootstrap.py,sha256=PBIStzaEuhAl0xBd0Ap6oUaWMDwvz5bZEWo6h2SFru0,5073
|
254
255
|
metaflow/plugins/pypi/conda_decorator.py,sha256=RHFzVJ5mdd4WeXC5cKe92xHgK775g8m-vY4REKJu4YI,13525
|
255
|
-
metaflow/plugins/pypi/conda_environment.py,sha256=
|
256
|
+
metaflow/plugins/pypi/conda_environment.py,sha256=FeHwE9xkI3nYv_OZ_iFzMI2Z2s5BG7llFSi-ndPeId8,17867
|
256
257
|
metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
|
257
|
-
metaflow/plugins/pypi/pip.py,sha256=
|
258
|
+
metaflow/plugins/pypi/pip.py,sha256=paL-hbj5j-vcYOeSm04OiBix-TpoKMF7bRfGVdXfGZs,11685
|
258
259
|
metaflow/plugins/pypi/pypi_decorator.py,sha256=6-8KzJqduV5JBnvJwZTl-ZC3lNUZF1BxAYiO1npttiQ,5553
|
259
260
|
metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
|
260
|
-
metaflow/plugins/pypi/utils.py,sha256=
|
261
|
+
metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
|
261
262
|
metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
|
262
263
|
metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
|
263
264
|
metaflow/plugins/secrets/secrets_decorator.py,sha256=Y41XCOEGt7asxx6FEOBdKMqYFlU_jcA5u7erHN8DqJM,10514
|
@@ -293,9 +294,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
293
294
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
294
295
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
295
296
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
296
|
-
metaflow-2.10.
|
297
|
-
metaflow-2.10.
|
298
|
-
metaflow-2.10.
|
299
|
-
metaflow-2.10.
|
300
|
-
metaflow-2.10.
|
301
|
-
metaflow-2.10.
|
297
|
+
metaflow-2.10.11.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
298
|
+
metaflow-2.10.11.dist-info/METADATA,sha256=Nzv4M2QBD8aZpENBzwYCadi_bRFT2imd3cKRth40mIY,5827
|
299
|
+
metaflow-2.10.11.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
|
300
|
+
metaflow-2.10.11.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
301
|
+
metaflow-2.10.11.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
302
|
+
metaflow-2.10.11.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|