metaflow 2.8.1__py2.py3-none-any.whl → 2.8.3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/client/core.py +14 -4
- metaflow/cmd/configure_cmd.py +3 -3
- metaflow/cmd/main_cli.py +9 -14
- metaflow/current.py +15 -0
- metaflow/datastore/datastore_set.py +7 -7
- metaflow/datastore/flow_datastore.py +1 -2
- metaflow/extension_support/__init__.py +1 -0
- metaflow/extension_support/integrations.py +141 -0
- metaflow/integrations.py +29 -0
- metaflow/metaflow_config.py +21 -0
- metaflow/metaflow_environment.py +5 -4
- metaflow/package.py +1 -1
- metaflow/plugins/airflow/airflow.py +0 -1
- metaflow/plugins/argo/argo_workflows.py +2 -0
- metaflow/plugins/argo/argo_workflows_cli.py +11 -1
- metaflow/plugins/aws/aws_utils.py +6 -1
- metaflow/plugins/aws/batch/batch.py +30 -8
- metaflow/plugins/aws/batch/batch_cli.py +12 -0
- metaflow/plugins/aws/batch/batch_client.py +39 -2
- metaflow/plugins/aws/batch/batch_decorator.py +23 -0
- metaflow/plugins/aws/step_functions/step_functions.py +7 -4
- metaflow/plugins/aws/step_functions/step_functions_cli.py +11 -1
- metaflow/plugins/cards/card_modules/bundle.css +56 -56
- metaflow/plugins/cards/card_modules/convert_to_native_type.py +67 -5
- metaflow/plugins/cards/card_modules/main.js +14 -7
- metaflow/plugins/conda/conda_environment.py +2 -2
- metaflow/plugins/conda/conda_step_decorator.py +7 -1
- metaflow/plugins/datatools/s3/s3.py +2 -2
- metaflow/plugins/env_escape/communication/channel.py +1 -1
- metaflow/plugins/kubernetes/kubernetes.py +4 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +6 -2
- metaflow/plugins/kubernetes/kubernetes_job.py +17 -2
- metaflow/plugins/metadata/service.py +3 -2
- metaflow/runtime.py +5 -3
- metaflow/tutorials/02-statistics/README.md +4 -9
- metaflow/tutorials/02-statistics/stats.py +38 -11
- metaflow/tutorials/03-playlist-redux/playlist.py +24 -16
- metaflow/tutorials/04-playlist-plus/playlist.py +14 -23
- metaflow/tutorials/05-hello-cloud/README.md +45 -0
- metaflow/tutorials/{05-helloaws/helloaws.ipynb → 05-hello-cloud/hello-cloud.ipynb} +10 -5
- metaflow/tutorials/{05-helloaws/helloaws.py → 05-hello-cloud/hello-cloud.py} +11 -13
- metaflow/tutorials/06-statistics-redux/README.md +6 -29
- metaflow/tutorials/06-statistics-redux/stats.ipynb +2 -2
- metaflow/tutorials/07-worldview/README.md +3 -11
- metaflow/tutorials/07-worldview/worldview.ipynb +3 -3
- metaflow/tutorials/08-autopilot/README.md +10 -17
- metaflow/tutorials/08-autopilot/autopilot.ipynb +12 -7
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/METADATA +1 -6
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/RECORD +53 -51
- metaflow/tutorials/05-helloaws/README.md +0 -27
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/LICENSE +0 -0
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/WHEEL +0 -0
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/entry_points.txt +0 -0
- {metaflow-2.8.1.dist-info → metaflow-2.8.3.dist-info}/top_level.txt +0 -0
|
@@ -149,7 +149,12 @@ class BatchJob(object):
|
|
|
149
149
|
max_swap,
|
|
150
150
|
swappiness,
|
|
151
151
|
inferentia,
|
|
152
|
+
memory,
|
|
152
153
|
host_volumes,
|
|
154
|
+
use_tmpfs,
|
|
155
|
+
tmpfs_tempdir,
|
|
156
|
+
tmpfs_size,
|
|
157
|
+
tmpfs_path,
|
|
153
158
|
num_parallel,
|
|
154
159
|
):
|
|
155
160
|
# identify platform from any compute environment associated with the
|
|
@@ -251,7 +256,7 @@ class BatchJob(object):
|
|
|
251
256
|
if inferentia:
|
|
252
257
|
if not (isinstance(inferentia, (int, unicode, basestring))):
|
|
253
258
|
raise BatchJobException(
|
|
254
|
-
"
|
|
259
|
+
"Invalid inferentia value: ({}) (should be 0 or greater)".format(
|
|
255
260
|
inferentia
|
|
256
261
|
)
|
|
257
262
|
)
|
|
@@ -282,6 +287,29 @@ class BatchJob(object):
|
|
|
282
287
|
{"sourceVolume": name, "containerPath": host_path}
|
|
283
288
|
)
|
|
284
289
|
|
|
290
|
+
if use_tmpfs or (tmpfs_size and use_tmpfs is None):
|
|
291
|
+
if tmpfs_size:
|
|
292
|
+
if not (isinstance(tmpfs_size, (int, unicode, basestring))):
|
|
293
|
+
raise BatchJobException(
|
|
294
|
+
"Invalid tmpfs value: ({}) (should be 0 or greater)".format(
|
|
295
|
+
tmpfs_size
|
|
296
|
+
)
|
|
297
|
+
)
|
|
298
|
+
else:
|
|
299
|
+
# default tmpfs behavior - https://man7.org/linux/man-pages/man5/tmpfs.5.html
|
|
300
|
+
tmpfs_size = int(memory) / 2
|
|
301
|
+
|
|
302
|
+
job_definition["containerProperties"]["linuxParameters"]["tmpfs"] = [
|
|
303
|
+
{
|
|
304
|
+
"containerPath": tmpfs_path,
|
|
305
|
+
"size": int(tmpfs_size),
|
|
306
|
+
"mountOptions": [
|
|
307
|
+
# should map to rw, suid, dev, exec, auto, nouser, and async
|
|
308
|
+
"defaults"
|
|
309
|
+
],
|
|
310
|
+
}
|
|
311
|
+
]
|
|
312
|
+
|
|
285
313
|
self.num_parallel = num_parallel or 0
|
|
286
314
|
if self.num_parallel >= 1:
|
|
287
315
|
job_definition["type"] = "multinode"
|
|
@@ -343,7 +371,12 @@ class BatchJob(object):
|
|
|
343
371
|
max_swap,
|
|
344
372
|
swappiness,
|
|
345
373
|
inferentia,
|
|
374
|
+
memory,
|
|
346
375
|
host_volumes,
|
|
376
|
+
use_tmpfs,
|
|
377
|
+
tmpfs_tempdir,
|
|
378
|
+
tmpfs_size,
|
|
379
|
+
tmpfs_path,
|
|
347
380
|
num_parallel,
|
|
348
381
|
):
|
|
349
382
|
self.payload["jobDefinition"] = self._register_job_definition(
|
|
@@ -355,7 +388,12 @@ class BatchJob(object):
|
|
|
355
388
|
max_swap,
|
|
356
389
|
swappiness,
|
|
357
390
|
inferentia,
|
|
391
|
+
memory,
|
|
358
392
|
host_volumes,
|
|
393
|
+
use_tmpfs,
|
|
394
|
+
tmpfs_tempdir,
|
|
395
|
+
tmpfs_size,
|
|
396
|
+
tmpfs_path,
|
|
359
397
|
num_parallel,
|
|
360
398
|
)
|
|
361
399
|
return self
|
|
@@ -522,7 +560,6 @@ class TriableException(Exception):
|
|
|
522
560
|
|
|
523
561
|
|
|
524
562
|
class RunningJob(object):
|
|
525
|
-
|
|
526
563
|
NUM_RETRIES = 8
|
|
527
564
|
|
|
528
565
|
def __init__(self, id, client):
|
|
@@ -71,6 +71,16 @@ class BatchDecorator(StepDecorator):
|
|
|
71
71
|
A swappiness value of 0 causes swapping not to happen unless absolutely
|
|
72
72
|
necessary. A swappiness value of 100 causes pages to be swapped very
|
|
73
73
|
aggressively. Accepted values are whole numbers between 0 and 100.
|
|
74
|
+
use_tmpfs: bool, default: False
|
|
75
|
+
This enables an explicit tmpfs mount for this step.
|
|
76
|
+
tmpfs_tempdir: bool, default: True
|
|
77
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
|
78
|
+
tmpfs_size: int, optional
|
|
79
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
|
80
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
|
81
|
+
memory allocated for this step.
|
|
82
|
+
tmpfs_path: string, optional
|
|
83
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
|
74
84
|
inferentia : int, default: 0
|
|
75
85
|
Number of Inferentia chips required for this step.
|
|
76
86
|
"""
|
|
@@ -89,6 +99,10 @@ class BatchDecorator(StepDecorator):
|
|
|
89
99
|
"swappiness": None,
|
|
90
100
|
"inferentia": None,
|
|
91
101
|
"host_volumes": None,
|
|
102
|
+
"use_tmpfs": False,
|
|
103
|
+
"tmpfs_tempdir": True,
|
|
104
|
+
"tmpfs_size": None,
|
|
105
|
+
"tmpfs_path": "/metaflow_temp",
|
|
92
106
|
}
|
|
93
107
|
resource_defaults = {
|
|
94
108
|
"cpu": "1",
|
|
@@ -153,6 +167,10 @@ class BatchDecorator(StepDecorator):
|
|
|
153
167
|
"least 60 seconds for execution on AWS Batch.".format(step=step)
|
|
154
168
|
)
|
|
155
169
|
|
|
170
|
+
# Validate tmpfs_path. Batch requires this to be an absolute path
|
|
171
|
+
if self.attributes["tmpfs_path"] and self.attributes["tmpfs_path"][0] != "/":
|
|
172
|
+
raise BatchException("'tmpfs_path' needs to be an absolute path")
|
|
173
|
+
|
|
156
174
|
def runtime_init(self, flow, graph, package, run_id):
|
|
157
175
|
# Set some more internal state.
|
|
158
176
|
self.flow = flow
|
|
@@ -198,6 +216,11 @@ class BatchDecorator(StepDecorator):
|
|
|
198
216
|
self.metadata = metadata
|
|
199
217
|
self.task_datastore = task_datastore
|
|
200
218
|
|
|
219
|
+
# current.tempdir reflects the value of METAFLOW_TEMPDIR (the current working
|
|
220
|
+
# directory by default), or the value of tmpfs_path if tmpfs_tempdir=False.
|
|
221
|
+
if not self.attributes["tmpfs_tempdir"]:
|
|
222
|
+
current._update_env({"tempdir": self.attributes["tmpfs_path"]})
|
|
223
|
+
|
|
201
224
|
# task_pre_step may run locally if fallback is activated for @catch
|
|
202
225
|
# decorator. In that scenario, we skip collecting AWS Batch execution
|
|
203
226
|
# metadata. A rudimentary way to detect non-local execution is to
|
|
@@ -227,7 +227,6 @@ class StepFunctions(object):
|
|
|
227
227
|
return None
|
|
228
228
|
|
|
229
229
|
def _compile(self):
|
|
230
|
-
|
|
231
230
|
# Visit every node of the flow and recursively build the state machine.
|
|
232
231
|
def _visit(node, workflow, exit_node=None):
|
|
233
232
|
if node.parallel_foreach:
|
|
@@ -637,8 +636,7 @@ class StepFunctions(object):
|
|
|
637
636
|
env["METAFLOW_SFN_DYNAMO_DB_TABLE"] = SFN_DYNAMO_DB_TABLE
|
|
638
637
|
|
|
639
638
|
# It makes no sense to set env vars to None (shows up as "None" string)
|
|
640
|
-
|
|
641
|
-
del env
|
|
639
|
+
env = {k: v for k, v in env.items() if v is not None}
|
|
642
640
|
|
|
643
641
|
# Resolve AWS Batch resource requirements.
|
|
644
642
|
batch_deco = [deco for deco in node.decorators if deco.name == "batch"][0]
|
|
@@ -686,7 +684,12 @@ class StepFunctions(object):
|
|
|
686
684
|
shared_memory=resources["shared_memory"],
|
|
687
685
|
max_swap=resources["max_swap"],
|
|
688
686
|
swappiness=resources["swappiness"],
|
|
689
|
-
|
|
687
|
+
use_tmpfs=resources["use_tmpfs"],
|
|
688
|
+
tmpfs_tempdir=resources["tmpfs_tempdir"],
|
|
689
|
+
tmpfs_size=resources["tmpfs_size"],
|
|
690
|
+
tmpfs_path=resources["tmpfs_path"],
|
|
691
|
+
inferentia=resources["inferentia"],
|
|
692
|
+
env=env,
|
|
690
693
|
attrs=attrs,
|
|
691
694
|
host_volumes=resources["host_volumes"],
|
|
692
695
|
)
|
|
@@ -9,6 +9,7 @@ from metaflow import current, decorators, parameters, JSONType
|
|
|
9
9
|
from metaflow.metaflow_config import (
|
|
10
10
|
SERVICE_VERSION_CHECK,
|
|
11
11
|
SFN_STATE_MACHINE_PREFIX,
|
|
12
|
+
UI_URL,
|
|
12
13
|
)
|
|
13
14
|
from metaflow.exception import MetaflowException, MetaflowInternalError
|
|
14
15
|
from metaflow.package import MetaflowPackage
|
|
@@ -311,7 +312,6 @@ def make_flow(
|
|
|
311
312
|
def resolve_token(
|
|
312
313
|
name, token_prefix, obj, authorize, given_token, generate_new_token, is_project
|
|
313
314
|
):
|
|
314
|
-
|
|
315
315
|
# 1) retrieve the previous deployment, if one exists
|
|
316
316
|
workflow = StepFunctions.get_existing_deployment(name)
|
|
317
317
|
if workflow is None:
|
|
@@ -452,6 +452,16 @@ def trigger(obj, run_id_file=None, **kwargs):
|
|
|
452
452
|
bold=True,
|
|
453
453
|
)
|
|
454
454
|
|
|
455
|
+
run_url = (
|
|
456
|
+
"%s/%s/%s" % (UI_URL.rstrip("/"), obj.flow.name, run_id) if UI_URL else None
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
if run_url:
|
|
460
|
+
obj.echo(
|
|
461
|
+
"See the run in the UI at %s" % run_url,
|
|
462
|
+
bold=True,
|
|
463
|
+
)
|
|
464
|
+
|
|
455
465
|
|
|
456
466
|
@step_functions.command(help="List all runs of the workflow on AWS Step Functions.")
|
|
457
467
|
@click.option(
|
|
@@ -1,57 +1,4 @@
|
|
|
1
|
-
.container.svelte-teyund{width:100%;display:flex;flex-direction:column;position:relative}
|
|
2
|
-
https://prismjs.com/download.html#themes=prism&languages=clike+python */
|
|
3
|
-
code[class*=language-],pre[class*=language-]{color:#000;background:0 0;text-shadow:0 1px #fff;font-family:Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}code[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,pre[class*=language-]::-moz-selection{text-shadow:none;background:#b3d4fc}code[class*=language-] ::selection,code[class*=language-]::selection,pre[class*=language-] ::selection,pre[class*=language-]::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.boolean,.token.constant,.token.deleted,.token.number,.token.property,.token.symbol,.token.tag{color:#905}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#690}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url{color:#9a6e3a;background:hsla(0,0%,100%,.5)}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.class-name,.token.function{color:#dd4a68}.token.important,.token.regex,.token.variable{color:#e90}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}
|
|
4
|
-
@import url("https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap");
|
|
5
|
-
|
|
6
|
-
:root {
|
|
7
|
-
--bg: #ffffff;
|
|
8
|
-
--black: #333;
|
|
9
|
-
--blue: #0c66de;
|
|
10
|
-
--dk-grey: #767676;
|
|
11
|
-
--dk-primary: #ef863b;
|
|
12
|
-
--dk-secondary: #13172d;
|
|
13
|
-
--dk-tertiary: #0f426e;
|
|
14
|
-
--error: #cf483e;
|
|
15
|
-
--grey: rgba(0, 0, 0, 0.125);
|
|
16
|
-
--highlight: #f8d9d8;
|
|
17
|
-
--lt-blue: #4fa7ff;
|
|
18
|
-
--lt-grey: #f3f3f3;
|
|
19
|
-
--lt-lt-grey: #f9f9f9;
|
|
20
|
-
--lt-primary: #ffcb8b;
|
|
21
|
-
--lt-secondary: #434d81;
|
|
22
|
-
--lt-tertiary: #4189c9;
|
|
23
|
-
--primary: #faab4a;
|
|
24
|
-
--quadrary: #f8d9d8;
|
|
25
|
-
--secondary: #2e3454;
|
|
26
|
-
--tertiary: #2a679d;
|
|
27
|
-
--white: #ffffff;
|
|
28
|
-
|
|
29
|
-
--component-spacer: 3rem;
|
|
30
|
-
--aside-width: 20rem;
|
|
31
|
-
--embed-card-min-height: 12rem;
|
|
32
|
-
|
|
33
|
-
--mono-font: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono",
|
|
34
|
-
"Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro",
|
|
35
|
-
"Fira Mono", "Droid Sans Mono", "Courier New", monospace;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
html, body {
|
|
39
|
-
margin: 0;
|
|
40
|
-
min-height: 100vh;
|
|
41
|
-
overflow-y: visible;
|
|
42
|
-
padding: 0;
|
|
43
|
-
width: 100%;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
.card_app {
|
|
47
|
-
width: 100%;
|
|
48
|
-
min-height: 100vh;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
.embed .card_app {
|
|
52
|
-
min-height: var(--embed-card-min-height);
|
|
53
|
-
}
|
|
54
|
-
.mf-card * {
|
|
1
|
+
.container.svelte-teyund{width:100%;display:flex;flex-direction:column;position:relative}.mf-card * {
|
|
55
2
|
box-sizing: border-box;
|
|
56
3
|
}
|
|
57
4
|
|
|
@@ -155,7 +102,60 @@ html, body {
|
|
|
155
102
|
text-shadow: none;
|
|
156
103
|
user-select: auto;
|
|
157
104
|
}
|
|
158
|
-
|
|
105
|
+
/* PrismJS 1.25.0
|
|
106
|
+
https://prismjs.com/download.html#themes=prism&languages=clike+python */
|
|
107
|
+
code[class*=language-],pre[class*=language-]{color:#000;background:0 0;text-shadow:0 1px #fff;font-family:Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}code[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,pre[class*=language-]::-moz-selection{text-shadow:none;background:#b3d4fc}code[class*=language-] ::selection,code[class*=language-]::selection,pre[class*=language-] ::selection,pre[class*=language-]::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.boolean,.token.constant,.token.deleted,.token.number,.token.property,.token.symbol,.token.tag{color:#905}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#690}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url{color:#9a6e3a;background:hsla(0,0%,100%,.5)}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.class-name,.token.function{color:#dd4a68}.token.important,.token.regex,.token.variable{color:#e90}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}
|
|
108
|
+
@import url("https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap");
|
|
109
|
+
|
|
110
|
+
:root {
|
|
111
|
+
--bg: #ffffff;
|
|
112
|
+
--black: #333;
|
|
113
|
+
--blue: #0c66de;
|
|
114
|
+
--dk-grey: #767676;
|
|
115
|
+
--dk-primary: #ef863b;
|
|
116
|
+
--dk-secondary: #13172d;
|
|
117
|
+
--dk-tertiary: #0f426e;
|
|
118
|
+
--error: #cf483e;
|
|
119
|
+
--grey: rgba(0, 0, 0, 0.125);
|
|
120
|
+
--highlight: #f8d9d8;
|
|
121
|
+
--lt-blue: #4fa7ff;
|
|
122
|
+
--lt-grey: #f3f3f3;
|
|
123
|
+
--lt-lt-grey: #f9f9f9;
|
|
124
|
+
--lt-primary: #ffcb8b;
|
|
125
|
+
--lt-secondary: #434d81;
|
|
126
|
+
--lt-tertiary: #4189c9;
|
|
127
|
+
--primary: #faab4a;
|
|
128
|
+
--quadrary: #f8d9d8;
|
|
129
|
+
--secondary: #2e3454;
|
|
130
|
+
--tertiary: #2a679d;
|
|
131
|
+
--white: #ffffff;
|
|
132
|
+
|
|
133
|
+
--component-spacer: 3rem;
|
|
134
|
+
--aside-width: 20rem;
|
|
135
|
+
--embed-card-min-height: 12rem;
|
|
136
|
+
|
|
137
|
+
--mono-font: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono",
|
|
138
|
+
"Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro",
|
|
139
|
+
"Fira Mono", "Droid Sans Mono", "Courier New", monospace;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
html, body {
|
|
143
|
+
margin: 0;
|
|
144
|
+
min-height: 100vh;
|
|
145
|
+
overflow-y: visible;
|
|
146
|
+
padding: 0;
|
|
147
|
+
width: 100%;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
.card_app {
|
|
151
|
+
width: 100%;
|
|
152
|
+
min-height: 100vh;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
.embed .card_app {
|
|
156
|
+
min-height: var(--embed-card-min-height);
|
|
157
|
+
}
|
|
158
|
+
aside.svelte-1okdv0e{display:none;line-height:2;text-align:left}@media(min-width: 60rem){aside.svelte-1okdv0e{display:flex;flex-direction:column;height:100vh;justify-content:space-between;padding:2.5rem 0 1.5rem 1.5rem;position:fixed;width:var(--aside-width)}}.embed aside{display:none}aside ul{list-style-type:none}aside a, aside button, aside a:visited{text-decoration:none;cursor:pointer;font-weight:700;color:var(--black)}aside a:hover, aside button:hover{text-decoration:underline}.logoContainer svg{width:100%;max-width:140px;margin-bottom:3.75rem;height:auto}.mainContainer.svelte-13ho8jo{max-width:110rem}main.svelte-13ho8jo{flex:0 1 auto;max-width:100rem;padding:1.5rem}@media(min-width: 60rem){main.svelte-13ho8jo{margin-left:var(--aside-width)}}.embed main{margin:0 auto}.modal.svelte-1hhf5ym{align-items:center;background:rgba(0, 0, 0, 0.5);bottom:0;cursor:pointer;display:flex;height:100%;justify-content:center;left:0;overflow:hidden;position:fixed;right:0;top:0;width:100%;z-index:100}.modalContainer > *{background-color:white;border-radius:5px;cursor:default;flex:0 1 auto;padding:1rem;position:relative}.modal img{max-height:80vh !important}.cancelButton.svelte-1hhf5ym{color:white;cursor:pointer;font-size:2rem;position:absolute;right:1rem;top:1rem}.cancelButton.svelte-1hhf5ym:hover{color:var(--blue)}.nav.svelte-1kdpgko.svelte-1kdpgko{border-radius:0 0 5px 0;display:none;margin:0;top:0}ul.navList.svelte-1kdpgko.svelte-1kdpgko{list-style-type:none}ul.navList.svelte-1kdpgko ul.svelte-1kdpgko{margin:0.5rem 1rem 2rem}.navList.svelte-1kdpgko li.svelte-1kdpgko{display:block;margin:0}.navItem.svelte-1kdpgko li.svelte-1kdpgko:hover{color:var(--blue)}.pageId.svelte-1kdpgko.svelte-1kdpgko{display:block;border-bottom:1px solid var(--grey);padding:0 0.5rem;margin-bottom:1rem}@media(min-width: 60rem){.nav.svelte-1kdpgko.svelte-1kdpgko{display:block}ul.navList.svelte-1kdpgko.svelte-1kdpgko{text-align:left}.navList.svelte-1kdpgko li.svelte-1kdpgko{display:block;margin:0.5rem 0}}.page.svelte-v7ihqd:last-of-type{margin-bottom:var(--component-spacer)}.page:last-of-type section:last-of-type hr{display:none}.container.svelte-ubs992{width:100%;overflow:auto}table.svelte-ubs992{width:100%}.heading.svelte-17n0qr8{margin-bottom:1.5rem}.sectionItems.svelte-17n0qr8{display:block}.sectionItems .imageContainer{max-height:500px}.container.svelte-17n0qr8{scroll-margin:var(--component-spacer)}hr.svelte-17n0qr8{background:var(--grey);border:none;height:1px;margin:var(--component-spacer) 0;padding:0}@media(min-width: 60rem){.sectionItems.svelte-17n0qr8{display:grid;grid-gap:2rem}}header.svelte-1ugmt5d{margin-bottom:var(--component-spacer)}.subtitle.svelte-lu9pnn{font-size:1rem;text-align:left}.log.svelte-1jhmsu{background:var(--lt-grey) !important;font-size:0.9rem;padding:2rem}.title.svelte-117s0ws{text-align:left}figure.svelte-1x96yvr{background:var(--lt-grey);padding:1rem;border-radius:5px;text-align:center;margin:0 auto var(--component-spacer)}@media(min-width: 60rem){figure.svelte-1x96yvr{margin-bottom:0}}img.svelte-1x96yvr{max-width:100%;max-height:500px}.label.svelte-1x96yvr{font-weight:bold;margin:0.5rem 0}.description.svelte-1x96yvr{font-size:0.9rem;font-style:italic;text-align:center;margin:0.5rem 0}.idCell.svelte-pt8vzv{font-weight:bold;text-align:right;background:var(--lt-grey);width:12%}.codeCell.svelte-pt8vzv{text-align:left;user-select:all}td.svelte-gl9h79{text-align:left}td.labelColumn.svelte-gl9h79{text-align:right;background-color:var(--lt-grey);font-weight:700;width:12%;white-space:nowrap}.tableContainer.svelte-q3hq57{overflow:auto}th.svelte-q3hq57{position:sticky;top:-1px;z-index:2;white-space:nowrap;background:var(--white)}.stepwrapper.svelte-18aex7a{display:flex;align-items:center;flex-direction:column;width:100%;position:relative;min-width:var(--dag-step-width)}.childwrapper.svelte-18aex7a{display:flex;width:100%}.gap.svelte-18aex7a{height:var(--dag-gap)}:root {
|
|
159
159
|
--dag-border: #282828;
|
|
160
160
|
--dag-bg-static: var(--lt-grey);
|
|
161
161
|
--dag-bg-success: #a5d46a;
|
|
@@ -167,4 +167,4 @@ aside.svelte-1okdv0e{display:none;line-height:2;text-align:left}@media(min-width
|
|
|
167
167
|
--dag-step-width: 11.25rem;
|
|
168
168
|
--dag-selected: #ffd700;
|
|
169
169
|
}
|
|
170
|
-
.
|
|
170
|
+
.wrapper.svelte-117ceti.svelte-117ceti{position:relative;z-index:1}.step.svelte-117ceti.svelte-117ceti{font-size:0.75rem;padding:0.5rem;color:var(--dk-grey)}.rectangle.svelte-117ceti.svelte-117ceti{background-color:var(--dag-bg-static);border:1px solid var(--dag-border);box-sizing:border-box;position:relative;height:var(--dag-step-height);width:var(--dag-step-width)}.rectangle.error.svelte-117ceti.svelte-117ceti{background-color:var(--dag-bg-error)}.rectangle.success.svelte-117ceti.svelte-117ceti{background-color:var(--dag-bg-success)}.rectangle.running.svelte-117ceti.svelte-117ceti{background-color:var(--dag-bg-running)}.level.svelte-117ceti.svelte-117ceti{z-index:-1;filter:contrast(0.5);position:absolute}.inner.svelte-117ceti.svelte-117ceti{position:relative;height:100%;width:100%}.name.svelte-117ceti.svelte-117ceti{font-weight:bold;overflow:hidden;text-overflow:ellipsis;display:block}.description.svelte-117ceti.svelte-117ceti{position:absolute;max-height:4rem;bottom:0;left:0;right:0;overflow:hidden;-webkit-line-clamp:4;line-clamp:4;display:-webkit-box;-webkit-box-orient:vertical}.overflown.description.svelte-117ceti.svelte-117ceti{cursor:help}.current.svelte-117ceti .rectangle.svelte-117ceti{box-shadow:0 0 10px var(--dag-selected)}.levelstoshow.svelte-117ceti.svelte-117ceti{position:absolute;bottom:100%;right:0;font-size:0.75rem;font-weight:100;text-align:right}.connectorwrapper.svelte-1hyaq5f{transform-origin:0 0;position:absolute;z-index:0;min-width:var(--strokeWidth)}.flip.svelte-1hyaq5f{transform:scaleX(-1)}.path.svelte-1hyaq5f{--strokeWidth:0.5rem;--strokeColor:var(--dag-connector);--borderRadius:1.25rem;box-sizing:border-box}.straightLine.svelte-1hyaq5f{position:absolute;top:0;bottom:0;left:0;right:0;border-left:var(--strokeWidth) solid var(--strokeColor)}.topLeft.svelte-1hyaq5f{position:absolute;top:0;left:0;right:50%;bottom:calc(var(--dag-gap) / 2 - var(--strokeWidth) / 2);border-radius:0 0 0 var(--borderRadius);border-left:var(--strokeWidth) solid var(--strokeColor);border-bottom:var(--strokeWidth) solid var(--strokeColor)}.bottomRight.svelte-1hyaq5f{position:absolute;top:calc(100% - (var(--dag-gap) / 2 + var(--strokeWidth) / 2));left:50%;right:0;bottom:0;border-radius:0 var(--borderRadius) 0 0;border-top:var(--strokeWidth) solid var(--strokeColor);border-right:var(--strokeWidth) solid var(--strokeColor)}
|
|
@@ -295,6 +295,70 @@ class TaskToDict:
|
|
|
295
295
|
def _parse_range(self, data_object):
|
|
296
296
|
return self._get_repr().repr(data_object)
|
|
297
297
|
|
|
298
|
+
@staticmethod
|
|
299
|
+
def _parse_pandas_column(column_object):
|
|
300
|
+
# There are two types of parsing we do here.
|
|
301
|
+
# 1. We explicitly parse the types we know how to parse
|
|
302
|
+
# 2. We try to partially match a type name to the column's type.
|
|
303
|
+
# - We do this because `datetime64` can match `datetime64[ns]` and `datetime64[ns, UTC]`
|
|
304
|
+
# - We do this because period can match `period[D]` and `period[2D]` etc.
|
|
305
|
+
# - There are just too many types to explicitly parse so we go by this heuristic
|
|
306
|
+
# We have a default parser called `truncate_long_objects` which type casts any column to string
|
|
307
|
+
# and truncates it to 30 characters.
|
|
308
|
+
# If there is any form of TypeError or ValueError we set the column value to "Unsupported Type"
|
|
309
|
+
# We also set columns which are have null values to "null" strings
|
|
310
|
+
time_format = "%Y-%m-%dT%H:%M:%SZ"
|
|
311
|
+
truncate_long_objects = (
|
|
312
|
+
lambda x: x.astype("string").str.slice(0, 30) + "..."
|
|
313
|
+
if x.astype("string").str.len().max() > 30
|
|
314
|
+
else x.astype("string")
|
|
315
|
+
)
|
|
316
|
+
type_parser = {
|
|
317
|
+
"int64": lambda x: x,
|
|
318
|
+
"float64": lambda x: x,
|
|
319
|
+
"bool": lambda x: x,
|
|
320
|
+
"object": lambda x: truncate_long_objects(x.fillna("null")),
|
|
321
|
+
"category": truncate_long_objects,
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
partial_type_name_match_parsers = {
|
|
325
|
+
"complex": {
|
|
326
|
+
"complex": lambda x: x.astype("string"),
|
|
327
|
+
},
|
|
328
|
+
"datetime": {
|
|
329
|
+
"datetime64": lambda x: x.dt.strftime(time_format),
|
|
330
|
+
"timedelta": lambda x: x.dt.total_seconds(),
|
|
331
|
+
},
|
|
332
|
+
"interval": {
|
|
333
|
+
"interval": lambda x: x.astype("string"),
|
|
334
|
+
},
|
|
335
|
+
"period": {
|
|
336
|
+
"period": lambda x: x.astype("string"),
|
|
337
|
+
},
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
def _match_partial_type():
|
|
341
|
+
col_type = column_object.dtype
|
|
342
|
+
for _, type_parsers in partial_type_name_match_parsers.items():
|
|
343
|
+
for type_name, parser in type_parsers.items():
|
|
344
|
+
if type_name in str(col_type):
|
|
345
|
+
return parser(column_object)
|
|
346
|
+
return None
|
|
347
|
+
|
|
348
|
+
try:
|
|
349
|
+
col_type = str(column_object.dtype)
|
|
350
|
+
if col_type in type_parser:
|
|
351
|
+
return type_parser[col_type](column_object)
|
|
352
|
+
else:
|
|
353
|
+
parsed_col = _match_partial_type()
|
|
354
|
+
if parsed_col is not None:
|
|
355
|
+
return parsed_col
|
|
356
|
+
return truncate_long_objects(column_object)
|
|
357
|
+
except ValueError as e:
|
|
358
|
+
return "Unsupported type: {0}".format(col_type)
|
|
359
|
+
except TypeError as e:
|
|
360
|
+
return "Unsupported type: {0}".format(col_type)
|
|
361
|
+
|
|
298
362
|
def _parse_pandas_dataframe(self, data_object, truncate=True):
|
|
299
363
|
headers = list(data_object.columns)
|
|
300
364
|
data = data_object
|
|
@@ -302,18 +366,16 @@ class TaskToDict:
|
|
|
302
366
|
data = data_object.head()
|
|
303
367
|
index_column = data.index
|
|
304
368
|
time_format = "%Y-%m-%dT%H:%M:%SZ"
|
|
305
|
-
|
|
369
|
+
|
|
370
|
+
if "datetime64" in str(index_column.dtype):
|
|
306
371
|
if index_column.__class__.__name__ == "DatetimeIndex":
|
|
307
372
|
index_column = index_column.strftime(time_format)
|
|
308
373
|
else:
|
|
309
374
|
index_column = index_column.dt.strftime(time_format)
|
|
310
375
|
|
|
311
376
|
for col in data.columns:
|
|
312
|
-
|
|
313
|
-
if data[col].dtype == "datetime64[ns]":
|
|
314
|
-
data[col] = data[col].dt.strftime(time_format)
|
|
377
|
+
data[col] = self._parse_pandas_column(data[col])
|
|
315
378
|
|
|
316
|
-
data = data.astype(object).where(data.notnull(), None)
|
|
317
379
|
data_vals = data.values.tolist()
|
|
318
380
|
for row, idx in zip(data_vals, index_column.values.tolist()):
|
|
319
381
|
row.insert(0, idx)
|