metaflow 2.15.21__py2.py3-none-any.whl → 2.16.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/__init__.py +7 -1
- metaflow/cli.py +19 -1
- metaflow/cli_components/init_cmd.py +1 -0
- metaflow/cli_components/run_cmds.py +8 -2
- metaflow/client/core.py +22 -30
- metaflow/datastore/task_datastore.py +0 -1
- metaflow/debug.py +5 -0
- metaflow/decorators.py +236 -70
- metaflow/extension_support/__init__.py +15 -8
- metaflow/extension_support/_empty_file.py +2 -2
- metaflow/flowspec.py +92 -60
- metaflow/graph.py +24 -2
- metaflow/meta_files.py +13 -0
- metaflow/metadata_provider/metadata.py +7 -1
- metaflow/metaflow_config.py +5 -0
- metaflow/metaflow_environment.py +82 -25
- metaflow/metaflow_version.py +1 -1
- metaflow/package/__init__.py +664 -0
- metaflow/packaging_sys/__init__.py +870 -0
- metaflow/packaging_sys/backend.py +113 -0
- metaflow/packaging_sys/distribution_support.py +153 -0
- metaflow/packaging_sys/tar_backend.py +86 -0
- metaflow/packaging_sys/utils.py +91 -0
- metaflow/packaging_sys/v1.py +480 -0
- metaflow/plugins/airflow/airflow.py +5 -1
- metaflow/plugins/airflow/airflow_cli.py +16 -5
- metaflow/plugins/argo/argo_workflows.py +15 -4
- metaflow/plugins/argo/argo_workflows_cli.py +17 -4
- metaflow/plugins/aws/batch/batch.py +22 -3
- metaflow/plugins/aws/batch/batch_cli.py +3 -0
- metaflow/plugins/aws/batch/batch_decorator.py +13 -5
- metaflow/plugins/aws/step_functions/step_functions.py +4 -1
- metaflow/plugins/aws/step_functions/step_functions_cli.py +16 -4
- metaflow/plugins/cards/card_decorator.py +0 -5
- metaflow/plugins/kubernetes/kubernetes.py +8 -1
- metaflow/plugins/kubernetes/kubernetes_cli.py +3 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +13 -5
- metaflow/plugins/package_cli.py +25 -23
- metaflow/plugins/parallel_decorator.py +4 -2
- metaflow/plugins/pypi/bootstrap.py +8 -2
- metaflow/plugins/pypi/conda_decorator.py +39 -82
- metaflow/plugins/pypi/conda_environment.py +6 -2
- metaflow/plugins/pypi/pypi_decorator.py +4 -4
- metaflow/plugins/test_unbounded_foreach_decorator.py +2 -2
- metaflow/plugins/timeout_decorator.py +0 -1
- metaflow/plugins/uv/bootstrap.py +12 -1
- metaflow/plugins/uv/uv_environment.py +4 -2
- metaflow/pylint_wrapper.py +5 -1
- metaflow/runner/click_api.py +5 -4
- metaflow/runner/subprocess_manager.py +14 -2
- metaflow/runtime.py +37 -11
- metaflow/task.py +92 -7
- metaflow/user_configs/config_options.py +13 -8
- metaflow/user_configs/config_parameters.py +0 -4
- metaflow/user_decorators/__init__.py +0 -0
- metaflow/user_decorators/common.py +144 -0
- metaflow/user_decorators/mutable_flow.py +499 -0
- metaflow/user_decorators/mutable_step.py +424 -0
- metaflow/user_decorators/user_flow_decorator.py +264 -0
- metaflow/user_decorators/user_step_decorator.py +712 -0
- metaflow/util.py +4 -1
- metaflow/version.py +1 -1
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/METADATA +2 -2
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/RECORD +71 -60
- metaflow/info_file.py +0 -25
- metaflow/package.py +0 -203
- metaflow/user_configs/config_decorators.py +0 -568
- {metaflow-2.15.21.data → metaflow-2.16.1.data}/data/share/metaflow/devtools/Makefile +0 -0
- {metaflow-2.15.21.data → metaflow-2.16.1.data}/data/share/metaflow/devtools/Tiltfile +0 -0
- {metaflow-2.15.21.data → metaflow-2.16.1.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/WHEEL +0 -0
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/entry_points.txt +0 -0
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/licenses/LICENSE +0 -0
- {metaflow-2.15.21.dist-info → metaflow-2.16.1.dist-info}/top_level.txt +0 -0
metaflow/util.py
CHANGED
@@ -9,7 +9,6 @@ from io import BytesIO
|
|
9
9
|
from itertools import takewhile
|
10
10
|
import re
|
11
11
|
|
12
|
-
from metaflow.exception import MetaflowUnknownUser, MetaflowInternalError
|
13
12
|
|
14
13
|
try:
|
15
14
|
# python2
|
@@ -162,6 +161,8 @@ def get_username():
|
|
162
161
|
|
163
162
|
|
164
163
|
def resolve_identity_as_tuple():
|
164
|
+
from metaflow.exception import MetaflowUnknownUser
|
165
|
+
|
165
166
|
prod_token = os.environ.get("METAFLOW_PRODUCTION_TOKEN")
|
166
167
|
if prod_token:
|
167
168
|
return "production", prod_token
|
@@ -236,6 +237,8 @@ def get_object_package_version(obj):
|
|
236
237
|
|
237
238
|
|
238
239
|
def compress_list(lst, separator=",", rangedelim=":", zlibmarker="!", zlibmin=500):
|
240
|
+
from metaflow.exception import MetaflowInternalError
|
241
|
+
|
239
242
|
bad_items = [x for x in lst if separator in x or rangedelim in x or zlibmarker in x]
|
240
243
|
if bad_items:
|
241
244
|
raise MetaflowInternalError(
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.
|
1
|
+
metaflow_version = "2.16.1"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.16.1
|
4
4
|
Summary: Metaflow: More AI and ML, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.
|
29
|
+
Requires-Dist: metaflow-stubs==2.16.1; extra == "stubs"
|
30
30
|
Dynamic: author
|
31
31
|
Dynamic: author-email
|
32
32
|
Dynamic: classifier
|
@@ -1,43 +1,42 @@
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
2
|
-
metaflow/__init__.py,sha256=
|
2
|
+
metaflow/__init__.py,sha256=8V_NsmWtat2FnIUp5GenscY0vz4egOkIlpNjqwtTJnY,6051
|
3
3
|
metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
|
4
|
-
metaflow/cli.py,sha256=
|
4
|
+
metaflow/cli.py,sha256=WG_2HjYM5ETGl_Vw6nQmFxROKXHH3ZqK1WVmFbGp4r4,22453
|
5
5
|
metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
|
6
6
|
metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
8
|
-
metaflow/debug.py,sha256=
|
9
|
-
metaflow/decorators.py,sha256=
|
8
|
+
metaflow/debug.py,sha256=DDUYLWLAZ4Shg_uXz67BD2nRt8Gilemhlbv9YccKVt4,1656
|
9
|
+
metaflow/decorators.py,sha256=HXfD7NIm5WED7aziq8ul3-lkFgzf7BVVQ3PmocAs8Mw,30835
|
10
10
|
metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
|
11
11
|
metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
|
12
12
|
metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
|
13
|
-
metaflow/flowspec.py,sha256=
|
14
|
-
metaflow/graph.py,sha256=
|
13
|
+
metaflow/flowspec.py,sha256=hCONv0wlC-MgxNUDWzDtS_4ayr-0sqPNUdnpXJ75b1Y,37944
|
14
|
+
metaflow/graph.py,sha256=alPCQPexm-0hjquQgLYNhoe6PbXL_LXMg1L__G1DxYw,13183
|
15
15
|
metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
|
16
|
-
metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
|
17
16
|
metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
|
18
17
|
metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
|
19
|
-
metaflow/
|
18
|
+
metaflow/meta_files.py,sha256=vlgJHI8GJUKzXoxdrVoH8yyCF5bhFgwYemUgnyd1wgM,342
|
19
|
+
metaflow/metaflow_config.py,sha256=lsdfr_1eS4cP6g1OgLWePKzUIypUGVjNxffRhKlFWfA,24102
|
20
20
|
metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
|
21
21
|
metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
|
22
|
-
metaflow/metaflow_environment.py,sha256=
|
22
|
+
metaflow/metaflow_environment.py,sha256=YVq2j8cK6JjbmBkPXoTBNiAw29H1TVpDsnpVN-ZTBR4,11210
|
23
23
|
metaflow/metaflow_git.py,sha256=Pb_VtvQzcjpuuM7UfC2u5kz85EbPMUfspl2UrPWBQMM,3647
|
24
24
|
metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
|
25
|
-
metaflow/metaflow_version.py,sha256=
|
25
|
+
metaflow/metaflow_version.py,sha256=KJJAxhOMY28DaavMpvJUzvw-G6MI-29Fi2A6AEcQpok,7495
|
26
26
|
metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
|
27
27
|
metaflow/multicore_utils.py,sha256=yEo5T6Gemn4_vl8b6IOz7fsTUYtEyqa3AaKZgJY96Wc,4974
|
28
|
-
metaflow/package.py,sha256=yfwVMVB1mD-Sw94KwXNK3N-26YHoKMn6btrcgd67Izs,7845
|
29
28
|
metaflow/parameters.py,sha256=b3rS6P-TeEj2JgPEKaiy0ys1B_JtOGJ6XM0KkY2layc,18649
|
30
29
|
metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
|
31
30
|
metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
32
|
-
metaflow/pylint_wrapper.py,sha256=
|
33
|
-
metaflow/runtime.py,sha256=
|
31
|
+
metaflow/pylint_wrapper.py,sha256=tJjmdsgtbHTCqg_oA6fV6SbWq_3V5XUgE9xH0zJ1CGU,3004
|
32
|
+
metaflow/runtime.py,sha256=KwHKZ6kNZuC8HgL_r0-gTi9Y-GPAhTz0tNXydQFjZO4,77617
|
34
33
|
metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
|
35
|
-
metaflow/task.py,sha256=
|
34
|
+
metaflow/task.py,sha256=1DslU1CHoEeFIilABQvk4Zbpl-yCdZMEkECnv2LQAZg,34356
|
36
35
|
metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
37
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
38
|
-
metaflow/util.py,sha256=
|
37
|
+
metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
|
39
38
|
metaflow/vendor.py,sha256=EDZokNMrx1PU07jNMiWFMFtC7TL03pMXZ1kKn13k-2g,5139
|
40
|
-
metaflow/version.py,sha256=
|
39
|
+
metaflow/version.py,sha256=UyMOYxhUmaZP3zt8IK5HPTbd1c0Z1QPVTghR2Ezbjr4,28
|
41
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
42
41
|
metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
|
43
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -136,12 +135,12 @@ metaflow/_vendor/v3_7/typeguard/_utils.py,sha256=Bj8D17zny8yTan8CNG6HhlL8Gb-BOMG
|
|
136
135
|
metaflow/_vendor/v3_7/typeguard/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
137
136
|
metaflow/cli_components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
138
137
|
metaflow/cli_components/dump_cmd.py,sha256=SZEX51BWNd1o3H2uHDkYA8KRvou5X8g5rTwpdu5vnNQ,2704
|
139
|
-
metaflow/cli_components/init_cmd.py,sha256=
|
140
|
-
metaflow/cli_components/run_cmds.py,sha256=
|
138
|
+
metaflow/cli_components/init_cmd.py,sha256=AIkQHNlMRsmf8fxXwXQSEWTuv_9nxNY82-IdWsgGjEs,1554
|
139
|
+
metaflow/cli_components/run_cmds.py,sha256=xULZQ2UrxLNsWjQIZd38EbOGNBw8UJT7w_T19UbS_fg,11555
|
141
140
|
metaflow/cli_components/step_cmd.py,sha256=zGJgTv7wxrv34nWDi__CHaC2eS6kItR95EdVGJX803w,4766
|
142
141
|
metaflow/cli_components/utils.py,sha256=gpoDociadjnJD7MuiJup_MDR02ZJjjleejr0jPBu29c,6057
|
143
142
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
144
|
-
metaflow/client/core.py,sha256=
|
143
|
+
metaflow/client/core.py,sha256=h_sPTG36U2UPiZRRwL-oQ5EAeJ6ijfXaibhinb22Hj0,83548
|
145
144
|
metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
|
146
145
|
metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
147
146
|
metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
|
@@ -160,21 +159,28 @@ metaflow/datastore/datastore_storage.py,sha256=7V43QuiWDQ_Q4oHw9y7Z7X9lYj3GI-LV1
|
|
160
159
|
metaflow/datastore/exceptions.py,sha256=r7Ab5FvHIzyFh6kwiptA1lO5nLqWg0xRBoeYGefvapA,373
|
161
160
|
metaflow/datastore/flow_datastore.py,sha256=rDMEHdYwub1PwLp2uaK-8CHdd8hiwxqeELXzsUfuqZs,10250
|
162
161
|
metaflow/datastore/inputs.py,sha256=i43dXr2xvgtsgKMO9allgCR18bk80GeayeQFyUTH36w,449
|
163
|
-
metaflow/datastore/task_datastore.py,sha256=
|
164
|
-
metaflow/extension_support/__init__.py,sha256=
|
165
|
-
metaflow/extension_support/_empty_file.py,sha256=
|
162
|
+
metaflow/datastore/task_datastore.py,sha256=bEti1X5rvKBQykfvsoAnmHXel_itZbI5MeLrEpWPHPQ,35059
|
163
|
+
metaflow/extension_support/__init__.py,sha256=Y0DoVZ1Eh4W7aO9DMIXrI8xUoJFP40Oux2gVWv-xoYY,52488
|
164
|
+
metaflow/extension_support/_empty_file.py,sha256=vz61sSExf5DZH3JCqdfwkp7l_NrJR8HV175kG82yUas,133
|
166
165
|
metaflow/extension_support/cmd.py,sha256=hk8iBUUINqvKCDxInKgWpum8ThiRZtHSJP7qBASHzl8,5711
|
167
166
|
metaflow/extension_support/integrations.py,sha256=AWAh-AZ-vo9IxuAVEjGw3s8p_NMm2DKHYx10oC51gPU,5506
|
168
167
|
metaflow/extension_support/plugins.py,sha256=gl7NbIJLJyLTb5LELsj1D9paQip6t6Lqz6Rhmvqvyrw,11286
|
169
168
|
metaflow/metadata_provider/__init__.py,sha256=FZNSnz26VB_m18DQG8mup6-Gfl7r1U6lRMljJBp3VAM,64
|
170
169
|
metaflow/metadata_provider/heartbeat.py,sha256=42mQo6wOHdFuaCh426uV6Kn8swe7e5I3gqA_G7cI_LA,3127
|
171
|
-
metaflow/metadata_provider/metadata.py,sha256=
|
170
|
+
metaflow/metadata_provider/metadata.py,sha256=zu96WqyeIDTtOcSnV8iFQrpwJVzMjs9ealOwzL8c8IE,28417
|
172
171
|
metaflow/metadata_provider/util.py,sha256=lYoQKbqoTM1iZChgyVWN-gX-HyM9tt9bXEMJexY9XmM,1723
|
173
172
|
metaflow/mflog/__init__.py,sha256=TkR9ny_JYvNCWJTdLiHsbLSLc9cUvzAzpDuHLdG8nkA,6020
|
174
173
|
metaflow/mflog/mflog.py,sha256=VebXxqitOtNAs7VJixnNfziO_i_urG7bsJ5JiB5IXgY,4370
|
175
174
|
metaflow/mflog/save_logs.py,sha256=4p1OwozsHJBslOzAf0wUq2XPMNpEOZWM68MgWzh_jJY,2330
|
176
175
|
metaflow/mflog/save_logs_periodically.py,sha256=2Uvk9hi-zlCqXxOQoXmmjH1SCugfw6eG6w70WgfI-ho,1256
|
177
176
|
metaflow/mflog/tee.py,sha256=wTER15qeHuiRpCkOqo-bd-r3Gj-EVlf3IvWRCA4beW4,887
|
177
|
+
metaflow/package/__init__.py,sha256=O2fCTo7Ktm4TnjA6blkepUWs8sf3fTY1q5y8I5nk8Ak,25962
|
178
|
+
metaflow/packaging_sys/__init__.py,sha256=VIDa-J_UNyD-Axf5Iuom6SUssQfqAcTq4ezHU6_uqIM,31529
|
179
|
+
metaflow/packaging_sys/backend.py,sha256=b3OX42Io8xP5l56zhXZxi-VLqXBr-B2q8AM7cWZvmMw,3104
|
180
|
+
metaflow/packaging_sys/distribution_support.py,sha256=VvikZBCH8N1TBZZ2Twk8jH1brmiinKWCD3y_aFqBsIw,4937
|
181
|
+
metaflow/packaging_sys/tar_backend.py,sha256=EYZD5iGEzPoO4L6IQhmrZC1QlaOPV471SBKyOYBS2XU,2593
|
182
|
+
metaflow/packaging_sys/utils.py,sha256=x8SVglJvY5mIAilS7MqZi2PpMr6IEyi6RCg3l8hN3G0,2972
|
183
|
+
metaflow/packaging_sys/v1.py,sha256=_YlVPR7oSYt7B8dHOS0Fb6ZPrxaSZMSQyRhdN8CL8ZY,20483
|
178
184
|
metaflow/plugins/__init__.py,sha256=yFxjJOlnfap7tQMNgSgaso2tl_zr1BcWL7KoUKk4c9Y,8617
|
179
185
|
metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn2GUev0,4061
|
180
186
|
metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
|
@@ -182,18 +188,18 @@ metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6
|
|
182
188
|
metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt_lAZu6UjAcUA,587
|
183
189
|
metaflow/plugins/events_decorator.py,sha256=T_YSK-DlgZhd3ge9PlpTRNaMi15GK0tKZMZl1NdV9DQ,24403
|
184
190
|
metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
|
185
|
-
metaflow/plugins/package_cli.py,sha256=
|
186
|
-
metaflow/plugins/parallel_decorator.py,sha256=
|
191
|
+
metaflow/plugins/package_cli.py,sha256=4OIBmuSSmQ6utWbeMln9HzQXnC9UYm3SilsHzf_sqbo,2002
|
192
|
+
metaflow/plugins/parallel_decorator.py,sha256=cUv_CrMwdItDqQScp5bg18JteSS6p0Ms70mCzD5YcNU,9180
|
187
193
|
metaflow/plugins/project_decorator.py,sha256=uhwsguEj7OM_E2OnY1ap3MoGocQHeywuJSa-qPuWn-U,7592
|
188
194
|
metaflow/plugins/resources_decorator.py,sha256=AtoOwg4mHYHYthg-CAfbfam-QiT0ViuDLDoukoDvF6Q,1347
|
189
195
|
metaflow/plugins/retry_decorator.py,sha256=tz_2Tq6GLg3vjDBZp0KKVTk3ADlCvqaWTSf7blmFdUw,1548
|
190
196
|
metaflow/plugins/storage_executor.py,sha256=FqAgR0-L9MuqN8fRtTe4jjUfJL9lqt6fQkYaglAjRbk,6137
|
191
197
|
metaflow/plugins/tag_cli.py,sha256=10039-0DUF0cmhudoDNrRGLWq8tCGQJ7tBsQAGAmkBQ,17549
|
192
|
-
metaflow/plugins/test_unbounded_foreach_decorator.py,sha256=
|
193
|
-
metaflow/plugins/timeout_decorator.py,sha256=
|
198
|
+
metaflow/plugins/test_unbounded_foreach_decorator.py,sha256=4FA7IFSJgyf4yAyEre7yBpj4VNeNhv8RHTJohAKBAXM,5988
|
199
|
+
metaflow/plugins/timeout_decorator.py,sha256=tKTUn9pUdX7l_AC7gydtBho0W2NfHdvFq6HUbxB0bHo,3563
|
194
200
|
metaflow/plugins/airflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
195
|
-
metaflow/plugins/airflow/airflow.py,sha256=
|
196
|
-
metaflow/plugins/airflow/airflow_cli.py,sha256=
|
201
|
+
metaflow/plugins/airflow/airflow.py,sha256=EEbQZk451p8ksvCiYjAUWg3d0h6Dm3N53Yhok0a2WgE,32778
|
202
|
+
metaflow/plugins/airflow/airflow_cli.py,sha256=qOWmHAe4Z5KKp-5EX-qtSh9lWNF6mB0Ze7lA_LqRaCg,15188
|
197
203
|
metaflow/plugins/airflow/airflow_decorator.py,sha256=IWT6M9gga8t65FR4Wi7pIZvOupk3hE75B5NRg9tMEps,1781
|
198
204
|
metaflow/plugins/airflow/airflow_utils.py,sha256=dvRllfQeOWfDUseFnOocIGaL3gRI_A7cEHnC1w01vfk,28905
|
199
205
|
metaflow/plugins/airflow/dag.py,sha256=zYV3QsyqGIOxgipbiEb4dX-r6aippNbXjuT6Jt2s4xI,129
|
@@ -207,8 +213,8 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
|
|
207
213
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
208
214
|
metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
|
209
215
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
210
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
211
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
216
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=v2_FiimUW-qSG_IBMhJjdtrguHsRCCTz6i-qEJ7-Ysc,187152
|
217
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=pfqYH0nqkgmkDcz3EPIz5yR_PSdd3CfLb92wzplS8fY,38810
|
212
218
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
|
213
219
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
214
220
|
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=7OiapcIM_r-aBkuIobhofgLC5NRJHC-p9bvBmxvhqoM,12500
|
@@ -220,10 +226,10 @@ metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
|
|
220
226
|
metaflow/plugins/aws/aws_client.py,sha256=BTiLMXa1agjja-N73oWinaOZHs-lGPbfKJG8CqdRgaU,4287
|
221
227
|
metaflow/plugins/aws/aws_utils.py,sha256=gdpbmme342bDT7uNAzyAlDZUV7VhvS2Bi82nvIXnoNY,7512
|
222
228
|
metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
223
|
-
metaflow/plugins/aws/batch/batch.py,sha256=
|
224
|
-
metaflow/plugins/aws/batch/batch_cli.py,sha256=
|
229
|
+
metaflow/plugins/aws/batch/batch.py,sha256=Psk-H6GspqZvk4n3_4UADAYAugULDGSQijqUBLyjiy4,18222
|
230
|
+
metaflow/plugins/aws/batch/batch_cli.py,sha256=Mfl0XFLia1QEcULfTS5nFEK-IXevALQr6G8XU0NAgdQ,11914
|
225
231
|
metaflow/plugins/aws/batch/batch_client.py,sha256=J50RMEXeEXFe5RqNUM1HN22BuDQFYFVQ4FSMOK55VWY,28798
|
226
|
-
metaflow/plugins/aws/batch/batch_decorator.py,sha256=
|
232
|
+
metaflow/plugins/aws/batch/batch_decorator.py,sha256=aynWtEj1K9U--wODGLJNfHXX-24NyFxNPyMqArK4JZw,18340
|
227
233
|
metaflow/plugins/aws/secrets_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
228
234
|
metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py,sha256=bBrGw4gRcKX9SLD8iKqPm_S_Zw5Y6F8AjxP6jPbkPpI,8136
|
229
235
|
metaflow/plugins/aws/step_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -232,8 +238,8 @@ metaflow/plugins/aws/step_functions/event_bridge_client.py,sha256=U9-tqKdih4KR-Z
|
|
232
238
|
metaflow/plugins/aws/step_functions/production_token.py,sha256=rREx9djJzKYDiGhPCZ919pSpfrBCYuhSL5WlwnAojNM,1890
|
233
239
|
metaflow/plugins/aws/step_functions/schedule_decorator.py,sha256=Ab1rW8O_no4HNZm4__iBmFDCDW0Z8-TgK4lnxHHA6HI,1940
|
234
240
|
metaflow/plugins/aws/step_functions/set_batch_environment.py,sha256=ibiGWFHDjKcLfprH3OsX-g2M9lUsh6J-bp7v2cdLhD4,1294
|
235
|
-
metaflow/plugins/aws/step_functions/step_functions.py,sha256=
|
236
|
-
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=
|
241
|
+
metaflow/plugins/aws/step_functions/step_functions.py,sha256=Svz232OA5UewY5W9gfDk-xGqiWyCNPkuUwlPxe9g5j8,53577
|
242
|
+
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=tLIfDwgdcfBjkjmQMNgVjXY85HoDZNA6lNcOtZZZA1A,26495
|
237
243
|
metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
|
238
244
|
metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=jzDHYmgU_XvLffZDazR_1viow_1qQFblx9UKyjtoM_0,3788
|
239
245
|
metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=JKYtDhKivtXUWPklprZFzkqezh14loGDmk8mNk6QtpI,3714
|
@@ -251,7 +257,7 @@ metaflow/plugins/cards/card_cli.py,sha256=h-ib-SXjJjAm-jSQC4pEmVcsM4M3lljSjOkZBx
|
|
251
257
|
metaflow/plugins/cards/card_client.py,sha256=30dFBoC3axc261GeV7QCIs_V1OHhRtS31S0wEWsjw90,9501
|
252
258
|
metaflow/plugins/cards/card_creator.py,sha256=Da4LOkRY3IJNcGQn1V6zlSdgVjgiFm6XMcsKOe_6v70,8784
|
253
259
|
metaflow/plugins/cards/card_datastore.py,sha256=bPLjBFWNlAgIjONAb0IjYXMmp9vMYlh3EYhRjAlDA0U,12714
|
254
|
-
metaflow/plugins/cards/card_decorator.py,sha256=
|
260
|
+
metaflow/plugins/cards/card_decorator.py,sha256=lu_m95tZRIkPp1wMlcT80aZRCbjuFapyuPXuelFCMAc,14073
|
255
261
|
metaflow/plugins/cards/card_resolver.py,sha256=bjyujYpGUFbLJNwXNGHlHhL4f-gVdVKebl7XW1vWDtE,717
|
256
262
|
metaflow/plugins/cards/card_server.py,sha256=DHv0RcepaPULWbkDahiEMrU5A281Cfb0DvHLUYd8JsU,11772
|
257
263
|
metaflow/plugins/cards/component_serializer.py,sha256=kQ5umqSQne6bTgx8V8JNQbiz-kH2Ntu0VK3ZHxywXww,36912
|
@@ -319,10 +325,10 @@ metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRm
|
|
319
325
|
metaflow/plugins/gcp/includefile_support.py,sha256=OQO0IVWv4ObboL0VqEZwcDOyj9ORLdur66JToxQ84vU,3887
|
320
326
|
metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
321
327
|
metaflow/plugins/kubernetes/kube_utils.py,sha256=jdFMGbEmIow-oli26v31W9CmbZXigx06b3D_xIobpk0,4140
|
322
|
-
metaflow/plugins/kubernetes/kubernetes.py,sha256=
|
323
|
-
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=
|
328
|
+
metaflow/plugins/kubernetes/kubernetes.py,sha256=2nwRfw2GZ_ESJslQEgGoBP7NodINhKnTWCoxJOn8MME,30412
|
329
|
+
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qtWTQqp8i-hTKAA0RcJ_qeOuD8TieN3B5vuyYdnvEP4,14425
|
324
330
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
|
325
|
-
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=
|
331
|
+
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=j5yoLJGowqXiOf4SCaus3qpOUQAyrFo9U3TRXxvoh6o,32428
|
326
332
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=0QyEyi6XusHq5M7RqPY4ypQXrsQac-eG2O14b7inPOo,32277
|
327
333
|
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=ZZU5vsBe67NmGuxgXw6clf7kKRST7867AW6_t3fCD5g,43065
|
328
334
|
metaflow/plugins/kubernetes/spot_metadata_cli.py,sha256=an0nWCxgflmqIPBCBrlb4m3DereDFFJBLt-KKhqcHc8,1670
|
@@ -331,13 +337,13 @@ metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uz
|
|
331
337
|
metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtShH64yaXZQ5i9Ym7g,24390
|
332
338
|
metaflow/plugins/metadata_providers/service.py,sha256=5UlK0R5M9_nq2J6MgJgCZwqAC3bEsofFbglq1K4p4QI,22942
|
333
339
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
334
|
-
metaflow/plugins/pypi/bootstrap.py,sha256=
|
335
|
-
metaflow/plugins/pypi/conda_decorator.py,sha256=
|
336
|
-
metaflow/plugins/pypi/conda_environment.py,sha256=
|
340
|
+
metaflow/plugins/pypi/bootstrap.py,sha256=8EWBdwOp5moXkTfLadn3ZOtPXoGftjOFD-c2W_rn77c,14998
|
341
|
+
metaflow/plugins/pypi/conda_decorator.py,sha256=ie0ftOcozloj_qdASOxYvycb-Zr1GJHedzEDnp6cl2w,13984
|
342
|
+
metaflow/plugins/pypi/conda_environment.py,sha256=j6QaIWmP50K2veN6Sywa04S2hfByR1YMqy7OyZpwvYs,24925
|
337
343
|
metaflow/plugins/pypi/micromamba.py,sha256=UltfY8NmLphfZ-AbpaMFIdxIeOXLdTYDrMrabvPrYVU,17352
|
338
344
|
metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
|
339
345
|
metaflow/plugins/pypi/pip.py,sha256=WhPyA18RoVT40sqbZRJdCEij4euL9PZwLfm1SrAKqmU,14333
|
340
|
-
metaflow/plugins/pypi/pypi_decorator.py,sha256=
|
346
|
+
metaflow/plugins/pypi/pypi_decorator.py,sha256=sBxmDH3WIWVfJDsSauiMy58fevOJuJeLSuS6MnIpw44,7312
|
341
347
|
metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
|
342
348
|
metaflow/plugins/pypi/utils.py,sha256=W8OhDrhz7YIE-2MSSN5Rj7AmESwsN5YDmdnro152J4w,3551
|
343
349
|
metaflow/plugins/secrets/__init__.py,sha256=hKwDnaMAG7Rjqv-chN4OjWy6G2dkncRjK_wDewthvL8,349
|
@@ -347,16 +353,16 @@ metaflow/plugins/secrets/secrets_func.py,sha256=zjDJdJLiKJfcvKzzkDbnbQCPUoCmr8q4
|
|
347
353
|
metaflow/plugins/secrets/secrets_spec.py,sha256=rZCBd7Til5laiXCaYLz4izwvM8aQ3__wgBkB-c8X4fA,4251
|
348
354
|
metaflow/plugins/secrets/utils.py,sha256=Pxx1iiYRitKQIwJeCnTtV4-IrACMI5aSaUkefKF9E7Y,2669
|
349
355
|
metaflow/plugins/uv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
350
|
-
metaflow/plugins/uv/bootstrap.py,sha256=
|
351
|
-
metaflow/plugins/uv/uv_environment.py,sha256=
|
356
|
+
metaflow/plugins/uv/bootstrap.py,sha256=awFHAiZANkAlvHWCm3G1UbhVxEjJB8buoZoBa5GiKJc,4365
|
357
|
+
metaflow/plugins/uv/uv_environment.py,sha256=AYZICrBEq3Bv-taXktJwu9DhKFxNooPFwlcH379EYMs,2719
|
352
358
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
353
|
-
metaflow/runner/click_api.py,sha256=
|
359
|
+
metaflow/runner/click_api.py,sha256=DSxa5A0C_IHNug7fZlLpD_N99F_skDcAjTRx5YRMylY,23756
|
354
360
|
metaflow/runner/deployer.py,sha256=U-hwf4gVzwUlXgnkfTW3y1daGXvo5eP4HTQZwb-vS0g,11058
|
355
361
|
metaflow/runner/deployer_impl.py,sha256=zTING0_fwP44JcGo69DuNrVut5KqdBVzYOM7MYTZgIY,7049
|
356
362
|
metaflow/runner/metaflow_runner.py,sha256=uo3BzcAfZ67VT_f-TPe5ZHiWHn6uuojWusOMGksvX14,18178
|
357
363
|
metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
|
358
364
|
metaflow/runner/nbrun.py,sha256=LhJu-Teoi7wTkNxg0kpNPVXFxH_9P4lvtp0ysMEIFJ8,7299
|
359
|
-
metaflow/runner/subprocess_manager.py,sha256=
|
365
|
+
metaflow/runner/subprocess_manager.py,sha256=0nAXG1PM7KXSIpu6ECeNMv5EfrqCymlIS6k25crnIBA,22792
|
360
366
|
metaflow/runner/utils.py,sha256=fU4vPazBdi6ATAUW_DaBAQeVslRwrLT8Pn9s5wav3gg,10350
|
361
367
|
metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
|
362
368
|
metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
|
@@ -395,15 +401,20 @@ metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuV
|
|
395
401
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
396
402
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
397
403
|
metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
398
|
-
metaflow/user_configs/
|
399
|
-
metaflow/user_configs/
|
400
|
-
metaflow/
|
401
|
-
metaflow
|
402
|
-
metaflow
|
403
|
-
metaflow
|
404
|
-
metaflow
|
405
|
-
metaflow
|
406
|
-
metaflow-2.
|
407
|
-
metaflow-2.
|
408
|
-
metaflow-2.
|
409
|
-
metaflow-2.
|
404
|
+
metaflow/user_configs/config_options.py,sha256=d3hKA6WRPe21PdTl7sBnxIp5sE6zBpRtgAGx7hWkkfw,21380
|
405
|
+
metaflow/user_configs/config_parameters.py,sha256=Loa5wu3vIs0SLyGhbOo8b88nWgCuZ09k24EqC_lI7n4,20890
|
406
|
+
metaflow/user_decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
407
|
+
metaflow/user_decorators/common.py,sha256=0u9NRLQ95TfJCjWVEOGT4MJ9WoQgAMBM9kJ2gJGlVjk,5362
|
408
|
+
metaflow/user_decorators/mutable_flow.py,sha256=y2FCTQVjTTeiptsztD26jdkU_LY_Z_kzMjTkDajo2rc,19303
|
409
|
+
metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
|
410
|
+
metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
|
411
|
+
metaflow/user_decorators/user_step_decorator.py,sha256=JYNGXONWCpzwn-_bF5WiAkof4Ii9tRS4xdK8ojSxG6M,26007
|
412
|
+
metaflow-2.16.1.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
|
413
|
+
metaflow-2.16.1.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
|
414
|
+
metaflow-2.16.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
|
415
|
+
metaflow-2.16.1.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
416
|
+
metaflow-2.16.1.dist-info/METADATA,sha256=juqAkLZQtmv1VhBrGrFeunOHFyoVK9rUyLQyCNDNdvw,6740
|
417
|
+
metaflow-2.16.1.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
|
418
|
+
metaflow-2.16.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
|
419
|
+
metaflow-2.16.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
420
|
+
metaflow-2.16.1.dist-info/RECORD,,
|
metaflow/info_file.py
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
import json
|
2
|
-
|
3
|
-
from os import path
|
4
|
-
|
5
|
-
CURRENT_DIRECTORY = path.dirname(path.abspath(__file__))
|
6
|
-
INFO_FILE = path.join(path.dirname(CURRENT_DIRECTORY), "INFO")
|
7
|
-
|
8
|
-
_info_file_content = None
|
9
|
-
_info_file_present = None
|
10
|
-
|
11
|
-
|
12
|
-
def read_info_file():
|
13
|
-
global _info_file_content
|
14
|
-
global _info_file_present
|
15
|
-
if _info_file_present is None:
|
16
|
-
_info_file_present = path.exists(INFO_FILE)
|
17
|
-
if _info_file_present:
|
18
|
-
try:
|
19
|
-
with open(INFO_FILE, "r", encoding="utf-8") as contents:
|
20
|
-
_info_file_content = json.load(contents)
|
21
|
-
except IOError:
|
22
|
-
pass
|
23
|
-
if _info_file_present:
|
24
|
-
return _info_file_content
|
25
|
-
return None
|
metaflow/package.py
DELETED
@@ -1,203 +0,0 @@
|
|
1
|
-
import importlib
|
2
|
-
import os
|
3
|
-
import sys
|
4
|
-
import tarfile
|
5
|
-
import time
|
6
|
-
import json
|
7
|
-
from io import BytesIO
|
8
|
-
|
9
|
-
from .user_configs.config_parameters import CONFIG_FILE, dump_config_values
|
10
|
-
from .extension_support import EXT_PKG, package_mfext_all
|
11
|
-
from .metaflow_config import DEFAULT_PACKAGE_SUFFIXES
|
12
|
-
from .exception import MetaflowException
|
13
|
-
from .util import to_unicode
|
14
|
-
from . import R
|
15
|
-
from .info_file import INFO_FILE
|
16
|
-
|
17
|
-
DEFAULT_SUFFIXES_LIST = DEFAULT_PACKAGE_SUFFIXES.split(",")
|
18
|
-
METAFLOW_SUFFIXES_LIST = [".py", ".html", ".css", ".js"]
|
19
|
-
|
20
|
-
|
21
|
-
class NonUniqueFileNameToFilePathMappingException(MetaflowException):
|
22
|
-
headline = "Non Unique file path for a file name included in code package"
|
23
|
-
|
24
|
-
def __init__(self, filename, file_paths, lineno=None):
|
25
|
-
msg = (
|
26
|
-
"Filename %s included in the code package includes multiple different paths for the same name : %s.\n"
|
27
|
-
"The `filename` in the `add_to_package` decorator hook requires a unique `file_path` to `file_name` mapping"
|
28
|
-
% (filename, ", ".join(file_paths))
|
29
|
-
)
|
30
|
-
super().__init__(msg=msg, lineno=lineno)
|
31
|
-
|
32
|
-
|
33
|
-
# this is os.walk(follow_symlinks=True) with cycle detection
|
34
|
-
def walk_without_cycles(top_root):
|
35
|
-
seen = set()
|
36
|
-
|
37
|
-
def _recurse(root):
|
38
|
-
for parent, dirs, files in os.walk(root):
|
39
|
-
for d in dirs:
|
40
|
-
path = os.path.join(parent, d)
|
41
|
-
if os.path.islink(path):
|
42
|
-
# Breaking loops: never follow the same symlink twice
|
43
|
-
#
|
44
|
-
# NOTE: this also means that links to sibling links are
|
45
|
-
# not followed. In this case:
|
46
|
-
#
|
47
|
-
# x -> y
|
48
|
-
# y -> oo
|
49
|
-
# oo/real_file
|
50
|
-
#
|
51
|
-
# real_file is only included twice, not three times
|
52
|
-
reallink = os.path.realpath(path)
|
53
|
-
if reallink not in seen:
|
54
|
-
seen.add(reallink)
|
55
|
-
for x in _recurse(path):
|
56
|
-
yield x
|
57
|
-
yield parent, files
|
58
|
-
|
59
|
-
for x in _recurse(top_root):
|
60
|
-
yield x
|
61
|
-
|
62
|
-
|
63
|
-
class MetaflowPackage(object):
|
64
|
-
def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST):
|
65
|
-
self.suffixes = list(set().union(suffixes, DEFAULT_SUFFIXES_LIST))
|
66
|
-
self.environment = environment
|
67
|
-
self.metaflow_root = os.path.dirname(__file__)
|
68
|
-
|
69
|
-
self.flow_name = flow.name
|
70
|
-
self._flow = flow
|
71
|
-
self.create_time = time.time()
|
72
|
-
environment.init_environment(echo)
|
73
|
-
for step in flow:
|
74
|
-
for deco in step.decorators:
|
75
|
-
deco.package_init(flow, step.__name__, environment)
|
76
|
-
self.blob = self._make()
|
77
|
-
|
78
|
-
def _walk(self, root, exclude_hidden=True, suffixes=None):
|
79
|
-
if suffixes is None:
|
80
|
-
suffixes = []
|
81
|
-
root = to_unicode(root) # handle files/folder with non ascii chars
|
82
|
-
prefixlen = len("%s/" % os.path.dirname(root))
|
83
|
-
for (
|
84
|
-
path,
|
85
|
-
files,
|
86
|
-
) in walk_without_cycles(root):
|
87
|
-
if exclude_hidden and "/." in path:
|
88
|
-
continue
|
89
|
-
# path = path[2:] # strip the ./ prefix
|
90
|
-
# if path and (path[0] == '.' or './' in path):
|
91
|
-
# continue
|
92
|
-
for fname in files:
|
93
|
-
if (fname[0] == "." and fname in suffixes) or (
|
94
|
-
fname[0] != "."
|
95
|
-
and any(fname.endswith(suffix) for suffix in suffixes)
|
96
|
-
):
|
97
|
-
p = os.path.join(path, fname)
|
98
|
-
yield p, p[prefixlen:]
|
99
|
-
|
100
|
-
def path_tuples(self):
|
101
|
-
"""
|
102
|
-
Returns list of (path, arcname) to be added to the job package, where
|
103
|
-
`arcname` is the alternative name for the file in the package.
|
104
|
-
"""
|
105
|
-
# We want the following contents in the tarball
|
106
|
-
# Metaflow package itself
|
107
|
-
for path_tuple in self._walk(
|
108
|
-
self.metaflow_root, exclude_hidden=False, suffixes=METAFLOW_SUFFIXES_LIST
|
109
|
-
):
|
110
|
-
yield path_tuple
|
111
|
-
|
112
|
-
# Metaflow extensions; for now, we package *all* extensions but this may change
|
113
|
-
# at a later date; it is possible to call `package_mfext_package` instead of
|
114
|
-
# `package_mfext_all` but in that case, make sure to also add a
|
115
|
-
# metaflow_extensions/__init__.py file to properly "close" the metaflow_extensions
|
116
|
-
# package and prevent other extensions from being loaded that may be
|
117
|
-
# present in the rest of the system
|
118
|
-
for path_tuple in package_mfext_all():
|
119
|
-
yield path_tuple
|
120
|
-
|
121
|
-
# Any custom packages exposed via decorators
|
122
|
-
deco_module_paths = {}
|
123
|
-
for step in self._flow:
|
124
|
-
for deco in step.decorators:
|
125
|
-
for path_tuple in deco.add_to_package():
|
126
|
-
file_path, file_name = path_tuple
|
127
|
-
# Check if the path is not duplicated as
|
128
|
-
# many steps can have the same packages being imported
|
129
|
-
if file_name not in deco_module_paths:
|
130
|
-
deco_module_paths[file_name] = file_path
|
131
|
-
yield path_tuple
|
132
|
-
elif deco_module_paths[file_name] != file_path:
|
133
|
-
raise NonUniqueFileNameToFilePathMappingException(
|
134
|
-
file_name, [deco_module_paths[file_name], file_path]
|
135
|
-
)
|
136
|
-
|
137
|
-
# the package folders for environment
|
138
|
-
for path_tuple in self.environment.add_to_package():
|
139
|
-
yield path_tuple
|
140
|
-
if R.use_r():
|
141
|
-
# the R working directory
|
142
|
-
for path_tuple in self._walk(
|
143
|
-
"%s/" % R.working_dir(), suffixes=self.suffixes
|
144
|
-
):
|
145
|
-
yield path_tuple
|
146
|
-
# the R package
|
147
|
-
for path_tuple in R.package_paths():
|
148
|
-
yield path_tuple
|
149
|
-
else:
|
150
|
-
# the user's working directory
|
151
|
-
flowdir = os.path.dirname(os.path.abspath(sys.argv[0])) + "/"
|
152
|
-
for path_tuple in self._walk(flowdir, suffixes=self.suffixes):
|
153
|
-
yield path_tuple
|
154
|
-
|
155
|
-
def _add_configs(self, tar):
|
156
|
-
buf = BytesIO()
|
157
|
-
buf.write(json.dumps(dump_config_values(self._flow)).encode("utf-8"))
|
158
|
-
self._add_file(tar, os.path.basename(CONFIG_FILE), buf)
|
159
|
-
|
160
|
-
def _add_info(self, tar):
|
161
|
-
buf = BytesIO()
|
162
|
-
buf.write(
|
163
|
-
json.dumps(
|
164
|
-
self.environment.get_environment_info(include_ext_info=True)
|
165
|
-
).encode("utf-8")
|
166
|
-
)
|
167
|
-
self._add_file(tar, os.path.basename(INFO_FILE), buf)
|
168
|
-
|
169
|
-
@staticmethod
|
170
|
-
def _add_file(tar, filename, buf):
|
171
|
-
info = tarfile.TarInfo(filename)
|
172
|
-
buf.seek(0)
|
173
|
-
info.size = len(buf.getvalue())
|
174
|
-
# Setting this default to Dec 3, 2019
|
175
|
-
info.mtime = 1575360000
|
176
|
-
tar.addfile(info, buf)
|
177
|
-
|
178
|
-
def _make(self):
|
179
|
-
def no_mtime(tarinfo):
|
180
|
-
# a modification time change should not change the hash of
|
181
|
-
# the package. Only content modifications will.
|
182
|
-
# Setting this default to Dec 3, 2019
|
183
|
-
tarinfo.mtime = 1575360000
|
184
|
-
return tarinfo
|
185
|
-
|
186
|
-
buf = BytesIO()
|
187
|
-
with tarfile.open(
|
188
|
-
fileobj=buf, mode="w:gz", compresslevel=3, dereference=True
|
189
|
-
) as tar:
|
190
|
-
self._add_info(tar)
|
191
|
-
self._add_configs(tar)
|
192
|
-
for path, arcname in self.path_tuples():
|
193
|
-
tar.add(path, arcname=arcname, recursive=False, filter=no_mtime)
|
194
|
-
|
195
|
-
blob = bytearray(buf.getvalue())
|
196
|
-
blob[4:8] = [0] * 4 # Reset 4 bytes from offset 4 to account for ts
|
197
|
-
return blob
|
198
|
-
|
199
|
-
def __str__(self):
|
200
|
-
return "<code package for flow %s (created @ %s)>" % (
|
201
|
-
self.flow_name,
|
202
|
-
time.strftime("%a, %d %b %Y %H:%M:%S", self.create_time),
|
203
|
-
)
|