pybiolib 1.1.1881__py3-none-any.whl → 1.1.2193__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biolib/__init__.py +11 -4
- biolib/_data_record/data_record.py +278 -0
- biolib/_internal/data_record/__init__.py +1 -1
- biolib/_internal/data_record/data_record.py +95 -151
- biolib/_internal/data_record/remote_storage_endpoint.py +18 -7
- biolib/_internal/file_utils.py +77 -0
- biolib/_internal/fuse_mount/__init__.py +1 -0
- biolib/_internal/fuse_mount/experiment_fuse_mount.py +209 -0
- biolib/_internal/http_client.py +29 -9
- biolib/_internal/lfs/__init__.py +1 -0
- biolib/_internal/libs/__init__.py +1 -0
- biolib/_internal/libs/fusepy/__init__.py +1257 -0
- biolib/_internal/push_application.py +1 -1
- biolib/_internal/runtime.py +2 -56
- biolib/_internal/types/__init__.py +4 -0
- biolib/_internal/types/app.py +9 -0
- biolib/_internal/types/data_record.py +40 -0
- biolib/_internal/types/experiment.py +10 -0
- biolib/_internal/types/resource.py +14 -0
- biolib/_internal/types/typing.py +7 -0
- biolib/_runtime/runtime.py +80 -0
- biolib/api/__init__.py +1 -0
- biolib/api/client.py +39 -17
- biolib/app/app.py +34 -71
- biolib/biolib_api_client/api_client.py +9 -2
- biolib/biolib_api_client/app_types.py +2 -2
- biolib/biolib_api_client/biolib_job_api.py +6 -0
- biolib/biolib_api_client/job_types.py +4 -4
- biolib/biolib_api_client/lfs_types.py +8 -2
- biolib/biolib_binary_format/remote_endpoints.py +12 -10
- biolib/biolib_binary_format/utils.py +23 -3
- biolib/cli/auth.py +1 -1
- biolib/cli/data_record.py +43 -6
- biolib/cli/lfs.py +10 -6
- biolib/compute_node/cloud_utils/cloud_utils.py +13 -16
- biolib/compute_node/job_worker/executors/docker_executor.py +126 -108
- biolib/compute_node/job_worker/job_storage.py +3 -4
- biolib/compute_node/job_worker/job_worker.py +25 -15
- biolib/compute_node/remote_host_proxy.py +61 -84
- biolib/compute_node/webserver/webserver_types.py +0 -1
- biolib/experiments/experiment.py +75 -44
- biolib/jobs/job.py +98 -19
- biolib/jobs/job_result.py +46 -21
- biolib/jobs/types.py +1 -1
- biolib/runtime/__init__.py +2 -1
- biolib/sdk/__init__.py +18 -7
- biolib/typing_utils.py +2 -7
- biolib/user/sign_in.py +2 -2
- biolib/utils/seq_util.py +38 -35
- {pybiolib-1.1.1881.dist-info → pybiolib-1.1.2193.dist-info}/METADATA +1 -1
- {pybiolib-1.1.1881.dist-info → pybiolib-1.1.2193.dist-info}/RECORD +55 -44
- biolib/experiments/types.py +0 -9
- biolib/lfs/__init__.py +0 -4
- biolib/lfs/utils.py +0 -153
- /biolib/{lfs → _internal/lfs}/cache.py +0 -0
- {pybiolib-1.1.1881.dist-info → pybiolib-1.1.2193.dist-info}/LICENSE +0 -0
- {pybiolib-1.1.1881.dist-info → pybiolib-1.1.2193.dist-info}/WHEEL +0 -0
- {pybiolib-1.1.1881.dist-info → pybiolib-1.1.2193.dist-info}/entry_points.txt +0 -0
@@ -1,51 +1,66 @@
|
|
1
1
|
LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
|
2
2
|
README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
|
3
|
-
biolib/__init__.py,sha256=
|
3
|
+
biolib/__init__.py,sha256=_tThyzISH81yS9KXP_X3qEiKXmsIp5XOBcJIODfLVnc,4338
|
4
|
+
biolib/_data_record/data_record.py,sha256=CoyYRse5VdUBhQzzPfR9BkytgOsM-IZxkfMX1kyRnPk,12589
|
4
5
|
biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
-
biolib/_internal/data_record/__init__.py,sha256=
|
6
|
-
biolib/_internal/data_record/data_record.py,sha256=
|
7
|
-
biolib/_internal/data_record/remote_storage_endpoint.py,sha256=
|
8
|
-
biolib/_internal/
|
9
|
-
biolib/_internal/
|
10
|
-
biolib/_internal/
|
6
|
+
biolib/_internal/data_record/__init__.py,sha256=fGdME6JGRU_2VxpJbYpGXYndjN-feUkmKY4fuMyq3cg,76
|
7
|
+
biolib/_internal/data_record/data_record.py,sha256=YmaAABR57goDCE8-rKb2j0FPMSbDtRPCm_HhT3mM074,4299
|
8
|
+
biolib/_internal/data_record/remote_storage_endpoint.py,sha256=eCptuZ4DMAPnaNCVDvpWXwXGI6Jac9U1N5dqU8Cj95Q,1732
|
9
|
+
biolib/_internal/file_utils.py,sha256=4jT6j7bB21c0JNn5BfnyWQib_zt0CVtJ_TiOFOStRcE,2604
|
10
|
+
biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
|
11
|
+
biolib/_internal/fuse_mount/experiment_fuse_mount.py,sha256=08aUdEq_bvqLBft_gSLjOClKDy5sBnMts1RfJf7AP_U,7012
|
12
|
+
biolib/_internal/http_client.py,sha256=DdooXei93JKGYGV4aQmzue_oFzvHkozg2UCxgk9dfDM,5081
|
13
|
+
biolib/_internal/lfs/__init__.py,sha256=gSWo_xg61UniYgD7yNYxeT4I9uaXBCBSi3_nmZjnPpE,35
|
14
|
+
biolib/_internal/lfs/cache.py,sha256=pQS2np21rdJ6I3DpoOutnzPHpLOZgUIS8TMltUJk_k4,2226
|
15
|
+
biolib/_internal/libs/__init__.py,sha256=Jdf4tNPqe_oIIf6zYml6TiqhL_02Vyqwge6IELrAFhw,98
|
16
|
+
biolib/_internal/libs/fusepy/__init__.py,sha256=AWDzNFS-XV_5yKb0Qx7kggIhPzq1nj_BZS5y2Nso08k,41944
|
17
|
+
biolib/_internal/push_application.py,sha256=8P7eXvySn7CRp5XBDkO3xjTGixS8g7-jD-_iwzM_XDI,10020
|
18
|
+
biolib/_internal/runtime.py,sha256=9pZ3s3L7LGxdqOgnHh1KK3Jjyn_9MjhQmKHI-6hMT3U,448
|
19
|
+
biolib/_internal/types/__init__.py,sha256=11ZucS8jKeLGAAswXyKI7FH2KLHd6T9Sh8ZK2Ar3jlk,152
|
20
|
+
biolib/_internal/types/app.py,sha256=Mz2QGD_jESX-K9JYnLWPo4YA__Q_1FQQTk9pvidCohU,118
|
21
|
+
biolib/_internal/types/data_record.py,sha256=AHoIiwVqeHj0HozQxFRAyxk-d3XJgLWno4ic1z9eTrQ,865
|
22
|
+
biolib/_internal/types/experiment.py,sha256=D94iBdn2nS92lRW-TOs1a2WKXJD5ZtmzL4ypggKX2ys,176
|
23
|
+
biolib/_internal/types/resource.py,sha256=G-vPkZoe4Um6FPxsQZtRzAlbSW5sDW4NFkbjn21I3V4,372
|
24
|
+
biolib/_internal/types/typing.py,sha256=D4EKKEe7kDx0K6lJi-H_XLtk-8w6nu2fdqn9bvzI-Xo,288
|
11
25
|
biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
|
12
|
-
biolib/
|
13
|
-
biolib/api/
|
26
|
+
biolib/_runtime/runtime.py,sha256=daYxzIpRoW4k-HJFu2BMXeylYSlCXn3-SqdSriCFnKw,2770
|
27
|
+
biolib/api/__init__.py,sha256=mQ4u8FijqyLzjYMezMUUbbBGNB3iFmkNdjXnWPZ7Jlw,138
|
28
|
+
biolib/api/client.py,sha256=FRpdH5aI187b_I_4HUNi680v4iOP65z5f2RcUo8D8MA,3559
|
14
29
|
biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
|
15
|
-
biolib/app/app.py,sha256=
|
30
|
+
biolib/app/app.py,sha256=P2RwaDAskUHzlciuTJUroqUocRwoyOLT6YbgMyCRRDI,8484
|
16
31
|
biolib/app/search_apps.py,sha256=K4a41f5XIWth2BWI7OffASgIsD0ko8elCax8YL2igaY,1470
|
17
32
|
biolib/biolib_api_client/__init__.py,sha256=E5EMa19wJoblwSdQPYrxc_BtIeRsAuO0L_jQweWw-Yk,182
|
18
|
-
biolib/biolib_api_client/api_client.py,sha256=
|
19
|
-
biolib/biolib_api_client/app_types.py,sha256=
|
33
|
+
biolib/biolib_api_client/api_client.py,sha256=ciNx4ybpyKG5LEf4KQdGEz13r0jTxImyQat4_HDecD0,7373
|
34
|
+
biolib/biolib_api_client/app_types.py,sha256=FxSr4UqfnMhLe34p8bm02wsC3g1Jz8iaing5tRKDOQI,2442
|
20
35
|
biolib/biolib_api_client/auth.py,sha256=kjm0ZHnH3I8so3su2sZbBxNHYp-ZUdrZ5lwQ0K36RSw,949
|
21
36
|
biolib/biolib_api_client/biolib_app_api.py,sha256=DndlVxrNTes6DOaWyMINLGZQCRMWVvR7gwt5HVlyf5Y,4240
|
22
|
-
biolib/biolib_api_client/biolib_job_api.py,sha256=
|
37
|
+
biolib/biolib_api_client/biolib_job_api.py,sha256=7bKfav3-12ewXkEUoLdCmbWdebW8148kxfGJW9SsXZI,7125
|
23
38
|
biolib/biolib_api_client/common_types.py,sha256=RH-1KNHqUF-EkTpfPOSTt5Mq1GPdfju_cqXDesscO1I,123
|
24
|
-
biolib/biolib_api_client/job_types.py,sha256=
|
25
|
-
biolib/biolib_api_client/lfs_types.py,sha256=
|
39
|
+
biolib/biolib_api_client/job_types.py,sha256=Dl4NhU2xpgpXV-7YIoDf6WL63SLR5bni55OX8x5539M,1300
|
40
|
+
biolib/biolib_api_client/lfs_types.py,sha256=joZWP6-sa5_Ug_6xIp5fHAgEo_bqLE3rbleQocZtDcg,339
|
26
41
|
biolib/biolib_api_client/user_state.py,sha256=XcgWV-MgVk88mIlMmnu8yHxMu8OCaw8o0tk7TVo5Hcg,637
|
27
42
|
biolib/biolib_binary_format/__init__.py,sha256=HMl5SdX_VUWE4OQzi4Jf_yFvC7b0bSPOGPHYi9dWM2Q,185
|
28
43
|
biolib/biolib_binary_format/base_bbf_package.py,sha256=vxRV4iKy0dXeDOlFWnMFI0hGnDBYDH5Cgh5gAfuObt8,959
|
29
44
|
biolib/biolib_binary_format/file_in_container.py,sha256=j1eEPRxf_ew8I6G8sDiiZZxn4Wx1ppagfM9K8zTDG4U,3591
|
30
45
|
biolib/biolib_binary_format/module_input.py,sha256=led2QhHeec_ymBPw5uEn3_3vJKI-1T8zrFQGqwEWLMY,2788
|
31
46
|
biolib/biolib_binary_format/module_output_v2.py,sha256=J5ZO5gCSeudpE12EVDrjYrNTS2DwgszY-SVXT7Qjuyg,5913
|
32
|
-
biolib/biolib_binary_format/remote_endpoints.py,sha256=
|
47
|
+
biolib/biolib_binary_format/remote_endpoints.py,sha256=V48mwOj3eAQAKp-8DjtWUdEKUyC0WKc1pEiKTmtjrJY,1651
|
33
48
|
biolib/biolib_binary_format/remote_stream_seeker.py,sha256=uyi6kJBU1C1DWfiuR0kRUQIY7nalG7ocgwgngd3Ul4U,1999
|
34
49
|
biolib/biolib_binary_format/saved_job.py,sha256=nFHVFRNTNcAFGODLSiBntCtMk55QKwreUq6qLX80dI4,1125
|
35
50
|
biolib/biolib_binary_format/stdout_and_stderr.py,sha256=WfUUJFFCBrtfXjuWIaRPiWCpuBLxfko68oxoTKhrwx8,1023
|
36
51
|
biolib/biolib_binary_format/system_exception.py,sha256=T3iL4_cSHAHim3RSDPS8Xyb1mfteaJBZonSXuRltc28,853
|
37
52
|
biolib/biolib_binary_format/system_status_update.py,sha256=aOELuQ0k-GtpaZTUxYd0GFomP_OInmrK585y6fuQuKE,1191
|
38
|
-
biolib/biolib_binary_format/utils.py,sha256=
|
53
|
+
biolib/biolib_binary_format/utils.py,sha256=ra_plrh_Z10u98O2gW9uW2qzscQZCfq91SOznmDTY64,5170
|
39
54
|
biolib/biolib_docker_client/__init__.py,sha256=aBfA6mtWSI5dBEfNNMD6bIZzCPloW4ghKm0wqQiljdo,1481
|
40
55
|
biolib/biolib_download_container.py,sha256=8TmBV8iv3bCvkNlHa1SSsc4zl0wX_eaxhfnW5rvFIh8,1779
|
41
56
|
biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
|
42
57
|
biolib/biolib_logging.py,sha256=J3E5H_LL5k6ZUim2C8gqN7E6lCBZMTpO4tnMpOPwG9U,2854
|
43
58
|
biolib/cli/__init__.py,sha256=0v3c_J-U0k46c5ZWeQjLG_kTaKDJm81LBxQpDO2B_aI,1286
|
44
|
-
biolib/cli/auth.py,sha256=
|
45
|
-
biolib/cli/data_record.py,sha256=
|
59
|
+
biolib/cli/auth.py,sha256=rpWGmXs6Fz6CGrO9K8ibPRszOdXG78Vig_boKaVCD9A,2082
|
60
|
+
biolib/cli/data_record.py,sha256=08JbZkFWKMo0PrnhhG0jQEKnNW7pPLti9cOw8s1TWfI,3344
|
46
61
|
biolib/cli/download_container.py,sha256=HIZVHOPmslGE5M2Dsp9r2cCkAEJx__vcsDz5Wt5LRos,483
|
47
62
|
biolib/cli/init.py,sha256=wQOfii_au-d30Hp7DdH-WVw-WVraKvA_zY4za1w7DE8,821
|
48
|
-
biolib/cli/lfs.py,sha256=
|
63
|
+
biolib/cli/lfs.py,sha256=z2qHUwink85mv9yDgifbVKkVwuyknGhMDTfly_gLKJM,4151
|
49
64
|
biolib/cli/push.py,sha256=TFi7O9tJ3zFe0VmtVTV3Vh9_xIMHnrc41xxcaBKU46g,813
|
50
65
|
biolib/cli/run.py,sha256=BbvXLQ-XibjQ71Y2d4URMH_8dflNVwM0i3TIWhw_u_c,1634
|
51
66
|
biolib/cli/runtime.py,sha256=Xv-nrma5xX8NidWcvbUKcUvuN5TCarZa4A8mPVmF-z0,361
|
@@ -53,60 +68,56 @@ biolib/cli/start.py,sha256=rg8VVY8rboFhf1iQo3zE3WA5oh_R1VWWfYJEO1gMReY,1737
|
|
53
68
|
biolib/compute_node/.gitignore,sha256=GZdZ4g7HftqfOfasFpBC5zV1YQAbht1a7EzcXD6f3zg,45
|
54
69
|
biolib/compute_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
55
70
|
biolib/compute_node/cloud_utils/__init__.py,sha256=VZSScLqaz5tg_gpMvWgwkAu9Qf-vgW_QHRoDOaAmU44,67
|
56
|
-
biolib/compute_node/cloud_utils/cloud_utils.py,sha256=
|
71
|
+
biolib/compute_node/cloud_utils/cloud_utils.py,sha256=_iaKelsmQLLwbDKVXZMXPFZayZPH9iHXc4NISFP9uzk,7462
|
57
72
|
biolib/compute_node/job_worker/__init__.py,sha256=ipdPWaABKYrltxny15e2kK8PWdEE7VzXbkKK6wM_zDk,71
|
58
73
|
biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSIiw4s90pNn83Netoo,4830
|
59
74
|
biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
|
60
75
|
biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
|
61
76
|
biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
|
62
|
-
biolib/compute_node/job_worker/executors/docker_executor.py,sha256=
|
77
|
+
biolib/compute_node/job_worker/executors/docker_executor.py,sha256=2H7GooL0oAifPcbie0unatB4fRoHyqbsr6S91uagc_g,27952
|
63
78
|
biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
|
64
79
|
biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
65
80
|
biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
|
66
81
|
biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
|
67
82
|
biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
|
68
|
-
biolib/compute_node/job_worker/job_storage.py,sha256=
|
69
|
-
biolib/compute_node/job_worker/job_worker.py,sha256=
|
83
|
+
biolib/compute_node/job_worker/job_storage.py,sha256=LNkklckDLbYgCHsK5FGrEK75Kw-H4f4JcTCAtuE9His,4035
|
84
|
+
biolib/compute_node/job_worker/job_worker.py,sha256=fuWoYJo9HOqLmWl8yeCXh0mhT4ebbkrWac-BVb58khs,28842
|
70
85
|
biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
|
71
86
|
biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
|
72
87
|
biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
|
73
88
|
biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO33vWtG60xg,1282
|
74
|
-
biolib/compute_node/remote_host_proxy.py,sha256=
|
89
|
+
biolib/compute_node/remote_host_proxy.py,sha256=CNWJLXXYm8DGujxEJIsg1wUKFoSgU0nhwdmjMn5gelE,14690
|
75
90
|
biolib/compute_node/socker_listener_thread.py,sha256=T5_UikA3MB9bD5W_dckYLPTgixh72vKUlgbBvj9dbM0,1601
|
76
91
|
biolib/compute_node/socket_sender_thread.py,sha256=YgamPHeUm2GjMFGx8qk-99WlZhEs-kAb3q_2O6qByig,971
|
77
92
|
biolib/compute_node/utils.py,sha256=M7i_WTyxbFM3Lri9RWZ_8FeQNYrQIWpKGLfp2I55oeY,4677
|
78
93
|
biolib/compute_node/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
79
94
|
biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLUXWo_vHFV-FIwlb8s8tacKmGHvh93qc,914
|
80
95
|
biolib/compute_node/webserver/webserver.py,sha256=15PkRyhtdtSgFDxa0z78aPO4ciZURsFqJYi-HtUmZF8,6494
|
81
|
-
biolib/compute_node/webserver/webserver_types.py,sha256=
|
96
|
+
biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
|
82
97
|
biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
|
83
98
|
biolib/compute_node/webserver/worker_thread.py,sha256=26tG73TADnOcXsAr7Iyf6smrLlCqB4x-vvmpUb8WqnA,11569
|
84
99
|
biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
85
|
-
biolib/experiments/experiment.py,sha256=
|
86
|
-
biolib/experiments/types.py,sha256=n9GxdFA7cLMfHvLLqLmZzX31ELeSSkMXFoEEdFsdWGY,171
|
100
|
+
biolib/experiments/experiment.py,sha256=jIRixmQm3Gq9YdJ3I0-rE1vFukXqq6U4zXehFOJ1yZk,7614
|
87
101
|
biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
|
88
|
-
biolib/jobs/job.py,sha256=
|
89
|
-
biolib/jobs/job_result.py,sha256=
|
90
|
-
biolib/jobs/types.py,sha256=
|
91
|
-
biolib/
|
92
|
-
biolib/
|
93
|
-
biolib/lfs/utils.py,sha256=HSs7F2wXklYhhv5tabfaeC5noXJyxRjbGD5IhWOVdxs,5918
|
94
|
-
biolib/runtime/__init__.py,sha256=x1Ivydtu9TFTaX-Cofg_kGA-TI0zLon-ccrFiiVgBok,492
|
95
|
-
biolib/sdk/__init__.py,sha256=77IBthMbwgmYymjQwLP4ny1X2ikvI1I0ocP6Eyzfyaw,1766
|
102
|
+
biolib/jobs/job.py,sha256=OfG8cLd3AjGjiMWRlJRZdVVbLsRWSX-OM5nxJhR6mPQ,19136
|
103
|
+
biolib/jobs/job_result.py,sha256=rALHiKYNaC9lHi_JJqBob1RubzNLwG9Z386kwRJjd2M,5885
|
104
|
+
biolib/jobs/types.py,sha256=qhadtH2KDC2WUOOqPiwke0YgtQY4FtuB71Stekq1k48,970
|
105
|
+
biolib/runtime/__init__.py,sha256=MlRepA11n2H-3plB5rzWyyHK2JmP6PiaP3i6x3vt0mg,506
|
106
|
+
biolib/sdk/__init__.py,sha256=amVp_jMxi2nqCcTsmL2aKUNGCAH3Yk4EzAnps9d1VH8,1928
|
96
107
|
biolib/tables.py,sha256=acH7VjwAbadLo8P84FSnKEZxCTVsF5rEg9VPuxElNs8,872
|
97
108
|
biolib/templates/__init__.py,sha256=Yx62sSyDCDesRQDQgmbDsLpfgEh93fWE8r9u4g2azXk,36
|
98
109
|
biolib/templates/example_app.py,sha256=EB3E3RT4SeO_ii5nVQqJpi5KDGNE_huF1ub-e5ZFveE,715
|
99
|
-
biolib/typing_utils.py,sha256=
|
110
|
+
biolib/typing_utils.py,sha256=ntzrlyTkUaO2OtccLYzCAGztGdca0WT5fikJUmSkT-Y,148
|
100
111
|
biolib/user/__init__.py,sha256=Db5wtxLfFz3ID9TULSSTo77csw9tO6RtxMRvV5cqKEE,39
|
101
|
-
biolib/user/sign_in.py,sha256=
|
112
|
+
biolib/user/sign_in.py,sha256=XTAmRPKfmg7VAaB8cT5wcmfxoPXeHqY8LmDiADF7zbw,2064
|
102
113
|
biolib/utils/__init__.py,sha256=fwjciJyJicvYyZcVTzfDBgD0SKY13DeXqvTeG4qZIy8,5548
|
103
114
|
biolib/utils/app_uri.py,sha256=Yq_-_VGugQhMMo6mM5f0G9yNlLkr0WK4j0Nrf3FE4xQ,2171
|
104
115
|
biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3100
|
105
116
|
biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
|
106
|
-
biolib/utils/seq_util.py,sha256=
|
117
|
+
biolib/utils/seq_util.py,sha256=ZQFcaE37B2dtucN2zDjOmdya_X0ITc1zBFZJNQY13XA,5183
|
107
118
|
biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
|
108
|
-
pybiolib-1.1.
|
109
|
-
pybiolib-1.1.
|
110
|
-
pybiolib-1.1.
|
111
|
-
pybiolib-1.1.
|
112
|
-
pybiolib-1.1.
|
119
|
+
pybiolib-1.1.2193.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
|
120
|
+
pybiolib-1.1.2193.dist-info/METADATA,sha256=NCFsHsCnbnOhw1O2L-polZfFa2pNdtA89TfBW7f_o7E,1508
|
121
|
+
pybiolib-1.1.2193.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
122
|
+
pybiolib-1.1.2193.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
|
123
|
+
pybiolib-1.1.2193.dist-info/RECORD,,
|
biolib/experiments/types.py
DELETED
biolib/lfs/__init__.py
DELETED
biolib/lfs/utils.py
DELETED
@@ -1,153 +0,0 @@
|
|
1
|
-
import io
|
2
|
-
import os
|
3
|
-
import zipfile as zf
|
4
|
-
from pathlib import Path
|
5
|
-
|
6
|
-
from biolib import utils, api
|
7
|
-
from biolib.biolib_api_client import BiolibApiClient
|
8
|
-
from biolib.biolib_api_client.lfs_types import LargeFileSystem, LargeFileSystemVersion
|
9
|
-
from biolib.biolib_logging import logger
|
10
|
-
from biolib.biolib_errors import BioLibError
|
11
|
-
from biolib.typing_utils import List, Tuple, Iterator, Optional
|
12
|
-
from biolib.utils.app_uri import parse_app_uri
|
13
|
-
|
14
|
-
|
15
|
-
def get_files_and_size_of_directory(directory: str) -> Tuple[List[str], int]:
|
16
|
-
data_size = 0
|
17
|
-
file_list: List[str] = []
|
18
|
-
|
19
|
-
for path, _, files in os.walk(directory):
|
20
|
-
for file in files:
|
21
|
-
file_path = os.path.join(path, file)
|
22
|
-
if os.path.islink(file_path):
|
23
|
-
continue # skip symlinks
|
24
|
-
|
25
|
-
relative_file_path = file_path[len(directory) + 1:] # +1 to remove starting slash
|
26
|
-
file_list.append(relative_file_path)
|
27
|
-
data_size += os.path.getsize(file_path)
|
28
|
-
|
29
|
-
return file_list, data_size
|
30
|
-
|
31
|
-
|
32
|
-
def get_iterable_zip_stream(files: List[str], chunk_size: int) -> Iterator[bytes]:
|
33
|
-
class ChunkedIOBuffer(io.RawIOBase):
|
34
|
-
def __init__(self, chunk_size: int):
|
35
|
-
super().__init__()
|
36
|
-
self.chunk_size = chunk_size
|
37
|
-
self.tmp_data = bytearray()
|
38
|
-
|
39
|
-
def get_buffer_size(self):
|
40
|
-
return len(self.tmp_data)
|
41
|
-
|
42
|
-
def read_chunk(self):
|
43
|
-
chunk = bytes(self.tmp_data[:self.chunk_size])
|
44
|
-
self.tmp_data = self.tmp_data[self.chunk_size:]
|
45
|
-
return chunk
|
46
|
-
|
47
|
-
def write(self, data):
|
48
|
-
data_length = len(data)
|
49
|
-
self.tmp_data += data
|
50
|
-
return data_length
|
51
|
-
|
52
|
-
# create chunked buffer to hold data temporarily
|
53
|
-
io_buffer = ChunkedIOBuffer(chunk_size)
|
54
|
-
|
55
|
-
# create zip writer that will write to the io buffer
|
56
|
-
zip_writer = zf.ZipFile(io_buffer, mode='w') # type: ignore
|
57
|
-
|
58
|
-
for file_path in files:
|
59
|
-
# generate zip info and prepare zip pointer for writing
|
60
|
-
z_info = zf.ZipInfo.from_file(file_path)
|
61
|
-
zip_pointer = zip_writer.open(z_info, mode='w')
|
62
|
-
if Path(file_path).is_file():
|
63
|
-
# read file chunk by chunk
|
64
|
-
with open(file_path, 'br') as file_pointer:
|
65
|
-
while True:
|
66
|
-
chunk = file_pointer.read(chunk_size)
|
67
|
-
if len(chunk) == 0:
|
68
|
-
break
|
69
|
-
# write the chunk to the zip
|
70
|
-
zip_pointer.write(chunk)
|
71
|
-
# if writing the chunk caused us to go over chunk_size, flush it
|
72
|
-
if io_buffer.get_buffer_size() > chunk_size:
|
73
|
-
yield io_buffer.read_chunk()
|
74
|
-
|
75
|
-
zip_pointer.close()
|
76
|
-
|
77
|
-
# flush any remaining data in the stream (e.g. zip file meta data)
|
78
|
-
zip_writer.close()
|
79
|
-
while True:
|
80
|
-
chunk = io_buffer.read_chunk()
|
81
|
-
if len(chunk) == 0:
|
82
|
-
break
|
83
|
-
yield chunk
|
84
|
-
|
85
|
-
|
86
|
-
def create_large_file_system(lfs_uri: str) -> str:
|
87
|
-
BiolibApiClient.assert_is_signed_in(authenticated_action_description='create a Large File System')
|
88
|
-
|
89
|
-
uri_parsed = parse_app_uri(lfs_uri)
|
90
|
-
response = api.client.post(
|
91
|
-
path='/lfs/',
|
92
|
-
data={
|
93
|
-
'account_handle': uri_parsed['account_handle_normalized'],
|
94
|
-
'name': uri_parsed['app_name'],
|
95
|
-
},
|
96
|
-
)
|
97
|
-
lfs: LargeFileSystem = response.json()
|
98
|
-
logger.info(f"Successfully created new Large File System '{lfs['uri']}'")
|
99
|
-
return lfs['uri']
|
100
|
-
|
101
|
-
|
102
|
-
def push_large_file_system(lfs_uri: str, input_dir: str, chunk_size_in_mb: Optional[int] = None) -> str:
|
103
|
-
BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Large File System')
|
104
|
-
|
105
|
-
if not os.path.isdir(input_dir):
|
106
|
-
raise BioLibError(f'Could not find folder at {input_dir}')
|
107
|
-
|
108
|
-
if os.path.realpath(input_dir) == '/':
|
109
|
-
raise BioLibError('Pushing your root directory is not possible')
|
110
|
-
|
111
|
-
original_working_dir = os.getcwd()
|
112
|
-
os.chdir(input_dir)
|
113
|
-
files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
|
114
|
-
|
115
|
-
if data_size_in_bytes > 4_500_000_000_000:
|
116
|
-
raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
|
117
|
-
|
118
|
-
min_chunk_size_bytes = 10_000_000
|
119
|
-
chunk_size_in_bytes: int
|
120
|
-
if chunk_size_in_mb:
|
121
|
-
chunk_size_in_bytes = chunk_size_in_mb * 1_000_000 # Convert megabytes to bytes
|
122
|
-
if chunk_size_in_bytes < min_chunk_size_bytes:
|
123
|
-
logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
|
124
|
-
chunk_size_in_bytes = min_chunk_size_bytes
|
125
|
-
else:
|
126
|
-
# Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
|
127
|
-
chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
|
128
|
-
|
129
|
-
data_size_in_mb = round(data_size_in_bytes / 10 ** 6)
|
130
|
-
print(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
|
131
|
-
|
132
|
-
response = api.client.post(path='/lfs/versions/', data={'resource_uri': lfs_uri})
|
133
|
-
lfs_version: LargeFileSystemVersion = response.json()
|
134
|
-
iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
|
135
|
-
|
136
|
-
multipart_uploader = utils.MultiPartUploader(
|
137
|
-
use_process_pool=True,
|
138
|
-
get_presigned_upload_url_request=dict(
|
139
|
-
headers=None,
|
140
|
-
requires_biolib_auth=True,
|
141
|
-
path=f"/lfs/versions/{lfs_version['uuid']}/presigned_upload_url/",
|
142
|
-
),
|
143
|
-
complete_upload_request=dict(
|
144
|
-
headers=None,
|
145
|
-
requires_biolib_auth=True,
|
146
|
-
path=f"/lfs/versions/{lfs_version['uuid']}/complete_upload/",
|
147
|
-
),
|
148
|
-
)
|
149
|
-
|
150
|
-
multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
|
151
|
-
os.chdir(original_working_dir)
|
152
|
-
logger.info(f"Successfully pushed a new LFS version '{lfs_version['uri']}'")
|
153
|
-
return lfs_version['uri']
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|