fractal-server 1.4.3a0__py3-none-any.whl → 1.4.3a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/db/__init__.py +36 -25
  3. fractal_server/app/routes/admin.py +8 -8
  4. fractal_server/app/routes/api/v1/_aux_functions.py +3 -5
  5. fractal_server/app/routes/api/v1/dataset.py +24 -23
  6. fractal_server/app/routes/api/v1/job.py +7 -7
  7. fractal_server/app/routes/api/v1/project.py +14 -19
  8. fractal_server/app/routes/api/v1/task.py +6 -6
  9. fractal_server/app/routes/api/v1/task_collection.py +12 -126
  10. fractal_server/app/routes/api/v1/workflow.py +13 -13
  11. fractal_server/app/routes/api/v1/workflowtask.py +5 -5
  12. fractal_server/app/routes/auth.py +2 -2
  13. fractal_server/app/runner/__init__.py +0 -1
  14. fractal_server/app/schemas/__init__.py +1 -0
  15. fractal_server/app/schemas/applyworkflow.py +5 -9
  16. fractal_server/app/schemas/task_collection.py +2 -10
  17. fractal_server/app/security/__init__.py +3 -3
  18. fractal_server/config.py +14 -0
  19. fractal_server/tasks/_TaskCollectPip.py +103 -0
  20. fractal_server/tasks/__init__.py +3 -1
  21. fractal_server/tasks/background_operations.py +384 -0
  22. fractal_server/tasks/endpoint_operations.py +167 -0
  23. fractal_server/tasks/utils.py +86 -0
  24. {fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/METADATA +1 -1
  25. {fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/RECORD +28 -25
  26. fractal_server/tasks/collection.py +0 -556
  27. {fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/LICENSE +0 -0
  28. {fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/WHEEL +0 -0
  29. {fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/entry_points.txt +0 -0
@@ -1,8 +1,8 @@
1
- fractal_server/__init__.py,sha256=c7VnyctolFPHRpW0iJqGeGZ7qOEUv4CzGL4vKUeJoKI,24
1
+ fractal_server/__init__.py,sha256=RMjEp0Q4tXgUXaLH-zfGlvDz08V2TCgRIy3M-r1RY0Y,24
2
2
  fractal_server/__main__.py,sha256=znijcImbcEC4P26ICOhEJ9VY3_5vWdMwQcl-WP25sYA,2202
3
3
  fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
4
4
  fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
- fractal_server/app/db/__init__.py,sha256=OIio0fPE35xC5V9Vom_25-NmUz_vuZNmlJnDZpp1ZnM,3742
5
+ fractal_server/app/db/__init__.py,sha256=igHvpGsQJ60i33WQucGQfRRYvgNeFWahtji1GlIIPe4,4058
6
6
  fractal_server/app/models/__init__.py,sha256=RuxWH8fsmkTWsjLhYjrxSt-mvk74coCilAQlX2Q6OO0,353
7
7
  fractal_server/app/models/dataset.py,sha256=nydU9syGVXSVuj3sTsVXIiU2vhTUrdwcUZipM-p00GY,2000
8
8
  fractal_server/app/models/job.py,sha256=t0O9EKGQO4aPuTtc_N9SzLF2vrc-pevjsHumLeCPvM8,3287
@@ -13,23 +13,23 @@ fractal_server/app/models/state.py,sha256=rSTjYPfPZntEfdQudKp6yu5vsdyfHA7nMYNRIB
13
13
  fractal_server/app/models/task.py,sha256=APndtea9A7EF7TtpVK8kWapBM01a6nk3FFCrQbbioI8,2632
14
14
  fractal_server/app/models/workflow.py,sha256=B6v3qqNDb6hvAyDN63n5vkemNueR2aH6zpwSGLlcRNE,3933
15
15
  fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
16
- fractal_server/app/routes/admin.py,sha256=71I4alHKObLWfhoRJ_6wtCuGepsdaqBEyOkaRjmaoZg,10272
16
+ fractal_server/app/routes/admin.py,sha256=zc9wu0CMwmBP2cYADGpdG_-4BvL1iAnTlqeu1lcpWEw,10320
17
17
  fractal_server/app/routes/api/__init__.py,sha256=EVyZrEq3I_1643QGTPCC5lgCp4xH_auYbrFfogTm4pc,315
18
18
  fractal_server/app/routes/api/v1/__init__.py,sha256=V4nhYyMIqhlJxbotLTYikq_ghb6KID0ZKOOYaOq7C-g,944
19
- fractal_server/app/routes/api/v1/_aux_functions.py,sha256=_kbJprGMsAFvg982L0WEwY5seJ5LR323ClX1ogRidWI,12027
20
- fractal_server/app/routes/api/v1/dataset.py,sha256=W1bNCt4ZEIcL0aTSizTQlxskmQp01k0G8bV1neFDows,16196
21
- fractal_server/app/routes/api/v1/job.py,sha256=5WpqEZ31Kl4ZyU5Ws1Y0ySy1Jk7-TDsIMKIQvD9BwI8,4740
22
- fractal_server/app/routes/api/v1/project.py,sha256=hT_qa0JHQVP9xD_QJH7EQmWSrdE_wlg4ZaC1A2VyXis,14419
23
- fractal_server/app/routes/api/v1/task.py,sha256=ZSw6aXe0NjO02x2k8X6-ZN3eM_V7Ec-NduY52G-ypXI,5576
24
- fractal_server/app/routes/api/v1/task_collection.py,sha256=7-kdQ07ByDwEjQCDhF0LPufqMixmlKhLvb9X63AVBf0,11864
25
- fractal_server/app/routes/api/v1/workflow.py,sha256=nWmNykN_CuNINoi-FDIVh5S9mI6Pdst7mLQDMvvriH4,10831
26
- fractal_server/app/routes/api/v1/workflowtask.py,sha256=nf15BmyJYfJGX4s2Z_RSxLzqmrymgkLNuFrlhLW1nwA,5520
27
- fractal_server/app/routes/auth.py,sha256=aBR0ZICrbJpOgPO972HN5lJ1YqfkrO0avwdrSRvI9aQ,4873
19
+ fractal_server/app/routes/api/v1/_aux_functions.py,sha256=BP_rdwciA3PLy7ipugcvkKT1pxQSg6uAF7rZirDwBZU,11968
20
+ fractal_server/app/routes/api/v1/dataset.py,sha256=4zKZKgX4cAdP_yTuOvFHXvdX9OiayHlrBZlXdOAOi8c,16265
21
+ fractal_server/app/routes/api/v1/job.py,sha256=6AZ-dRQK4roocZZntHGHPNSKtK-WL9xX3apmruVTTj4,4782
22
+ fractal_server/app/routes/api/v1/project.py,sha256=NTn3IluB9Lk7L2x7ybxHGifqjBXglrLd7G7FVd_NeZM,14331
23
+ fractal_server/app/routes/api/v1/task.py,sha256=FtGfqhapIOuGj5gxHYXabm2jU4L71h5_5-0VnuzZQ0g,5612
24
+ fractal_server/app/routes/api/v1/task_collection.py,sha256=zKkKd-3hne16hYCaopySvkj1l8HOfWozgjHsQaceGN8,8340
25
+ fractal_server/app/routes/api/v1/workflow.py,sha256=3dfFBUh0qJ_h4zMEsRgPit7g2Nu7v0CczeyfVA_Q4Fw,10864
26
+ fractal_server/app/routes/api/v1/workflowtask.py,sha256=9QrsnZatai4PXvRgD7gfT-8QGRu787-2wenN_6gfYuo,5550
27
+ fractal_server/app/routes/auth.py,sha256=Xv80iqdyfY3lyicYs2Y8B6zEDEnyUu_H6_6psYtv3R4,4885
28
28
  fractal_server/app/routes/aux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
29
  fractal_server/app/routes/aux/_job.py,sha256=whx2G9PCCt-Hw_lgsZa1ECQlhDKNq4eHvwqgpgvBgwg,1246
30
30
  fractal_server/app/routes/aux/_runner.py,sha256=psW6fsoo_VrAHrD5UQPbqFYikCp0m16VRymC-U1yUTk,675
31
31
  fractal_server/app/runner/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
32
- fractal_server/app/runner/__init__.py,sha256=CR-ajmLzbNS8-QIAujoYsOSh2hSFmzKxEHxVmhjqfL0,13702
32
+ fractal_server/app/runner/__init__.py,sha256=ByYDW7C2qw7s7oxsFdBT1b5wO_qFkH_6EPyDPHxJ5DM,13659
33
33
  fractal_server/app/runner/_common.py,sha256=WwFyhqAEe9SHNCrlJq8is-1RF_LZOF9RqawK0JXMy4g,22672
34
34
  fractal_server/app/runner/_local/__init__.py,sha256=gHsilCnT9VkqVbKpnEIZCnx4BuDydWcKneeWHWb2410,6799
35
35
  fractal_server/app/runner/_local/_local_config.py,sha256=-oNTsjEUmytHlsYpWfw2CrPvSxDFeEhZSdQvI_wf3Mk,3245
@@ -46,9 +46,9 @@ fractal_server/app/runner/_slurm/executor.py,sha256=UOXduxOi_LkmtvlU-Xd3CKJj07gT
46
46
  fractal_server/app/runner/_slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
47
47
  fractal_server/app/runner/common.py,sha256=nz0ZuIro0iwZm-OV-e-Y-PrtgKcLK0d7BrzebWyEWEk,9496
48
48
  fractal_server/app/runner/handle_failed_job.py,sha256=Kov_Ha1rcPNdoLuQx8Dq4fz7s2naR25ce4oQaUy-7TI,4653
49
- fractal_server/app/schemas/__init__.py,sha256=tTUrAQ492v_w7Ab_oa2dq3pMjcmROaBJRg-9bt6mGEk,1969
49
+ fractal_server/app/schemas/__init__.py,sha256=vjGKGMM45ywNClHV5KZ2u9eGLCa4p7i6ueQqCGtPcSk,2010
50
50
  fractal_server/app/schemas/_validators.py,sha256=dsLMEZ3fdY3NGFodeKdWPizsf1Ifxoz1oGL2_FVYBiE,2114
51
- fractal_server/app/schemas/applyworkflow.py,sha256=s3-B_18nu0JTlxMbfqgUMYORUKye50pPJIBdkIeniGA,4206
51
+ fractal_server/app/schemas/applyworkflow.py,sha256=79gDiEMvUNJx5SODzz7ha1vStxd2zXG5T6fNX2rAuUY,4015
52
52
  fractal_server/app/schemas/dataset.py,sha256=Qgkn_qDI4FRfKRvRF-IzxKSqbzspkGX1M1kN3lFb9Ec,3225
53
53
  fractal_server/app/schemas/dumps.py,sha256=GPeTeg2yoQCPgaQoj_jHA8Lnt1fL_j4QBOb0IKH0lv8,1296
54
54
  fractal_server/app/schemas/json_schemas/manifest.json,sha256=yXYKHbYXPYSkSXMTLfTpfCUGBtmQuPTk1xuSXscdba4,1787
@@ -56,11 +56,11 @@ fractal_server/app/schemas/manifest.py,sha256=xxTd39dAXMK9Ox1y-p3gbyg0zd5udW99pV
56
56
  fractal_server/app/schemas/project.py,sha256=zAOWyr6UVqyCn6UOAt7Ulx8J7b7IEsp2cCBN_UyNxdc,1046
57
57
  fractal_server/app/schemas/state.py,sha256=cBco_ViYRvlbWjoeycHAQyADGuiebIntJjf6xdbyod8,549
58
58
  fractal_server/app/schemas/task.py,sha256=2TBE5Ne9tO_-a2-Es0PRXMT8ZddSInTOPMor7u8-gx0,3671
59
- fractal_server/app/schemas/task_collection.py,sha256=DPdBPL96e1q88pZeXmjd7e7iRZ_xPkYI_npF4sekF6o,3158
59
+ fractal_server/app/schemas/task_collection.py,sha256=nkbW076pB0wWYyWkFpplyLBBEWufAP6buYAmEupWV6I,3044
60
60
  fractal_server/app/schemas/user.py,sha256=rE8WgBz-ceVUs0Sz2ZwcjUrSTZTnS0ys5SBtD2XD9r8,3113
61
61
  fractal_server/app/schemas/workflow.py,sha256=DKKsKVMwUeZXjLEMviLvPKhNgSya9GKt5rOMS6oJEC0,4374
62
- fractal_server/app/security/__init__.py,sha256=d1BjGxz0FWZO6p6TVUt78aprvpzLFnQsstIrW55JPcY,11185
63
- fractal_server/config.py,sha256=2HB74iFQV4MDH7Ub66uke532G6YoyBWmLYfbfYApJqc,14457
62
+ fractal_server/app/security/__init__.py,sha256=wxosoHc3mJYPCdPMyWnRD8w_2OgnKYp2aDkdmwrZh5k,11203
63
+ fractal_server/config.py,sha256=CpKlEfY5sl7snBJGXvTFoFnfMst2hPQtPlJccrGzm8w,15021
64
64
  fractal_server/logger.py,sha256=keri8i960WHT8Zz9Rm2MwfnrA2dw9TsrfCmojqtGDLs,4562
65
65
  fractal_server/main.py,sha256=dyU9jGCJCAGiTF0veggAbcMivs3I6U98341tkSxdcpg,3303
66
66
  fractal_server/migrations/README,sha256=4rQvyDfqodGhpJw74VYijRmgFP49ji5chyEemWGHsuw,59
@@ -83,11 +83,14 @@ fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py
83
83
  fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
84
84
  fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
85
85
  fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
86
- fractal_server/tasks/__init__.py,sha256=Wzuxf5EoH1v0fYzRpAZHG_S-Z9f6DmbIsuSvllBCGvc,72
87
- fractal_server/tasks/collection.py,sha256=POKvQyS5G5ySybH0r0v21I_ZQ5AREe9kAqr_uFfGyaU,17627
86
+ fractal_server/tasks/_TaskCollectPip.py,sha256=Y1YPu0YB0z5abmwyWvBhFVIkP8ORv6lxihg8Q5zsY9I,3765
87
+ fractal_server/tasks/__init__.py,sha256=k5bhaUOXRrSQSik_riqTDQlWgNHzHMR92AIwmyBrIlw,176
88
+ fractal_server/tasks/background_operations.py,sha256=GiDIE4s3tVkjJbUle7rSzQsldiFnABes8Vm2zii1WdY,12744
89
+ fractal_server/tasks/endpoint_operations.py,sha256=PC94y_sNajyGxNFsgxNGB8FDZF8MuCxquL6l63FJeY4,5549
90
+ fractal_server/tasks/utils.py,sha256=-j8T1VBbjTt5fjP2XdIcs0nBwSkYyuv_yLI1troBg9Q,2274
88
91
  fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
89
- fractal_server-1.4.3a0.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
90
- fractal_server-1.4.3a0.dist-info/METADATA,sha256=lqQNo3aIYAac78m6-5EGliZhXUJJzYccec6ZR0vgxb4,4225
91
- fractal_server-1.4.3a0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
92
- fractal_server-1.4.3a0.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
93
- fractal_server-1.4.3a0.dist-info/RECORD,,
92
+ fractal_server-1.4.3a1.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
93
+ fractal_server-1.4.3a1.dist-info/METADATA,sha256=ZMYga-bgZoG5MfBegVWkTySm23A70yqt0_c7gW4ucvQ,4225
94
+ fractal_server-1.4.3a1.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
95
+ fractal_server-1.4.3a1.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
96
+ fractal_server-1.4.3a1.dist-info/RECORD,,
@@ -1,556 +0,0 @@
1
- # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
2
- # University of Zurich
3
- #
4
- # Original authors:
5
- # Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
6
- # Tommaso Comparin <tommaso.comparin@exact-lab.it>
7
- # Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
8
- #
9
- # This file is part of Fractal and was originally developed by eXact lab S.r.l.
10
- # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
11
- # Institute for Biomedical Research and Pelkmans Lab from the University of
12
- # Zurich.
13
- """
14
- This module takes care of installing tasks so that fractal can execute them
15
-
16
- Tasks are installed under `Settings.FRACTAL_TASKS_DIR/{username}`, with
17
- `username = ".fractal"`.
18
- """
19
- import json
20
- import shutil
21
- import sys
22
- from pathlib import Path
23
- from typing import Optional
24
- from typing import Union
25
- from zipfile import ZipFile
26
-
27
- from pydantic import root_validator
28
-
29
- from ..app.schemas import ManifestV1
30
- from ..app.schemas import TaskCollectPip
31
- from ..app.schemas import TaskCollectStatus
32
- from ..app.schemas import TaskCreate
33
- from ..config import get_settings
34
- from ..logger import get_logger
35
- from ..syringe import Inject
36
- from ..utils import execute_command
37
-
38
-
39
- FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
40
-
41
-
42
- def slugify_task_name(task_name: str) -> str:
43
- return task_name.replace(" ", "_").lower()
44
-
45
-
46
- def get_python_interpreter(version: Optional[str] = None) -> str:
47
- """
48
- Return the path to the python interpreter
49
-
50
- Args:
51
- version: Python version
52
-
53
- Raises:
54
- ValueError: If the python version requested is not available on the
55
- host.
56
-
57
- Returns:
58
- interpreter: string representing the python executable or its path
59
- """
60
- if version:
61
- interpreter = shutil.which(f"python{version}")
62
- if not interpreter:
63
- raise ValueError(
64
- f"Python version {version} not available on host."
65
- )
66
- else:
67
- interpreter = sys.executable
68
-
69
- return interpreter
70
-
71
-
72
- def get_absolute_venv_path(venv_path: Path) -> Path:
73
- """
74
- Note:
75
- In Python 3.9 it would be safer to do:
76
-
77
- if venv_path.is_relative_to(settings.FRACTAL_TASKS_DIR):
78
- package_path = venv_path
79
- else:
80
- package_path = settings.FRACTAL_TASKS_DIR / venv_path
81
- """
82
- if venv_path.is_absolute():
83
- package_path = venv_path
84
- else:
85
- settings = Inject(get_settings)
86
- package_path = settings.FRACTAL_TASKS_DIR / venv_path
87
- return package_path
88
-
89
-
90
- def get_collection_path(base: Path) -> Path:
91
- return base / "collection.json"
92
-
93
-
94
- def get_log_path(base: Path) -> Path:
95
- return base / "collection.log"
96
-
97
-
98
- def get_collection_log(venv_path: Path) -> str:
99
- package_path = get_absolute_venv_path(venv_path)
100
- log_path = get_log_path(package_path)
101
- log = log_path.open().read()
102
- return log
103
-
104
-
105
- def get_collection_data(venv_path: Path) -> TaskCollectStatus:
106
- package_path = get_absolute_venv_path(venv_path)
107
- collection_path = get_collection_path(package_path)
108
- with collection_path.open() as f:
109
- data = json.load(f)
110
- return TaskCollectStatus(**data)
111
-
112
-
113
- class _TaskCollectPip(TaskCollectPip):
114
- """
115
- Internal TaskCollectPip schema
116
-
117
- Differences with its parent class (`TaskCollectPip`):
118
-
119
- 1. We check if the package corresponds to a path in the filesystem, and
120
- whether it exists (via new validator `check_local_package`, new
121
- method `is_local_package` and new attribute `package_path`).
122
- 2. We include an additional `package_manifest` attribute.
123
- 3. We expose an additional attribute `package_name`, which is filled
124
- during task collection.
125
- """
126
-
127
- package_name: Optional[str] = None
128
- package_path: Optional[Path] = None
129
- package_manifest: Optional[ManifestV1] = None
130
-
131
- @property
132
- def is_local_package(self) -> bool:
133
- return bool(self.package_path)
134
-
135
- @root_validator(pre=True)
136
- def check_local_package(cls, values):
137
- """
138
- Checks if package corresponds to an existing path on the filesystem
139
-
140
- In this case, the user is providing directly a package file, rather
141
- than a remote one from PyPI. We set the `package_path` attribute and
142
- get the actual package name and version from the package file name.
143
- """
144
- if "/" in values["package"]:
145
- package_path = Path(values["package"])
146
- if not package_path.is_absolute():
147
- raise ValueError("Package path must be absolute")
148
- if package_path.exists():
149
- values["package_path"] = package_path
150
- (
151
- values["package"],
152
- values["version"],
153
- *_,
154
- ) = package_path.name.split("-")
155
- else:
156
- raise ValueError(f"Package {package_path} does not exist.")
157
- return values
158
-
159
- @property
160
- def package_source(self):
161
- if not self.package_name or not self.package_version:
162
- raise ValueError(
163
- "Cannot construct `package_source` property with "
164
- f"{self.package_name=} and {self.package_version=}."
165
- )
166
- if self.is_local_package:
167
- collection_type = "pip_local"
168
- else:
169
- collection_type = "pip_remote"
170
-
171
- package_extras = self.package_extras or ""
172
- if self.python_version:
173
- python_version = f"py{self.python_version}"
174
- else:
175
- python_version = "" # FIXME: can we allow this?
176
-
177
- source = ":".join(
178
- (
179
- collection_type,
180
- self.package_name,
181
- self.package_version,
182
- package_extras,
183
- python_version,
184
- )
185
- )
186
- return source
187
-
188
- def check(self):
189
- """
190
- Verify that the package has all attributes that are needed to continue
191
- with task collection
192
- """
193
- if not self.package_name:
194
- raise ValueError("`package_name` attribute is not set")
195
- if not self.package_version:
196
- raise ValueError("`package_version` attribute is not set")
197
- if not self.package_manifest:
198
- raise ValueError("`package_manifest` attribute is not set")
199
-
200
-
201
- def create_package_dir_pip(
202
- *,
203
- task_pkg: _TaskCollectPip,
204
- create: bool = True,
205
- ) -> Path:
206
- """
207
- Create venv folder for a task package and return corresponding Path object
208
- """
209
- settings = Inject(get_settings)
210
- user = FRACTAL_PUBLIC_TASK_SUBDIR
211
- if task_pkg.package_version is None:
212
- raise ValueError(
213
- f"Cannot create venv folder for package `{task_pkg.package}` "
214
- "with `version=None`."
215
- )
216
- package_dir = f"{task_pkg.package}{task_pkg.package_version}"
217
- venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
218
- if create:
219
- venv_path.mkdir(exist_ok=False, parents=True)
220
- return venv_path
221
-
222
-
223
- async def download_package(
224
- *,
225
- task_pkg: _TaskCollectPip,
226
- dest: Union[str, Path],
227
- ):
228
- """
229
- Download package to destination
230
- """
231
- interpreter = get_python_interpreter(version=task_pkg.python_version)
232
- pip = f"{interpreter} -m pip"
233
- version = (
234
- f"=={task_pkg.package_version}" if task_pkg.package_version else ""
235
- )
236
- package_and_version = f"{task_pkg.package}{version}"
237
- cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
238
- stdout = await execute_command(command=cmd, cwd=Path("."))
239
- pkg_file = next(
240
- line.split()[-1] for line in stdout.split("\n") if "Saved" in line
241
- )
242
- return Path(pkg_file)
243
-
244
-
245
- def _load_manifest_from_wheel(
246
- path: Path, wheel: ZipFile, logger_name: Optional[str] = None
247
- ) -> ManifestV1:
248
- logger = get_logger(logger_name)
249
- namelist = wheel.namelist()
250
- try:
251
- manifest = next(
252
- name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
253
- )
254
- except StopIteration:
255
- msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
256
- logger.error(msg)
257
- raise ValueError(msg)
258
- with wheel.open(manifest) as manifest_fd:
259
- manifest_dict = json.load(manifest_fd)
260
- manifest_version = str(manifest_dict["manifest_version"])
261
- if manifest_version == "1":
262
- pkg_manifest = ManifestV1(**manifest_dict)
263
- return pkg_manifest
264
- else:
265
- msg = f"Manifest version {manifest_version=} not supported"
266
- logger.error(msg)
267
- raise ValueError(msg)
268
-
269
-
270
- def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
271
- """
272
- Inspect task package to extract version, name and manifest
273
-
274
- Note that this only works with wheel files, which have a well-defined
275
- dist-info section. If we need to generalize to to tar.gz archives, we would
276
- need to go and look for `PKG-INFO`.
277
-
278
- Note: package name is normalized by replacing `{-,.}` with `_`.
279
-
280
- Args:
281
- path: Path
282
- the path in which the package is saved
283
-
284
- Returns:
285
- version_manifest: A dictionary containing `version`, the version of the
286
- pacakge, and `manifest`, the Fractal manifest object relative to the
287
- tasks.
288
- """
289
-
290
- logger = get_logger(logger_name)
291
-
292
- if not path.as_posix().endswith(".whl"):
293
- raise ValueError(
294
- f"Only wheel packages are supported, given {path.as_posix()}."
295
- )
296
-
297
- with ZipFile(path) as wheel:
298
- namelist = wheel.namelist()
299
-
300
- # Read and validate task manifest
301
- logger.debug(f"Now reading manifest for {path.as_posix()}")
302
- pkg_manifest = _load_manifest_from_wheel(
303
- path, wheel, logger_name=logger_name
304
- )
305
- logger.debug("Manifest read correctly.")
306
-
307
- # Read package name and version from *.dist-info/METADATA
308
- logger.debug(
309
- f"Now reading package name and version for {path.as_posix()}"
310
- )
311
- metadata = next(
312
- name for name in namelist if "dist-info/METADATA" in name
313
- )
314
- with wheel.open(metadata) as metadata_fd:
315
- meta = metadata_fd.read().decode("utf-8")
316
- pkg_name = next(
317
- line.split()[-1]
318
- for line in meta.splitlines()
319
- if line.startswith("Name")
320
- )
321
- pkg_version = next(
322
- line.split()[-1]
323
- for line in meta.splitlines()
324
- if line.startswith("Version")
325
- )
326
- logger.debug("Package name and version read correctly.")
327
-
328
- # Normalize package name:
329
- pkg_name = pkg_name.replace("-", "_").replace(".", "_")
330
-
331
- info = dict(
332
- pkg_name=pkg_name,
333
- pkg_version=pkg_version,
334
- pkg_manifest=pkg_manifest,
335
- )
336
- return info
337
-
338
-
339
- async def create_package_environment_pip(
340
- *,
341
- task_pkg: _TaskCollectPip,
342
- venv_path: Path,
343
- logger_name: str,
344
- ) -> list[TaskCreate]:
345
- """
346
- Create environment, install package, and prepare task list
347
- """
348
-
349
- logger = get_logger(logger_name)
350
-
351
- # Only proceed if package, version and manifest attributes are set
352
- task_pkg.check()
353
-
354
- try:
355
-
356
- logger.debug("Creating venv and installing package")
357
- python_bin, package_root = await _create_venv_install_package(
358
- path=venv_path,
359
- task_pkg=task_pkg,
360
- logger_name=logger_name,
361
- )
362
- logger.debug("Venv creation and package installation ended correctly.")
363
-
364
- # Prepare task_list with appropriate metadata
365
- logger.debug("Creating task list from manifest")
366
- task_list = []
367
- for t in task_pkg.package_manifest.task_list:
368
- # Fill in attributes for TaskCreate
369
- task_executable = package_root / t.executable
370
- cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
371
- task_name_slug = slugify_task_name(t.name)
372
- task_source = f"{task_pkg.package_source}:{task_name_slug}"
373
- if not task_executable.exists():
374
- raise FileNotFoundError(
375
- f"Cannot find executable `{task_executable}` "
376
- f"for task `{t.name}`"
377
- )
378
- manifest = task_pkg.package_manifest
379
- if manifest.has_args_schemas:
380
- additional_attrs = dict(
381
- args_schema_version=manifest.args_schema_version
382
- )
383
- else:
384
- additional_attrs = {}
385
- this_task = TaskCreate(
386
- **t.dict(),
387
- command=cmd,
388
- version=task_pkg.package_version,
389
- **additional_attrs,
390
- source=task_source,
391
- )
392
- task_list.append(this_task)
393
- logger.debug("Task list created correctly")
394
- except Exception as e:
395
- logger.error("Task manifest loading failed")
396
- raise e
397
- return task_list
398
-
399
-
400
- async def _create_venv_install_package(
401
- *,
402
- task_pkg: _TaskCollectPip,
403
- path: Path,
404
- logger_name: str,
405
- ) -> tuple[Path, Path]:
406
- """Create venv and install package
407
-
408
- Args:
409
- path: the directory in which to create the environment
410
- task_pkg: object containing the different metadata required to install
411
- the package
412
-
413
- Returns:
414
- python_bin: path to venv's python interpreter
415
- package_root: the location of the package manifest
416
- """
417
- python_bin = await _init_venv(
418
- path=path,
419
- python_version=task_pkg.python_version,
420
- logger_name=logger_name,
421
- )
422
- package_root = await _pip_install(
423
- venv_path=path, task_pkg=task_pkg, logger_name=logger_name
424
- )
425
- return python_bin, package_root
426
-
427
-
428
- async def _init_venv(
429
- *,
430
- path: Path,
431
- python_version: Optional[str] = None,
432
- logger_name: str,
433
- ) -> Path:
434
- """
435
- Set a virtual environment at `path/venv`
436
-
437
- Args:
438
- path : Path
439
- path to directory in which to set up the virtual environment
440
- python_version : default=None
441
- Python version the virtual environment will be based upon
442
-
443
- Returns:
444
- python_bin : Path
445
- path to python interpreter
446
- """
447
- interpreter = get_python_interpreter(version=python_version)
448
- await execute_command(
449
- cwd=path,
450
- command=f"{interpreter} -m venv venv",
451
- logger_name=logger_name,
452
- )
453
- return path / "venv/bin/python"
454
-
455
-
456
- async def _pip_install(
457
- venv_path: Path,
458
- task_pkg: _TaskCollectPip,
459
- logger_name: str,
460
- ) -> Path:
461
- """
462
- Install package in venv
463
-
464
- Returns:
465
- package_root : Path
466
- the location of the package manifest
467
- """
468
-
469
- logger = get_logger(logger_name)
470
-
471
- pip = venv_path / "venv/bin/pip"
472
-
473
- extras = f"[{task_pkg.package_extras}]" if task_pkg.package_extras else ""
474
-
475
- if task_pkg.is_local_package:
476
- pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
477
- else:
478
- version_string = (
479
- f"=={task_pkg.package_version}" if task_pkg.package_version else ""
480
- )
481
- pip_install_str = f"{task_pkg.package}{extras}{version_string}"
482
-
483
- cmd_install = f"{pip} install {pip_install_str}"
484
- cmd_inspect = f"{pip} show {task_pkg.package}"
485
-
486
- await execute_command(
487
- cwd=venv_path,
488
- command=f"{pip} install --upgrade pip",
489
- logger_name=logger_name,
490
- )
491
- await execute_command(
492
- cwd=venv_path, command=cmd_install, logger_name=logger_name
493
- )
494
- if task_pkg.pinned_package_versions:
495
- for (
496
- pinned_pkg_name,
497
- pinned_pkg_version,
498
- ) in task_pkg.pinned_package_versions.items():
499
-
500
- logger.debug(
501
- "Specific version required: "
502
- f"{pinned_pkg_name}=={pinned_pkg_version}"
503
- )
504
- logger.debug(
505
- "Preliminary check: verify that "
506
- f"{pinned_pkg_version} is already installed"
507
- )
508
- stdout_inspect = await execute_command(
509
- cwd=venv_path,
510
- command=f"{pip} show {pinned_pkg_name}",
511
- logger_name=logger_name,
512
- )
513
- current_version = next(
514
- line.split()[-1]
515
- for line in stdout_inspect.split("\n")
516
- if line.startswith("Version:")
517
- )
518
- if current_version != pinned_pkg_version:
519
- logger.debug(
520
- f"Currently installed version of {pinned_pkg_name} "
521
- f"({current_version}) differs from pinned version "
522
- f"({pinned_pkg_version}); "
523
- f"install version {pinned_pkg_version}."
524
- )
525
- await execute_command(
526
- cwd=venv_path,
527
- command=(
528
- f"{pip} install "
529
- f"{pinned_pkg_name}=={pinned_pkg_version}"
530
- ),
531
- logger_name=logger_name,
532
- )
533
- else:
534
- logger.debug(
535
- f"Currently installed version of {pinned_pkg_name} "
536
- f"({current_version}) already matches the pinned version."
537
- )
538
-
539
- # Extract package installation path from `pip show`
540
- stdout_inspect = await execute_command(
541
- cwd=venv_path, command=cmd_inspect, logger_name=logger_name
542
- )
543
-
544
- location = Path(
545
- next(
546
- line.split()[-1]
547
- for line in stdout_inspect.split("\n")
548
- if line.startswith("Location:")
549
- )
550
- )
551
- package_root = location / task_pkg.package.replace("-", "_")
552
- if not package_root.exists():
553
- raise RuntimeError(
554
- "Could not determine package installation location."
555
- )
556
- return package_root