fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/db/__init__.py +0 -1
  3. fractal_server/app/models/__init__.py +6 -8
  4. fractal_server/app/models/linkuserproject.py +9 -0
  5. fractal_server/app/models/security.py +6 -0
  6. fractal_server/app/models/v1/__init__.py +12 -0
  7. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  8. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  9. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  10. fractal_server/app/models/{state.py → v1/state.py} +2 -2
  11. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  12. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  13. fractal_server/app/models/v2/__init__.py +22 -0
  14. fractal_server/app/models/v2/collection_state.py +21 -0
  15. fractal_server/app/models/v2/dataset.py +54 -0
  16. fractal_server/app/models/v2/job.py +51 -0
  17. fractal_server/app/models/v2/project.py +30 -0
  18. fractal_server/app/models/v2/task.py +93 -0
  19. fractal_server/app/models/v2/workflow.py +35 -0
  20. fractal_server/app/models/v2/workflowtask.py +49 -0
  21. fractal_server/app/routes/admin/__init__.py +0 -0
  22. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  23. fractal_server/app/routes/admin/v2.py +309 -0
  24. fractal_server/app/routes/api/v1/__init__.py +7 -7
  25. fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
  26. fractal_server/app/routes/api/v1/dataset.py +48 -41
  27. fractal_server/app/routes/api/v1/job.py +14 -14
  28. fractal_server/app/routes/api/v1/project.py +30 -27
  29. fractal_server/app/routes/api/v1/task.py +26 -16
  30. fractal_server/app/routes/api/v1/task_collection.py +28 -16
  31. fractal_server/app/routes/api/v1/workflow.py +28 -28
  32. fractal_server/app/routes/api/v1/workflowtask.py +11 -11
  33. fractal_server/app/routes/api/v2/__init__.py +34 -0
  34. fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
  35. fractal_server/app/routes/api/v2/dataset.py +293 -0
  36. fractal_server/app/routes/api/v2/images.py +279 -0
  37. fractal_server/app/routes/api/v2/job.py +200 -0
  38. fractal_server/app/routes/api/v2/project.py +186 -0
  39. fractal_server/app/routes/api/v2/status.py +150 -0
  40. fractal_server/app/routes/api/v2/submit.py +210 -0
  41. fractal_server/app/routes/api/v2/task.py +222 -0
  42. fractal_server/app/routes/api/v2/task_collection.py +239 -0
  43. fractal_server/app/routes/api/v2/task_legacy.py +59 -0
  44. fractal_server/app/routes/api/v2/workflow.py +380 -0
  45. fractal_server/app/routes/api/v2/workflowtask.py +265 -0
  46. fractal_server/app/routes/aux/_job.py +2 -2
  47. fractal_server/app/runner/__init__.py +0 -379
  48. fractal_server/app/runner/async_wrap.py +27 -0
  49. fractal_server/app/runner/components.py +5 -0
  50. fractal_server/app/runner/exceptions.py +129 -0
  51. fractal_server/app/runner/executors/__init__.py +0 -0
  52. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  53. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  54. fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
  55. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
  56. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  57. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +42 -1
  58. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +46 -27
  59. fractal_server/app/runner/filenames.py +6 -0
  60. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  61. fractal_server/app/runner/task_files.py +103 -0
  62. fractal_server/app/runner/v1/__init__.py +366 -0
  63. fractal_server/app/runner/{_common.py → v1/_common.py} +56 -111
  64. fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
  65. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  66. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  67. fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
  68. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
  69. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  70. fractal_server/app/runner/v1/common.py +117 -0
  71. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  72. fractal_server/app/runner/v2/__init__.py +336 -0
  73. fractal_server/app/runner/v2/_local/__init__.py +162 -0
  74. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  75. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  76. fractal_server/app/runner/v2/_local/executor.py +100 -0
  77. fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
  78. fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
  79. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
  80. fractal_server/app/runner/v2/deduplicate_list.py +23 -0
  81. fractal_server/app/runner/v2/handle_failed_job.py +165 -0
  82. fractal_server/app/runner/v2/merge_outputs.py +38 -0
  83. fractal_server/app/runner/v2/runner.py +343 -0
  84. fractal_server/app/runner/v2/runner_functions.py +374 -0
  85. fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
  86. fractal_server/app/runner/v2/task_interface.py +62 -0
  87. fractal_server/app/runner/v2/v1_compat.py +31 -0
  88. fractal_server/app/schemas/__init__.py +1 -42
  89. fractal_server/app/schemas/_validators.py +28 -5
  90. fractal_server/app/schemas/v1/__init__.py +36 -0
  91. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  92. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  93. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  94. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  95. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  96. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  97. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  98. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  99. fractal_server/app/schemas/v2/__init__.py +37 -0
  100. fractal_server/app/schemas/v2/dataset.py +126 -0
  101. fractal_server/app/schemas/v2/dumps.py +87 -0
  102. fractal_server/app/schemas/v2/job.py +114 -0
  103. fractal_server/app/schemas/v2/manifest.py +159 -0
  104. fractal_server/app/schemas/v2/project.py +34 -0
  105. fractal_server/app/schemas/v2/status.py +16 -0
  106. fractal_server/app/schemas/v2/task.py +151 -0
  107. fractal_server/app/schemas/v2/task_collection.py +109 -0
  108. fractal_server/app/schemas/v2/workflow.py +79 -0
  109. fractal_server/app/schemas/v2/workflowtask.py +208 -0
  110. fractal_server/config.py +13 -10
  111. fractal_server/images/__init__.py +4 -0
  112. fractal_server/images/models.py +136 -0
  113. fractal_server/images/tools.py +84 -0
  114. fractal_server/main.py +11 -3
  115. fractal_server/migrations/env.py +0 -2
  116. fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
  117. fractal_server/tasks/__init__.py +0 -5
  118. fractal_server/tasks/endpoint_operations.py +13 -19
  119. fractal_server/tasks/utils.py +35 -0
  120. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  121. fractal_server/tasks/v1/__init__.py +0 -0
  122. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
  123. fractal_server/tasks/v1/get_collection_data.py +14 -0
  124. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  125. fractal_server/tasks/v2/__init__.py +0 -0
  126. fractal_server/tasks/v2/background_operations.py +381 -0
  127. fractal_server/tasks/v2/get_collection_data.py +14 -0
  128. fractal_server/urls.py +13 -0
  129. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/METADATA +11 -12
  130. fractal_server-2.0.0.dist-info/RECORD +169 -0
  131. fractal_server/app/runner/_slurm/.gitignore +0 -2
  132. fractal_server/app/runner/common.py +0 -307
  133. fractal_server/app/schemas/json_schemas/manifest.json +0 -81
  134. fractal_server-1.4.6.dist-info/RECORD +0 -97
  135. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  136. /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
  137. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
  138. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
  139. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,245 @@
1
+ """v2
2
+
3
+ Revision ID: 5bf02391cfef
4
+ Revises: 9fd26a2b0de4
5
+ Create Date: 2024-04-18 10:35:19.067833
6
+
7
+ """
8
+ import sqlalchemy as sa
9
+ import sqlmodel
10
+ from alembic import op
11
+
12
+
13
+ # revision identifiers, used by Alembic.
14
+ revision = "5bf02391cfef"
15
+ down_revision = "9fd26a2b0de4"
16
+ branch_labels = None
17
+ depends_on = None
18
+
19
+
20
+ def upgrade() -> None:
21
+ # ### commands auto generated by Alembic - please adjust! ###
22
+ op.create_table(
23
+ "collectionstatev2",
24
+ sa.Column("id", sa.Integer(), nullable=False),
25
+ sa.Column("data", sa.JSON(), nullable=True),
26
+ sa.Column("timestamp", sa.DateTime(timezone=True), nullable=True),
27
+ sa.PrimaryKeyConstraint("id"),
28
+ )
29
+ op.create_table(
30
+ "projectv2",
31
+ sa.Column("id", sa.Integer(), nullable=False),
32
+ sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
33
+ sa.Column(
34
+ "timestamp_created", sa.DateTime(timezone=True), nullable=False
35
+ ),
36
+ sa.PrimaryKeyConstraint("id"),
37
+ )
38
+ op.create_table(
39
+ "taskv2",
40
+ sa.Column("id", sa.Integer(), nullable=False),
41
+ sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
42
+ sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
43
+ sa.Column(
44
+ "command_non_parallel",
45
+ sqlmodel.sql.sqltypes.AutoString(),
46
+ nullable=True,
47
+ ),
48
+ sa.Column(
49
+ "command_parallel",
50
+ sqlmodel.sql.sqltypes.AutoString(),
51
+ nullable=True,
52
+ ),
53
+ sa.Column(
54
+ "source", sqlmodel.sql.sqltypes.AutoString(), nullable=False
55
+ ),
56
+ sa.Column(
57
+ "meta_non_parallel", sa.JSON(), server_default="{}", nullable=False
58
+ ),
59
+ sa.Column(
60
+ "meta_parallel", sa.JSON(), server_default="{}", nullable=False
61
+ ),
62
+ sa.Column("owner", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
63
+ sa.Column(
64
+ "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
65
+ ),
66
+ sa.Column("args_schema_non_parallel", sa.JSON(), nullable=True),
67
+ sa.Column("args_schema_parallel", sa.JSON(), nullable=True),
68
+ sa.Column(
69
+ "args_schema_version",
70
+ sqlmodel.sql.sqltypes.AutoString(),
71
+ nullable=True,
72
+ ),
73
+ sa.Column(
74
+ "docs_info", sqlmodel.sql.sqltypes.AutoString(), nullable=True
75
+ ),
76
+ sa.Column(
77
+ "docs_link", sqlmodel.sql.sqltypes.AutoString(), nullable=True
78
+ ),
79
+ sa.Column("input_types", sa.JSON(), nullable=True),
80
+ sa.Column("output_types", sa.JSON(), nullable=True),
81
+ sa.PrimaryKeyConstraint("id"),
82
+ sa.UniqueConstraint("source"),
83
+ )
84
+ op.create_table(
85
+ "datasetv2",
86
+ sa.Column("id", sa.Integer(), nullable=False),
87
+ sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
88
+ sa.Column("project_id", sa.Integer(), nullable=False),
89
+ sa.Column("history", sa.JSON(), server_default="[]", nullable=False),
90
+ sa.Column(
91
+ "timestamp_created", sa.DateTime(timezone=True), nullable=False
92
+ ),
93
+ sa.Column(
94
+ "zarr_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=False
95
+ ),
96
+ sa.Column("images", sa.JSON(), server_default="[]", nullable=False),
97
+ sa.Column(
98
+ "filters",
99
+ sa.JSON(),
100
+ server_default='{"attributes": {}, "types": {}}',
101
+ nullable=False,
102
+ ),
103
+ sa.ForeignKeyConstraint(
104
+ ["project_id"],
105
+ ["projectv2.id"],
106
+ ),
107
+ sa.PrimaryKeyConstraint("id"),
108
+ )
109
+ op.create_table(
110
+ "linkuserprojectv2",
111
+ sa.Column("project_id", sa.Integer(), nullable=False),
112
+ sa.Column("user_id", sa.Integer(), nullable=False),
113
+ sa.ForeignKeyConstraint(
114
+ ["project_id"],
115
+ ["projectv2.id"],
116
+ ),
117
+ sa.ForeignKeyConstraint(
118
+ ["user_id"],
119
+ ["user_oauth.id"],
120
+ ),
121
+ sa.PrimaryKeyConstraint("project_id", "user_id"),
122
+ )
123
+ op.create_table(
124
+ "workflowv2",
125
+ sa.Column("id", sa.Integer(), nullable=False),
126
+ sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
127
+ sa.Column("project_id", sa.Integer(), nullable=False),
128
+ sa.Column(
129
+ "timestamp_created", sa.DateTime(timezone=True), nullable=False
130
+ ),
131
+ sa.ForeignKeyConstraint(
132
+ ["project_id"],
133
+ ["projectv2.id"],
134
+ ),
135
+ sa.PrimaryKeyConstraint("id"),
136
+ )
137
+ op.create_table(
138
+ "jobv2",
139
+ sa.Column("id", sa.Integer(), nullable=False),
140
+ sa.Column("project_id", sa.Integer(), nullable=True),
141
+ sa.Column("workflow_id", sa.Integer(), nullable=True),
142
+ sa.Column("dataset_id", sa.Integer(), nullable=True),
143
+ sa.Column(
144
+ "user_email", sqlmodel.sql.sqltypes.AutoString(), nullable=False
145
+ ),
146
+ sa.Column(
147
+ "slurm_account", sqlmodel.sql.sqltypes.AutoString(), nullable=True
148
+ ),
149
+ sa.Column("dataset_dump", sa.JSON(), nullable=False),
150
+ sa.Column("workflow_dump", sa.JSON(), nullable=False),
151
+ sa.Column("project_dump", sa.JSON(), nullable=False),
152
+ sa.Column(
153
+ "worker_init", sqlmodel.sql.sqltypes.AutoString(), nullable=True
154
+ ),
155
+ sa.Column(
156
+ "working_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
157
+ ),
158
+ sa.Column(
159
+ "working_dir_user",
160
+ sqlmodel.sql.sqltypes.AutoString(),
161
+ nullable=True,
162
+ ),
163
+ sa.Column("first_task_index", sa.Integer(), nullable=False),
164
+ sa.Column("last_task_index", sa.Integer(), nullable=False),
165
+ sa.Column(
166
+ "start_timestamp", sa.DateTime(timezone=True), nullable=False
167
+ ),
168
+ sa.Column("end_timestamp", sa.DateTime(timezone=True), nullable=True),
169
+ sa.Column(
170
+ "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
171
+ ),
172
+ sa.Column("log", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
173
+ sa.ForeignKeyConstraint(
174
+ ["dataset_id"],
175
+ ["datasetv2.id"],
176
+ ),
177
+ sa.ForeignKeyConstraint(
178
+ ["project_id"],
179
+ ["projectv2.id"],
180
+ ),
181
+ sa.ForeignKeyConstraint(
182
+ ["workflow_id"],
183
+ ["workflowv2.id"],
184
+ ),
185
+ sa.PrimaryKeyConstraint("id"),
186
+ )
187
+ op.create_table(
188
+ "workflowtaskv2",
189
+ sa.Column("id", sa.Integer(), nullable=False),
190
+ sa.Column("workflow_id", sa.Integer(), nullable=False),
191
+ sa.Column("order", sa.Integer(), nullable=True),
192
+ sa.Column("meta_parallel", sa.JSON(), nullable=True),
193
+ sa.Column("meta_non_parallel", sa.JSON(), nullable=True),
194
+ sa.Column("args_parallel", sa.JSON(), nullable=True),
195
+ sa.Column("args_non_parallel", sa.JSON(), nullable=True),
196
+ sa.Column(
197
+ "input_filters",
198
+ sa.JSON(),
199
+ server_default='{"attributes": {}, "types": {}}',
200
+ nullable=False,
201
+ ),
202
+ sa.Column("is_legacy_task", sa.Boolean(), nullable=False),
203
+ sa.Column(
204
+ "task_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
205
+ ),
206
+ sa.Column("task_id", sa.Integer(), nullable=True),
207
+ sa.Column("task_legacy_id", sa.Integer(), nullable=True),
208
+ sa.ForeignKeyConstraint(
209
+ ["task_id"],
210
+ ["taskv2.id"],
211
+ ),
212
+ sa.ForeignKeyConstraint(
213
+ ["task_legacy_id"],
214
+ ["task.id"],
215
+ ),
216
+ sa.ForeignKeyConstraint(
217
+ ["workflow_id"],
218
+ ["workflowv2.id"],
219
+ ),
220
+ sa.PrimaryKeyConstraint("id"),
221
+ )
222
+ op.add_column(
223
+ "task",
224
+ sa.Column(
225
+ "is_v2_compatible",
226
+ sa.Boolean(),
227
+ server_default=sa.text("false"),
228
+ nullable=False,
229
+ ),
230
+ )
231
+ # ### end Alembic commands ###
232
+
233
+
234
+ def downgrade() -> None:
235
+ # ### commands auto generated by Alembic - please adjust! ###
236
+ op.drop_column("task", "is_v2_compatible")
237
+ op.drop_table("workflowtaskv2")
238
+ op.drop_table("jobv2")
239
+ op.drop_table("workflowv2")
240
+ op.drop_table("linkuserprojectv2")
241
+ op.drop_table("datasetv2")
242
+ op.drop_table("taskv2")
243
+ op.drop_table("projectv2")
244
+ op.drop_table("collectionstatev2")
245
+ # ### end Alembic commands ###
@@ -1,8 +1,3 @@
1
1
  """
2
2
  `tasks` module
3
-
4
- Submodules:
5
- * [Endpoint functions](background_operations)
6
- * [Background-task functions](endpoint_operations)
7
- * [Auxiliary functions](utils)
8
3
  """
@@ -4,35 +4,26 @@ from typing import Optional
4
4
  from typing import Union
5
5
  from zipfile import ZipFile
6
6
 
7
- from fractal_server.app.schemas import ManifestV1
8
- from fractal_server.app.schemas import TaskCollectStatus
7
+ from .utils import _normalize_package_name
8
+ from .utils import get_python_interpreter
9
+ from .v1._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV1
10
+ from .v2._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV2
11
+ from fractal_server.app.schemas.v1 import ManifestV1
12
+ from fractal_server.app.schemas.v2 import ManifestV2
9
13
  from fractal_server.config import get_settings
10
14
  from fractal_server.logger import get_logger
11
15
  from fractal_server.syringe import Inject
12
- from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
13
- from fractal_server.tasks.utils import _normalize_package_name
14
- from fractal_server.tasks.utils import get_absolute_venv_path
15
- from fractal_server.tasks.utils import get_collection_path
16
- from fractal_server.tasks.utils import get_python_interpreter
17
16
  from fractal_server.utils import execute_command
18
17
 
19
18
 
20
19
  FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
21
20
 
22
21
 
23
- def get_collection_data(venv_path: Path) -> TaskCollectStatus:
24
- package_path = get_absolute_venv_path(venv_path)
25
- collection_path = get_collection_path(package_path)
26
- with collection_path.open() as f:
27
- data = json.load(f)
28
- return TaskCollectStatus(**data)
29
-
30
-
31
22
  async def download_package(
32
23
  *,
33
- task_pkg: _TaskCollectPip,
24
+ task_pkg: Union[_TaskCollectPipV1, _TaskCollectPipV2],
34
25
  dest: Union[str, Path],
35
- ):
26
+ ) -> Path:
36
27
  """
37
28
  Download package to destination
38
29
  """
@@ -52,7 +43,7 @@ async def download_package(
52
43
 
53
44
  def _load_manifest_from_wheel(
54
45
  path: Path, wheel: ZipFile, logger_name: Optional[str] = None
55
- ) -> ManifestV1:
46
+ ) -> Union[ManifestV1, ManifestV2]:
56
47
  logger = get_logger(logger_name)
57
48
  namelist = wheel.namelist()
58
49
  try:
@@ -69,6 +60,9 @@ def _load_manifest_from_wheel(
69
60
  if manifest_version == "1":
70
61
  pkg_manifest = ManifestV1(**manifest_dict)
71
62
  return pkg_manifest
63
+ elif manifest_version == "2":
64
+ pkg_manifest = ManifestV2(**manifest_dict)
65
+ return pkg_manifest
72
66
  else:
73
67
  msg = f"Manifest version {manifest_version=} not supported"
74
68
  logger.error(msg)
@@ -146,7 +140,7 @@ def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
146
140
 
147
141
  def create_package_dir_pip(
148
142
  *,
149
- task_pkg: _TaskCollectPip,
143
+ task_pkg: Union[_TaskCollectPipV1, _TaskCollectPipV2],
150
144
  create: bool = True,
151
145
  ) -> Path:
152
146
  """
@@ -5,7 +5,9 @@ from pathlib import Path
5
5
  from typing import Optional
6
6
 
7
7
  from fractal_server.config import get_settings
8
+ from fractal_server.logger import get_logger
8
9
  from fractal_server.syringe import Inject
10
+ from fractal_server.utils import execute_command
9
11
 
10
12
  COLLECTION_FILENAME = "collection.json"
11
13
  COLLECTION_LOG_FILENAME = "collection.log"
@@ -84,3 +86,36 @@ def _normalize_package_name(name: str) -> str:
84
86
  The normalized package name.
85
87
  """
86
88
  return re.sub(r"[-_.]+", "-", name).lower()
89
+
90
+
91
+ async def _init_venv(
92
+ *,
93
+ path: Path,
94
+ python_version: Optional[str] = None,
95
+ logger_name: str,
96
+ ) -> Path:
97
+ """
98
+ Set a virtual environment at `path/venv`
99
+
100
+ Args:
101
+ path : Path
102
+ path to directory in which to set up the virtual environment
103
+ python_version : default=None
104
+ Python version the virtual environment will be based upon
105
+
106
+ Returns:
107
+ python_bin : Path
108
+ path to python interpreter
109
+ """
110
+ logger = get_logger(logger_name)
111
+ logger.debug(f"[_init_venv] {path=}")
112
+ interpreter = get_python_interpreter(version=python_version)
113
+ logger.debug(f"[_init_venv] {interpreter=}")
114
+ await execute_command(
115
+ cwd=path,
116
+ command=f"{interpreter} -m venv venv",
117
+ logger_name=logger_name,
118
+ )
119
+ python_bin = path / "venv/bin/python"
120
+ logger.debug(f"[_init_venv] {python_bin=}")
121
+ return python_bin
@@ -3,11 +3,11 @@ from typing import Optional
3
3
 
4
4
  from pydantic import root_validator
5
5
 
6
- from fractal_server.app.schemas import ManifestV1
7
- from fractal_server.app.schemas import TaskCollectPip
6
+ from fractal_server.app.schemas.v1 import ManifestV1
7
+ from fractal_server.app.schemas.v1 import TaskCollectPipV1
8
8
 
9
9
 
10
- class _TaskCollectPip(TaskCollectPip):
10
+ class _TaskCollectPip(TaskCollectPipV1):
11
11
  """
12
12
  Internal TaskCollectPip schema
13
13
 
File without changes
@@ -5,61 +5,27 @@ is used as a background task for the task-collection endpoint.
5
5
  import json
6
6
  from pathlib import Path
7
7
  from shutil import rmtree as shell_rmtree
8
- from typing import Optional
9
8
 
9
+ from ..utils import _init_venv
10
+ from ..utils import _normalize_package_name
11
+ from ..utils import get_collection_log
12
+ from ..utils import get_collection_path
13
+ from ..utils import get_log_path
14
+ from ..utils import slugify_task_name
15
+ from ._TaskCollectPip import _TaskCollectPip
10
16
  from fractal_server.app.db import DBSyncSession
11
17
  from fractal_server.app.db import get_sync_db
12
- from fractal_server.app.models import State
13
- from fractal_server.app.models import Task
14
- from fractal_server.app.schemas import TaskCollectStatus
15
- from fractal_server.app.schemas import TaskCreate
16
- from fractal_server.app.schemas import TaskRead
18
+ from fractal_server.app.models.v1 import State
19
+ from fractal_server.app.models.v1 import Task
20
+ from fractal_server.app.schemas.v1 import TaskCollectStatusV1
21
+ from fractal_server.app.schemas.v1 import TaskCreateV1
22
+ from fractal_server.app.schemas.v1 import TaskReadV1
17
23
  from fractal_server.logger import close_logger
18
24
  from fractal_server.logger import get_logger
19
25
  from fractal_server.logger import set_logger
20
- from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
21
- from fractal_server.tasks.utils import _normalize_package_name
22
- from fractal_server.tasks.utils import get_collection_log
23
- from fractal_server.tasks.utils import get_collection_path
24
- from fractal_server.tasks.utils import get_log_path
25
- from fractal_server.tasks.utils import get_python_interpreter
26
- from fractal_server.tasks.utils import slugify_task_name
27
26
  from fractal_server.utils import execute_command
28
27
 
29
28
 
30
- async def _init_venv(
31
- *,
32
- path: Path,
33
- python_version: Optional[str] = None,
34
- logger_name: str,
35
- ) -> Path:
36
- """
37
- Set a virtual environment at `path/venv`
38
-
39
- Args:
40
- path : Path
41
- path to directory in which to set up the virtual environment
42
- python_version : default=None
43
- Python version the virtual environment will be based upon
44
-
45
- Returns:
46
- python_bin : Path
47
- path to python interpreter
48
- """
49
- logger = get_logger(logger_name)
50
- logger.debug(f"[_init_venv] {path=}")
51
- interpreter = get_python_interpreter(version=python_version)
52
- logger.debug(f"[_init_venv] {interpreter=}")
53
- await execute_command(
54
- cwd=path,
55
- command=f"{interpreter} -m venv venv",
56
- logger_name=logger_name,
57
- )
58
- python_bin = path / "venv/bin/python"
59
- logger.debug(f"[_init_venv] {python_bin=}")
60
- return python_bin
61
-
62
-
63
29
  async def _pip_install(
64
30
  venv_path: Path,
65
31
  task_pkg: _TaskCollectPip,
@@ -218,7 +184,7 @@ async def create_package_environment_pip(
218
184
  task_pkg: _TaskCollectPip,
219
185
  venv_path: Path,
220
186
  logger_name: str,
221
- ) -> list[TaskCreate]:
187
+ ) -> list[TaskCreateV1]:
222
188
  """
223
189
  Create environment, install package, and prepare task list
224
190
  """
@@ -263,7 +229,7 @@ async def create_package_environment_pip(
263
229
  )
264
230
  else:
265
231
  additional_attrs = {}
266
- this_task = TaskCreate(
232
+ this_task = TaskCreateV1(
267
233
  **t.dict(),
268
234
  command=cmd,
269
235
  version=task_pkg.package_version,
@@ -279,7 +245,7 @@ async def create_package_environment_pip(
279
245
 
280
246
 
281
247
  async def _insert_tasks(
282
- task_list: list[TaskCreate],
248
+ task_list: list[TaskCreateV1],
283
249
  db: DBSyncSession,
284
250
  ) -> list[Task]:
285
251
  """
@@ -319,7 +285,7 @@ async def background_collect_pip(
319
285
 
320
286
  with next(get_sync_db()) as db:
321
287
  state: State = db.get(State, state_id)
322
- data = TaskCollectStatus(**state.data)
288
+ data = TaskCollectStatusV1(**state.data)
323
289
  data.info = None
324
290
 
325
291
  try:
@@ -347,9 +313,11 @@ async def background_collect_pip(
347
313
  # finalise
348
314
  logger.debug("Task-collection status: finalising")
349
315
  collection_path = get_collection_path(venv_path)
350
- data.task_list = [TaskRead(**task.model_dump()) for task in tasks]
316
+ data.task_list = [
317
+ TaskReadV1(**task.model_dump()) for task in tasks
318
+ ]
351
319
  with collection_path.open("w") as f:
352
- json.dump(data.sanitised_dict(), f)
320
+ json.dump(data.sanitised_dict(), f, indent=2)
353
321
 
354
322
  # Update DB
355
323
  data.status = "OK"
@@ -0,0 +1,14 @@
1
+ import json
2
+ from pathlib import Path
3
+
4
+ from fractal_server.app.schemas.v1 import TaskCollectStatusV1
5
+ from fractal_server.tasks.utils import get_absolute_venv_path
6
+ from fractal_server.tasks.utils import get_collection_path
7
+
8
+
9
+ def get_collection_data(venv_path: Path) -> TaskCollectStatusV1:
10
+ package_path = get_absolute_venv_path(venv_path)
11
+ collection_path = get_collection_path(package_path)
12
+ with collection_path.open() as f:
13
+ data = json.load(f)
14
+ return TaskCollectStatusV1(**data)
@@ -0,0 +1,103 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+
4
+ from pydantic import root_validator
5
+
6
+ from fractal_server.app.schemas.v2 import ManifestV2
7
+ from fractal_server.app.schemas.v2 import TaskCollectPipV2
8
+
9
+
10
+ class _TaskCollectPip(TaskCollectPipV2):
11
+ """
12
+ Internal TaskCollectPip schema
13
+
14
+ Differences with its parent class (`TaskCollectPip`):
15
+
16
+ 1. We check if the package corresponds to a path in the filesystem, and
17
+ whether it exists (via new validator `check_local_package`, new
18
+ method `is_local_package` and new attribute `package_path`).
19
+ 2. We include an additional `package_manifest` attribute.
20
+ 3. We expose an additional attribute `package_name`, which is filled
21
+ during task collection.
22
+ """
23
+
24
+ package_name: Optional[str] = None
25
+ package_path: Optional[Path] = None
26
+ package_manifest: Optional[ManifestV2] = None
27
+
28
+ @property
29
+ def is_local_package(self) -> bool:
30
+ return bool(self.package_path)
31
+
32
+ @root_validator(pre=True)
33
+ def check_local_package(cls, values):
34
+ """
35
+ Checks if package corresponds to an existing path on the filesystem
36
+
37
+ In this case, the user is providing directly a package file, rather
38
+ than a remote one from PyPI. We set the `package_path` attribute and
39
+ get the actual package name and version from the package file name.
40
+ """
41
+ if "/" in values["package"]:
42
+ package_path = Path(values["package"])
43
+ if not package_path.is_absolute():
44
+ raise ValueError("Package path must be absolute")
45
+ if package_path.exists():
46
+ values["package_path"] = package_path
47
+ (
48
+ values["package"],
49
+ values["version"],
50
+ *_,
51
+ ) = package_path.name.split("-")
52
+ else:
53
+ raise ValueError(f"Package {package_path} does not exist.")
54
+ return values
55
+
56
+ @property
57
+ def package_source(self) -> str:
58
+ """
59
+ NOTE: As of PR #1188 in `fractal-server`, the attribute
60
+ `self.package_name` is normalized; this means e.g. that `_` is
61
+ replaced by `-`. To guarantee backwards compatibility with
62
+ `Task.source` attributes created before this change, we still replace
63
+ `-` with `_` upon generation of the `source` attribute, in this
64
+ method.
65
+ """
66
+ if not self.package_name or not self.package_version:
67
+ raise ValueError(
68
+ "Cannot construct `package_source` property with "
69
+ f"{self.package_name=} and {self.package_version=}."
70
+ )
71
+ if self.is_local_package:
72
+ collection_type = "pip_local"
73
+ else:
74
+ collection_type = "pip_remote"
75
+
76
+ package_extras = self.package_extras or ""
77
+ if self.python_version:
78
+ python_version = f"py{self.python_version}"
79
+ else:
80
+ python_version = "" # FIXME: can we allow this?
81
+
82
+ source = ":".join(
83
+ (
84
+ collection_type,
85
+ self.package_name.replace("-", "_"), # see method docstring
86
+ self.package_version,
87
+ package_extras,
88
+ python_version,
89
+ )
90
+ )
91
+ return source
92
+
93
+ def check(self):
94
+ """
95
+ Verify that the package has all attributes that are needed to continue
96
+ with task collection
97
+ """
98
+ if not self.package_name:
99
+ raise ValueError("`package_name` attribute is not set")
100
+ if not self.package_version:
101
+ raise ValueError("`package_version` attribute is not set")
102
+ if not self.package_manifest:
103
+ raise ValueError("`package_manifest` attribute is not set")
File without changes