fractal-server 2.18.0__py3-none-any.whl → 2.18.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +1 -2
- fractal_server/app/models/security.py +5 -7
- fractal_server/app/models/v2/job.py +2 -13
- fractal_server/app/models/v2/resource.py +0 -13
- fractal_server/app/routes/admin/v2/__init__.py +12 -10
- fractal_server/app/routes/admin/v2/job.py +15 -15
- fractal_server/app/routes/admin/v2/task.py +7 -7
- fractal_server/app/routes/admin/v2/task_group.py +12 -14
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +20 -20
- fractal_server/app/routes/api/__init__.py +9 -0
- fractal_server/app/routes/api/v2/__init__.py +49 -47
- fractal_server/app/routes/api/v2/_aux_functions.py +47 -22
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +4 -4
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +2 -2
- fractal_server/app/routes/api/v2/dataset.py +60 -66
- fractal_server/app/routes/api/v2/history.py +5 -7
- fractal_server/app/routes/api/v2/job.py +12 -12
- fractal_server/app/routes/api/v2/project.py +11 -11
- fractal_server/app/routes/api/v2/status_legacy.py +29 -15
- fractal_server/app/routes/api/v2/submit.py +66 -65
- fractal_server/app/routes/api/v2/task.py +17 -15
- fractal_server/app/routes/api/v2/task_collection.py +18 -18
- fractal_server/app/routes/api/v2/task_collection_custom.py +13 -11
- fractal_server/app/routes/api/v2/task_collection_pixi.py +9 -9
- fractal_server/app/routes/api/v2/task_group.py +18 -18
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +26 -26
- fractal_server/app/routes/api/v2/task_version_update.py +5 -5
- fractal_server/app/routes/api/v2/workflow.py +18 -18
- fractal_server/app/routes/api/v2/workflow_import.py +11 -11
- fractal_server/app/routes/api/v2/workflowtask.py +37 -10
- fractal_server/app/routes/auth/_aux_auth.py +0 -100
- fractal_server/app/routes/auth/current_user.py +63 -0
- fractal_server/app/routes/auth/group.py +30 -1
- fractal_server/app/routes/auth/router.py +0 -2
- fractal_server/app/routes/auth/users.py +0 -9
- fractal_server/app/schemas/user.py +12 -29
- fractal_server/app/schemas/user_group.py +15 -0
- fractal_server/app/schemas/v2/__init__.py +48 -48
- fractal_server/app/schemas/v2/dataset.py +13 -35
- fractal_server/app/schemas/v2/dumps.py +9 -9
- fractal_server/app/schemas/v2/job.py +11 -11
- fractal_server/app/schemas/v2/project.py +3 -3
- fractal_server/app/schemas/v2/resource.py +4 -13
- fractal_server/app/schemas/v2/status_legacy.py +3 -3
- fractal_server/app/schemas/v2/task.py +6 -6
- fractal_server/app/schemas/v2/task_collection.py +4 -4
- fractal_server/app/schemas/v2/task_group.py +16 -16
- fractal_server/app/schemas/v2/workflow.py +16 -16
- fractal_server/app/schemas/v2/workflowtask.py +14 -14
- fractal_server/app/security/__init__.py +1 -1
- fractal_server/app/shutdown.py +6 -6
- fractal_server/config/__init__.py +6 -0
- fractal_server/config/_data.py +79 -0
- fractal_server/config/_main.py +1 -6
- fractal_server/images/models.py +2 -1
- fractal_server/main.py +11 -72
- fractal_server/runner/config/_slurm.py +0 -2
- fractal_server/runner/executors/slurm_common/slurm_config.py +0 -1
- fractal_server/runner/v2/_local.py +3 -4
- fractal_server/runner/v2/_slurm_ssh.py +3 -4
- fractal_server/runner/v2/_slurm_sudo.py +3 -4
- fractal_server/runner/v2/runner.py +17 -36
- fractal_server/runner/v2/runner_functions.py +14 -11
- fractal_server/runner/v2/submit_workflow.py +9 -22
- fractal_server/tasks/v2/local/_utils.py +2 -2
- fractal_server/tasks/v2/local/collect.py +6 -5
- fractal_server/tasks/v2/local/collect_pixi.py +6 -5
- fractal_server/tasks/v2/local/deactivate.py +7 -7
- fractal_server/tasks/v2/local/deactivate_pixi.py +3 -3
- fractal_server/tasks/v2/local/delete.py +5 -5
- fractal_server/tasks/v2/local/reactivate.py +5 -5
- fractal_server/tasks/v2/local/reactivate_pixi.py +5 -5
- fractal_server/tasks/v2/ssh/collect.py +5 -5
- fractal_server/tasks/v2/ssh/collect_pixi.py +5 -5
- fractal_server/tasks/v2/ssh/deactivate.py +7 -7
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +2 -2
- fractal_server/tasks/v2/ssh/delete.py +5 -5
- fractal_server/tasks/v2/ssh/reactivate.py +5 -5
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +5 -5
- fractal_server/tasks/v2/utils_background.py +7 -7
- fractal_server/tasks/v2/utils_database.py +5 -5
- fractal_server/types/__init__.py +0 -22
- fractal_server/types/validators/__init__.py +0 -3
- fractal_server/types/validators/_common_validators.py +0 -32
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a1.dist-info}/METADATA +1 -1
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a1.dist-info}/RECORD +90 -95
- fractal_server/app/routes/auth/viewer_paths.py +0 -43
- fractal_server/data_migrations/2_18_0.py +0 -30
- fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +0 -60
- fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +0 -39
- fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +0 -40
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a1.dist-info}/WHEEL +0 -0
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a1.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.18.0.dist-info → fractal_server-2.18.0a1.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from fastapi import APIRouter
|
|
2
2
|
from fastapi import Depends
|
|
3
|
+
from fastapi import HTTPException
|
|
4
|
+
from fastapi import status
|
|
3
5
|
|
|
4
6
|
from fractal_server.app.db import AsyncSession
|
|
5
7
|
from fractal_server.app.db import get_async_db
|
|
@@ -7,8 +9,8 @@ from fractal_server.app.models import UserOAuth
|
|
|
7
9
|
from fractal_server.app.models.v2 import JobV2
|
|
8
10
|
from fractal_server.app.routes.auth import current_user_act_ver_prof
|
|
9
11
|
from fractal_server.app.schemas.v2.sharing import ProjectPermissions
|
|
10
|
-
from fractal_server.app.schemas.v2.status_legacy import
|
|
11
|
-
from fractal_server.app.schemas.v2.status_legacy import
|
|
12
|
+
from fractal_server.app.schemas.v2.status_legacy import LegacyStatusReadV2
|
|
13
|
+
from fractal_server.app.schemas.v2.status_legacy import WorkflowTaskStatusTypeV2
|
|
12
14
|
from fractal_server.logger import set_logger
|
|
13
15
|
|
|
14
16
|
from ._aux_functions import _get_dataset_check_access
|
|
@@ -22,7 +24,7 @@ logger = set_logger(__name__)
|
|
|
22
24
|
|
|
23
25
|
@router.get(
|
|
24
26
|
"/project/{project_id}/status-legacy/",
|
|
25
|
-
response_model=
|
|
27
|
+
response_model=LegacyStatusReadV2,
|
|
26
28
|
)
|
|
27
29
|
async def get_workflowtask_status(
|
|
28
30
|
project_id: int,
|
|
@@ -30,7 +32,7 @@ async def get_workflowtask_status(
|
|
|
30
32
|
workflow_id: int,
|
|
31
33
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
32
34
|
db: AsyncSession = Depends(get_async_db),
|
|
33
|
-
) ->
|
|
35
|
+
) -> LegacyStatusReadV2 | None:
|
|
34
36
|
"""
|
|
35
37
|
Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
|
|
36
38
|
on a given `DatasetV2`.
|
|
@@ -62,12 +64,24 @@ async def get_workflowtask_status(
|
|
|
62
64
|
# Check whether there exists a submitted job associated to this
|
|
63
65
|
# workflow/dataset pair. If it does exist, it will be used later.
|
|
64
66
|
# If there are multiple jobs, raise an error.
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
)
|
|
70
|
-
|
|
67
|
+
stm = _get_submitted_jobs_statement()
|
|
68
|
+
stm = stm.where(JobV2.dataset_id == dataset_id)
|
|
69
|
+
stm = stm.where(JobV2.workflow_id == workflow_id)
|
|
70
|
+
res = await db.execute(stm)
|
|
71
|
+
running_jobs = res.scalars().all()
|
|
72
|
+
if len(running_jobs) == 0:
|
|
73
|
+
running_job = None
|
|
74
|
+
elif len(running_jobs) == 1:
|
|
75
|
+
running_job = running_jobs[0]
|
|
76
|
+
else:
|
|
77
|
+
string_ids = str([job.id for job in running_jobs])[1:-1]
|
|
78
|
+
raise HTTPException(
|
|
79
|
+
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
80
|
+
detail=(
|
|
81
|
+
f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
|
|
82
|
+
f" is linked to multiple active jobs: {string_ids}."
|
|
83
|
+
),
|
|
84
|
+
)
|
|
71
85
|
|
|
72
86
|
# Initialize empty dictionary for WorkflowTaskV2 status
|
|
73
87
|
workflow_tasks_status_dict: dict = {}
|
|
@@ -102,18 +116,18 @@ async def get_workflowtask_status(
|
|
|
102
116
|
]
|
|
103
117
|
try:
|
|
104
118
|
first_submitted_index = running_job_statuses.index(
|
|
105
|
-
|
|
119
|
+
WorkflowTaskStatusTypeV2.SUBMITTED
|
|
106
120
|
)
|
|
107
121
|
except ValueError:
|
|
108
122
|
logger.warning(
|
|
109
123
|
f"Job {running_job.id} is submitted but its task list does not"
|
|
110
|
-
f" contain a {
|
|
124
|
+
f" contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
|
|
111
125
|
)
|
|
112
126
|
first_submitted_index = 0
|
|
113
127
|
|
|
114
128
|
for wftask in running_job_wftasks[first_submitted_index:]:
|
|
115
129
|
workflow_tasks_status_dict[wftask.id] = (
|
|
116
|
-
|
|
130
|
+
WorkflowTaskStatusTypeV2.SUBMITTED
|
|
117
131
|
)
|
|
118
132
|
|
|
119
133
|
# The last workflow task that is included in the submitted job is also
|
|
@@ -143,7 +157,7 @@ async def get_workflowtask_status(
|
|
|
143
157
|
# If a wftask ID was not found, ignore it and continue
|
|
144
158
|
continue
|
|
145
159
|
clean_workflow_tasks_status_dict[str(wf_task.id)] = wf_task_status
|
|
146
|
-
if wf_task_status ==
|
|
160
|
+
if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
|
|
147
161
|
# Starting from the beginning of `workflow.task_list`, stop the
|
|
148
162
|
# first time that you hit a failed job
|
|
149
163
|
break
|
|
@@ -152,5 +166,5 @@ async def get_workflowtask_status(
|
|
|
152
166
|
# first time that you hit `last_valid_wftask_id``
|
|
153
167
|
break
|
|
154
168
|
|
|
155
|
-
response_body =
|
|
169
|
+
response_body = LegacyStatusReadV2(status=clean_workflow_tasks_status_dict)
|
|
156
170
|
return response_body
|
|
@@ -9,7 +9,6 @@ from fastapi import HTTPException
|
|
|
9
9
|
from fastapi import Request
|
|
10
10
|
from fastapi import status
|
|
11
11
|
from sqlmodel import select
|
|
12
|
-
from sqlmodel import update
|
|
13
12
|
|
|
14
13
|
from fractal_server.app.db import AsyncSession
|
|
15
14
|
from fractal_server.app.db import get_async_db
|
|
@@ -24,9 +23,9 @@ from fractal_server.app.routes.auth import current_user_act_ver_prof
|
|
|
24
23
|
from fractal_server.app.routes.aux.validate_user_profile import (
|
|
25
24
|
validate_user_profile,
|
|
26
25
|
)
|
|
27
|
-
from fractal_server.app.schemas.v2 import
|
|
28
|
-
from fractal_server.app.schemas.v2 import
|
|
29
|
-
from fractal_server.app.schemas.v2 import
|
|
26
|
+
from fractal_server.app.schemas.v2 import JobCreateV2
|
|
27
|
+
from fractal_server.app.schemas.v2 import JobReadV2
|
|
28
|
+
from fractal_server.app.schemas.v2 import JobStatusTypeV2
|
|
30
29
|
from fractal_server.app.schemas.v2 import ResourceType
|
|
31
30
|
from fractal_server.app.schemas.v2.sharing import ProjectPermissions
|
|
32
31
|
from fractal_server.config import get_settings
|
|
@@ -39,7 +38,7 @@ from fractal_server.syringe import Inject
|
|
|
39
38
|
|
|
40
39
|
from ._aux_functions import _get_dataset_check_access
|
|
41
40
|
from ._aux_functions import _get_workflow_check_access
|
|
42
|
-
from ._aux_functions import
|
|
41
|
+
from ._aux_functions import clean_app_job_list_v2
|
|
43
42
|
from ._aux_functions_tasks import _check_type_filters_compatibility
|
|
44
43
|
|
|
45
44
|
FRACTAL_CACHE_DIR = ".fractal_cache"
|
|
@@ -50,27 +49,29 @@ logger = set_logger(__name__)
|
|
|
50
49
|
@router.post(
|
|
51
50
|
"/project/{project_id}/job/submit/",
|
|
52
51
|
status_code=status.HTTP_202_ACCEPTED,
|
|
53
|
-
response_model=
|
|
52
|
+
response_model=JobReadV2,
|
|
54
53
|
)
|
|
55
|
-
async def
|
|
54
|
+
async def apply_workflow(
|
|
56
55
|
project_id: int,
|
|
57
56
|
workflow_id: int,
|
|
58
57
|
dataset_id: int,
|
|
59
|
-
job_create:
|
|
58
|
+
job_create: JobCreateV2,
|
|
60
59
|
background_tasks: BackgroundTasks,
|
|
61
60
|
request: Request,
|
|
62
61
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
63
62
|
db: AsyncSession = Depends(get_async_db),
|
|
64
|
-
) ->
|
|
65
|
-
# Remove non-submitted
|
|
63
|
+
) -> JobReadV2 | None:
|
|
64
|
+
# Remove non-submitted V2 jobs from the app state when the list grows
|
|
66
65
|
# beyond a threshold
|
|
67
|
-
# NOTE: this may lead to a race condition on `app.state.
|
|
68
|
-
# requests take place at the same time and `
|
|
66
|
+
# NOTE: this may lead to a race condition on `app.state.jobsV2` if two
|
|
67
|
+
# requests take place at the same time and `clean_app_job_list_v2` is
|
|
69
68
|
# somewhat slow.
|
|
70
69
|
settings = Inject(get_settings)
|
|
71
|
-
if len(request.app.state.
|
|
72
|
-
new_jobs_list = await
|
|
73
|
-
|
|
70
|
+
if len(request.app.state.jobsV2) > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH:
|
|
71
|
+
new_jobs_list = await clean_app_job_list_v2(
|
|
72
|
+
db, request.app.state.jobsV2
|
|
73
|
+
)
|
|
74
|
+
request.app.state.jobsV2 = new_jobs_list
|
|
74
75
|
|
|
75
76
|
output = await _get_dataset_check_access(
|
|
76
77
|
project_id=project_id,
|
|
@@ -146,15 +147,35 @@ async def submit_job(
|
|
|
146
147
|
user=user,
|
|
147
148
|
db=db,
|
|
148
149
|
)
|
|
149
|
-
|
|
150
|
+
|
|
151
|
+
# Check that no other job with the same dataset_id is SUBMITTED
|
|
152
|
+
stm = (
|
|
153
|
+
select(JobV2)
|
|
154
|
+
.where(JobV2.dataset_id == dataset_id)
|
|
155
|
+
.where(JobV2.status == JobStatusTypeV2.SUBMITTED)
|
|
156
|
+
)
|
|
157
|
+
res = await db.execute(stm)
|
|
158
|
+
if res.scalars().all():
|
|
150
159
|
raise HTTPException(
|
|
151
160
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
152
161
|
detail=(
|
|
153
|
-
f"
|
|
154
|
-
"new job submissions."
|
|
162
|
+
f"Dataset {dataset_id} is already in use in submitted job(s)."
|
|
155
163
|
),
|
|
156
164
|
)
|
|
157
165
|
|
|
166
|
+
if job_create.slurm_account is not None:
|
|
167
|
+
if job_create.slurm_account not in user.slurm_accounts:
|
|
168
|
+
raise HTTPException(
|
|
169
|
+
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
170
|
+
detail=(
|
|
171
|
+
f"SLURM account '{job_create.slurm_account}' is not "
|
|
172
|
+
"among those available to the current user"
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
if len(user.slurm_accounts) > 0:
|
|
177
|
+
job_create.slurm_account = user.slurm_accounts[0]
|
|
178
|
+
|
|
158
179
|
# User appropriate FractalSSH object
|
|
159
180
|
if resource.type == ResourceType.SLURM_SSH:
|
|
160
181
|
ssh_config = dict(
|
|
@@ -177,35 +198,6 @@ async def submit_job(
|
|
|
177
198
|
else:
|
|
178
199
|
fractal_ssh = None
|
|
179
200
|
|
|
180
|
-
# Assign `job_create.slurm_account`
|
|
181
|
-
if job_create.slurm_account is not None:
|
|
182
|
-
if job_create.slurm_account not in user.slurm_accounts:
|
|
183
|
-
raise HTTPException(
|
|
184
|
-
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
185
|
-
detail=(
|
|
186
|
-
f"SLURM account '{job_create.slurm_account}' is not "
|
|
187
|
-
"among those available to the current user"
|
|
188
|
-
),
|
|
189
|
-
)
|
|
190
|
-
else:
|
|
191
|
-
if len(user.slurm_accounts) > 0:
|
|
192
|
-
job_create.slurm_account = user.slurm_accounts[0]
|
|
193
|
-
|
|
194
|
-
# Check that no other job with the same dataset_id is SUBMITTED
|
|
195
|
-
stm = (
|
|
196
|
-
select(JobV2)
|
|
197
|
-
.where(JobV2.dataset_id == dataset_id)
|
|
198
|
-
.where(JobV2.status == JobStatusType.SUBMITTED)
|
|
199
|
-
)
|
|
200
|
-
res = await db.execute(stm)
|
|
201
|
-
if res.scalars().all():
|
|
202
|
-
raise HTTPException(
|
|
203
|
-
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
204
|
-
detail=(
|
|
205
|
-
f"Dataset {dataset_id} is already in use in submitted job(s)."
|
|
206
|
-
),
|
|
207
|
-
)
|
|
208
|
-
|
|
209
201
|
# Add new Job object to DB
|
|
210
202
|
job = JobV2(
|
|
211
203
|
project_id=project_id,
|
|
@@ -229,31 +221,38 @@ async def submit_job(
|
|
|
229
221
|
await db.refresh(job)
|
|
230
222
|
|
|
231
223
|
# Update TaskGroupV2.timestamp_last_used
|
|
232
|
-
await db.execute(
|
|
233
|
-
|
|
234
|
-
.where(TaskGroupV2.id.in_(used_task_group_ids))
|
|
235
|
-
.values(timestamp_last_used=job.start_timestamp)
|
|
224
|
+
res = await db.execute(
|
|
225
|
+
select(TaskGroupV2).where(TaskGroupV2.id.in_(used_task_group_ids))
|
|
236
226
|
)
|
|
227
|
+
used_task_groups = res.scalars().all()
|
|
228
|
+
for used_task_group in used_task_groups:
|
|
229
|
+
used_task_group.timestamp_last_used = job.start_timestamp
|
|
230
|
+
db.add(used_task_group)
|
|
237
231
|
await db.commit()
|
|
238
232
|
|
|
239
|
-
# Define
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
# Define server-side and user-side job directories
|
|
243
|
-
timestamp_string = job.start_timestamp.strftime(r"%Y%m%d_%H%M%S")
|
|
244
|
-
working_dir = Path(resource.jobs_local_dir) / (
|
|
233
|
+
# Define server-side job directory
|
|
234
|
+
timestamp_string = job.start_timestamp.strftime("%Y%m%d_%H%M%S")
|
|
235
|
+
WORKFLOW_DIR_LOCAL = Path(resource.jobs_local_dir) / (
|
|
245
236
|
f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
|
|
246
237
|
f"_{timestamp_string}"
|
|
247
238
|
)
|
|
239
|
+
|
|
240
|
+
# Define user-side job directory
|
|
241
|
+
cache_dir = Path(user.project_dir, FRACTAL_CACHE_DIR)
|
|
248
242
|
match resource.type:
|
|
249
243
|
case ResourceType.LOCAL:
|
|
250
|
-
|
|
244
|
+
WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
|
|
251
245
|
case ResourceType.SLURM_SUDO:
|
|
252
|
-
|
|
246
|
+
WORKFLOW_DIR_REMOTE = cache_dir / WORKFLOW_DIR_LOCAL.name
|
|
253
247
|
case ResourceType.SLURM_SSH:
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
248
|
+
WORKFLOW_DIR_REMOTE = Path(
|
|
249
|
+
profile.jobs_remote_dir,
|
|
250
|
+
WORKFLOW_DIR_LOCAL.name,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# Update job folders in the db
|
|
254
|
+
job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
|
|
255
|
+
job.working_dir_user = WORKFLOW_DIR_REMOTE.as_posix()
|
|
257
256
|
await db.merge(job)
|
|
258
257
|
await db.commit()
|
|
259
258
|
|
|
@@ -269,9 +268,11 @@ async def submit_job(
|
|
|
269
268
|
resource=resource,
|
|
270
269
|
profile=profile,
|
|
271
270
|
)
|
|
272
|
-
request.app.state.
|
|
271
|
+
request.app.state.jobsV2.append(job.id)
|
|
273
272
|
logger.info(
|
|
274
|
-
f"
|
|
275
|
-
f"
|
|
273
|
+
f"Current worker's pid is {os.getpid()}. "
|
|
274
|
+
f"Current status of worker job's list "
|
|
275
|
+
f"{request.app.state.jobsV2}"
|
|
276
276
|
)
|
|
277
|
+
await db.close()
|
|
277
278
|
return job
|
|
@@ -25,11 +25,11 @@ from fractal_server.app.models import UserOAuth
|
|
|
25
25
|
from fractal_server.app.models.v2 import TaskGroupV2
|
|
26
26
|
from fractal_server.app.models.v2 import TaskV2
|
|
27
27
|
from fractal_server.app.routes.auth import current_user_act_ver_prof
|
|
28
|
-
from fractal_server.app.schemas.v2 import
|
|
29
|
-
from fractal_server.app.schemas.v2 import
|
|
30
|
-
from fractal_server.app.schemas.v2 import
|
|
28
|
+
from fractal_server.app.schemas.v2 import TaskCreateV2
|
|
29
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
30
|
+
from fractal_server.app.schemas.v2 import TaskReadV2
|
|
31
31
|
from fractal_server.app.schemas.v2 import TaskType
|
|
32
|
-
from fractal_server.app.schemas.v2 import
|
|
32
|
+
from fractal_server.app.schemas.v2 import TaskUpdateV2
|
|
33
33
|
from fractal_server.logger import set_logger
|
|
34
34
|
|
|
35
35
|
router = APIRouter()
|
|
@@ -37,7 +37,7 @@ router = APIRouter()
|
|
|
37
37
|
logger = set_logger(__name__)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
@router.get("/", response_model=list[
|
|
40
|
+
@router.get("/", response_model=list[TaskReadV2])
|
|
41
41
|
async def get_list_task(
|
|
42
42
|
args_schema: bool = True,
|
|
43
43
|
category: str | None = None,
|
|
@@ -45,7 +45,7 @@ async def get_list_task(
|
|
|
45
45
|
author: str | None = None,
|
|
46
46
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
47
47
|
db: AsyncSession = Depends(get_async_db),
|
|
48
|
-
) -> list[
|
|
48
|
+
) -> list[TaskReadV2]:
|
|
49
49
|
"""
|
|
50
50
|
Get list of available tasks
|
|
51
51
|
"""
|
|
@@ -86,12 +86,12 @@ async def get_list_task(
|
|
|
86
86
|
return task_list
|
|
87
87
|
|
|
88
88
|
|
|
89
|
-
@router.get("/{task_id}/", response_model=
|
|
89
|
+
@router.get("/{task_id}/", response_model=TaskReadV2)
|
|
90
90
|
async def get_task(
|
|
91
91
|
task_id: int,
|
|
92
92
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
93
93
|
db: AsyncSession = Depends(get_async_db),
|
|
94
|
-
) ->
|
|
94
|
+
) -> TaskReadV2:
|
|
95
95
|
"""
|
|
96
96
|
Get info on a specific task
|
|
97
97
|
"""
|
|
@@ -99,13 +99,13 @@ async def get_task(
|
|
|
99
99
|
return task
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
@router.patch("/{task_id}/", response_model=
|
|
102
|
+
@router.patch("/{task_id}/", response_model=TaskReadV2)
|
|
103
103
|
async def patch_task(
|
|
104
104
|
task_id: int,
|
|
105
|
-
task_update:
|
|
105
|
+
task_update: TaskUpdateV2,
|
|
106
106
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
107
107
|
db: AsyncSession = Depends(get_async_db),
|
|
108
|
-
) ->
|
|
108
|
+
) -> TaskReadV2 | None:
|
|
109
109
|
"""
|
|
110
110
|
Edit a specific task (restricted to task owner)
|
|
111
111
|
"""
|
|
@@ -137,14 +137,16 @@ async def patch_task(
|
|
|
137
137
|
return db_task
|
|
138
138
|
|
|
139
139
|
|
|
140
|
-
@router.post(
|
|
140
|
+
@router.post(
|
|
141
|
+
"/", response_model=TaskReadV2, status_code=status.HTTP_201_CREATED
|
|
142
|
+
)
|
|
141
143
|
async def create_task(
|
|
142
|
-
task:
|
|
144
|
+
task: TaskCreateV2,
|
|
143
145
|
user_group_id: int | None = None,
|
|
144
146
|
private: bool = False,
|
|
145
147
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
146
148
|
db: AsyncSession = Depends(get_async_db),
|
|
147
|
-
) ->
|
|
149
|
+
) -> TaskReadV2 | None:
|
|
148
150
|
"""
|
|
149
151
|
Create a new task
|
|
150
152
|
"""
|
|
@@ -209,7 +211,7 @@ async def create_task(
|
|
|
209
211
|
resource_id=resource_id,
|
|
210
212
|
active=True,
|
|
211
213
|
task_list=[db_task],
|
|
212
|
-
origin=
|
|
214
|
+
origin=TaskGroupV2OriginEnum.OTHER,
|
|
213
215
|
version=db_task.version,
|
|
214
216
|
pkg_name=pkg_name,
|
|
215
217
|
)
|
|
@@ -25,12 +25,12 @@ from fractal_server.app.routes.aux.validate_user_profile import (
|
|
|
25
25
|
)
|
|
26
26
|
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
|
27
27
|
from fractal_server.app.schemas.v2 import ResourceType
|
|
28
|
-
from fractal_server.app.schemas.v2 import
|
|
29
|
-
from fractal_server.app.schemas.v2 import
|
|
30
|
-
from fractal_server.app.schemas.v2 import
|
|
31
|
-
from fractal_server.app.schemas.v2 import
|
|
32
|
-
from fractal_server.app.schemas.v2 import
|
|
33
|
-
from fractal_server.app.schemas.v2 import
|
|
28
|
+
from fractal_server.app.schemas.v2 import TaskCollectPipV2
|
|
29
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
30
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
31
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
|
32
|
+
from fractal_server.app.schemas.v2 import TaskGroupCreateV2Strict
|
|
33
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
34
34
|
from fractal_server.logger import reset_logger_handlers
|
|
35
35
|
from fractal_server.logger import set_logger
|
|
36
36
|
from fractal_server.tasks.v2.local.collect import collect_local
|
|
@@ -59,9 +59,9 @@ class CollectionRequestData(BaseModel):
|
|
|
59
59
|
Validate form data _and_ wheel file.
|
|
60
60
|
"""
|
|
61
61
|
|
|
62
|
-
task_collect:
|
|
62
|
+
task_collect: TaskCollectPipV2
|
|
63
63
|
file: UploadFile | None = None
|
|
64
|
-
origin:
|
|
64
|
+
origin: TaskGroupV2OriginEnum
|
|
65
65
|
|
|
66
66
|
@model_validator(mode="before")
|
|
67
67
|
@classmethod
|
|
@@ -75,7 +75,7 @@ class CollectionRequestData(BaseModel):
|
|
|
75
75
|
raise ValueError(
|
|
76
76
|
"When no `file` is provided, `package` is required."
|
|
77
77
|
)
|
|
78
|
-
values["origin"] =
|
|
78
|
+
values["origin"] = TaskGroupV2OriginEnum.PYPI
|
|
79
79
|
else:
|
|
80
80
|
if package is not None:
|
|
81
81
|
raise ValueError(
|
|
@@ -87,7 +87,7 @@ class CollectionRequestData(BaseModel):
|
|
|
87
87
|
"Cannot set `package_version` when `file` is "
|
|
88
88
|
f"provided (given package_version='{package_version}')."
|
|
89
89
|
)
|
|
90
|
-
values["origin"] =
|
|
90
|
+
values["origin"] = TaskGroupV2OriginEnum.WHEELFILE
|
|
91
91
|
|
|
92
92
|
for forbidden_char in FORBIDDEN_CHAR_WHEEL:
|
|
93
93
|
if forbidden_char in file.filename:
|
|
@@ -125,7 +125,7 @@ def parse_request_data(
|
|
|
125
125
|
else None
|
|
126
126
|
)
|
|
127
127
|
# Validate and coerce form data
|
|
128
|
-
task_collect_pip =
|
|
128
|
+
task_collect_pip = TaskCollectPipV2(
|
|
129
129
|
package=package,
|
|
130
130
|
package_version=package_version,
|
|
131
131
|
package_extras=package_extras,
|
|
@@ -150,7 +150,7 @@ def parse_request_data(
|
|
|
150
150
|
|
|
151
151
|
@router.post(
|
|
152
152
|
"/collect/pip/",
|
|
153
|
-
response_model=
|
|
153
|
+
response_model=TaskGroupActivityV2Read,
|
|
154
154
|
)
|
|
155
155
|
async def collect_tasks_pip(
|
|
156
156
|
response: Response,
|
|
@@ -160,7 +160,7 @@ async def collect_tasks_pip(
|
|
|
160
160
|
user_group_id: int | None = None,
|
|
161
161
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
162
162
|
db: AsyncSession = Depends(get_async_db),
|
|
163
|
-
) ->
|
|
163
|
+
) -> TaskGroupActivityV2Read:
|
|
164
164
|
"""
|
|
165
165
|
Task-collection endpoint
|
|
166
166
|
"""
|
|
@@ -221,7 +221,7 @@ async def collect_tasks_pip(
|
|
|
221
221
|
wheel_file = None
|
|
222
222
|
|
|
223
223
|
# Set pkg_name, version, origin and archive_path
|
|
224
|
-
if request_data.origin ==
|
|
224
|
+
if request_data.origin == TaskGroupV2OriginEnum.WHEELFILE:
|
|
225
225
|
try:
|
|
226
226
|
wheel_filename = request_data.file.filename
|
|
227
227
|
wheel_info = _parse_wheel_filename(wheel_filename)
|
|
@@ -242,7 +242,7 @@ async def collect_tasks_pip(
|
|
|
242
242
|
wheel_info["distribution"]
|
|
243
243
|
)
|
|
244
244
|
task_group_attrs["version"] = wheel_info["version"]
|
|
245
|
-
elif request_data.origin ==
|
|
245
|
+
elif request_data.origin == TaskGroupV2OriginEnum.PYPI:
|
|
246
246
|
pkg_name = task_collect.package
|
|
247
247
|
task_group_attrs["pkg_name"] = normalize_package_name(pkg_name)
|
|
248
248
|
latest_version = await get_package_version_from_pypi(
|
|
@@ -278,7 +278,7 @@ async def collect_tasks_pip(
|
|
|
278
278
|
|
|
279
279
|
# Validate TaskGroupV2 attributes
|
|
280
280
|
try:
|
|
281
|
-
|
|
281
|
+
TaskGroupCreateV2Strict(**task_group_attrs)
|
|
282
282
|
except ValidationError as e:
|
|
283
283
|
raise HTTPException(
|
|
284
284
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
@@ -328,8 +328,8 @@ async def collect_tasks_pip(
|
|
|
328
328
|
task_group_activity = TaskGroupActivityV2(
|
|
329
329
|
user_id=task_group.user_id,
|
|
330
330
|
taskgroupv2_id=task_group.id,
|
|
331
|
-
status=
|
|
332
|
-
action=
|
|
331
|
+
status=TaskGroupActivityStatusV2.PENDING,
|
|
332
|
+
action=TaskGroupActivityActionV2.COLLECT,
|
|
333
333
|
pkg_name=task_group.pkg_name,
|
|
334
334
|
version=task_group.version,
|
|
335
335
|
)
|
|
@@ -17,11 +17,11 @@ from fractal_server.app.routes.aux.validate_user_profile import (
|
|
|
17
17
|
validate_user_profile,
|
|
18
18
|
)
|
|
19
19
|
from fractal_server.app.schemas.v2 import ResourceType
|
|
20
|
-
from fractal_server.app.schemas.v2 import
|
|
21
|
-
from fractal_server.app.schemas.v2 import
|
|
22
|
-
from fractal_server.app.schemas.v2 import
|
|
23
|
-
from fractal_server.app.schemas.v2 import
|
|
24
|
-
from fractal_server.app.schemas.v2 import
|
|
20
|
+
from fractal_server.app.schemas.v2 import TaskCollectCustomV2
|
|
21
|
+
from fractal_server.app.schemas.v2 import TaskCreateV2
|
|
22
|
+
from fractal_server.app.schemas.v2 import TaskGroupCreateV2
|
|
23
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
24
|
+
from fractal_server.app.schemas.v2 import TaskReadV2
|
|
25
25
|
from fractal_server.logger import set_logger
|
|
26
26
|
from fractal_server.string_tools import validate_cmd
|
|
27
27
|
from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
|
|
@@ -38,14 +38,16 @@ router = APIRouter()
|
|
|
38
38
|
logger = set_logger(__name__)
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
@router.post(
|
|
41
|
+
@router.post(
|
|
42
|
+
"/collect/custom/", status_code=201, response_model=list[TaskReadV2]
|
|
43
|
+
)
|
|
42
44
|
async def collect_task_custom(
|
|
43
|
-
task_collect:
|
|
45
|
+
task_collect: TaskCollectCustomV2,
|
|
44
46
|
private: bool = False,
|
|
45
47
|
user_group_id: int | None = None,
|
|
46
48
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
47
49
|
db: AsyncSession = Depends(get_async_db),
|
|
48
|
-
) -> list[
|
|
50
|
+
) -> list[TaskReadV2]:
|
|
49
51
|
# Get validated resource and profile
|
|
50
52
|
resource, profile = await validate_user_profile(user=user, db=db)
|
|
51
53
|
resource_id = resource.id
|
|
@@ -137,7 +139,7 @@ async def collect_task_custom(
|
|
|
137
139
|
else:
|
|
138
140
|
package_root = Path(task_collect.package_root)
|
|
139
141
|
|
|
140
|
-
task_list: list[
|
|
142
|
+
task_list: list[TaskCreateV2] = prepare_tasks_metadata(
|
|
141
143
|
package_manifest=task_collect.manifest,
|
|
142
144
|
python_bin=Path(task_collect.python_interpreter),
|
|
143
145
|
package_root=package_root,
|
|
@@ -146,14 +148,14 @@ async def collect_task_custom(
|
|
|
146
148
|
|
|
147
149
|
# Prepare task-group attributes
|
|
148
150
|
task_group_attrs = dict(
|
|
149
|
-
origin=
|
|
151
|
+
origin=TaskGroupV2OriginEnum.OTHER,
|
|
150
152
|
pkg_name=task_collect.label,
|
|
151
153
|
user_id=user.id,
|
|
152
154
|
user_group_id=user_group_id,
|
|
153
155
|
version=task_collect.version,
|
|
154
156
|
resource_id=resource_id,
|
|
155
157
|
)
|
|
156
|
-
|
|
158
|
+
TaskGroupCreateV2(**task_group_attrs)
|
|
157
159
|
|
|
158
160
|
# Verify non-duplication constraints
|
|
159
161
|
await _verify_non_duplication_user_constraint(
|
|
@@ -33,10 +33,10 @@ from fractal_server.app.routes.aux.validate_user_profile import (
|
|
|
33
33
|
)
|
|
34
34
|
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
|
35
35
|
from fractal_server.app.schemas.v2 import ResourceType
|
|
36
|
-
from fractal_server.app.schemas.v2 import
|
|
37
|
-
from fractal_server.app.schemas.v2 import
|
|
38
|
-
from fractal_server.app.schemas.v2 import
|
|
39
|
-
from fractal_server.app.schemas.v2.task_group import
|
|
36
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
37
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
38
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
|
39
|
+
from fractal_server.app.schemas.v2.task_group import TaskGroupV2OriginEnum
|
|
40
40
|
from fractal_server.logger import set_logger
|
|
41
41
|
from fractal_server.tasks.v2.local import collect_local_pixi
|
|
42
42
|
from fractal_server.tasks.v2.ssh import collect_ssh_pixi
|
|
@@ -74,7 +74,7 @@ def validate_pkgname_and_version(filename: str) -> tuple[str, str]:
|
|
|
74
74
|
@router.post(
|
|
75
75
|
"/collect/pixi/",
|
|
76
76
|
status_code=202,
|
|
77
|
-
response_model=
|
|
77
|
+
response_model=TaskGroupActivityV2Read,
|
|
78
78
|
)
|
|
79
79
|
async def collect_task_pixi(
|
|
80
80
|
response: Response,
|
|
@@ -85,7 +85,7 @@ async def collect_task_pixi(
|
|
|
85
85
|
user_group_id: int | None = None,
|
|
86
86
|
user: UserOAuth = Depends(current_user_act_ver_prof),
|
|
87
87
|
db: AsyncSession = Depends(get_async_db),
|
|
88
|
-
) ->
|
|
88
|
+
) -> TaskGroupActivityV2Read:
|
|
89
89
|
# Get validated resource and profile
|
|
90
90
|
resource, profile = await validate_user_profile(user=user, db=db)
|
|
91
91
|
resource_id = resource.id
|
|
@@ -136,7 +136,7 @@ async def collect_task_pixi(
|
|
|
136
136
|
user_id=user.id,
|
|
137
137
|
user_group_id=user_group_id,
|
|
138
138
|
resource_id=resource_id,
|
|
139
|
-
origin=
|
|
139
|
+
origin=TaskGroupV2OriginEnum.PIXI,
|
|
140
140
|
pixi_version=pixi_version,
|
|
141
141
|
pkg_name=pkg_name,
|
|
142
142
|
version=version,
|
|
@@ -178,8 +178,8 @@ async def collect_task_pixi(
|
|
|
178
178
|
task_group_activity = TaskGroupActivityV2(
|
|
179
179
|
user_id=task_group.user_id,
|
|
180
180
|
taskgroupv2_id=task_group.id,
|
|
181
|
-
status=
|
|
182
|
-
action=
|
|
181
|
+
status=TaskGroupActivityStatusV2.PENDING,
|
|
182
|
+
action=TaskGroupActivityActionV2.COLLECT,
|
|
183
183
|
pkg_name=task_group.pkg_name,
|
|
184
184
|
version=task_group.version,
|
|
185
185
|
)
|