fractal-server 2.8.1__py3-none-any.whl → 2.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/db/__init__.py +2 -35
- fractal_server/app/models/v2/__init__.py +3 -3
- fractal_server/app/models/v2/task.py +0 -72
- fractal_server/app/models/v2/task_group.py +113 -0
- fractal_server/app/routes/admin/v1.py +13 -30
- fractal_server/app/routes/admin/v2/__init__.py +4 -0
- fractal_server/app/routes/admin/v2/job.py +13 -24
- fractal_server/app/routes/admin/v2/task.py +13 -0
- fractal_server/app/routes/admin/v2/task_group.py +75 -14
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +267 -0
- fractal_server/app/routes/api/v1/project.py +7 -19
- fractal_server/app/routes/api/v2/__init__.py +11 -2
- fractal_server/app/routes/api/v2/{_aux_functions_task_collection.py → _aux_functions_task_lifecycle.py} +83 -0
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +27 -17
- fractal_server/app/routes/api/v2/submit.py +19 -24
- fractal_server/app/routes/api/v2/task_collection.py +33 -65
- fractal_server/app/routes/api/v2/task_collection_custom.py +3 -3
- fractal_server/app/routes/api/v2/task_group.py +86 -14
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +272 -0
- fractal_server/app/routes/api/v2/workflow.py +1 -1
- fractal_server/app/routes/api/v2/workflow_import.py +2 -2
- fractal_server/app/routes/auth/current_user.py +60 -17
- fractal_server/app/routes/auth/group.py +67 -39
- fractal_server/app/routes/auth/users.py +97 -99
- fractal_server/app/routes/aux/__init__.py +20 -0
- fractal_server/app/runner/executors/slurm/_slurm_config.py +0 -17
- fractal_server/app/runner/executors/slurm/ssh/executor.py +49 -204
- fractal_server/app/runner/executors/slurm/sudo/executor.py +26 -109
- fractal_server/app/runner/executors/slurm/utils_executors.py +58 -0
- fractal_server/app/runner/v2/_local_experimental/executor.py +2 -1
- fractal_server/app/schemas/_validators.py +0 -15
- fractal_server/app/schemas/user.py +16 -10
- fractal_server/app/schemas/user_group.py +0 -11
- fractal_server/app/schemas/v1/applyworkflow.py +0 -8
- fractal_server/app/schemas/v1/dataset.py +0 -5
- fractal_server/app/schemas/v1/project.py +0 -5
- fractal_server/app/schemas/v1/state.py +0 -5
- fractal_server/app/schemas/v1/workflow.py +0 -5
- fractal_server/app/schemas/v2/__init__.py +4 -2
- fractal_server/app/schemas/v2/dataset.py +0 -6
- fractal_server/app/schemas/v2/job.py +0 -8
- fractal_server/app/schemas/v2/project.py +0 -5
- fractal_server/app/schemas/v2/task_collection.py +0 -21
- fractal_server/app/schemas/v2/task_group.py +59 -8
- fractal_server/app/schemas/v2/workflow.py +0 -5
- fractal_server/app/security/__init__.py +17 -0
- fractal_server/config.py +61 -59
- fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +117 -0
- fractal_server/ssh/_fabric.py +156 -83
- fractal_server/tasks/utils.py +2 -12
- fractal_server/tasks/v2/local/__init__.py +3 -0
- fractal_server/tasks/v2/local/_utils.py +70 -0
- fractal_server/tasks/v2/local/collect.py +291 -0
- fractal_server/tasks/v2/local/deactivate.py +218 -0
- fractal_server/tasks/v2/local/reactivate.py +159 -0
- fractal_server/tasks/v2/ssh/__init__.py +3 -0
- fractal_server/tasks/v2/ssh/_utils.py +87 -0
- fractal_server/tasks/v2/ssh/collect.py +311 -0
- fractal_server/tasks/v2/ssh/deactivate.py +253 -0
- fractal_server/tasks/v2/ssh/reactivate.py +202 -0
- fractal_server/tasks/v2/templates/{_2_preliminary_pip_operations.sh → 1_create_venv.sh} +6 -7
- fractal_server/tasks/v2/templates/{_3_pip_install.sh → 2_pip_install.sh} +8 -1
- fractal_server/tasks/v2/templates/{_4_pip_freeze.sh → 3_pip_freeze.sh} +0 -7
- fractal_server/tasks/v2/templates/{_5_pip_show.sh → 4_pip_show.sh} +5 -6
- fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +10 -0
- fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +35 -0
- fractal_server/tasks/v2/utils_background.py +42 -127
- fractal_server/tasks/v2/utils_templates.py +32 -2
- fractal_server/utils.py +4 -2
- fractal_server/zip_tools.py +21 -4
- {fractal_server-2.8.1.dist-info → fractal_server-2.9.0.dist-info}/METADATA +3 -5
- {fractal_server-2.8.1.dist-info → fractal_server-2.9.0.dist-info}/RECORD +77 -64
- fractal_server/app/models/v2/collection_state.py +0 -22
- fractal_server/tasks/v2/collection_local.py +0 -357
- fractal_server/tasks/v2/collection_ssh.py +0 -352
- fractal_server/tasks/v2/templates/_1_create_venv.sh +0 -42
- /fractal_server/tasks/v2/{database_operations.py → utils_database.py} +0 -0
- {fractal_server-2.8.1.dist-info → fractal_server-2.9.0.dist-info}/LICENSE +0 -0
- {fractal_server-2.8.1.dist-info → fractal_server-2.9.0.dist-info}/WHEEL +0 -0
- {fractal_server-2.8.1.dist-info → fractal_server-2.9.0.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,4 @@
|
|
1
|
+
from datetime import datetime
|
1
2
|
from typing import Optional
|
2
3
|
|
3
4
|
from fastapi import APIRouter
|
@@ -12,13 +13,17 @@ from sqlmodel import select
|
|
12
13
|
from fractal_server.app.db import AsyncSession
|
13
14
|
from fractal_server.app.db import get_async_db
|
14
15
|
from fractal_server.app.models import UserOAuth
|
15
|
-
from fractal_server.app.models.v2 import
|
16
|
+
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
16
17
|
from fractal_server.app.models.v2 import TaskGroupV2
|
17
18
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
18
19
|
from fractal_server.app.routes.auth import current_active_superuser
|
19
20
|
from fractal_server.app.routes.auth._aux_auth import (
|
20
21
|
_verify_user_belongs_to_group,
|
21
22
|
)
|
23
|
+
from fractal_server.app.routes.aux import _raise_if_naive_datetime
|
24
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
25
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
26
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
22
27
|
from fractal_server.app.schemas.v2 import TaskGroupReadV2
|
23
28
|
from fractal_server.app.schemas.v2 import TaskGroupUpdateV2
|
24
29
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
@@ -29,6 +34,44 @@ router = APIRouter()
|
|
29
34
|
logger = set_logger(__name__)
|
30
35
|
|
31
36
|
|
37
|
+
@router.get("/activity/", response_model=list[TaskGroupActivityV2Read])
|
38
|
+
async def get_task_group_activity_list(
|
39
|
+
task_group_activity_id: Optional[int] = None,
|
40
|
+
user_id: Optional[int] = None,
|
41
|
+
taskgroupv2_id: Optional[int] = None,
|
42
|
+
pkg_name: Optional[str] = None,
|
43
|
+
status: Optional[TaskGroupActivityStatusV2] = None,
|
44
|
+
action: Optional[TaskGroupActivityActionV2] = None,
|
45
|
+
timestamp_started_min: Optional[datetime] = None,
|
46
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
47
|
+
db: AsyncSession = Depends(get_async_db),
|
48
|
+
) -> list[TaskGroupActivityV2Read]:
|
49
|
+
|
50
|
+
_raise_if_naive_datetime(timestamp_started_min)
|
51
|
+
|
52
|
+
stm = select(TaskGroupActivityV2)
|
53
|
+
if task_group_activity_id is not None:
|
54
|
+
stm = stm.where(TaskGroupActivityV2.id == task_group_activity_id)
|
55
|
+
if user_id:
|
56
|
+
stm = stm.where(TaskGroupActivityV2.user_id == user_id)
|
57
|
+
if taskgroupv2_id:
|
58
|
+
stm = stm.where(TaskGroupActivityV2.taskgroupv2_id == taskgroupv2_id)
|
59
|
+
if pkg_name:
|
60
|
+
stm = stm.where(TaskGroupActivityV2.pkg_name.icontains(pkg_name))
|
61
|
+
if status:
|
62
|
+
stm = stm.where(TaskGroupActivityV2.status == status)
|
63
|
+
if action:
|
64
|
+
stm = stm.where(TaskGroupActivityV2.action == action)
|
65
|
+
if timestamp_started_min is not None:
|
66
|
+
stm = stm.where(
|
67
|
+
TaskGroupActivityV2.timestamp_started >= timestamp_started_min
|
68
|
+
)
|
69
|
+
|
70
|
+
res = await db.execute(stm)
|
71
|
+
activities = res.scalars().all()
|
72
|
+
return activities
|
73
|
+
|
74
|
+
|
32
75
|
@router.get("/{task_group_id}/", response_model=TaskGroupReadV2)
|
33
76
|
async def query_task_group(
|
34
77
|
task_group_id: int,
|
@@ -53,16 +96,26 @@ async def query_task_group_list(
|
|
53
96
|
active: Optional[bool] = None,
|
54
97
|
pkg_name: Optional[str] = None,
|
55
98
|
origin: Optional[TaskGroupV2OriginEnum] = None,
|
99
|
+
timestamp_last_used_min: Optional[datetime] = None,
|
100
|
+
timestamp_last_used_max: Optional[datetime] = None,
|
56
101
|
user: UserOAuth = Depends(current_active_superuser),
|
57
102
|
db: AsyncSession = Depends(get_async_db),
|
58
103
|
) -> list[TaskGroupReadV2]:
|
59
104
|
|
60
105
|
stm = select(TaskGroupV2)
|
61
106
|
|
107
|
+
_raise_if_naive_datetime(
|
108
|
+
timestamp_last_used_max,
|
109
|
+
timestamp_last_used_min,
|
110
|
+
)
|
111
|
+
|
62
112
|
if user_group_id is not None and private is True:
|
63
113
|
raise HTTPException(
|
64
114
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
65
|
-
detail=
|
115
|
+
detail=(
|
116
|
+
"Cannot get task groups with both "
|
117
|
+
f"{user_group_id=} and {private=}."
|
118
|
+
),
|
66
119
|
)
|
67
120
|
if user_id is not None:
|
68
121
|
stm = stm.where(TaskGroupV2.user_id == user_id)
|
@@ -82,6 +135,14 @@ async def query_task_group_list(
|
|
82
135
|
stm = stm.where(TaskGroupV2.origin == origin)
|
83
136
|
if pkg_name is not None:
|
84
137
|
stm = stm.where(TaskGroupV2.pkg_name.icontains(pkg_name))
|
138
|
+
if timestamp_last_used_min is not None:
|
139
|
+
stm = stm.where(
|
140
|
+
TaskGroupV2.timestamp_last_used >= timestamp_last_used_min
|
141
|
+
)
|
142
|
+
if timestamp_last_used_max is not None:
|
143
|
+
stm = stm.where(
|
144
|
+
TaskGroupV2.timestamp_last_used <= timestamp_last_used_max
|
145
|
+
)
|
85
146
|
|
86
147
|
res = await db.execute(stm)
|
87
148
|
task_groups_list = res.scalars().all()
|
@@ -139,22 +200,22 @@ async def delete_task_group(
|
|
139
200
|
detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
|
140
201
|
)
|
141
202
|
|
142
|
-
# Cascade operations: set foreign-keys to null for
|
143
|
-
# are in relationship with the current TaskGroupV2
|
144
|
-
logger.debug("Start of cascade operations on
|
145
|
-
stm = select(
|
146
|
-
|
203
|
+
# Cascade operations: set foreign-keys to null for TaskGroupActivityV2
|
204
|
+
# which are in relationship with the current TaskGroupV2
|
205
|
+
logger.debug("Start of cascade operations on TaskGroupActivityV2.")
|
206
|
+
stm = select(TaskGroupActivityV2).where(
|
207
|
+
TaskGroupActivityV2.taskgroupv2_id == task_group_id
|
147
208
|
)
|
148
209
|
res = await db.execute(stm)
|
149
|
-
|
150
|
-
for
|
210
|
+
task_group_activity_list = res.scalars().all()
|
211
|
+
for task_group_activity in task_group_activity_list:
|
151
212
|
logger.debug(
|
152
|
-
f"Setting
|
153
|
-
"to None."
|
213
|
+
f"Setting TaskGroupActivityV2[{task_group_activity.id}]"
|
214
|
+
".taskgroupv2_id to None."
|
154
215
|
)
|
155
|
-
|
156
|
-
db.add(
|
157
|
-
logger.debug("End of cascade operations on
|
216
|
+
task_group_activity.taskgroupv2_id = None
|
217
|
+
db.add(task_group_activity)
|
218
|
+
logger.debug("End of cascade operations on TaskGroupActivityV2.")
|
158
219
|
|
159
220
|
await db.delete(task_group)
|
160
221
|
await db.commit()
|
@@ -0,0 +1,267 @@
|
|
1
|
+
from fastapi import APIRouter
|
2
|
+
from fastapi import BackgroundTasks
|
3
|
+
from fastapi import Depends
|
4
|
+
from fastapi import HTTPException
|
5
|
+
from fastapi import Request
|
6
|
+
from fastapi import Response
|
7
|
+
from fastapi import status
|
8
|
+
|
9
|
+
from fractal_server.app.db import AsyncSession
|
10
|
+
from fractal_server.app.db import get_async_db
|
11
|
+
from fractal_server.app.models import UserOAuth
|
12
|
+
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
13
|
+
from fractal_server.app.routes.api.v2._aux_functions_task_lifecycle import (
|
14
|
+
check_no_ongoing_activity,
|
15
|
+
)
|
16
|
+
from fractal_server.app.routes.api.v2._aux_functions_task_lifecycle import (
|
17
|
+
check_no_submitted_job,
|
18
|
+
)
|
19
|
+
from fractal_server.app.routes.api.v2._aux_functions_tasks import (
|
20
|
+
_get_task_group_or_404,
|
21
|
+
)
|
22
|
+
from fractal_server.app.routes.auth import current_active_superuser
|
23
|
+
from fractal_server.app.routes.aux.validate_user_settings import (
|
24
|
+
validate_user_settings,
|
25
|
+
)
|
26
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
27
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
28
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
29
|
+
from fractal_server.app.schemas.v2 import TaskGroupReadV2
|
30
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
31
|
+
from fractal_server.config import get_settings
|
32
|
+
from fractal_server.logger import set_logger
|
33
|
+
from fractal_server.syringe import Inject
|
34
|
+
from fractal_server.tasks.v2.local import deactivate_local
|
35
|
+
from fractal_server.tasks.v2.local import reactivate_local
|
36
|
+
from fractal_server.tasks.v2.ssh import deactivate_ssh
|
37
|
+
from fractal_server.tasks.v2.ssh import reactivate_ssh
|
38
|
+
from fractal_server.utils import get_timestamp
|
39
|
+
|
40
|
+
router = APIRouter()
|
41
|
+
|
42
|
+
logger = set_logger(__name__)
|
43
|
+
|
44
|
+
|
45
|
+
@router.post(
|
46
|
+
"/{task_group_id}/deactivate/",
|
47
|
+
response_model=TaskGroupActivityV2Read,
|
48
|
+
)
|
49
|
+
async def deactivate_task_group(
|
50
|
+
task_group_id: int,
|
51
|
+
background_tasks: BackgroundTasks,
|
52
|
+
response: Response,
|
53
|
+
request: Request,
|
54
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
55
|
+
db: AsyncSession = Depends(get_async_db),
|
56
|
+
) -> TaskGroupReadV2:
|
57
|
+
"""
|
58
|
+
Deactivate task-group venv
|
59
|
+
"""
|
60
|
+
task_group = await _get_task_group_or_404(
|
61
|
+
task_group_id=task_group_id, db=db
|
62
|
+
)
|
63
|
+
|
64
|
+
# Check that task-group is active
|
65
|
+
if not task_group.active:
|
66
|
+
raise HTTPException(
|
67
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
68
|
+
detail=(
|
69
|
+
f"Cannot deactivate a task group with {task_group.active=}."
|
70
|
+
),
|
71
|
+
)
|
72
|
+
|
73
|
+
# Check no other activity is ongoing
|
74
|
+
await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
|
75
|
+
|
76
|
+
# Check no submitted jobs use tasks from this task group
|
77
|
+
await check_no_submitted_job(task_group_id=task_group.id, db=db)
|
78
|
+
|
79
|
+
# Shortcut for task-group with origin="other"
|
80
|
+
if task_group.origin == TaskGroupV2OriginEnum.OTHER:
|
81
|
+
task_group.active = False
|
82
|
+
task_group_activity = TaskGroupActivityV2(
|
83
|
+
user_id=task_group.user_id,
|
84
|
+
taskgroupv2_id=task_group.id,
|
85
|
+
status=TaskGroupActivityStatusV2.OK,
|
86
|
+
action=TaskGroupActivityActionV2.DEACTIVATE,
|
87
|
+
pkg_name=task_group.pkg_name,
|
88
|
+
version=(task_group.version or "N/A"),
|
89
|
+
log=(
|
90
|
+
f"Task group has {task_group.origin=}, set "
|
91
|
+
"task_group.active to False and exit."
|
92
|
+
),
|
93
|
+
timestamp_started=get_timestamp(),
|
94
|
+
timestamp_ended=get_timestamp(),
|
95
|
+
)
|
96
|
+
db.add(task_group)
|
97
|
+
db.add(task_group_activity)
|
98
|
+
await db.commit()
|
99
|
+
response.status_code = status.HTTP_202_ACCEPTED
|
100
|
+
return task_group_activity
|
101
|
+
|
102
|
+
task_group_activity = TaskGroupActivityV2(
|
103
|
+
user_id=task_group.user_id,
|
104
|
+
taskgroupv2_id=task_group.id,
|
105
|
+
status=TaskGroupActivityStatusV2.PENDING,
|
106
|
+
action=TaskGroupActivityActionV2.DEACTIVATE,
|
107
|
+
pkg_name=task_group.pkg_name,
|
108
|
+
version=task_group.version,
|
109
|
+
timestamp_started=get_timestamp(),
|
110
|
+
)
|
111
|
+
db.add(task_group_activity)
|
112
|
+
await db.commit()
|
113
|
+
|
114
|
+
# Submit background task
|
115
|
+
settings = Inject(get_settings)
|
116
|
+
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
117
|
+
# Validate user settings (backend-specific)
|
118
|
+
user = await db.get(UserOAuth, task_group.user_id)
|
119
|
+
user_settings = await validate_user_settings(
|
120
|
+
user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
|
121
|
+
)
|
122
|
+
# User appropriate FractalSSH object
|
123
|
+
ssh_credentials = dict(
|
124
|
+
user=user_settings.ssh_username,
|
125
|
+
host=user_settings.ssh_host,
|
126
|
+
key_path=user_settings.ssh_private_key_path,
|
127
|
+
)
|
128
|
+
fractal_ssh_list = request.app.state.fractal_ssh_list
|
129
|
+
fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
|
130
|
+
|
131
|
+
background_tasks.add_task(
|
132
|
+
deactivate_ssh,
|
133
|
+
task_group_id=task_group.id,
|
134
|
+
task_group_activity_id=task_group_activity.id,
|
135
|
+
fractal_ssh=fractal_ssh,
|
136
|
+
tasks_base_dir=user_settings.ssh_tasks_dir,
|
137
|
+
)
|
138
|
+
else:
|
139
|
+
background_tasks.add_task(
|
140
|
+
deactivate_local,
|
141
|
+
task_group_id=task_group.id,
|
142
|
+
task_group_activity_id=task_group_activity.id,
|
143
|
+
)
|
144
|
+
|
145
|
+
logger.debug(
|
146
|
+
"Admin task group deactivation endpoint: start deactivate "
|
147
|
+
"and return task_group_activity"
|
148
|
+
)
|
149
|
+
response.status_code = status.HTTP_202_ACCEPTED
|
150
|
+
return task_group_activity
|
151
|
+
|
152
|
+
|
153
|
+
@router.post(
|
154
|
+
"/{task_group_id}/reactivate/",
|
155
|
+
response_model=TaskGroupActivityV2Read,
|
156
|
+
)
|
157
|
+
async def reactivate_task_group(
|
158
|
+
task_group_id: int,
|
159
|
+
background_tasks: BackgroundTasks,
|
160
|
+
response: Response,
|
161
|
+
request: Request,
|
162
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
163
|
+
db: AsyncSession = Depends(get_async_db),
|
164
|
+
) -> TaskGroupReadV2:
|
165
|
+
"""
|
166
|
+
Deactivate task-group venv
|
167
|
+
"""
|
168
|
+
|
169
|
+
task_group = await _get_task_group_or_404(
|
170
|
+
task_group_id=task_group_id, db=db
|
171
|
+
)
|
172
|
+
|
173
|
+
# Check that task-group is not active
|
174
|
+
if task_group.active:
|
175
|
+
raise HTTPException(
|
176
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
177
|
+
detail=(
|
178
|
+
f"Cannot reactivate a task group with {task_group.active=}."
|
179
|
+
),
|
180
|
+
)
|
181
|
+
|
182
|
+
# Check no other activity is ongoing
|
183
|
+
await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
|
184
|
+
|
185
|
+
# Check no submitted jobs use tasks from this task group
|
186
|
+
await check_no_submitted_job(task_group_id=task_group.id, db=db)
|
187
|
+
|
188
|
+
# Shortcut for task-group with origin="other"
|
189
|
+
if task_group.origin == TaskGroupV2OriginEnum.OTHER:
|
190
|
+
task_group.active = True
|
191
|
+
task_group_activity = TaskGroupActivityV2(
|
192
|
+
user_id=task_group.user_id,
|
193
|
+
taskgroupv2_id=task_group.id,
|
194
|
+
status=TaskGroupActivityStatusV2.OK,
|
195
|
+
action=TaskGroupActivityActionV2.REACTIVATE,
|
196
|
+
pkg_name=task_group.pkg_name,
|
197
|
+
version=(task_group.version or "N/A"),
|
198
|
+
log=(
|
199
|
+
f"Task group has {task_group.origin=}, set "
|
200
|
+
"task_group.active to True and exit."
|
201
|
+
),
|
202
|
+
timestamp_started=get_timestamp(),
|
203
|
+
timestamp_ended=get_timestamp(),
|
204
|
+
)
|
205
|
+
db.add(task_group)
|
206
|
+
db.add(task_group_activity)
|
207
|
+
await db.commit()
|
208
|
+
response.status_code = status.HTTP_202_ACCEPTED
|
209
|
+
return task_group_activity
|
210
|
+
|
211
|
+
if task_group.pip_freeze is None:
|
212
|
+
raise HTTPException(
|
213
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
214
|
+
detail=(
|
215
|
+
"Cannot reactivate a task group with "
|
216
|
+
f"{task_group.pip_freeze=}."
|
217
|
+
),
|
218
|
+
)
|
219
|
+
|
220
|
+
task_group_activity = TaskGroupActivityV2(
|
221
|
+
user_id=task_group.user_id,
|
222
|
+
taskgroupv2_id=task_group.id,
|
223
|
+
status=TaskGroupActivityStatusV2.PENDING,
|
224
|
+
action=TaskGroupActivityActionV2.REACTIVATE,
|
225
|
+
pkg_name=task_group.pkg_name,
|
226
|
+
version=task_group.version,
|
227
|
+
timestamp_started=get_timestamp(),
|
228
|
+
)
|
229
|
+
db.add(task_group_activity)
|
230
|
+
await db.commit()
|
231
|
+
|
232
|
+
# Submit background task
|
233
|
+
settings = Inject(get_settings)
|
234
|
+
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
235
|
+
# Validate user settings (backend-specific)
|
236
|
+
user = await db.get(UserOAuth, task_group.user_id)
|
237
|
+
user_settings = await validate_user_settings(
|
238
|
+
user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
|
239
|
+
)
|
240
|
+
# Use appropriate FractalSSH object
|
241
|
+
ssh_credentials = dict(
|
242
|
+
user=user_settings.ssh_username,
|
243
|
+
host=user_settings.ssh_host,
|
244
|
+
key_path=user_settings.ssh_private_key_path,
|
245
|
+
)
|
246
|
+
fractal_ssh_list = request.app.state.fractal_ssh_list
|
247
|
+
fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
|
248
|
+
|
249
|
+
background_tasks.add_task(
|
250
|
+
reactivate_ssh,
|
251
|
+
task_group_id=task_group.id,
|
252
|
+
task_group_activity_id=task_group_activity.id,
|
253
|
+
fractal_ssh=fractal_ssh,
|
254
|
+
tasks_base_dir=user_settings.ssh_tasks_dir,
|
255
|
+
)
|
256
|
+
else:
|
257
|
+
background_tasks.add_task(
|
258
|
+
reactivate_local,
|
259
|
+
task_group_id=task_group.id,
|
260
|
+
task_group_activity_id=task_group_activity.id,
|
261
|
+
)
|
262
|
+
logger.debug(
|
263
|
+
"Admin task group reactivation endpoint: start reactivate "
|
264
|
+
"and return task_group_activity"
|
265
|
+
)
|
266
|
+
response.status_code = status.HTTP_202_ACCEPTED
|
267
|
+
return task_group_activity
|
@@ -1,5 +1,5 @@
|
|
1
|
+
import json
|
1
2
|
import os
|
2
|
-
from datetime import datetime
|
3
3
|
from datetime import timedelta
|
4
4
|
from datetime import timezone
|
5
5
|
from typing import Optional
|
@@ -50,10 +50,6 @@ router = APIRouter()
|
|
50
50
|
logger = set_logger(__name__)
|
51
51
|
|
52
52
|
|
53
|
-
def _encode_as_utc(dt: datetime):
|
54
|
-
return dt.replace(tzinfo=timezone.utc).isoformat()
|
55
|
-
|
56
|
-
|
57
53
|
@router.get("/", response_model=list[ProjectReadV1])
|
58
54
|
async def get_list_project(
|
59
55
|
user: UserOAuth = Depends(current_active_user),
|
@@ -393,33 +389,25 @@ async def apply_workflow(
|
|
393
389
|
workflow_id=workflow_id,
|
394
390
|
user_email=user.email,
|
395
391
|
input_dataset_dump=dict(
|
396
|
-
**
|
397
|
-
exclude={"resource_list", "history"
|
392
|
+
**json.loads(
|
393
|
+
input_dataset.json(exclude={"resource_list", "history"})
|
398
394
|
),
|
399
|
-
timestamp_created=_encode_as_utc(input_dataset.timestamp_created),
|
400
395
|
resource_list=[
|
401
396
|
resource.model_dump()
|
402
397
|
for resource in input_dataset.resource_list
|
403
398
|
],
|
404
399
|
),
|
405
400
|
output_dataset_dump=dict(
|
406
|
-
**
|
407
|
-
exclude={"resource_list", "history"
|
401
|
+
**json.loads(
|
402
|
+
output_dataset.json(exclude={"resource_list", "history"})
|
408
403
|
),
|
409
|
-
timestamp_created=_encode_as_utc(output_dataset.timestamp_created),
|
410
404
|
resource_list=[
|
411
405
|
resource.model_dump()
|
412
406
|
for resource in output_dataset.resource_list
|
413
407
|
],
|
414
408
|
),
|
415
|
-
workflow_dump=
|
416
|
-
|
417
|
-
timestamp_created=_encode_as_utc(workflow.timestamp_created),
|
418
|
-
),
|
419
|
-
project_dump=dict(
|
420
|
-
**project.model_dump(exclude={"user_list", "timestamp_created"}),
|
421
|
-
timestamp_created=_encode_as_utc(project.timestamp_created),
|
422
|
-
),
|
409
|
+
workflow_dump=json.loads(workflow.json(exclude={"task_list"})),
|
410
|
+
project_dump=json.loads(project.json(exclude={"user_list"})),
|
423
411
|
**apply_workflow.dict(),
|
424
412
|
)
|
425
413
|
|
@@ -13,6 +13,7 @@ from .task import router as task_router_v2
|
|
13
13
|
from .task_collection import router as task_collection_router_v2
|
14
14
|
from .task_collection_custom import router as task_collection_router_v2_custom
|
15
15
|
from .task_group import router as task_group_router_v2
|
16
|
+
from .task_group_lifecycle import router as task_group_lifecycle_router_v2
|
16
17
|
from .workflow import router as workflow_router_v2
|
17
18
|
from .workflow_import import router as workflow_import_router_v2
|
18
19
|
from .workflowtask import router as workflowtask_router_v2
|
@@ -31,13 +32,21 @@ router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
|
|
31
32
|
|
32
33
|
settings = Inject(get_settings)
|
33
34
|
router_api_v2.include_router(
|
34
|
-
task_collection_router_v2,
|
35
|
+
task_collection_router_v2,
|
36
|
+
prefix="/task",
|
37
|
+
tags=["V2 Task Lifecycle"],
|
35
38
|
)
|
36
39
|
router_api_v2.include_router(
|
37
40
|
task_collection_router_v2_custom,
|
38
41
|
prefix="/task",
|
39
|
-
tags=["V2 Task
|
42
|
+
tags=["V2 Task Lifecycle"],
|
43
|
+
)
|
44
|
+
router_api_v2.include_router(
|
45
|
+
task_group_lifecycle_router_v2,
|
46
|
+
prefix="/task-group",
|
47
|
+
tags=["V2 Task Lifecycle"],
|
40
48
|
)
|
49
|
+
|
41
50
|
router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
|
42
51
|
router_api_v2.include_router(
|
43
52
|
task_group_router_v2, prefix="/task-group", tags=["V2 TaskGroup"]
|
@@ -4,7 +4,17 @@ from fastapi import HTTPException
|
|
4
4
|
from fastapi import status
|
5
5
|
from httpx import AsyncClient
|
6
6
|
from httpx import TimeoutException
|
7
|
+
from sqlmodel import func
|
8
|
+
from sqlmodel import select
|
7
9
|
|
10
|
+
from fractal_server.app.db import AsyncSession
|
11
|
+
from fractal_server.app.models.v2 import JobV2
|
12
|
+
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
13
|
+
from fractal_server.app.models.v2 import TaskV2
|
14
|
+
from fractal_server.app.models.v2 import WorkflowTaskV2
|
15
|
+
from fractal_server.app.models.v2 import WorkflowV2
|
16
|
+
from fractal_server.app.schemas.v2 import JobStatusTypeV2
|
17
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
8
18
|
from fractal_server.logger import set_logger
|
9
19
|
|
10
20
|
|
@@ -122,3 +132,76 @@ async def get_package_version_from_pypi(
|
|
122
132
|
# Case 3: `version` is unset and we use latest
|
123
133
|
logger.info(f"No version requested, returning {latest_version=}.")
|
124
134
|
return latest_version
|
135
|
+
|
136
|
+
|
137
|
+
async def check_no_ongoing_activity(
|
138
|
+
*,
|
139
|
+
task_group_id: int,
|
140
|
+
db: AsyncSession,
|
141
|
+
) -> None:
|
142
|
+
"""
|
143
|
+
Find ongoing activities for the same task group.
|
144
|
+
|
145
|
+
Arguments:
|
146
|
+
task_group_id:
|
147
|
+
db:
|
148
|
+
"""
|
149
|
+
# DB query
|
150
|
+
stm = (
|
151
|
+
select(TaskGroupActivityV2)
|
152
|
+
.where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
|
153
|
+
.where(TaskGroupActivityV2.status == TaskGroupActivityStatusV2.ONGOING)
|
154
|
+
)
|
155
|
+
res = await db.execute(stm)
|
156
|
+
ongoing_activities = res.scalars().all()
|
157
|
+
|
158
|
+
if ongoing_activities == []:
|
159
|
+
# All good, exit
|
160
|
+
return
|
161
|
+
|
162
|
+
msg = "Found ongoing activities for the same task-group:"
|
163
|
+
for ind, activity in enumerate(ongoing_activities):
|
164
|
+
msg = (
|
165
|
+
f"{msg}\n{ind + 1}) "
|
166
|
+
f"Action={activity.action}, "
|
167
|
+
f"status={activity.status}, "
|
168
|
+
f"timestamp_started={activity.timestamp_started}."
|
169
|
+
)
|
170
|
+
raise HTTPException(
|
171
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
172
|
+
detail=msg,
|
173
|
+
)
|
174
|
+
|
175
|
+
|
176
|
+
async def check_no_submitted_job(
|
177
|
+
*,
|
178
|
+
task_group_id: int,
|
179
|
+
db: AsyncSession,
|
180
|
+
) -> None:
|
181
|
+
"""
|
182
|
+
Find submitted jobs which include tasks from a given task group.
|
183
|
+
|
184
|
+
Arguments:
|
185
|
+
task_id_list: List of TaskV2 IDs
|
186
|
+
db: Database session
|
187
|
+
"""
|
188
|
+
stm = (
|
189
|
+
select(func.count(JobV2.id))
|
190
|
+
.join(WorkflowV2, JobV2.workflow_id == WorkflowV2.id)
|
191
|
+
.join(WorkflowTaskV2, WorkflowTaskV2.workflow_id == WorkflowV2.id)
|
192
|
+
.join(TaskV2, WorkflowTaskV2.task_id == TaskV2.id)
|
193
|
+
.where(WorkflowTaskV2.order >= JobV2.first_task_index)
|
194
|
+
.where(WorkflowTaskV2.order <= JobV2.last_task_index)
|
195
|
+
.where(JobV2.status == JobStatusTypeV2.SUBMITTED)
|
196
|
+
.where(TaskV2.taskgroupv2_id == task_group_id)
|
197
|
+
)
|
198
|
+
res = await db.execute(stm)
|
199
|
+
num_submitted_jobs = res.scalar()
|
200
|
+
if num_submitted_jobs > 0:
|
201
|
+
raise HTTPException(
|
202
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
203
|
+
detail=(
|
204
|
+
f"Cannot act on task group because {num_submitted_jobs} "
|
205
|
+
"submitted jobs use its tasks."
|
206
|
+
),
|
207
|
+
)
|
@@ -13,7 +13,7 @@ from fractal_server.app.db import AsyncSession
|
|
13
13
|
from fractal_server.app.models import LinkUserGroup
|
14
14
|
from fractal_server.app.models import UserGroup
|
15
15
|
from fractal_server.app.models import UserOAuth
|
16
|
-
from fractal_server.app.models.v2 import
|
16
|
+
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
17
17
|
from fractal_server.app.models.v2 import TaskGroupV2
|
18
18
|
from fractal_server.app.models.v2 import TaskV2
|
19
19
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
@@ -21,6 +21,7 @@ from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
|
|
21
21
|
from fractal_server.app.routes.auth._aux_auth import (
|
22
22
|
_verify_user_belongs_to_group,
|
23
23
|
)
|
24
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
24
25
|
from fractal_server.logger import set_logger
|
25
26
|
|
26
27
|
logger = set_logger(__name__)
|
@@ -219,27 +220,32 @@ async def _get_valid_user_group_id(
|
|
219
220
|
return user_group_id
|
220
221
|
|
221
222
|
|
222
|
-
async def
|
223
|
-
|
223
|
+
async def _get_collection_task_group_activity_status_message(
|
224
|
+
task_group_id: int,
|
225
|
+
db: AsyncSession,
|
224
226
|
) -> str:
|
227
|
+
|
225
228
|
res = await db.execute(
|
226
|
-
select(
|
227
|
-
|
228
|
-
)
|
229
|
+
select(TaskGroupActivityV2)
|
230
|
+
.where(TaskGroupActivityV2.taskgroupv2_id == task_group_id)
|
231
|
+
.where(TaskGroupActivityV2.action == TaskGroupActivityActionV2.COLLECT)
|
229
232
|
)
|
230
|
-
|
231
|
-
if len(
|
233
|
+
task_group_activity_list = res.scalars().all()
|
234
|
+
if len(task_group_activity_list) > 1:
|
232
235
|
msg = (
|
233
|
-
"
|
234
|
-
|
235
|
-
f"
|
236
|
+
"\nWarning: "
|
237
|
+
"Expected only one TaskGroupActivityV2 associated to TaskGroup "
|
238
|
+
f"{task_group_id}, found {len(task_group_activity_list)} "
|
239
|
+
f"(IDs: {[tga.id for tga in task_group_activity_list]})."
|
236
240
|
"Warning: this should have not happened, please contact an admin."
|
237
241
|
)
|
238
|
-
elif len(
|
242
|
+
elif len(task_group_activity_list) == 1:
|
239
243
|
msg = (
|
240
|
-
|
241
|
-
|
242
|
-
f"
|
244
|
+
"\nNote:"
|
245
|
+
"There exists another task-group collection "
|
246
|
+
f"(activity ID={task_group_activity_list[0].id}) for "
|
247
|
+
f"this task group (ID={task_group_id}), with status "
|
248
|
+
f"'{task_group_activity_list[0].status}'."
|
243
249
|
)
|
244
250
|
else:
|
245
251
|
msg = ""
|
@@ -273,7 +279,9 @@ async def _verify_non_duplication_user_constraint(
|
|
273
279
|
"This should have not happened: please contact an admin."
|
274
280
|
),
|
275
281
|
)
|
276
|
-
state_msg = await
|
282
|
+
state_msg = await _get_collection_task_group_activity_status_message(
|
283
|
+
duplicate[0].id, db
|
284
|
+
)
|
277
285
|
raise HTTPException(
|
278
286
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
279
287
|
detail=(
|
@@ -313,7 +321,9 @@ async def _verify_non_duplication_group_constraint(
|
|
313
321
|
"This should have not happened: please contact an admin."
|
314
322
|
),
|
315
323
|
)
|
316
|
-
state_msg = await
|
324
|
+
state_msg = await _get_collection_task_group_activity_status_message(
|
325
|
+
duplicate[0].id, db
|
326
|
+
)
|
317
327
|
raise HTTPException(
|
318
328
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
319
329
|
detail=(
|