fractal-server 2.13.0__py3-none-any.whl → 2.14.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/history/__init__.py +4 -0
- fractal_server/app/history/image_updates.py +142 -0
- fractal_server/app/history/status_enum.py +16 -0
- fractal_server/app/models/v2/__init__.py +9 -1
- fractal_server/app/models/v2/accounting.py +35 -0
- fractal_server/app/models/v2/history.py +53 -0
- fractal_server/app/routes/admin/v2/__init__.py +4 -0
- fractal_server/app/routes/admin/v2/accounting.py +108 -0
- fractal_server/app/routes/admin/v2/impersonate.py +35 -0
- fractal_server/app/routes/admin/v2/job.py +5 -13
- fractal_server/app/routes/admin/v2/task_group.py +4 -12
- fractal_server/app/routes/api/v2/__init__.py +2 -2
- fractal_server/app/routes/api/v2/_aux_functions.py +78 -0
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +3 -3
- fractal_server/app/routes/api/v2/dataset.py +12 -9
- fractal_server/app/routes/api/v2/history.py +247 -0
- fractal_server/app/routes/api/v2/submit.py +1 -0
- fractal_server/app/routes/api/v2/task_group.py +2 -5
- fractal_server/app/routes/api/v2/workflow.py +18 -3
- fractal_server/app/routes/api/v2/workflowtask.py +22 -0
- fractal_server/app/routes/aux/__init__.py +0 -20
- fractal_server/app/runner/executors/base_runner.py +114 -0
- fractal_server/app/runner/{v2/_local → executors/local}/_local_config.py +3 -3
- fractal_server/app/runner/executors/local/_submit_setup.py +54 -0
- fractal_server/app/runner/executors/local/runner.py +200 -0
- fractal_server/app/runner/executors/{slurm → slurm_common}/_batching.py +1 -1
- fractal_server/app/runner/executors/{slurm → slurm_common}/_slurm_config.py +3 -3
- fractal_server/app/runner/{v2/_slurm_ssh → executors/slurm_common}/_submit_setup.py +13 -12
- fractal_server/app/runner/{v2/_slurm_common → executors/slurm_common}/get_slurm_config.py +9 -15
- fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_executor_wait_thread.py +1 -1
- fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_slurm_job.py +1 -1
- fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/executor.py +13 -14
- fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_check_jobs_status.py +11 -9
- fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_executor_wait_thread.py +3 -3
- fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_subprocess_run_as_user.py +2 -68
- fractal_server/app/runner/executors/slurm_sudo/runner.py +632 -0
- fractal_server/app/runner/task_files.py +70 -96
- fractal_server/app/runner/v2/__init__.py +9 -19
- fractal_server/app/runner/v2/_local.py +84 -0
- fractal_server/app/runner/v2/{_slurm_ssh/__init__.py → _slurm_ssh.py} +12 -13
- fractal_server/app/runner/v2/{_slurm_sudo/__init__.py → _slurm_sudo.py} +12 -12
- fractal_server/app/runner/v2/runner.py +106 -31
- fractal_server/app/runner/v2/runner_functions.py +88 -64
- fractal_server/app/runner/v2/runner_functions_low_level.py +20 -20
- fractal_server/app/schemas/v2/__init__.py +1 -0
- fractal_server/app/schemas/v2/accounting.py +18 -0
- fractal_server/app/schemas/v2/dataset.py +0 -17
- fractal_server/app/schemas/v2/history.py +23 -0
- fractal_server/config.py +58 -52
- fractal_server/migrations/versions/8223fcef886c_image_status.py +63 -0
- fractal_server/migrations/versions/87cd72a537a2_add_historyitem_table.py +68 -0
- fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +57 -0
- fractal_server/tasks/v2/utils_background.py +1 -1
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/METADATA +1 -1
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/RECORD +66 -55
- fractal_server/app/routes/api/v2/status.py +0 -168
- fractal_server/app/runner/executors/slurm/sudo/executor.py +0 -1281
- fractal_server/app/runner/v2/_local/__init__.py +0 -129
- fractal_server/app/runner/v2/_local/_submit_setup.py +0 -52
- fractal_server/app/runner/v2/_local/executor.py +0 -100
- fractal_server/app/runner/v2/_slurm_sudo/_submit_setup.py +0 -83
- fractal_server/app/runner/v2/handle_failed_job.py +0 -59
- /fractal_server/app/runner/executors/{slurm → local}/__init__.py +0 -0
- /fractal_server/app/runner/executors/{slurm/ssh → slurm_common}/__init__.py +0 -0
- /fractal_server/app/runner/executors/{_job_states.py → slurm_common/_job_states.py} +0 -0
- /fractal_server/app/runner/executors/{slurm → slurm_common}/remote.py +0 -0
- /fractal_server/app/runner/executors/{slurm → slurm_common}/utils_executors.py +0 -0
- /fractal_server/app/runner/executors/{slurm/sudo → slurm_ssh}/__init__.py +0 -0
- /fractal_server/app/runner/{v2/_slurm_common → executors/slurm_sudo}/__init__.py +0 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/LICENSE +0 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/entry_points.txt +0 -0
@@ -1,168 +0,0 @@
|
|
1
|
-
from typing import Optional
|
2
|
-
|
3
|
-
from fastapi import APIRouter
|
4
|
-
from fastapi import Depends
|
5
|
-
from fastapi import HTTPException
|
6
|
-
from fastapi import status
|
7
|
-
|
8
|
-
from .....logger import set_logger
|
9
|
-
from ....db import AsyncSession
|
10
|
-
from ....db import get_async_db
|
11
|
-
from ....models.v2 import JobV2
|
12
|
-
from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
|
13
|
-
from ....schemas.v2.status import StatusReadV2
|
14
|
-
from ._aux_functions import _get_dataset_check_owner
|
15
|
-
from ._aux_functions import _get_submitted_jobs_statement
|
16
|
-
from ._aux_functions import _get_workflow_check_owner
|
17
|
-
from fractal_server.app.models import UserOAuth
|
18
|
-
from fractal_server.app.routes.auth import current_active_user
|
19
|
-
|
20
|
-
router = APIRouter()
|
21
|
-
|
22
|
-
logger = set_logger(__name__)
|
23
|
-
|
24
|
-
|
25
|
-
@router.get(
|
26
|
-
"/project/{project_id}/status/",
|
27
|
-
response_model=StatusReadV2,
|
28
|
-
)
|
29
|
-
async def get_workflowtask_status(
|
30
|
-
project_id: int,
|
31
|
-
dataset_id: int,
|
32
|
-
workflow_id: int,
|
33
|
-
user: UserOAuth = Depends(current_active_user),
|
34
|
-
db: AsyncSession = Depends(get_async_db),
|
35
|
-
) -> Optional[StatusReadV2]:
|
36
|
-
"""
|
37
|
-
Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
|
38
|
-
on a given `DatasetV2`.
|
39
|
-
|
40
|
-
*NOTE*: the current endpoint is not guaranteed to provide consistent
|
41
|
-
results if the workflow task list is modified in a non-trivial way
|
42
|
-
(that is, by adding intermediate tasks, removing tasks, or changing their
|
43
|
-
order). See fractal-server GitHub issues: 793, 1083.
|
44
|
-
"""
|
45
|
-
# Get the dataset DB entry
|
46
|
-
output = await _get_dataset_check_owner(
|
47
|
-
project_id=project_id,
|
48
|
-
dataset_id=dataset_id,
|
49
|
-
user_id=user.id,
|
50
|
-
db=db,
|
51
|
-
)
|
52
|
-
dataset = output["dataset"]
|
53
|
-
|
54
|
-
# Get the workflow DB entry
|
55
|
-
workflow = await _get_workflow_check_owner(
|
56
|
-
project_id=project_id,
|
57
|
-
workflow_id=workflow_id,
|
58
|
-
user_id=user.id,
|
59
|
-
db=db,
|
60
|
-
)
|
61
|
-
|
62
|
-
# Check whether there exists a submitted job associated to this
|
63
|
-
# workflow/dataset pair. If it does exist, it will be used later.
|
64
|
-
# If there are multiple jobs, raise an error.
|
65
|
-
stm = _get_submitted_jobs_statement()
|
66
|
-
stm = stm.where(JobV2.dataset_id == dataset_id)
|
67
|
-
stm = stm.where(JobV2.workflow_id == workflow_id)
|
68
|
-
res = await db.execute(stm)
|
69
|
-
running_jobs = res.scalars().all()
|
70
|
-
if len(running_jobs) == 0:
|
71
|
-
running_job = None
|
72
|
-
elif len(running_jobs) == 1:
|
73
|
-
running_job = running_jobs[0]
|
74
|
-
else:
|
75
|
-
string_ids = str([job.id for job in running_jobs])[1:-1]
|
76
|
-
raise HTTPException(
|
77
|
-
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
78
|
-
detail=(
|
79
|
-
f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
|
80
|
-
f" is linked to multiple active jobs: {string_ids}."
|
81
|
-
),
|
82
|
-
)
|
83
|
-
|
84
|
-
# Initialize empty dictionary for WorkflowTaskV2 status
|
85
|
-
workflow_tasks_status_dict: dict = {}
|
86
|
-
|
87
|
-
# Lowest priority: read status from DB, which corresponds to jobs that are
|
88
|
-
# not running
|
89
|
-
history = dataset.history
|
90
|
-
for history_item in history:
|
91
|
-
wftask_id = history_item["workflowtask"]["id"]
|
92
|
-
wftask_status = history_item["status"]
|
93
|
-
workflow_tasks_status_dict[wftask_id] = wftask_status
|
94
|
-
|
95
|
-
if running_job is None:
|
96
|
-
# If no job is running, the chronological-last history item is also the
|
97
|
-
# positional-last workflow task to be included in the response.
|
98
|
-
if len(history) > 0:
|
99
|
-
last_valid_wftask_id = history[-1]["workflowtask"]["id"]
|
100
|
-
else:
|
101
|
-
last_valid_wftask_id = None
|
102
|
-
else:
|
103
|
-
# If a job is running, then gather more up-to-date information
|
104
|
-
|
105
|
-
# Mid priority: Set all WorkflowTask's that are part of the running job
|
106
|
-
# as "submitted"
|
107
|
-
start = running_job.first_task_index
|
108
|
-
end = running_job.last_task_index + 1
|
109
|
-
|
110
|
-
running_job_wftasks = workflow.task_list[start:end]
|
111
|
-
running_job_statuses = [
|
112
|
-
workflow_tasks_status_dict.get(wft.id, None)
|
113
|
-
for wft in running_job_wftasks
|
114
|
-
]
|
115
|
-
try:
|
116
|
-
first_submitted_index = running_job_statuses.index(
|
117
|
-
WorkflowTaskStatusTypeV2.SUBMITTED
|
118
|
-
)
|
119
|
-
except ValueError:
|
120
|
-
logger.warning(
|
121
|
-
f"Job {running_job.id} is submitted but its task list does "
|
122
|
-
f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
|
123
|
-
)
|
124
|
-
first_submitted_index = 0
|
125
|
-
|
126
|
-
for wftask in running_job_wftasks[first_submitted_index:]:
|
127
|
-
workflow_tasks_status_dict[
|
128
|
-
wftask.id
|
129
|
-
] = WorkflowTaskStatusTypeV2.SUBMITTED
|
130
|
-
|
131
|
-
# The last workflow task that is included in the submitted job is also
|
132
|
-
# the positional-last workflow task to be included in the response.
|
133
|
-
try:
|
134
|
-
last_valid_wftask_id = workflow.task_list[end - 1].id
|
135
|
-
except IndexError as e:
|
136
|
-
logger.warning(
|
137
|
-
f"Handled IndexError in `get_workflowtask_status` ({str(e)})."
|
138
|
-
)
|
139
|
-
logger.warning(
|
140
|
-
"Additional information: "
|
141
|
-
f"{running_job.first_task_index=}; "
|
142
|
-
f"{running_job.last_task_index=}; "
|
143
|
-
f"{len(workflow.task_list)=}; "
|
144
|
-
f"{dataset_id=}; "
|
145
|
-
f"{workflow_id=}."
|
146
|
-
)
|
147
|
-
last_valid_wftask_id = None
|
148
|
-
logger.warning(f"Now setting {last_valid_wftask_id=}.")
|
149
|
-
|
150
|
-
# Based on previously-gathered information, clean up the response body
|
151
|
-
clean_workflow_tasks_status_dict = {}
|
152
|
-
for wf_task in workflow.task_list:
|
153
|
-
wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
|
154
|
-
if wf_task_status is None:
|
155
|
-
# If a wftask ID was not found, ignore it and continue
|
156
|
-
continue
|
157
|
-
clean_workflow_tasks_status_dict[str(wf_task.id)] = wf_task_status
|
158
|
-
if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
|
159
|
-
# Starting from the beginning of `workflow.task_list`, stop the
|
160
|
-
# first time that you hit a failed job
|
161
|
-
break
|
162
|
-
if wf_task.id == last_valid_wftask_id:
|
163
|
-
# Starting from the beginning of `workflow.task_list`, stop the
|
164
|
-
# first time that you hit `last_valid_wftask_id``
|
165
|
-
break
|
166
|
-
|
167
|
-
response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
|
168
|
-
return response_body
|