fractal-server 2.13.0__py3-none-any.whl → 2.14.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/history/__init__.py +4 -0
  3. fractal_server/app/history/image_updates.py +142 -0
  4. fractal_server/app/history/status_enum.py +16 -0
  5. fractal_server/app/models/v2/__init__.py +9 -1
  6. fractal_server/app/models/v2/accounting.py +35 -0
  7. fractal_server/app/models/v2/history.py +53 -0
  8. fractal_server/app/routes/admin/v2/__init__.py +4 -0
  9. fractal_server/app/routes/admin/v2/accounting.py +108 -0
  10. fractal_server/app/routes/admin/v2/impersonate.py +35 -0
  11. fractal_server/app/routes/admin/v2/job.py +5 -13
  12. fractal_server/app/routes/admin/v2/task_group.py +4 -12
  13. fractal_server/app/routes/api/v2/__init__.py +2 -2
  14. fractal_server/app/routes/api/v2/_aux_functions.py +78 -0
  15. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +3 -3
  16. fractal_server/app/routes/api/v2/dataset.py +12 -9
  17. fractal_server/app/routes/api/v2/history.py +247 -0
  18. fractal_server/app/routes/api/v2/submit.py +1 -0
  19. fractal_server/app/routes/api/v2/task_group.py +2 -5
  20. fractal_server/app/routes/api/v2/workflow.py +18 -3
  21. fractal_server/app/routes/api/v2/workflowtask.py +22 -0
  22. fractal_server/app/routes/aux/__init__.py +0 -20
  23. fractal_server/app/runner/executors/base_runner.py +114 -0
  24. fractal_server/app/runner/{v2/_local → executors/local}/_local_config.py +3 -3
  25. fractal_server/app/runner/executors/local/_submit_setup.py +54 -0
  26. fractal_server/app/runner/executors/local/runner.py +200 -0
  27. fractal_server/app/runner/executors/{slurm → slurm_common}/_batching.py +1 -1
  28. fractal_server/app/runner/executors/{slurm → slurm_common}/_slurm_config.py +3 -3
  29. fractal_server/app/runner/{v2/_slurm_ssh → executors/slurm_common}/_submit_setup.py +13 -12
  30. fractal_server/app/runner/{v2/_slurm_common → executors/slurm_common}/get_slurm_config.py +9 -15
  31. fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_executor_wait_thread.py +1 -1
  32. fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_slurm_job.py +1 -1
  33. fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/executor.py +13 -14
  34. fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_check_jobs_status.py +11 -9
  35. fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_executor_wait_thread.py +3 -3
  36. fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_subprocess_run_as_user.py +2 -68
  37. fractal_server/app/runner/executors/slurm_sudo/runner.py +632 -0
  38. fractal_server/app/runner/task_files.py +70 -96
  39. fractal_server/app/runner/v2/__init__.py +9 -19
  40. fractal_server/app/runner/v2/_local.py +84 -0
  41. fractal_server/app/runner/v2/{_slurm_ssh/__init__.py → _slurm_ssh.py} +12 -13
  42. fractal_server/app/runner/v2/{_slurm_sudo/__init__.py → _slurm_sudo.py} +12 -12
  43. fractal_server/app/runner/v2/runner.py +106 -31
  44. fractal_server/app/runner/v2/runner_functions.py +88 -64
  45. fractal_server/app/runner/v2/runner_functions_low_level.py +20 -20
  46. fractal_server/app/schemas/v2/__init__.py +1 -0
  47. fractal_server/app/schemas/v2/accounting.py +18 -0
  48. fractal_server/app/schemas/v2/dataset.py +0 -17
  49. fractal_server/app/schemas/v2/history.py +23 -0
  50. fractal_server/config.py +58 -52
  51. fractal_server/migrations/versions/8223fcef886c_image_status.py +63 -0
  52. fractal_server/migrations/versions/87cd72a537a2_add_historyitem_table.py +68 -0
  53. fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +57 -0
  54. fractal_server/tasks/v2/utils_background.py +1 -1
  55. {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/METADATA +1 -1
  56. {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/RECORD +66 -55
  57. fractal_server/app/routes/api/v2/status.py +0 -168
  58. fractal_server/app/runner/executors/slurm/sudo/executor.py +0 -1281
  59. fractal_server/app/runner/v2/_local/__init__.py +0 -129
  60. fractal_server/app/runner/v2/_local/_submit_setup.py +0 -52
  61. fractal_server/app/runner/v2/_local/executor.py +0 -100
  62. fractal_server/app/runner/v2/_slurm_sudo/_submit_setup.py +0 -83
  63. fractal_server/app/runner/v2/handle_failed_job.py +0 -59
  64. /fractal_server/app/runner/executors/{slurm → local}/__init__.py +0 -0
  65. /fractal_server/app/runner/executors/{slurm/ssh → slurm_common}/__init__.py +0 -0
  66. /fractal_server/app/runner/executors/{_job_states.py → slurm_common/_job_states.py} +0 -0
  67. /fractal_server/app/runner/executors/{slurm → slurm_common}/remote.py +0 -0
  68. /fractal_server/app/runner/executors/{slurm → slurm_common}/utils_executors.py +0 -0
  69. /fractal_server/app/runner/executors/{slurm/sudo → slurm_ssh}/__init__.py +0 -0
  70. /fractal_server/app/runner/{v2/_slurm_common → executors/slurm_sudo}/__init__.py +0 -0
  71. {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/LICENSE +0 -0
  72. {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/WHEEL +0 -0
  73. {fractal_server-2.13.0.dist-info → fractal_server-2.14.0a0.dist-info}/entry_points.txt +0 -0
@@ -1,168 +0,0 @@
1
- from typing import Optional
2
-
3
- from fastapi import APIRouter
4
- from fastapi import Depends
5
- from fastapi import HTTPException
6
- from fastapi import status
7
-
8
- from .....logger import set_logger
9
- from ....db import AsyncSession
10
- from ....db import get_async_db
11
- from ....models.v2 import JobV2
12
- from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
13
- from ....schemas.v2.status import StatusReadV2
14
- from ._aux_functions import _get_dataset_check_owner
15
- from ._aux_functions import _get_submitted_jobs_statement
16
- from ._aux_functions import _get_workflow_check_owner
17
- from fractal_server.app.models import UserOAuth
18
- from fractal_server.app.routes.auth import current_active_user
19
-
20
- router = APIRouter()
21
-
22
- logger = set_logger(__name__)
23
-
24
-
25
- @router.get(
26
- "/project/{project_id}/status/",
27
- response_model=StatusReadV2,
28
- )
29
- async def get_workflowtask_status(
30
- project_id: int,
31
- dataset_id: int,
32
- workflow_id: int,
33
- user: UserOAuth = Depends(current_active_user),
34
- db: AsyncSession = Depends(get_async_db),
35
- ) -> Optional[StatusReadV2]:
36
- """
37
- Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
38
- on a given `DatasetV2`.
39
-
40
- *NOTE*: the current endpoint is not guaranteed to provide consistent
41
- results if the workflow task list is modified in a non-trivial way
42
- (that is, by adding intermediate tasks, removing tasks, or changing their
43
- order). See fractal-server GitHub issues: 793, 1083.
44
- """
45
- # Get the dataset DB entry
46
- output = await _get_dataset_check_owner(
47
- project_id=project_id,
48
- dataset_id=dataset_id,
49
- user_id=user.id,
50
- db=db,
51
- )
52
- dataset = output["dataset"]
53
-
54
- # Get the workflow DB entry
55
- workflow = await _get_workflow_check_owner(
56
- project_id=project_id,
57
- workflow_id=workflow_id,
58
- user_id=user.id,
59
- db=db,
60
- )
61
-
62
- # Check whether there exists a submitted job associated to this
63
- # workflow/dataset pair. If it does exist, it will be used later.
64
- # If there are multiple jobs, raise an error.
65
- stm = _get_submitted_jobs_statement()
66
- stm = stm.where(JobV2.dataset_id == dataset_id)
67
- stm = stm.where(JobV2.workflow_id == workflow_id)
68
- res = await db.execute(stm)
69
- running_jobs = res.scalars().all()
70
- if len(running_jobs) == 0:
71
- running_job = None
72
- elif len(running_jobs) == 1:
73
- running_job = running_jobs[0]
74
- else:
75
- string_ids = str([job.id for job in running_jobs])[1:-1]
76
- raise HTTPException(
77
- status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
78
- detail=(
79
- f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
80
- f" is linked to multiple active jobs: {string_ids}."
81
- ),
82
- )
83
-
84
- # Initialize empty dictionary for WorkflowTaskV2 status
85
- workflow_tasks_status_dict: dict = {}
86
-
87
- # Lowest priority: read status from DB, which corresponds to jobs that are
88
- # not running
89
- history = dataset.history
90
- for history_item in history:
91
- wftask_id = history_item["workflowtask"]["id"]
92
- wftask_status = history_item["status"]
93
- workflow_tasks_status_dict[wftask_id] = wftask_status
94
-
95
- if running_job is None:
96
- # If no job is running, the chronological-last history item is also the
97
- # positional-last workflow task to be included in the response.
98
- if len(history) > 0:
99
- last_valid_wftask_id = history[-1]["workflowtask"]["id"]
100
- else:
101
- last_valid_wftask_id = None
102
- else:
103
- # If a job is running, then gather more up-to-date information
104
-
105
- # Mid priority: Set all WorkflowTask's that are part of the running job
106
- # as "submitted"
107
- start = running_job.first_task_index
108
- end = running_job.last_task_index + 1
109
-
110
- running_job_wftasks = workflow.task_list[start:end]
111
- running_job_statuses = [
112
- workflow_tasks_status_dict.get(wft.id, None)
113
- for wft in running_job_wftasks
114
- ]
115
- try:
116
- first_submitted_index = running_job_statuses.index(
117
- WorkflowTaskStatusTypeV2.SUBMITTED
118
- )
119
- except ValueError:
120
- logger.warning(
121
- f"Job {running_job.id} is submitted but its task list does "
122
- f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
123
- )
124
- first_submitted_index = 0
125
-
126
- for wftask in running_job_wftasks[first_submitted_index:]:
127
- workflow_tasks_status_dict[
128
- wftask.id
129
- ] = WorkflowTaskStatusTypeV2.SUBMITTED
130
-
131
- # The last workflow task that is included in the submitted job is also
132
- # the positional-last workflow task to be included in the response.
133
- try:
134
- last_valid_wftask_id = workflow.task_list[end - 1].id
135
- except IndexError as e:
136
- logger.warning(
137
- f"Handled IndexError in `get_workflowtask_status` ({str(e)})."
138
- )
139
- logger.warning(
140
- "Additional information: "
141
- f"{running_job.first_task_index=}; "
142
- f"{running_job.last_task_index=}; "
143
- f"{len(workflow.task_list)=}; "
144
- f"{dataset_id=}; "
145
- f"{workflow_id=}."
146
- )
147
- last_valid_wftask_id = None
148
- logger.warning(f"Now setting {last_valid_wftask_id=}.")
149
-
150
- # Based on previously-gathered information, clean up the response body
151
- clean_workflow_tasks_status_dict = {}
152
- for wf_task in workflow.task_list:
153
- wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
154
- if wf_task_status is None:
155
- # If a wftask ID was not found, ignore it and continue
156
- continue
157
- clean_workflow_tasks_status_dict[str(wf_task.id)] = wf_task_status
158
- if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
159
- # Starting from the beginning of `workflow.task_list`, stop the
160
- # first time that you hit a failed job
161
- break
162
- if wf_task.id == last_valid_wftask_id:
163
- # Starting from the beginning of `workflow.task_list`, stop the
164
- # first time that you hit `last_valid_wftask_id``
165
- break
166
-
167
- response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
168
- return response_body