fractal-server 2.13.0__py3-none-any.whl → 2.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +3 -1
- fractal_server/app/models/linkusergroup.py +6 -2
- fractal_server/app/models/v2/__init__.py +11 -1
- fractal_server/app/models/v2/accounting.py +35 -0
- fractal_server/app/models/v2/dataset.py +1 -11
- fractal_server/app/models/v2/history.py +78 -0
- fractal_server/app/models/v2/job.py +10 -3
- fractal_server/app/models/v2/task_group.py +2 -2
- fractal_server/app/models/v2/workflow.py +1 -1
- fractal_server/app/models/v2/workflowtask.py +1 -1
- fractal_server/app/routes/admin/v2/__init__.py +4 -0
- fractal_server/app/routes/admin/v2/accounting.py +98 -0
- fractal_server/app/routes/admin/v2/impersonate.py +35 -0
- fractal_server/app/routes/admin/v2/job.py +5 -13
- fractal_server/app/routes/admin/v2/task.py +1 -1
- fractal_server/app/routes/admin/v2/task_group.py +4 -29
- fractal_server/app/routes/api/__init__.py +1 -1
- fractal_server/app/routes/api/v2/__init__.py +8 -2
- fractal_server/app/routes/api/v2/_aux_functions.py +66 -0
- fractal_server/app/routes/api/v2/_aux_functions_history.py +166 -0
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +3 -3
- fractal_server/app/routes/api/v2/dataset.py +0 -17
- fractal_server/app/routes/api/v2/history.py +544 -0
- fractal_server/app/routes/api/v2/images.py +31 -43
- fractal_server/app/routes/api/v2/job.py +30 -0
- fractal_server/app/routes/api/v2/project.py +1 -53
- fractal_server/app/routes/api/v2/{status.py → status_legacy.py} +6 -6
- fractal_server/app/routes/api/v2/submit.py +17 -14
- fractal_server/app/routes/api/v2/task.py +3 -10
- fractal_server/app/routes/api/v2/task_collection_custom.py +4 -9
- fractal_server/app/routes/api/v2/task_group.py +2 -22
- fractal_server/app/routes/api/v2/verify_image_types.py +61 -0
- fractal_server/app/routes/api/v2/workflow.py +28 -69
- fractal_server/app/routes/api/v2/workflowtask.py +53 -50
- fractal_server/app/routes/auth/group.py +0 -16
- fractal_server/app/routes/auth/oauth.py +5 -3
- fractal_server/app/routes/aux/__init__.py +0 -20
- fractal_server/app/routes/pagination.py +47 -0
- fractal_server/app/runner/components.py +0 -3
- fractal_server/app/runner/compress_folder.py +57 -29
- fractal_server/app/runner/exceptions.py +4 -0
- fractal_server/app/runner/executors/base_runner.py +157 -0
- fractal_server/app/runner/{v2/_local/_local_config.py → executors/local/get_local_config.py} +7 -9
- fractal_server/app/runner/executors/local/runner.py +248 -0
- fractal_server/app/runner/executors/{slurm → slurm_common}/_batching.py +1 -1
- fractal_server/app/runner/executors/{slurm → slurm_common}/_slurm_config.py +9 -7
- fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py +868 -0
- fractal_server/app/runner/{v2/_slurm_common → executors/slurm_common}/get_slurm_config.py +48 -17
- fractal_server/app/runner/executors/{slurm → slurm_common}/remote.py +36 -47
- fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py +134 -0
- fractal_server/app/runner/executors/slurm_ssh/runner.py +268 -0
- fractal_server/app/runner/executors/slurm_sudo/__init__.py +0 -0
- fractal_server/app/runner/executors/{slurm/sudo → slurm_sudo}/_subprocess_run_as_user.py +2 -83
- fractal_server/app/runner/executors/slurm_sudo/runner.py +193 -0
- fractal_server/app/runner/extract_archive.py +1 -3
- fractal_server/app/runner/task_files.py +134 -87
- fractal_server/app/runner/v2/__init__.py +0 -395
- fractal_server/app/runner/v2/_local.py +88 -0
- fractal_server/app/runner/v2/{_slurm_ssh/__init__.py → _slurm_ssh.py} +22 -19
- fractal_server/app/runner/v2/{_slurm_sudo/__init__.py → _slurm_sudo.py} +19 -15
- fractal_server/app/runner/v2/db_tools.py +119 -0
- fractal_server/app/runner/v2/runner.py +219 -98
- fractal_server/app/runner/v2/runner_functions.py +491 -189
- fractal_server/app/runner/v2/runner_functions_low_level.py +40 -43
- fractal_server/app/runner/v2/submit_workflow.py +358 -0
- fractal_server/app/runner/v2/task_interface.py +31 -0
- fractal_server/app/schemas/_validators.py +13 -24
- fractal_server/app/schemas/user.py +10 -7
- fractal_server/app/schemas/user_settings.py +9 -21
- fractal_server/app/schemas/v2/__init__.py +10 -1
- fractal_server/app/schemas/v2/accounting.py +18 -0
- fractal_server/app/schemas/v2/dataset.py +12 -94
- fractal_server/app/schemas/v2/dumps.py +26 -9
- fractal_server/app/schemas/v2/history.py +80 -0
- fractal_server/app/schemas/v2/job.py +15 -8
- fractal_server/app/schemas/v2/manifest.py +14 -7
- fractal_server/app/schemas/v2/project.py +9 -7
- fractal_server/app/schemas/v2/status_legacy.py +35 -0
- fractal_server/app/schemas/v2/task.py +72 -77
- fractal_server/app/schemas/v2/task_collection.py +14 -32
- fractal_server/app/schemas/v2/task_group.py +10 -9
- fractal_server/app/schemas/v2/workflow.py +10 -11
- fractal_server/app/schemas/v2/workflowtask.py +2 -21
- fractal_server/app/security/__init__.py +3 -3
- fractal_server/app/security/signup_email.py +2 -2
- fractal_server/config.py +91 -90
- fractal_server/images/tools.py +23 -0
- fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +50 -0
- fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +250 -0
- fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +57 -0
- fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +41 -0
- fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +36 -0
- fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +39 -0
- fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +120 -0
- fractal_server/ssh/_fabric.py +28 -14
- fractal_server/tasks/v2/local/collect.py +2 -2
- fractal_server/tasks/v2/ssh/collect.py +2 -2
- fractal_server/tasks/v2/templates/2_pip_install.sh +1 -1
- fractal_server/tasks/v2/templates/4_pip_show.sh +1 -1
- fractal_server/tasks/v2/utils_background.py +1 -20
- fractal_server/tasks/v2/utils_database.py +30 -17
- fractal_server/tasks/v2/utils_templates.py +6 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0.dist-info}/METADATA +4 -4
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0.dist-info}/RECORD +114 -99
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0.dist-info}/WHEEL +1 -1
- fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py +0 -126
- fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py +0 -116
- fractal_server/app/runner/executors/slurm/ssh/executor.py +0 -1386
- fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py +0 -71
- fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py +0 -130
- fractal_server/app/runner/executors/slurm/sudo/executor.py +0 -1281
- fractal_server/app/runner/v2/_local/__init__.py +0 -129
- fractal_server/app/runner/v2/_local/_submit_setup.py +0 -52
- fractal_server/app/runner/v2/_local/executor.py +0 -100
- fractal_server/app/runner/v2/_slurm_ssh/_submit_setup.py +0 -83
- fractal_server/app/runner/v2/_slurm_sudo/_submit_setup.py +0 -83
- fractal_server/app/runner/v2/handle_failed_job.py +0 -59
- fractal_server/app/schemas/v2/status.py +0 -16
- /fractal_server/app/{runner/executors/slurm → history}/__init__.py +0 -0
- /fractal_server/app/runner/executors/{slurm/ssh → local}/__init__.py +0 -0
- /fractal_server/app/runner/executors/{slurm/sudo → slurm_common}/__init__.py +0 -0
- /fractal_server/app/runner/executors/{_job_states.py → slurm_common/_job_states.py} +0 -0
- /fractal_server/app/runner/executors/{slurm → slurm_common}/utils_executors.py +0 -0
- /fractal_server/app/runner/{v2/_slurm_common → executors/slurm_ssh}/__init__.py +0 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0.dist-info}/LICENSE +0 -0
- {fractal_server-2.13.0.dist-info → fractal_server-2.14.0.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.
|
1
|
+
__VERSION__ = "2.14.0"
|
fractal_server/__main__.py
CHANGED
@@ -123,7 +123,9 @@ def set_db(skip_init_data: bool = False):
|
|
123
123
|
asyncio.run(
|
124
124
|
_create_first_user(
|
125
125
|
email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
|
126
|
-
password=
|
126
|
+
password=(
|
127
|
+
settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()
|
128
|
+
),
|
127
129
|
username=settings.FRACTAL_DEFAULT_ADMIN_USERNAME,
|
128
130
|
is_superuser=True,
|
129
131
|
is_verified=True,
|
@@ -13,8 +13,12 @@ class LinkUserGroup(SQLModel, table=True):
|
|
13
13
|
Crossing table between User and UserGroup
|
14
14
|
"""
|
15
15
|
|
16
|
-
group_id: int = Field(
|
17
|
-
|
16
|
+
group_id: int = Field(
|
17
|
+
foreign_key="usergroup.id", primary_key=True, ondelete="CASCADE"
|
18
|
+
)
|
19
|
+
user_id: int = Field(
|
20
|
+
foreign_key="user_oauth.id", primary_key=True, ondelete="CASCADE"
|
21
|
+
)
|
18
22
|
|
19
23
|
timestamp_created: datetime = Field(
|
20
24
|
default_factory=get_timestamp,
|
@@ -2,7 +2,12 @@
|
|
2
2
|
v2 `models` module
|
3
3
|
"""
|
4
4
|
from ..linkuserproject import LinkUserProjectV2
|
5
|
+
from .accounting import AccountingRecord
|
6
|
+
from .accounting import AccountingRecordSlurm
|
5
7
|
from .dataset import DatasetV2
|
8
|
+
from .history import HistoryImageCache
|
9
|
+
from .history import HistoryRun
|
10
|
+
from .history import HistoryUnit
|
6
11
|
from .job import JobV2
|
7
12
|
from .project import ProjectV2
|
8
13
|
from .task import TaskV2
|
@@ -12,6 +17,8 @@ from .workflow import WorkflowV2
|
|
12
17
|
from .workflowtask import WorkflowTaskV2
|
13
18
|
|
14
19
|
__all__ = [
|
20
|
+
"AccountingRecord",
|
21
|
+
"AccountingRecordSlurm",
|
15
22
|
"LinkUserProjectV2",
|
16
23
|
"DatasetV2",
|
17
24
|
"JobV2",
|
@@ -19,6 +26,9 @@ __all__ = [
|
|
19
26
|
"TaskGroupV2",
|
20
27
|
"TaskGroupActivityV2",
|
21
28
|
"TaskV2",
|
22
|
-
"WorkflowTaskV2",
|
23
29
|
"WorkflowV2",
|
30
|
+
"WorkflowTaskV2",
|
31
|
+
"HistoryRun",
|
32
|
+
"HistoryUnit",
|
33
|
+
"HistoryImageCache",
|
24
34
|
]
|
@@ -0,0 +1,35 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from sqlalchemy import Column
|
5
|
+
from sqlalchemy import Integer
|
6
|
+
from sqlalchemy.dialects.postgresql import ARRAY
|
7
|
+
from sqlalchemy.types import DateTime
|
8
|
+
from sqlmodel import Field
|
9
|
+
from sqlmodel import SQLModel
|
10
|
+
|
11
|
+
from ....utils import get_timestamp
|
12
|
+
|
13
|
+
|
14
|
+
class AccountingRecord(SQLModel, table=True):
|
15
|
+
id: Optional[int] = Field(default=None, primary_key=True)
|
16
|
+
user_id: int = Field(foreign_key="user_oauth.id", nullable=False)
|
17
|
+
timestamp: datetime = Field(
|
18
|
+
default_factory=get_timestamp,
|
19
|
+
sa_column=Column(DateTime(timezone=True), nullable=False),
|
20
|
+
)
|
21
|
+
num_tasks: int
|
22
|
+
num_new_images: int
|
23
|
+
|
24
|
+
|
25
|
+
class AccountingRecordSlurm(SQLModel, table=True):
|
26
|
+
id: Optional[int] = Field(default=None, primary_key=True)
|
27
|
+
user_id: int = Field(foreign_key="user_oauth.id", nullable=False)
|
28
|
+
timestamp: datetime = Field(
|
29
|
+
default_factory=get_timestamp,
|
30
|
+
sa_column=Column(DateTime(timezone=True), nullable=False),
|
31
|
+
)
|
32
|
+
slurm_job_ids: list[int] = Field(
|
33
|
+
default_factory=list,
|
34
|
+
sa_column=Column(ARRAY(Integer)),
|
35
|
+
)
|
@@ -11,7 +11,6 @@ from sqlmodel import Relationship
|
|
11
11
|
from sqlmodel import SQLModel
|
12
12
|
|
13
13
|
from ....utils import get_timestamp
|
14
|
-
from fractal_server.images.models import AttributeFiltersType
|
15
14
|
|
16
15
|
|
17
16
|
class DatasetV2(SQLModel, table=True):
|
@@ -20,7 +19,7 @@ class DatasetV2(SQLModel, table=True):
|
|
20
19
|
id: Optional[int] = Field(default=None, primary_key=True)
|
21
20
|
name: str
|
22
21
|
|
23
|
-
project_id: int = Field(foreign_key="projectv2.id")
|
22
|
+
project_id: int = Field(foreign_key="projectv2.id", ondelete="CASCADE")
|
24
23
|
project: "ProjectV2" = Relationship( # noqa: F821
|
25
24
|
sa_relationship_kwargs=dict(lazy="selectin"),
|
26
25
|
)
|
@@ -34,20 +33,11 @@ class DatasetV2(SQLModel, table=True):
|
|
34
33
|
sa_column=Column(DateTime(timezone=True), nullable=False),
|
35
34
|
)
|
36
35
|
|
37
|
-
# New in V2
|
38
|
-
|
39
36
|
zarr_dir: str
|
40
37
|
images: list[dict[str, Any]] = Field(
|
41
38
|
sa_column=Column(JSON, server_default="[]", nullable=False)
|
42
39
|
)
|
43
40
|
|
44
|
-
type_filters: dict[str, bool] = Field(
|
45
|
-
sa_column=Column(JSON, nullable=False, server_default="{}")
|
46
|
-
)
|
47
|
-
attribute_filters: AttributeFiltersType = Field(
|
48
|
-
sa_column=Column(JSON, nullable=False, server_default="{}")
|
49
|
-
)
|
50
|
-
|
51
41
|
@property
|
52
42
|
def image_zarr_urls(self) -> list[str]:
|
53
43
|
return [image["zarr_url"] for image in self.images]
|
@@ -0,0 +1,78 @@
|
|
1
|
+
from datetime import datetime
|
2
|
+
from typing import Any
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from pydantic import ConfigDict
|
6
|
+
from sqlalchemy import Column
|
7
|
+
from sqlalchemy import String
|
8
|
+
from sqlalchemy.dialects.postgresql import ARRAY
|
9
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
10
|
+
from sqlalchemy.types import DateTime
|
11
|
+
from sqlmodel import Field
|
12
|
+
from sqlmodel import SQLModel
|
13
|
+
|
14
|
+
from ....utils import get_timestamp
|
15
|
+
|
16
|
+
|
17
|
+
class HistoryRun(SQLModel, table=True):
|
18
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
19
|
+
|
20
|
+
id: Optional[int] = Field(default=None, primary_key=True)
|
21
|
+
dataset_id: int = Field(
|
22
|
+
foreign_key="datasetv2.id",
|
23
|
+
ondelete="CASCADE",
|
24
|
+
)
|
25
|
+
workflowtask_id: Optional[int] = Field(
|
26
|
+
foreign_key="workflowtaskv2.id",
|
27
|
+
default=None,
|
28
|
+
ondelete="SET NULL",
|
29
|
+
)
|
30
|
+
job_id: int = Field(foreign_key="jobv2.id")
|
31
|
+
|
32
|
+
workflowtask_dump: dict[str, Any] = Field(
|
33
|
+
sa_column=Column(JSONB, nullable=False),
|
34
|
+
)
|
35
|
+
task_group_dump: dict[str, Any] = Field(
|
36
|
+
sa_column=Column(JSONB, nullable=False),
|
37
|
+
)
|
38
|
+
|
39
|
+
timestamp_started: datetime = Field(
|
40
|
+
sa_column=Column(DateTime(timezone=True), nullable=False),
|
41
|
+
default_factory=get_timestamp,
|
42
|
+
)
|
43
|
+
status: str
|
44
|
+
num_available_images: int
|
45
|
+
|
46
|
+
|
47
|
+
class HistoryUnit(SQLModel, table=True):
|
48
|
+
id: Optional[int] = Field(default=None, primary_key=True)
|
49
|
+
history_run_id: int = Field(
|
50
|
+
foreign_key="historyrun.id",
|
51
|
+
ondelete="CASCADE",
|
52
|
+
)
|
53
|
+
|
54
|
+
logfile: str
|
55
|
+
status: str
|
56
|
+
zarr_urls: list[str] = Field(
|
57
|
+
sa_column=Column(ARRAY(String)),
|
58
|
+
default_factory=list,
|
59
|
+
)
|
60
|
+
|
61
|
+
|
62
|
+
class HistoryImageCache(SQLModel, table=True):
|
63
|
+
zarr_url: str = Field(primary_key=True)
|
64
|
+
dataset_id: int = Field(
|
65
|
+
primary_key=True,
|
66
|
+
foreign_key="datasetv2.id",
|
67
|
+
ondelete="CASCADE",
|
68
|
+
)
|
69
|
+
workflowtask_id: int = Field(
|
70
|
+
primary_key=True,
|
71
|
+
foreign_key="workflowtaskv2.id",
|
72
|
+
ondelete="CASCADE",
|
73
|
+
)
|
74
|
+
|
75
|
+
latest_history_unit_id: int = Field(
|
76
|
+
foreign_key="historyunit.id",
|
77
|
+
ondelete="CASCADE",
|
78
|
+
)
|
@@ -18,11 +18,15 @@ class JobV2(SQLModel, table=True):
|
|
18
18
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
19
19
|
|
20
20
|
id: Optional[int] = Field(default=None, primary_key=True)
|
21
|
-
project_id: Optional[int] = Field(
|
21
|
+
project_id: Optional[int] = Field(
|
22
|
+
foreign_key="projectv2.id", default=None, ondelete="SET NULL"
|
23
|
+
)
|
22
24
|
workflow_id: Optional[int] = Field(
|
23
|
-
foreign_key="workflowv2.id", default=None
|
25
|
+
foreign_key="workflowv2.id", default=None, ondelete="SET NULL"
|
26
|
+
)
|
27
|
+
dataset_id: Optional[int] = Field(
|
28
|
+
foreign_key="datasetv2.id", default=None, ondelete="SET NULL"
|
24
29
|
)
|
25
|
-
dataset_id: Optional[int] = Field(foreign_key="datasetv2.id", default=None)
|
26
30
|
|
27
31
|
user_email: str = Field(nullable=False)
|
28
32
|
slurm_account: Optional[str] = None
|
@@ -56,3 +60,6 @@ class JobV2(SQLModel, table=True):
|
|
56
60
|
attribute_filters: AttributeFiltersType = Field(
|
57
61
|
sa_column=Column(JSON, nullable=False, server_default="{}")
|
58
62
|
)
|
63
|
+
type_filters: dict[str, bool] = Field(
|
64
|
+
sa_column=Column(JSON, nullable=False, server_default="{}")
|
65
|
+
)
|
@@ -23,7 +23,7 @@ class TaskGroupV2(SQLModel, table=True):
|
|
23
23
|
|
24
24
|
user_id: int = Field(foreign_key="user_oauth.id")
|
25
25
|
user_group_id: Optional[int] = Field(
|
26
|
-
foreign_key="usergroup.id", default=None
|
26
|
+
foreign_key="usergroup.id", default=None, ondelete="SET NULL"
|
27
27
|
)
|
28
28
|
|
29
29
|
origin: str
|
@@ -100,7 +100,7 @@ class TaskGroupActivityV2(SQLModel, table=True):
|
|
100
100
|
id: Optional[int] = Field(default=None, primary_key=True)
|
101
101
|
user_id: int = Field(foreign_key="user_oauth.id")
|
102
102
|
taskgroupv2_id: Optional[int] = Field(
|
103
|
-
default=None, foreign_key="taskgroupv2.id"
|
103
|
+
default=None, foreign_key="taskgroupv2.id", ondelete="SET NULL"
|
104
104
|
)
|
105
105
|
timestamp_started: datetime = Field(
|
106
106
|
default_factory=get_timestamp,
|
@@ -16,7 +16,7 @@ class WorkflowV2(SQLModel, table=True):
|
|
16
16
|
|
17
17
|
id: Optional[int] = Field(default=None, primary_key=True)
|
18
18
|
name: str
|
19
|
-
project_id: int = Field(foreign_key="projectv2.id")
|
19
|
+
project_id: int = Field(foreign_key="projectv2.id", ondelete="CASCADE")
|
20
20
|
project: "ProjectV2" = Relationship( # noqa: F821
|
21
21
|
sa_relationship_kwargs=dict(lazy="selectin"),
|
22
22
|
)
|
@@ -16,7 +16,7 @@ class WorkflowTaskV2(SQLModel, table=True):
|
|
16
16
|
|
17
17
|
id: Optional[int] = Field(default=None, primary_key=True)
|
18
18
|
|
19
|
-
workflow_id: int = Field(foreign_key="workflowv2.id")
|
19
|
+
workflow_id: int = Field(foreign_key="workflowv2.id", ondelete="CASCADE")
|
20
20
|
order: Optional[int] = None
|
21
21
|
meta_parallel: Optional[dict[str, Any]] = Field(
|
22
22
|
sa_column=Column(JSON), default=None
|
@@ -3,6 +3,8 @@
|
|
3
3
|
"""
|
4
4
|
from fastapi import APIRouter
|
5
5
|
|
6
|
+
from .accounting import router as accounting_router
|
7
|
+
from .impersonate import router as impersonate_router
|
6
8
|
from .job import router as job_router
|
7
9
|
from .project import router as project_router
|
8
10
|
from .task import router as task_router
|
@@ -11,6 +13,7 @@ from .task_group_lifecycle import router as task_group_lifecycle_router
|
|
11
13
|
|
12
14
|
router_admin_v2 = APIRouter()
|
13
15
|
|
16
|
+
router_admin_v2.include_router(accounting_router, prefix="/accounting")
|
14
17
|
router_admin_v2.include_router(job_router, prefix="/job")
|
15
18
|
router_admin_v2.include_router(project_router, prefix="/project")
|
16
19
|
router_admin_v2.include_router(task_router, prefix="/task")
|
@@ -18,3 +21,4 @@ router_admin_v2.include_router(task_group_router, prefix="/task-group")
|
|
18
21
|
router_admin_v2.include_router(
|
19
22
|
task_group_lifecycle_router, prefix="/task-group"
|
20
23
|
)
|
24
|
+
router_admin_v2.include_router(impersonate_router, prefix="/impersonate")
|
@@ -0,0 +1,98 @@
|
|
1
|
+
from itertools import chain
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from fastapi import APIRouter
|
5
|
+
from fastapi import Depends
|
6
|
+
from fastapi.responses import JSONResponse
|
7
|
+
from pydantic import BaseModel
|
8
|
+
from pydantic.types import AwareDatetime
|
9
|
+
from sqlmodel import func
|
10
|
+
from sqlmodel import select
|
11
|
+
|
12
|
+
from fractal_server.app.db import AsyncSession
|
13
|
+
from fractal_server.app.db import get_async_db
|
14
|
+
from fractal_server.app.models import UserOAuth
|
15
|
+
from fractal_server.app.models.v2 import AccountingRecord
|
16
|
+
from fractal_server.app.models.v2 import AccountingRecordSlurm
|
17
|
+
from fractal_server.app.routes.auth import current_active_superuser
|
18
|
+
from fractal_server.app.routes.pagination import get_pagination_params
|
19
|
+
from fractal_server.app.routes.pagination import PaginationRequest
|
20
|
+
from fractal_server.app.routes.pagination import PaginationResponse
|
21
|
+
from fractal_server.app.schemas.v2 import AccountingRecordRead
|
22
|
+
|
23
|
+
|
24
|
+
class AccountingQuery(BaseModel):
|
25
|
+
user_id: Optional[int] = None
|
26
|
+
timestamp_min: Optional[AwareDatetime] = None
|
27
|
+
timestamp_max: Optional[AwareDatetime] = None
|
28
|
+
|
29
|
+
|
30
|
+
router = APIRouter()
|
31
|
+
|
32
|
+
|
33
|
+
@router.post("/", response_model=PaginationResponse[AccountingRecordRead])
|
34
|
+
async def query_accounting(
|
35
|
+
query: AccountingQuery,
|
36
|
+
# Dependencies
|
37
|
+
pagination: PaginationRequest = Depends(get_pagination_params),
|
38
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
39
|
+
db: AsyncSession = Depends(get_async_db),
|
40
|
+
) -> PaginationResponse[AccountingRecordRead]:
|
41
|
+
page = pagination.page
|
42
|
+
page_size = pagination.page_size
|
43
|
+
|
44
|
+
stm = select(AccountingRecord).order_by(AccountingRecord.id)
|
45
|
+
stm_count = select(func.count(AccountingRecord.id))
|
46
|
+
if query.user_id is not None:
|
47
|
+
stm = stm.where(AccountingRecord.user_id == query.user_id)
|
48
|
+
stm_count = stm_count.where(AccountingRecord.user_id == query.user_id)
|
49
|
+
if query.timestamp_min is not None:
|
50
|
+
stm = stm.where(AccountingRecord.timestamp >= query.timestamp_min)
|
51
|
+
stm_count = stm_count.where(
|
52
|
+
AccountingRecord.timestamp >= query.timestamp_min
|
53
|
+
)
|
54
|
+
if query.timestamp_max is not None:
|
55
|
+
stm = stm.where(AccountingRecord.timestamp <= query.timestamp_max)
|
56
|
+
stm_count = stm_count.where(
|
57
|
+
AccountingRecord.timestamp <= query.timestamp_max
|
58
|
+
)
|
59
|
+
|
60
|
+
res_total_count = await db.execute(stm_count)
|
61
|
+
total_count = res_total_count.scalar()
|
62
|
+
|
63
|
+
if page_size is not None:
|
64
|
+
stm = stm.offset((page - 1) * page_size).limit(page_size)
|
65
|
+
else:
|
66
|
+
page_size = total_count
|
67
|
+
|
68
|
+
res = await db.execute(stm)
|
69
|
+
records = res.scalars().all()
|
70
|
+
|
71
|
+
return PaginationResponse[AccountingRecordRead](
|
72
|
+
total_count=total_count,
|
73
|
+
page_size=page_size,
|
74
|
+
current_page=page,
|
75
|
+
items=[record.model_dump() for record in records],
|
76
|
+
)
|
77
|
+
|
78
|
+
|
79
|
+
@router.post("/slurm/")
|
80
|
+
async def query_accounting_slurm(
|
81
|
+
query: AccountingQuery,
|
82
|
+
# dependencies
|
83
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
84
|
+
db: AsyncSession = Depends(get_async_db),
|
85
|
+
) -> JSONResponse:
|
86
|
+
|
87
|
+
stm = select(AccountingRecordSlurm.slurm_job_ids)
|
88
|
+
if query.user_id is not None:
|
89
|
+
stm = stm.where(AccountingRecordSlurm.user_id == query.user_id)
|
90
|
+
if query.timestamp_min is not None:
|
91
|
+
stm = stm.where(AccountingRecordSlurm.timestamp >= query.timestamp_min)
|
92
|
+
if query.timestamp_max is not None:
|
93
|
+
stm = stm.where(AccountingRecordSlurm.timestamp <= query.timestamp_max)
|
94
|
+
|
95
|
+
res = await db.execute(stm)
|
96
|
+
nested_slurm_job_ids = res.scalars().all()
|
97
|
+
aggregated_slurm_job_ids = list(chain(*nested_slurm_job_ids))
|
98
|
+
return JSONResponse(content=aggregated_slurm_job_ids, status_code=200)
|
@@ -0,0 +1,35 @@
|
|
1
|
+
from fastapi import APIRouter
|
2
|
+
from fastapi import Depends
|
3
|
+
from fastapi.responses import JSONResponse
|
4
|
+
from fastapi_users.authentication import JWTStrategy
|
5
|
+
|
6
|
+
from fractal_server.app.db import AsyncSession
|
7
|
+
from fractal_server.app.db import get_async_db
|
8
|
+
from fractal_server.app.models import UserOAuth
|
9
|
+
from fractal_server.app.routes.auth import current_active_superuser
|
10
|
+
from fractal_server.app.routes.auth._aux_auth import _user_or_404
|
11
|
+
from fractal_server.config import get_settings
|
12
|
+
from fractal_server.syringe import Inject
|
13
|
+
|
14
|
+
router = APIRouter()
|
15
|
+
|
16
|
+
|
17
|
+
@router.get("/{user_id}/")
|
18
|
+
async def impersonate_user(
|
19
|
+
user_id: int,
|
20
|
+
superuser: UserOAuth = Depends(current_active_superuser),
|
21
|
+
db: AsyncSession = Depends(get_async_db),
|
22
|
+
) -> JSONResponse:
|
23
|
+
user = await _user_or_404(user_id, db)
|
24
|
+
|
25
|
+
settings = Inject(get_settings)
|
26
|
+
jwt_strategy = JWTStrategy(
|
27
|
+
secret=settings.JWT_SECRET_KEY, # type: ignore
|
28
|
+
lifetime_seconds=7200, # 2 hours
|
29
|
+
)
|
30
|
+
token = await jwt_strategy.write_token(user)
|
31
|
+
|
32
|
+
return JSONResponse(
|
33
|
+
content={"access_token": token, "token_type": "bearer"},
|
34
|
+
status_code=200,
|
35
|
+
)
|
@@ -1,4 +1,3 @@
|
|
1
|
-
from datetime import datetime
|
2
1
|
from pathlib import Path
|
3
2
|
from typing import Optional
|
4
3
|
|
@@ -8,6 +7,7 @@ from fastapi import HTTPException
|
|
8
7
|
from fastapi import Response
|
9
8
|
from fastapi import status
|
10
9
|
from fastapi.responses import StreamingResponse
|
10
|
+
from pydantic.types import AwareDatetime
|
11
11
|
from sqlmodel import select
|
12
12
|
|
13
13
|
from fractal_server.app.db import AsyncSession
|
@@ -16,7 +16,6 @@ from fractal_server.app.models import UserOAuth
|
|
16
16
|
from fractal_server.app.models.v2 import JobV2
|
17
17
|
from fractal_server.app.models.v2 import ProjectV2
|
18
18
|
from fractal_server.app.routes.auth import current_active_superuser
|
19
|
-
from fractal_server.app.routes.aux import _raise_if_naive_datetime
|
20
19
|
from fractal_server.app.routes.aux._job import _write_shutdown_file
|
21
20
|
from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
|
22
21
|
from fractal_server.app.runner.filenames import WORKFLOW_LOG_FILENAME
|
@@ -37,10 +36,10 @@ async def view_job(
|
|
37
36
|
dataset_id: Optional[int] = None,
|
38
37
|
workflow_id: Optional[int] = None,
|
39
38
|
status: Optional[JobStatusTypeV2] = None,
|
40
|
-
start_timestamp_min: Optional[
|
41
|
-
start_timestamp_max: Optional[
|
42
|
-
end_timestamp_min: Optional[
|
43
|
-
end_timestamp_max: Optional[
|
39
|
+
start_timestamp_min: Optional[AwareDatetime] = None,
|
40
|
+
start_timestamp_max: Optional[AwareDatetime] = None,
|
41
|
+
end_timestamp_min: Optional[AwareDatetime] = None,
|
42
|
+
end_timestamp_max: Optional[AwareDatetime] = None,
|
44
43
|
log: bool = True,
|
45
44
|
user: UserOAuth = Depends(current_active_superuser),
|
46
45
|
db: AsyncSession = Depends(get_async_db),
|
@@ -67,13 +66,6 @@ async def view_job(
|
|
67
66
|
`job.log` is set to `None`.
|
68
67
|
"""
|
69
68
|
|
70
|
-
_raise_if_naive_datetime(
|
71
|
-
start_timestamp_min,
|
72
|
-
start_timestamp_max,
|
73
|
-
end_timestamp_min,
|
74
|
-
end_timestamp_max,
|
75
|
-
)
|
76
|
-
|
77
69
|
stm = select(JobV2)
|
78
70
|
|
79
71
|
if id is not None:
|
@@ -68,7 +68,7 @@ async def query_tasks(
|
|
68
68
|
db: AsyncSession = Depends(get_async_db),
|
69
69
|
) -> list[TaskV2Info]:
|
70
70
|
"""
|
71
|
-
Query `TaskV2` table and get
|
71
|
+
Query `TaskV2` table and get information about related items
|
72
72
|
(WorkflowV2s and ProjectV2s)
|
73
73
|
|
74
74
|
Args:
|
@@ -1,4 +1,3 @@
|
|
1
|
-
from datetime import datetime
|
2
1
|
from typing import Optional
|
3
2
|
|
4
3
|
from fastapi import APIRouter
|
@@ -6,6 +5,7 @@ from fastapi import Depends
|
|
6
5
|
from fastapi import HTTPException
|
7
6
|
from fastapi import Response
|
8
7
|
from fastapi import status
|
8
|
+
from pydantic.types import AwareDatetime
|
9
9
|
from sqlalchemy.sql.operators import is_
|
10
10
|
from sqlalchemy.sql.operators import is_not
|
11
11
|
from sqlmodel import select
|
@@ -20,7 +20,6 @@ from fractal_server.app.routes.auth import current_active_superuser
|
|
20
20
|
from fractal_server.app.routes.auth._aux_auth import (
|
21
21
|
_verify_user_belongs_to_group,
|
22
22
|
)
|
23
|
-
from fractal_server.app.routes.aux import _raise_if_naive_datetime
|
24
23
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
25
24
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
26
25
|
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
@@ -42,13 +41,11 @@ async def get_task_group_activity_list(
|
|
42
41
|
pkg_name: Optional[str] = None,
|
43
42
|
status: Optional[TaskGroupActivityStatusV2] = None,
|
44
43
|
action: Optional[TaskGroupActivityActionV2] = None,
|
45
|
-
timestamp_started_min: Optional[
|
44
|
+
timestamp_started_min: Optional[AwareDatetime] = None,
|
46
45
|
superuser: UserOAuth = Depends(current_active_superuser),
|
47
46
|
db: AsyncSession = Depends(get_async_db),
|
48
47
|
) -> list[TaskGroupActivityV2Read]:
|
49
48
|
|
50
|
-
_raise_if_naive_datetime(timestamp_started_min)
|
51
|
-
|
52
49
|
stm = select(TaskGroupActivityV2)
|
53
50
|
if task_group_activity_id is not None:
|
54
51
|
stm = stm.where(TaskGroupActivityV2.id == task_group_activity_id)
|
@@ -96,19 +93,14 @@ async def query_task_group_list(
|
|
96
93
|
active: Optional[bool] = None,
|
97
94
|
pkg_name: Optional[str] = None,
|
98
95
|
origin: Optional[TaskGroupV2OriginEnum] = None,
|
99
|
-
timestamp_last_used_min: Optional[
|
100
|
-
timestamp_last_used_max: Optional[
|
96
|
+
timestamp_last_used_min: Optional[AwareDatetime] = None,
|
97
|
+
timestamp_last_used_max: Optional[AwareDatetime] = None,
|
101
98
|
user: UserOAuth = Depends(current_active_superuser),
|
102
99
|
db: AsyncSession = Depends(get_async_db),
|
103
100
|
) -> list[TaskGroupReadV2]:
|
104
101
|
|
105
102
|
stm = select(TaskGroupV2)
|
106
103
|
|
107
|
-
_raise_if_naive_datetime(
|
108
|
-
timestamp_last_used_max,
|
109
|
-
timestamp_last_used_min,
|
110
|
-
)
|
111
|
-
|
112
104
|
if user_group_id is not None and private is True:
|
113
105
|
raise HTTPException(
|
114
106
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
@@ -200,23 +192,6 @@ async def delete_task_group(
|
|
200
192
|
detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
|
201
193
|
)
|
202
194
|
|
203
|
-
# Cascade operations: set foreign-keys to null for TaskGroupActivityV2
|
204
|
-
# which are in relationship with the current TaskGroupV2
|
205
|
-
logger.debug("Start of cascade operations on TaskGroupActivityV2.")
|
206
|
-
stm = select(TaskGroupActivityV2).where(
|
207
|
-
TaskGroupActivityV2.taskgroupv2_id == task_group_id
|
208
|
-
)
|
209
|
-
res = await db.execute(stm)
|
210
|
-
task_group_activity_list = res.scalars().all()
|
211
|
-
for task_group_activity in task_group_activity_list:
|
212
|
-
logger.debug(
|
213
|
-
f"Setting TaskGroupActivityV2[{task_group_activity.id}]"
|
214
|
-
".taskgroupv2_id to None."
|
215
|
-
)
|
216
|
-
task_group_activity.taskgroupv2_id = None
|
217
|
-
db.add(task_group_activity)
|
218
|
-
logger.debug("End of cascade operations on TaskGroupActivityV2.")
|
219
|
-
|
220
195
|
await db.delete(task_group)
|
221
196
|
await db.commit()
|
222
197
|
|
@@ -4,16 +4,18 @@
|
|
4
4
|
from fastapi import APIRouter
|
5
5
|
|
6
6
|
from .dataset import router as dataset_router_v2
|
7
|
+
from .history import router as history_router_v2
|
7
8
|
from .images import router as images_routes_v2
|
8
9
|
from .job import router as job_router_v2
|
9
10
|
from .project import router as project_router_v2
|
10
|
-
from .
|
11
|
+
from .status_legacy import router as status_legacy_router_v2
|
11
12
|
from .submit import router as submit_job_router_v2
|
12
13
|
from .task import router as task_router_v2
|
13
14
|
from .task_collection import router as task_collection_router_v2
|
14
15
|
from .task_collection_custom import router as task_collection_router_v2_custom
|
15
16
|
from .task_group import router as task_group_router_v2
|
16
17
|
from .task_group_lifecycle import router as task_group_lifecycle_router_v2
|
18
|
+
from .verify_image_types import router as verify_image_types_router
|
17
19
|
from .workflow import router as workflow_router_v2
|
18
20
|
from .workflow_import import router as workflow_import_router_v2
|
19
21
|
from .workflowtask import router as workflowtask_router_v2
|
@@ -24,10 +26,15 @@ from fractal_server.syringe import Inject
|
|
24
26
|
router_api_v2 = APIRouter()
|
25
27
|
|
26
28
|
router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
|
29
|
+
router_api_v2.include_router(verify_image_types_router, tags=["V2 Job"])
|
27
30
|
router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
|
28
31
|
router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
|
29
32
|
router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
|
30
33
|
router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
|
34
|
+
router_api_v2.include_router(history_router_v2, tags=["V2 History"])
|
35
|
+
router_api_v2.include_router(
|
36
|
+
status_legacy_router_v2, tags=["V2 Status Legacy"]
|
37
|
+
)
|
31
38
|
|
32
39
|
|
33
40
|
settings = Inject(get_settings)
|
@@ -56,4 +63,3 @@ router_api_v2.include_router(
|
|
56
63
|
workflow_import_router_v2, tags=["V2 Workflow Import"]
|
57
64
|
)
|
58
65
|
router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
|
59
|
-
router_api_v2.include_router(status_router_v2, tags=["V2 Status"])
|