fractal-server 2.16.5__py3-none-any.whl → 2.17.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +129 -22
- fractal_server/app/db/__init__.py +9 -11
- fractal_server/app/models/security.py +7 -3
- fractal_server/app/models/user_settings.py +0 -4
- fractal_server/app/models/v2/__init__.py +4 -0
- fractal_server/app/models/v2/job.py +3 -4
- fractal_server/app/models/v2/profile.py +16 -0
- fractal_server/app/models/v2/project.py +3 -0
- fractal_server/app/models/v2/resource.py +130 -0
- fractal_server/app/models/v2/task_group.py +3 -0
- fractal_server/app/routes/admin/v2/__init__.py +4 -0
- fractal_server/app/routes/admin/v2/_aux_functions.py +55 -0
- fractal_server/app/routes/admin/v2/profile.py +86 -0
- fractal_server/app/routes/admin/v2/resource.py +229 -0
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +48 -82
- fractal_server/app/routes/api/__init__.py +26 -7
- fractal_server/app/routes/api/v2/_aux_functions.py +27 -1
- fractal_server/app/routes/api/v2/_aux_functions_history.py +2 -2
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +3 -3
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -7
- fractal_server/app/routes/api/v2/project.py +5 -1
- fractal_server/app/routes/api/v2/submit.py +32 -24
- fractal_server/app/routes/api/v2/task.py +5 -0
- fractal_server/app/routes/api/v2/task_collection.py +36 -47
- fractal_server/app/routes/api/v2/task_collection_custom.py +11 -5
- fractal_server/app/routes/api/v2/task_collection_pixi.py +34 -40
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +39 -82
- fractal_server/app/routes/api/v2/workflow_import.py +4 -3
- fractal_server/app/routes/auth/_aux_auth.py +3 -3
- fractal_server/app/routes/auth/current_user.py +45 -7
- fractal_server/app/routes/auth/oauth.py +1 -1
- fractal_server/app/routes/auth/users.py +9 -0
- fractal_server/app/routes/aux/_runner.py +2 -1
- fractal_server/app/routes/aux/validate_user_profile.py +62 -0
- fractal_server/app/routes/aux/validate_user_settings.py +12 -9
- fractal_server/app/schemas/user.py +20 -13
- fractal_server/app/schemas/user_settings.py +0 -4
- fractal_server/app/schemas/v2/__init__.py +11 -0
- fractal_server/app/schemas/v2/profile.py +72 -0
- fractal_server/app/schemas/v2/resource.py +117 -0
- fractal_server/app/security/__init__.py +6 -13
- fractal_server/app/security/signup_email.py +2 -2
- fractal_server/app/user_settings.py +2 -12
- fractal_server/config/__init__.py +23 -0
- fractal_server/config/_database.py +58 -0
- fractal_server/config/_email.py +170 -0
- fractal_server/config/_init_data.py +27 -0
- fractal_server/config/_main.py +216 -0
- fractal_server/config/_settings_config.py +7 -0
- fractal_server/images/tools.py +3 -3
- fractal_server/logger.py +3 -3
- fractal_server/main.py +14 -21
- fractal_server/migrations/versions/90f6508c6379_drop_useroauth_username.py +36 -0
- fractal_server/migrations/versions/a80ac5a352bf_resource_profile.py +195 -0
- fractal_server/runner/config/__init__.py +2 -0
- fractal_server/runner/config/_local.py +21 -0
- fractal_server/runner/config/_slurm.py +128 -0
- fractal_server/runner/config/slurm_mem_to_MB.py +63 -0
- fractal_server/runner/exceptions.py +4 -0
- fractal_server/runner/executors/base_runner.py +17 -7
- fractal_server/runner/executors/local/get_local_config.py +21 -86
- fractal_server/runner/executors/local/runner.py +48 -5
- fractal_server/runner/executors/slurm_common/_batching.py +2 -2
- fractal_server/runner/executors/slurm_common/base_slurm_runner.py +59 -25
- fractal_server/runner/executors/slurm_common/get_slurm_config.py +38 -54
- fractal_server/runner/executors/slurm_common/remote.py +1 -1
- fractal_server/runner/executors/slurm_common/{_slurm_config.py → slurm_config.py} +3 -254
- fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -1
- fractal_server/runner/executors/slurm_ssh/runner.py +12 -14
- fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +2 -2
- fractal_server/runner/executors/slurm_sudo/runner.py +12 -12
- fractal_server/runner/v2/_local.py +36 -21
- fractal_server/runner/v2/_slurm_ssh.py +40 -4
- fractal_server/runner/v2/_slurm_sudo.py +41 -11
- fractal_server/runner/v2/db_tools.py +1 -1
- fractal_server/runner/v2/runner.py +3 -11
- fractal_server/runner/v2/runner_functions.py +42 -28
- fractal_server/runner/v2/submit_workflow.py +87 -108
- fractal_server/runner/versions.py +8 -3
- fractal_server/ssh/_fabric.py +6 -6
- fractal_server/tasks/config/__init__.py +3 -0
- fractal_server/tasks/config/_pixi.py +127 -0
- fractal_server/tasks/config/_python.py +51 -0
- fractal_server/tasks/v2/local/_utils.py +7 -7
- fractal_server/tasks/v2/local/collect.py +13 -5
- fractal_server/tasks/v2/local/collect_pixi.py +26 -10
- fractal_server/tasks/v2/local/deactivate.py +7 -1
- fractal_server/tasks/v2/local/deactivate_pixi.py +5 -1
- fractal_server/tasks/v2/local/delete.py +4 -0
- fractal_server/tasks/v2/local/reactivate.py +13 -5
- fractal_server/tasks/v2/local/reactivate_pixi.py +27 -9
- fractal_server/tasks/v2/ssh/_pixi_slurm_ssh.py +11 -10
- fractal_server/tasks/v2/ssh/_utils.py +6 -7
- fractal_server/tasks/v2/ssh/collect.py +19 -12
- fractal_server/tasks/v2/ssh/collect_pixi.py +34 -16
- fractal_server/tasks/v2/ssh/deactivate.py +12 -8
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +14 -10
- fractal_server/tasks/v2/ssh/delete.py +12 -9
- fractal_server/tasks/v2/ssh/reactivate.py +18 -12
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +36 -17
- fractal_server/tasks/v2/templates/4_pip_show.sh +4 -6
- fractal_server/tasks/v2/utils_database.py +2 -2
- fractal_server/tasks/v2/utils_python_interpreter.py +8 -16
- fractal_server/tasks/v2/utils_templates.py +7 -10
- fractal_server/utils.py +1 -1
- {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/METADATA +5 -5
- {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/RECORD +112 -90
- {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/WHEEL +1 -1
- fractal_server/config.py +0 -906
- /fractal_server/{runner → app}/shutdown.py +0 -0
- {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info/licenses}/LICENSE +0 -0
|
@@ -9,14 +9,14 @@ from ..utils_pixi import parse_collect_stdout
|
|
|
9
9
|
from ..utils_pixi import SOURCE_DIR_NAME
|
|
10
10
|
from ._utils import edit_pyproject_toml_in_place_local
|
|
11
11
|
from fractal_server.app.db import get_sync_db
|
|
12
|
+
from fractal_server.app.models import Profile
|
|
13
|
+
from fractal_server.app.models import Resource
|
|
12
14
|
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
|
13
15
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
14
16
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
15
17
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
|
16
|
-
from fractal_server.config import get_settings
|
|
17
18
|
from fractal_server.logger import reset_logger_handlers
|
|
18
19
|
from fractal_server.logger import set_logger
|
|
19
|
-
from fractal_server.syringe import Inject
|
|
20
20
|
from fractal_server.tasks.utils import get_log_path
|
|
21
21
|
from fractal_server.tasks.v2.local._utils import _customize_and_run_template
|
|
22
22
|
from fractal_server.tasks.v2.local._utils import check_task_files_exist
|
|
@@ -34,12 +34,12 @@ from fractal_server.utils import get_timestamp
|
|
|
34
34
|
|
|
35
35
|
def collect_local_pixi(
|
|
36
36
|
*,
|
|
37
|
-
task_group_activity_id: int,
|
|
38
37
|
task_group_id: int,
|
|
38
|
+
task_group_activity_id: int,
|
|
39
39
|
tar_gz_file: FractalUploadedFile,
|
|
40
|
+
resource: Resource,
|
|
41
|
+
profile: Profile,
|
|
40
42
|
) -> None:
|
|
41
|
-
settings = Inject(get_settings)
|
|
42
|
-
|
|
43
43
|
LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
|
|
44
44
|
|
|
45
45
|
with TemporaryDirectory() as tmpdir:
|
|
@@ -90,7 +90,9 @@ def collect_local_pixi(
|
|
|
90
90
|
replacements={
|
|
91
91
|
(
|
|
92
92
|
"__PIXI_HOME__",
|
|
93
|
-
|
|
93
|
+
resource.tasks_pixi_config["versions"][
|
|
94
|
+
task_group.pixi_version
|
|
95
|
+
],
|
|
94
96
|
),
|
|
95
97
|
("__PACKAGE_DIR__", task_group.path),
|
|
96
98
|
("__TAR_GZ_PATH__", archive_path),
|
|
@@ -102,15 +104,27 @@ def collect_local_pixi(
|
|
|
102
104
|
("__FROZEN_OPTION__", ""),
|
|
103
105
|
(
|
|
104
106
|
"__TOKIO_WORKER_THREADS__",
|
|
105
|
-
str(
|
|
107
|
+
str(
|
|
108
|
+
resource.tasks_pixi_config[
|
|
109
|
+
"TOKIO_WORKER_THREADS"
|
|
110
|
+
]
|
|
111
|
+
),
|
|
106
112
|
),
|
|
107
113
|
(
|
|
108
114
|
"__PIXI_CONCURRENT_SOLVES__",
|
|
109
|
-
str(
|
|
115
|
+
str(
|
|
116
|
+
resource.tasks_pixi_config[
|
|
117
|
+
"PIXI_CONCURRENT_SOLVES"
|
|
118
|
+
]
|
|
119
|
+
),
|
|
110
120
|
),
|
|
111
121
|
(
|
|
112
122
|
"__PIXI_CONCURRENT_DOWNLOADS__",
|
|
113
|
-
str(
|
|
123
|
+
str(
|
|
124
|
+
resource.tasks_pixi_config[
|
|
125
|
+
"PIXI_CONCURRENT_DOWNLOADS"
|
|
126
|
+
]
|
|
127
|
+
),
|
|
114
128
|
),
|
|
115
129
|
},
|
|
116
130
|
script_dir=Path(
|
|
@@ -138,7 +152,9 @@ def collect_local_pixi(
|
|
|
138
152
|
# Simplify `pyproject.toml`
|
|
139
153
|
source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
|
|
140
154
|
pyproject_toml_path = Path(source_dir, "pyproject.toml")
|
|
141
|
-
edit_pyproject_toml_in_place_local(
|
|
155
|
+
edit_pyproject_toml_in_place_local(
|
|
156
|
+
pyproject_toml_path, resource=resource
|
|
157
|
+
)
|
|
142
158
|
|
|
143
159
|
# Run script 2
|
|
144
160
|
_customize_and_run_template(
|
|
@@ -9,6 +9,8 @@ from ..utils_background import get_activity_and_task_group
|
|
|
9
9
|
from ..utils_templates import get_collection_replacements
|
|
10
10
|
from ._utils import _customize_and_run_template
|
|
11
11
|
from fractal_server.app.db import get_sync_db
|
|
12
|
+
from fractal_server.app.models import Profile
|
|
13
|
+
from fractal_server.app.models import Resource
|
|
12
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
13
15
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
14
16
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
|
@@ -25,6 +27,8 @@ def deactivate_local(
|
|
|
25
27
|
*,
|
|
26
28
|
task_group_activity_id: int,
|
|
27
29
|
task_group_id: int,
|
|
30
|
+
resource: Resource,
|
|
31
|
+
profile: Profile,
|
|
28
32
|
) -> None:
|
|
29
33
|
"""
|
|
30
34
|
Deactivate a task group venv.
|
|
@@ -32,9 +36,10 @@ def deactivate_local(
|
|
|
32
36
|
This function is run as a background task, therefore exceptions must be
|
|
33
37
|
handled.
|
|
34
38
|
|
|
35
|
-
|
|
39
|
+
Args:
|
|
36
40
|
task_group_id:
|
|
37
41
|
task_group_activity_id:
|
|
42
|
+
resource:
|
|
38
43
|
"""
|
|
39
44
|
|
|
40
45
|
LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
|
|
@@ -85,6 +90,7 @@ def deactivate_local(
|
|
|
85
90
|
replacements = get_collection_replacements(
|
|
86
91
|
task_group=task_group,
|
|
87
92
|
python_bin="/not/applicable",
|
|
93
|
+
resource=resource,
|
|
88
94
|
)
|
|
89
95
|
|
|
90
96
|
# Prepare common arguments for _customize_and_run_template
|
|
@@ -7,6 +7,8 @@ from ..utils_background import fail_and_cleanup
|
|
|
7
7
|
from ..utils_background import get_activity_and_task_group
|
|
8
8
|
from ..utils_pixi import SOURCE_DIR_NAME
|
|
9
9
|
from fractal_server.app.db import get_sync_db
|
|
10
|
+
from fractal_server.app.models import Profile
|
|
11
|
+
from fractal_server.app.models import Resource
|
|
10
12
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
|
11
13
|
from fractal_server.logger import reset_logger_handlers
|
|
12
14
|
from fractal_server.logger import set_logger
|
|
@@ -19,6 +21,8 @@ def deactivate_local_pixi(
|
|
|
19
21
|
*,
|
|
20
22
|
task_group_activity_id: int,
|
|
21
23
|
task_group_id: int,
|
|
24
|
+
resource: Resource,
|
|
25
|
+
profile: Profile,
|
|
22
26
|
) -> None:
|
|
23
27
|
"""
|
|
24
28
|
Deactivate a pixi task group venv.
|
|
@@ -26,7 +30,7 @@ def deactivate_local_pixi(
|
|
|
26
30
|
This function is run as a background task, therefore exceptions must be
|
|
27
31
|
handled.
|
|
28
32
|
|
|
29
|
-
|
|
33
|
+
Args:
|
|
30
34
|
task_group_id:
|
|
31
35
|
task_group_activity_id:
|
|
32
36
|
"""
|
|
@@ -3,6 +3,8 @@ from pathlib import Path
|
|
|
3
3
|
from tempfile import TemporaryDirectory
|
|
4
4
|
|
|
5
5
|
from fractal_server.app.db import get_sync_db
|
|
6
|
+
from fractal_server.app.models import Profile
|
|
7
|
+
from fractal_server.app.models import Resource
|
|
6
8
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
7
9
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
8
10
|
from fractal_server.logger import reset_logger_handlers
|
|
@@ -21,6 +23,8 @@ def delete_local(
|
|
|
21
23
|
*,
|
|
22
24
|
task_group_activity_id: int,
|
|
23
25
|
task_group_id: int,
|
|
26
|
+
resource: Resource,
|
|
27
|
+
profile: Profile,
|
|
24
28
|
) -> None:
|
|
25
29
|
LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
|
|
26
30
|
|
|
@@ -9,6 +9,8 @@ from ..utils_background import get_activity_and_task_group
|
|
|
9
9
|
from ..utils_templates import get_collection_replacements
|
|
10
10
|
from ._utils import _customize_and_run_template
|
|
11
11
|
from fractal_server.app.db import get_sync_db
|
|
12
|
+
from fractal_server.app.models import Profile
|
|
13
|
+
from fractal_server.app.models import Resource
|
|
12
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
13
15
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
|
14
16
|
from fractal_server.logger import reset_logger_handlers
|
|
@@ -16,7 +18,7 @@ from fractal_server.logger import set_logger
|
|
|
16
18
|
from fractal_server.tasks.utils import get_log_path
|
|
17
19
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
|
18
20
|
from fractal_server.tasks.v2.utils_python_interpreter import (
|
|
19
|
-
|
|
21
|
+
get_python_interpreter,
|
|
20
22
|
)
|
|
21
23
|
from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
|
|
22
24
|
from fractal_server.utils import get_timestamp
|
|
@@ -26,6 +28,8 @@ def reactivate_local(
|
|
|
26
28
|
*,
|
|
27
29
|
task_group_activity_id: int,
|
|
28
30
|
task_group_id: int,
|
|
31
|
+
resource: Resource,
|
|
32
|
+
profile: Profile,
|
|
29
33
|
) -> None:
|
|
30
34
|
"""
|
|
31
35
|
Reactivate a task group venv.
|
|
@@ -33,9 +37,10 @@ def reactivate_local(
|
|
|
33
37
|
This function is run as a background task, therefore exceptions must be
|
|
34
38
|
handled.
|
|
35
39
|
|
|
36
|
-
|
|
40
|
+
Args:
|
|
37
41
|
task_group_id:
|
|
38
42
|
task_group_activity_id:
|
|
43
|
+
resource:
|
|
39
44
|
"""
|
|
40
45
|
|
|
41
46
|
LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
|
|
@@ -77,11 +82,14 @@ def reactivate_local(
|
|
|
77
82
|
activity = add_commit_refresh(obj=activity, db=db)
|
|
78
83
|
|
|
79
84
|
# Prepare replacements for templates
|
|
85
|
+
python_bin = get_python_interpreter(
|
|
86
|
+
python_version=task_group.python_version,
|
|
87
|
+
resource=resource,
|
|
88
|
+
)
|
|
80
89
|
replacements = get_collection_replacements(
|
|
81
90
|
task_group=task_group,
|
|
82
|
-
python_bin=
|
|
83
|
-
|
|
84
|
-
),
|
|
91
|
+
python_bin=python_bin,
|
|
92
|
+
resource=resource,
|
|
85
93
|
)
|
|
86
94
|
with open(f"{tmpdir}/pip_freeze.txt", "w") as f:
|
|
87
95
|
f.write(task_group.env_info)
|
|
@@ -9,12 +9,12 @@ from ..utils_background import get_activity_and_task_group
|
|
|
9
9
|
from ..utils_pixi import SOURCE_DIR_NAME
|
|
10
10
|
from ._utils import edit_pyproject_toml_in_place_local
|
|
11
11
|
from fractal_server.app.db import get_sync_db
|
|
12
|
+
from fractal_server.app.models import Profile
|
|
13
|
+
from fractal_server.app.models import Resource
|
|
12
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
13
15
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
|
14
|
-
from fractal_server.config import get_settings
|
|
15
16
|
from fractal_server.logger import reset_logger_handlers
|
|
16
17
|
from fractal_server.logger import set_logger
|
|
17
|
-
from fractal_server.syringe import Inject
|
|
18
18
|
from fractal_server.tasks.utils import get_log_path
|
|
19
19
|
from fractal_server.tasks.v2.local._utils import _customize_and_run_template
|
|
20
20
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
|
@@ -27,6 +27,8 @@ def reactivate_local_pixi(
|
|
|
27
27
|
*,
|
|
28
28
|
task_group_activity_id: int,
|
|
29
29
|
task_group_id: int,
|
|
30
|
+
resource: Resource,
|
|
31
|
+
profile: Profile,
|
|
30
32
|
) -> None:
|
|
31
33
|
"""
|
|
32
34
|
Reactivate a task group venv.
|
|
@@ -34,9 +36,10 @@ def reactivate_local_pixi(
|
|
|
34
36
|
This function is run as a background task, therefore exceptions must be
|
|
35
37
|
handled.
|
|
36
38
|
|
|
37
|
-
|
|
39
|
+
Args:
|
|
38
40
|
task_group_id:
|
|
39
41
|
task_group_activity_id:
|
|
42
|
+
resource:
|
|
40
43
|
"""
|
|
41
44
|
|
|
42
45
|
LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
|
|
@@ -76,12 +79,13 @@ def reactivate_local_pixi(
|
|
|
76
79
|
activity.status = TaskGroupActivityStatusV2.ONGOING
|
|
77
80
|
activity = add_commit_refresh(obj=activity, db=db)
|
|
78
81
|
|
|
79
|
-
settings = Inject(get_settings)
|
|
80
82
|
common_args = dict(
|
|
81
83
|
replacements={
|
|
82
84
|
(
|
|
83
85
|
"__PIXI_HOME__",
|
|
84
|
-
|
|
86
|
+
resource.tasks_pixi_config["versions"][
|
|
87
|
+
task_group.pixi_version
|
|
88
|
+
],
|
|
85
89
|
),
|
|
86
90
|
("__PACKAGE_DIR__", task_group.path),
|
|
87
91
|
("__TAR_GZ_PATH__", task_group.archive_path),
|
|
@@ -93,15 +97,27 @@ def reactivate_local_pixi(
|
|
|
93
97
|
("__FROZEN_OPTION__", "--frozen"),
|
|
94
98
|
(
|
|
95
99
|
"__TOKIO_WORKER_THREADS__",
|
|
96
|
-
str(
|
|
100
|
+
str(
|
|
101
|
+
resource.tasks_pixi_config[
|
|
102
|
+
"TOKIO_WORKER_THREADS"
|
|
103
|
+
]
|
|
104
|
+
),
|
|
97
105
|
),
|
|
98
106
|
(
|
|
99
107
|
"__PIXI_CONCURRENT_SOLVES__",
|
|
100
|
-
str(
|
|
108
|
+
str(
|
|
109
|
+
resource.tasks_pixi_config[
|
|
110
|
+
"PIXI_CONCURRENT_SOLVES"
|
|
111
|
+
]
|
|
112
|
+
),
|
|
101
113
|
),
|
|
102
114
|
(
|
|
103
115
|
"__PIXI_CONCURRENT_DOWNLOADS__",
|
|
104
|
-
str(
|
|
116
|
+
str(
|
|
117
|
+
resource.tasks_pixi_config[
|
|
118
|
+
"PIXI_CONCURRENT_DOWNLOADS"
|
|
119
|
+
]
|
|
120
|
+
),
|
|
105
121
|
),
|
|
106
122
|
},
|
|
107
123
|
script_dir=Path(
|
|
@@ -124,7 +140,9 @@ def reactivate_local_pixi(
|
|
|
124
140
|
|
|
125
141
|
# Simplify `pyproject.toml`
|
|
126
142
|
pyproject_toml_path = Path(source_dir, "pyproject.toml")
|
|
127
|
-
edit_pyproject_toml_in_place_local(
|
|
143
|
+
edit_pyproject_toml_in_place_local(
|
|
144
|
+
pyproject_toml_path, resource=resource
|
|
145
|
+
)
|
|
128
146
|
|
|
129
147
|
# Write pixi.lock into `source_dir`
|
|
130
148
|
logger.debug(f"start - writing {source_dir}/pixi.lock")
|
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import time
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
4
5
|
|
|
5
6
|
from sqlalchemy.orm import Session
|
|
6
7
|
|
|
7
8
|
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
8
|
-
from fractal_server.config import get_settings
|
|
9
|
-
from fractal_server.config import PixiSLURMConfig
|
|
10
9
|
from fractal_server.logger import get_logger
|
|
11
10
|
from fractal_server.ssh._fabric import FractalSSH
|
|
12
|
-
from fractal_server.
|
|
11
|
+
from fractal_server.tasks.config import PixiSLURMConfig
|
|
13
12
|
from fractal_server.tasks.v2.utils_background import add_commit_refresh
|
|
14
13
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
|
15
14
|
|
|
@@ -141,13 +140,14 @@ def _verify_success_file_exists(
|
|
|
141
140
|
def run_script_on_remote_slurm(
|
|
142
141
|
*,
|
|
143
142
|
script_paths: list[str],
|
|
144
|
-
slurm_config:
|
|
143
|
+
slurm_config: dict[str, Any],
|
|
145
144
|
fractal_ssh: FractalSSH,
|
|
146
145
|
logger_name: str,
|
|
147
146
|
log_file_path: Path,
|
|
148
147
|
prefix: str,
|
|
149
148
|
db: Session,
|
|
150
149
|
activity: TaskGroupActivityV2,
|
|
150
|
+
poll_interval: int,
|
|
151
151
|
):
|
|
152
152
|
"""
|
|
153
153
|
Run a `pixi install` script as a SLURM job.
|
|
@@ -156,8 +156,9 @@ def run_script_on_remote_slurm(
|
|
|
156
156
|
as a mechanism to propagate failure/errors.
|
|
157
157
|
"""
|
|
158
158
|
|
|
159
|
+
slurm_config_obj = PixiSLURMConfig(**slurm_config)
|
|
160
|
+
|
|
159
161
|
logger = get_logger(logger_name=logger_name)
|
|
160
|
-
settings = Inject(get_settings)
|
|
161
162
|
|
|
162
163
|
# (1) Prepare remote submission script
|
|
163
164
|
workdir_remote = _get_workdir_remote(script_paths)
|
|
@@ -169,10 +170,10 @@ def run_script_on_remote_slurm(
|
|
|
169
170
|
success_file_remote = os.path.join(workdir_remote, f"{prefix}-success.txt")
|
|
170
171
|
script_lines = [
|
|
171
172
|
"#!/bin/bash",
|
|
172
|
-
f"#SBATCH --partition={
|
|
173
|
-
f"#SBATCH --cpus-per-task={
|
|
174
|
-
f"#SBATCH --mem={
|
|
175
|
-
f"#SBATCH --time={
|
|
173
|
+
f"#SBATCH --partition={slurm_config_obj.partition}",
|
|
174
|
+
f"#SBATCH --cpus-per-task={slurm_config_obj.cpus}",
|
|
175
|
+
f"#SBATCH --mem={slurm_config_obj.mem}",
|
|
176
|
+
f"#SBATCH --time={slurm_config_obj.time}",
|
|
176
177
|
f"#SBATCH --err={stderr_remote}",
|
|
177
178
|
f"#SBATCH --out={stdout_remote}",
|
|
178
179
|
f"#SBATCH -D {workdir_remote}",
|
|
@@ -234,7 +235,7 @@ def run_script_on_remote_slurm(
|
|
|
234
235
|
logger.debug(f"Exit retrieval loop (state={new_state}).")
|
|
235
236
|
break
|
|
236
237
|
old_state = new_state
|
|
237
|
-
time.sleep(
|
|
238
|
+
time.sleep(poll_interval)
|
|
238
239
|
|
|
239
240
|
_verify_success_file_exists(
|
|
240
241
|
fractal_ssh=fractal_ssh,
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
|
-
from sqlalchemy.
|
|
4
|
+
from sqlalchemy.orm import Session
|
|
5
5
|
|
|
6
6
|
from ..utils_background import fail_and_cleanup
|
|
7
7
|
from ..utils_pixi import simplify_pyproject_toml
|
|
8
|
+
from fractal_server.app.models import Resource
|
|
8
9
|
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
9
10
|
from fractal_server.app.models.v2 import TaskGroupV2
|
|
10
|
-
from fractal_server.config import get_settings
|
|
11
11
|
from fractal_server.logger import get_logger
|
|
12
12
|
from fractal_server.logger import set_logger
|
|
13
13
|
from fractal_server.ssh._fabric import FractalSSH
|
|
14
|
-
from fractal_server.syringe import Inject
|
|
15
14
|
from fractal_server.tasks.v2.utils_templates import customize_template
|
|
16
15
|
|
|
17
16
|
logger = set_logger(__name__)
|
|
@@ -143,7 +142,7 @@ def check_ssh_or_fail_and_cleanup(
|
|
|
143
142
|
task_group_activity: TaskGroupActivityV2,
|
|
144
143
|
logger_name: str,
|
|
145
144
|
log_file_path: Path,
|
|
146
|
-
db:
|
|
145
|
+
db: Session,
|
|
147
146
|
) -> bool:
|
|
148
147
|
"""
|
|
149
148
|
Check SSH connection.
|
|
@@ -174,6 +173,7 @@ def edit_pyproject_toml_in_place_ssh(
|
|
|
174
173
|
*,
|
|
175
174
|
fractal_ssh: FractalSSH,
|
|
176
175
|
pyproject_toml_path: Path,
|
|
176
|
+
resource: Resource,
|
|
177
177
|
) -> None:
|
|
178
178
|
"""
|
|
179
179
|
Wrapper of `simplify_pyproject_toml`, with I/O.
|
|
@@ -185,11 +185,10 @@ def edit_pyproject_toml_in_place_ssh(
|
|
|
185
185
|
)
|
|
186
186
|
|
|
187
187
|
# Simplify contents
|
|
188
|
-
settings = Inject(get_settings)
|
|
189
188
|
new_pyproject_contents = simplify_pyproject_toml(
|
|
190
189
|
original_toml_string=pyproject_contents,
|
|
191
|
-
pixi_environment=
|
|
192
|
-
pixi_platform=
|
|
190
|
+
pixi_environment=resource.tasks_pixi_config["DEFAULT_ENVIRONMENT"],
|
|
191
|
+
pixi_platform=resource.tasks_pixi_config["DEFAULT_PLATFORM"],
|
|
193
192
|
)
|
|
194
193
|
# Write new `pyproject.toml`
|
|
195
194
|
fractal_ssh.write_remote_file(
|
|
@@ -8,6 +8,8 @@ from ..utils_background import prepare_tasks_metadata
|
|
|
8
8
|
from ..utils_database import create_db_tasks_and_update_task_group_sync
|
|
9
9
|
from ._utils import check_ssh_or_fail_and_cleanup
|
|
10
10
|
from fractal_server.app.db import get_sync_db
|
|
11
|
+
from fractal_server.app.models import Profile
|
|
12
|
+
from fractal_server.app.models import Resource
|
|
11
13
|
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
|
12
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
13
15
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
@@ -21,7 +23,7 @@ from fractal_server.tasks.v2.utils_background import add_commit_refresh
|
|
|
21
23
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
|
22
24
|
from fractal_server.tasks.v2.utils_package_names import compare_package_names
|
|
23
25
|
from fractal_server.tasks.v2.utils_python_interpreter import (
|
|
24
|
-
|
|
26
|
+
get_python_interpreter,
|
|
25
27
|
)
|
|
26
28
|
from fractal_server.tasks.v2.utils_templates import get_collection_replacements
|
|
27
29
|
from fractal_server.tasks.v2.utils_templates import (
|
|
@@ -35,8 +37,8 @@ def collect_ssh(
|
|
|
35
37
|
*,
|
|
36
38
|
task_group_id: int,
|
|
37
39
|
task_group_activity_id: int,
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
resource: Resource,
|
|
41
|
+
profile: Profile,
|
|
40
42
|
wheel_file: FractalUploadedFile | None = None,
|
|
41
43
|
) -> None:
|
|
42
44
|
"""
|
|
@@ -50,13 +52,11 @@ def collect_ssh(
|
|
|
50
52
|
https://github.com/encode/starlette/blob/master/starlette/background.py).
|
|
51
53
|
|
|
52
54
|
|
|
53
|
-
|
|
55
|
+
Args:
|
|
54
56
|
task_group_id:
|
|
55
57
|
task_group_activity_id:
|
|
56
58
|
ssh_config:
|
|
57
|
-
|
|
58
|
-
Only used as a `safe_root` in `remove_dir`, and typically set to
|
|
59
|
-
`user_settings.ssh_tasks_dir`.
|
|
59
|
+
resource:
|
|
60
60
|
wheel_file:
|
|
61
61
|
"""
|
|
62
62
|
|
|
@@ -81,7 +81,11 @@ def collect_ssh(
|
|
|
81
81
|
return
|
|
82
82
|
|
|
83
83
|
with SingleUseFractalSSH(
|
|
84
|
-
ssh_config=
|
|
84
|
+
ssh_config=SSHConfig(
|
|
85
|
+
host=resource.host,
|
|
86
|
+
user=profile.username,
|
|
87
|
+
key_path=profile.ssh_key_path,
|
|
88
|
+
),
|
|
85
89
|
logger_name=LOGGER_NAME,
|
|
86
90
|
) as fractal_ssh:
|
|
87
91
|
try:
|
|
@@ -144,11 +148,14 @@ def collect_ssh(
|
|
|
144
148
|
task_group.archive_path = archive_path
|
|
145
149
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
|
146
150
|
|
|
151
|
+
python_bin = get_python_interpreter(
|
|
152
|
+
python_version=task_group.python_version,
|
|
153
|
+
resource=resource,
|
|
154
|
+
)
|
|
147
155
|
replacements = get_collection_replacements(
|
|
148
156
|
task_group=task_group,
|
|
149
|
-
python_bin=
|
|
150
|
-
|
|
151
|
-
),
|
|
157
|
+
python_bin=python_bin,
|
|
158
|
+
resource=resource,
|
|
152
159
|
)
|
|
153
160
|
|
|
154
161
|
# Prepare common arguments for _customize_and_run_template
|
|
@@ -301,7 +308,7 @@ def collect_ssh(
|
|
|
301
308
|
)
|
|
302
309
|
fractal_ssh.remove_folder(
|
|
303
310
|
folder=task_group.path,
|
|
304
|
-
safe_root=
|
|
311
|
+
safe_root=profile.tasks_remote_dir,
|
|
305
312
|
)
|
|
306
313
|
logger.info(
|
|
307
314
|
f"Deleted remoted folder {task_group.path}"
|
|
@@ -12,16 +12,16 @@ from ._pixi_slurm_ssh import run_script_on_remote_slurm
|
|
|
12
12
|
from ._utils import check_ssh_or_fail_and_cleanup
|
|
13
13
|
from ._utils import edit_pyproject_toml_in_place_ssh
|
|
14
14
|
from fractal_server.app.db import get_sync_db
|
|
15
|
+
from fractal_server.app.models import Profile
|
|
16
|
+
from fractal_server.app.models import Resource
|
|
15
17
|
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
|
16
18
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
17
19
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
18
20
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
|
19
|
-
from fractal_server.config import get_settings
|
|
20
21
|
from fractal_server.logger import reset_logger_handlers
|
|
21
22
|
from fractal_server.logger import set_logger
|
|
22
23
|
from fractal_server.ssh._fabric import SingleUseFractalSSH
|
|
23
24
|
from fractal_server.ssh._fabric import SSHConfig
|
|
24
|
-
from fractal_server.syringe import Inject
|
|
25
25
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
|
|
26
26
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_send_template
|
|
27
27
|
from fractal_server.tasks.v2.utils_background import add_commit_refresh
|
|
@@ -34,9 +34,9 @@ def collect_ssh_pixi(
|
|
|
34
34
|
*,
|
|
35
35
|
task_group_id: int,
|
|
36
36
|
task_group_activity_id: int,
|
|
37
|
-
ssh_config: SSHConfig,
|
|
38
|
-
tasks_base_dir: str,
|
|
39
37
|
tar_gz_file: FractalUploadedFile,
|
|
38
|
+
resource: Resource,
|
|
39
|
+
profile: Profile,
|
|
40
40
|
) -> None:
|
|
41
41
|
"""
|
|
42
42
|
Collect a task package over SSH
|
|
@@ -49,13 +49,10 @@ def collect_ssh_pixi(
|
|
|
49
49
|
https://github.com/encode/starlette/blob/master/starlette/background.py).
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
|
|
52
|
+
Args:
|
|
53
53
|
task_group_id:
|
|
54
54
|
task_group_activity_id:
|
|
55
55
|
ssh_config:
|
|
56
|
-
tasks_base_dir:
|
|
57
|
-
Only used as a `safe_root` in `remove_dir`, and typically set to
|
|
58
|
-
`user_settings.ssh_tasks_dir`.
|
|
59
56
|
tar_gz_file:
|
|
60
57
|
"""
|
|
61
58
|
|
|
@@ -80,7 +77,11 @@ def collect_ssh_pixi(
|
|
|
80
77
|
return
|
|
81
78
|
|
|
82
79
|
with SingleUseFractalSSH(
|
|
83
|
-
ssh_config=
|
|
80
|
+
ssh_config=SSHConfig(
|
|
81
|
+
host=resource.host,
|
|
82
|
+
user=profile.username,
|
|
83
|
+
key_path=profile.ssh_key_path,
|
|
84
|
+
),
|
|
84
85
|
logger_name=LOGGER_NAME,
|
|
85
86
|
) as fractal_ssh:
|
|
86
87
|
try:
|
|
@@ -135,11 +136,12 @@ def collect_ssh_pixi(
|
|
|
135
136
|
task_group.archive_path = archive_path
|
|
136
137
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
|
137
138
|
|
|
138
|
-
settings = Inject(get_settings)
|
|
139
139
|
replacements = {
|
|
140
140
|
(
|
|
141
141
|
"__PIXI_HOME__",
|
|
142
|
-
|
|
142
|
+
resource.tasks_pixi_config["versions"][
|
|
143
|
+
task_group.pixi_version
|
|
144
|
+
],
|
|
143
145
|
),
|
|
144
146
|
("__PACKAGE_DIR__", task_group.path),
|
|
145
147
|
("__TAR_GZ_PATH__", task_group.archive_path),
|
|
@@ -151,15 +153,27 @@ def collect_ssh_pixi(
|
|
|
151
153
|
("__FROZEN_OPTION__", ""),
|
|
152
154
|
(
|
|
153
155
|
"__TOKIO_WORKER_THREADS__",
|
|
154
|
-
str(
|
|
156
|
+
str(
|
|
157
|
+
resource.tasks_pixi_config[
|
|
158
|
+
"TOKIO_WORKER_THREADS"
|
|
159
|
+
]
|
|
160
|
+
),
|
|
155
161
|
),
|
|
156
162
|
(
|
|
157
163
|
"__PIXI_CONCURRENT_SOLVES__",
|
|
158
|
-
str(
|
|
164
|
+
str(
|
|
165
|
+
resource.tasks_pixi_config[
|
|
166
|
+
"PIXI_CONCURRENT_SOLVES"
|
|
167
|
+
]
|
|
168
|
+
),
|
|
159
169
|
),
|
|
160
170
|
(
|
|
161
171
|
"__PIXI_CONCURRENT_DOWNLOADS__",
|
|
162
|
-
str(
|
|
172
|
+
str(
|
|
173
|
+
resource.tasks_pixi_config[
|
|
174
|
+
"PIXI_CONCURRENT_DOWNLOADS"
|
|
175
|
+
]
|
|
176
|
+
),
|
|
163
177
|
),
|
|
164
178
|
}
|
|
165
179
|
|
|
@@ -201,6 +215,7 @@ def collect_ssh_pixi(
|
|
|
201
215
|
edit_pyproject_toml_in_place_ssh(
|
|
202
216
|
fractal_ssh=fractal_ssh,
|
|
203
217
|
pyproject_toml_path=pyproject_toml_path,
|
|
218
|
+
resource=resource,
|
|
204
219
|
)
|
|
205
220
|
|
|
206
221
|
# Prepare scripts 2 and 3
|
|
@@ -232,13 +247,16 @@ def collect_ssh_pixi(
|
|
|
232
247
|
remote_script3_path,
|
|
233
248
|
f"chmod -R 755 {source_dir}",
|
|
234
249
|
],
|
|
235
|
-
slurm_config=
|
|
250
|
+
slurm_config=resource.tasks_pixi_config[
|
|
251
|
+
"SLURM_CONFIG"
|
|
252
|
+
],
|
|
236
253
|
fractal_ssh=fractal_ssh,
|
|
237
254
|
logger_name=LOGGER_NAME,
|
|
238
255
|
prefix=common_args["prefix"],
|
|
239
256
|
db=db,
|
|
240
257
|
activity=activity,
|
|
241
258
|
log_file_path=log_file_path,
|
|
259
|
+
poll_interval=resource.jobs_poll_interval,
|
|
242
260
|
)
|
|
243
261
|
activity.log = get_current_log(log_file_path)
|
|
244
262
|
activity = add_commit_refresh(obj=activity, db=db)
|
|
@@ -324,7 +342,7 @@ def collect_ssh_pixi(
|
|
|
324
342
|
)
|
|
325
343
|
fractal_ssh.remove_folder(
|
|
326
344
|
folder=task_group.path,
|
|
327
|
-
safe_root=
|
|
345
|
+
safe_root=profile.tasks_remote_dir,
|
|
328
346
|
)
|
|
329
347
|
logger.info(
|
|
330
348
|
f"Deleted remoted folder {task_group.path}"
|