fractal-server 2.14.16__py3-none-any.whl → 2.15.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/v2/task_group.py +17 -5
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +2 -2
- fractal_server/app/routes/api/v2/__init__.py +6 -0
- fractal_server/app/routes/api/v2/task_collection.py +3 -3
- fractal_server/app/routes/api/v2/task_collection_custom.py +2 -2
- fractal_server/app/routes/api/v2/task_collection_pixi.py +236 -0
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +26 -7
- fractal_server/app/schemas/v2/__init__.py +2 -1
- fractal_server/app/schemas/v2/dumps.py +1 -1
- fractal_server/app/schemas/v2/task_collection.py +1 -1
- fractal_server/app/schemas/v2/task_group.py +16 -5
- fractal_server/config.py +42 -0
- fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +53 -0
- fractal_server/ssh/_fabric.py +26 -0
- fractal_server/tasks/v2/local/__init__.py +3 -0
- fractal_server/tasks/v2/local/_utils.py +7 -2
- fractal_server/tasks/v2/local/collect.py +23 -24
- fractal_server/tasks/v2/local/collect_pixi.py +234 -0
- fractal_server/tasks/v2/local/deactivate.py +36 -39
- fractal_server/tasks/v2/local/deactivate_pixi.py +102 -0
- fractal_server/tasks/v2/local/reactivate.py +9 -16
- fractal_server/tasks/v2/local/reactivate_pixi.py +146 -0
- fractal_server/tasks/v2/ssh/__init__.py +3 -0
- fractal_server/tasks/v2/ssh/_utils.py +5 -5
- fractal_server/tasks/v2/ssh/collect.py +23 -28
- fractal_server/tasks/v2/ssh/collect_pixi.py +306 -0
- fractal_server/tasks/v2/ssh/deactivate.py +39 -45
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +128 -0
- fractal_server/tasks/v2/ssh/reactivate.py +8 -15
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +108 -0
- fractal_server/tasks/v2/templates/pixi_1_extract.sh +40 -0
- fractal_server/tasks/v2/templates/pixi_2_install.sh +48 -0
- fractal_server/tasks/v2/templates/pixi_3_post_install.sh +80 -0
- fractal_server/tasks/v2/utils_background.py +43 -8
- fractal_server/tasks/v2/utils_pixi.py +38 -0
- {fractal_server-2.14.16.dist-info → fractal_server-2.15.0a1.dist-info}/METADATA +1 -1
- {fractal_server-2.14.16.dist-info → fractal_server-2.15.0a1.dist-info}/RECORD +41 -29
- {fractal_server-2.14.16.dist-info → fractal_server-2.15.0a1.dist-info}/LICENSE +0 -0
- {fractal_server-2.14.16.dist-info → fractal_server-2.15.0a1.dist-info}/WHEEL +0 -0
- {fractal_server-2.14.16.dist-info → fractal_server-2.15.0a1.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.
|
1
|
+
__VERSION__ = "2.15.0a1"
|
@@ -29,8 +29,9 @@ class TaskGroupV2(SQLModel, table=True):
|
|
29
29
|
pkg_name: str
|
30
30
|
version: str | None = None
|
31
31
|
python_version: str | None = None
|
32
|
+
pixi_version: str | None = None
|
32
33
|
path: str | None = None
|
33
|
-
|
34
|
+
archive_path: str | None = None
|
34
35
|
pip_extras: str | None = None
|
35
36
|
pinned_package_versions: dict[str, str] = Field(
|
36
37
|
sa_column=Column(
|
@@ -40,7 +41,7 @@ class TaskGroupV2(SQLModel, table=True):
|
|
40
41
|
nullable=True,
|
41
42
|
),
|
42
43
|
)
|
43
|
-
|
44
|
+
env_info: str | None = None
|
44
45
|
venv_path: str | None = None
|
45
46
|
venv_size_in_kB: int | None = None
|
46
47
|
venv_file_number: int | None = None
|
@@ -66,15 +67,20 @@ class TaskGroupV2(SQLModel, table=True):
|
|
66
67
|
"""
|
67
68
|
Prepare string to be used in `python -m pip install`.
|
68
69
|
"""
|
70
|
+
if self.origin == "pixi":
|
71
|
+
raise ValueError(
|
72
|
+
f"Cannot call 'pip_install_string' if {self.origin=}."
|
73
|
+
)
|
74
|
+
|
69
75
|
extras = f"[{self.pip_extras}]" if self.pip_extras is not None else ""
|
70
76
|
|
71
|
-
if self.
|
72
|
-
return f"{self.
|
77
|
+
if self.archive_path is not None:
|
78
|
+
return f"{self.archive_path}{extras}"
|
73
79
|
else:
|
74
80
|
if self.version is None:
|
75
81
|
raise ValueError(
|
76
82
|
"Cannot run `pip_install_string` with "
|
77
|
-
f"{self.pkg_name=}, {self.
|
83
|
+
f"{self.pkg_name=}, {self.archive_path=}, {self.version=}."
|
78
84
|
)
|
79
85
|
return f"{self.pkg_name}{extras}=={self.version}"
|
80
86
|
|
@@ -83,6 +89,12 @@ class TaskGroupV2(SQLModel, table=True):
|
|
83
89
|
"""
|
84
90
|
Prepare string to be used in `python -m pip install`.
|
85
91
|
"""
|
92
|
+
if self.origin == "pixi":
|
93
|
+
raise ValueError(
|
94
|
+
"Cannot call 'pinned_package_versions_string' if "
|
95
|
+
f"{self.origin=}."
|
96
|
+
)
|
97
|
+
|
86
98
|
if self.pinned_package_versions is None:
|
87
99
|
return ""
|
88
100
|
output = " ".join(
|
@@ -207,12 +207,12 @@ async def reactivate_task_group(
|
|
207
207
|
response.status_code = status.HTTP_202_ACCEPTED
|
208
208
|
return task_group_activity
|
209
209
|
|
210
|
-
if task_group.
|
210
|
+
if task_group.env_info is None:
|
211
211
|
raise HTTPException(
|
212
212
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
213
213
|
detail=(
|
214
214
|
"Cannot reactivate a task group with "
|
215
|
-
f"{task_group.
|
215
|
+
f"{task_group.env_info=}."
|
216
216
|
),
|
217
217
|
)
|
218
218
|
|
@@ -14,6 +14,7 @@ from .submit import router as submit_job_router_v2
|
|
14
14
|
from .task import router as task_router_v2
|
15
15
|
from .task_collection import router as task_collection_router_v2
|
16
16
|
from .task_collection_custom import router as task_collection_router_v2_custom
|
17
|
+
from .task_collection_pixi import router as task_collection_pixi_router_v2
|
17
18
|
from .task_group import router as task_group_router_v2
|
18
19
|
from .task_group_lifecycle import router as task_group_lifecycle_router_v2
|
19
20
|
from .task_version_update import router as task_version_update_router_v2
|
@@ -49,6 +50,11 @@ router_api_v2.include_router(
|
|
49
50
|
prefix="/task",
|
50
51
|
tags=["V2 Task Lifecycle"],
|
51
52
|
)
|
53
|
+
router_api_v2.include_router(
|
54
|
+
task_collection_pixi_router_v2,
|
55
|
+
prefix="/task",
|
56
|
+
tags=["V2 Task Lifecycle"],
|
57
|
+
)
|
52
58
|
router_api_v2.include_router(
|
53
59
|
task_group_lifecycle_router_v2,
|
54
60
|
prefix="/task-group",
|
@@ -23,11 +23,11 @@ from .....syringe import Inject
|
|
23
23
|
from ....db import AsyncSession
|
24
24
|
from ....db import get_async_db
|
25
25
|
from ....models.v2 import TaskGroupV2
|
26
|
+
from ....schemas.v2 import FractalUploadedFile
|
26
27
|
from ....schemas.v2 import TaskCollectPipV2
|
27
28
|
from ....schemas.v2 import TaskGroupActivityStatusV2
|
28
29
|
from ....schemas.v2 import TaskGroupActivityV2Read
|
29
30
|
from ....schemas.v2 import TaskGroupCreateV2Strict
|
30
|
-
from ....schemas.v2 import WheelFile
|
31
31
|
from ...aux.validate_user_settings import validate_user_settings
|
32
32
|
from ._aux_functions_task_lifecycle import get_package_version_from_pypi
|
33
33
|
from ._aux_functions_tasks import _get_valid_user_group_id
|
@@ -208,13 +208,13 @@ async def collect_tasks_pip(
|
|
208
208
|
# Initialize wheel_file_content as None
|
209
209
|
wheel_file = None
|
210
210
|
|
211
|
-
# Set pkg_name, version, origin and
|
211
|
+
# Set pkg_name, version, origin and archive_path
|
212
212
|
if request_data.origin == TaskGroupV2OriginEnum.WHEELFILE:
|
213
213
|
try:
|
214
214
|
wheel_filename = request_data.file.filename
|
215
215
|
wheel_info = _parse_wheel_filename(wheel_filename)
|
216
216
|
wheel_file_content = await request_data.file.read()
|
217
|
-
wheel_file =
|
217
|
+
wheel_file = FractalUploadedFile(
|
218
218
|
filename=wheel_filename,
|
219
219
|
contents=wheel_file_content,
|
220
220
|
)
|
@@ -26,7 +26,7 @@ from fractal_server.logger import set_logger
|
|
26
26
|
from fractal_server.string_tools import validate_cmd
|
27
27
|
from fractal_server.syringe import Inject
|
28
28
|
from fractal_server.tasks.v2.utils_background import (
|
29
|
-
|
29
|
+
prepare_tasks_metadata,
|
30
30
|
)
|
31
31
|
from fractal_server.tasks.v2.utils_database import (
|
32
32
|
create_db_tasks_and_update_task_group_async,
|
@@ -138,7 +138,7 @@ async def collect_task_custom(
|
|
138
138
|
else:
|
139
139
|
package_root = Path(task_collect.package_root)
|
140
140
|
|
141
|
-
task_list: list[TaskCreateV2] =
|
141
|
+
task_list: list[TaskCreateV2] = prepare_tasks_metadata(
|
142
142
|
package_manifest=task_collect.manifest,
|
143
143
|
python_bin=Path(task_collect.python_interpreter),
|
144
144
|
package_root=package_root,
|
@@ -0,0 +1,236 @@
|
|
1
|
+
import os
|
2
|
+
from pathlib import Path
|
3
|
+
|
4
|
+
from fastapi import APIRouter
|
5
|
+
from fastapi import BackgroundTasks
|
6
|
+
from fastapi import Depends
|
7
|
+
from fastapi import Form
|
8
|
+
from fastapi import HTTPException
|
9
|
+
from fastapi import Request
|
10
|
+
from fastapi import Response
|
11
|
+
from fastapi import status
|
12
|
+
from fastapi import UploadFile
|
13
|
+
from pydantic import ValidationError
|
14
|
+
from sqlmodel import select
|
15
|
+
|
16
|
+
from fractal_server.app.db import AsyncSession
|
17
|
+
from fractal_server.app.db import get_async_db
|
18
|
+
from fractal_server.app.models import UserOAuth
|
19
|
+
from fractal_server.app.models.v2 import TaskGroupActivityV2
|
20
|
+
from fractal_server.app.models.v2 import TaskGroupV2
|
21
|
+
from fractal_server.app.routes.api.v2._aux_functions_tasks import (
|
22
|
+
_get_valid_user_group_id,
|
23
|
+
)
|
24
|
+
from fractal_server.app.routes.api.v2._aux_functions_tasks import (
|
25
|
+
_verify_non_duplication_group_constraint,
|
26
|
+
)
|
27
|
+
from fractal_server.app.routes.api.v2._aux_functions_tasks import (
|
28
|
+
_verify_non_duplication_user_constraint,
|
29
|
+
)
|
30
|
+
from fractal_server.app.routes.auth import current_active_verified_user
|
31
|
+
from fractal_server.app.routes.aux.validate_user_settings import (
|
32
|
+
validate_user_settings,
|
33
|
+
)
|
34
|
+
from fractal_server.app.schemas.v2 import FractalUploadedFile
|
35
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
36
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
37
|
+
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
38
|
+
from fractal_server.app.schemas.v2 import TaskGroupCreateV2StrictPixi
|
39
|
+
from fractal_server.app.schemas.v2.task_group import TaskGroupV2OriginEnum
|
40
|
+
from fractal_server.config import get_settings
|
41
|
+
from fractal_server.logger import set_logger
|
42
|
+
from fractal_server.ssh._fabric import SSHConfig
|
43
|
+
from fractal_server.syringe import Inject
|
44
|
+
from fractal_server.tasks.v2.local import collect_local_pixi
|
45
|
+
from fractal_server.tasks.v2.ssh import collect_ssh_pixi
|
46
|
+
from fractal_server.tasks.v2.utils_package_names import normalize_package_name
|
47
|
+
from fractal_server.types import NonEmptyStr
|
48
|
+
|
49
|
+
|
50
|
+
router = APIRouter()
|
51
|
+
|
52
|
+
logger = set_logger(__name__)
|
53
|
+
|
54
|
+
|
55
|
+
def validate_pkgname_and_version(filename: str) -> tuple[str, str]:
|
56
|
+
if not filename.endswith(".tar.gz"):
|
57
|
+
raise HTTPException(
|
58
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
59
|
+
detail=f"{filename=} does not end with '.tar.gz'.",
|
60
|
+
)
|
61
|
+
filename_splitted = filename.split("-")
|
62
|
+
if len(filename_splitted) != 2:
|
63
|
+
raise HTTPException(
|
64
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
65
|
+
detail=(
|
66
|
+
f"Invalid filename: '{filename}' must contain a single `-` "
|
67
|
+
"character, separating the package name from the version "
|
68
|
+
"(expected format: 'pkg_name-version')."
|
69
|
+
),
|
70
|
+
)
|
71
|
+
|
72
|
+
pkg_name = filename_splitted[0]
|
73
|
+
version = filename.removeprefix(f"{pkg_name}-").removesuffix(".tar.gz")
|
74
|
+
|
75
|
+
return normalize_package_name(pkg_name), version
|
76
|
+
|
77
|
+
|
78
|
+
@router.post(
|
79
|
+
"/collect/pixi/",
|
80
|
+
status_code=202,
|
81
|
+
response_model=TaskGroupActivityV2Read,
|
82
|
+
)
|
83
|
+
async def collect_task_pixi(
|
84
|
+
request: Request,
|
85
|
+
response: Response,
|
86
|
+
background_tasks: BackgroundTasks,
|
87
|
+
file: UploadFile,
|
88
|
+
pixi_version: NonEmptyStr | None = Form(None),
|
89
|
+
private: bool = False,
|
90
|
+
user_group_id: int | None = None,
|
91
|
+
user: UserOAuth = Depends(current_active_verified_user),
|
92
|
+
db: AsyncSession = Depends(get_async_db),
|
93
|
+
) -> TaskGroupActivityV2Read:
|
94
|
+
|
95
|
+
settings = Inject(get_settings)
|
96
|
+
# Check if Pixi is available
|
97
|
+
if settings.pixi is None:
|
98
|
+
raise HTTPException(
|
99
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
100
|
+
detail="Pixi task collection is not available.",
|
101
|
+
)
|
102
|
+
# Check if provided Pixi version is available. Use default if not provided
|
103
|
+
if pixi_version is None:
|
104
|
+
pixi_version = settings.pixi.default_version
|
105
|
+
else:
|
106
|
+
if pixi_version not in settings.pixi.versions:
|
107
|
+
raise HTTPException(
|
108
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
109
|
+
detail=(
|
110
|
+
f"Pixi version {pixi_version} is not available. Available "
|
111
|
+
f"versions: {list(settings.pixi.versions.keys())}"
|
112
|
+
),
|
113
|
+
)
|
114
|
+
|
115
|
+
pkg_name, version = validate_pkgname_and_version(file.filename)
|
116
|
+
tar_gz_content = await file.read()
|
117
|
+
tar_gz_file = FractalUploadedFile(
|
118
|
+
filename=file.filename,
|
119
|
+
contents=tar_gz_content,
|
120
|
+
)
|
121
|
+
|
122
|
+
user_group_id = await _get_valid_user_group_id(
|
123
|
+
user_group_id=user_group_id,
|
124
|
+
private=private,
|
125
|
+
user_id=user.id,
|
126
|
+
db=db,
|
127
|
+
)
|
128
|
+
|
129
|
+
user_settings = await validate_user_settings(
|
130
|
+
user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
|
131
|
+
)
|
132
|
+
|
133
|
+
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
134
|
+
base_tasks_path = user_settings.ssh_tasks_dir
|
135
|
+
else:
|
136
|
+
base_tasks_path = settings.FRACTAL_TASKS_DIR.as_posix()
|
137
|
+
task_group_path = (
|
138
|
+
Path(base_tasks_path) / str(user.id) / pkg_name / version
|
139
|
+
).as_posix()
|
140
|
+
|
141
|
+
task_group_attrs = dict(
|
142
|
+
user_id=user.id,
|
143
|
+
user_group_id=user_group_id,
|
144
|
+
origin=TaskGroupV2OriginEnum.PIXI,
|
145
|
+
pixi_version=pixi_version,
|
146
|
+
pkg_name=pkg_name,
|
147
|
+
version=version,
|
148
|
+
path=task_group_path,
|
149
|
+
)
|
150
|
+
try:
|
151
|
+
TaskGroupCreateV2StrictPixi(**task_group_attrs)
|
152
|
+
except ValidationError as e:
|
153
|
+
raise HTTPException(
|
154
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
155
|
+
detail=f"Invalid task-group object. Original error: {e}",
|
156
|
+
)
|
157
|
+
|
158
|
+
await _verify_non_duplication_user_constraint(
|
159
|
+
user_id=user.id,
|
160
|
+
pkg_name=task_group_attrs["pkg_name"],
|
161
|
+
version=task_group_attrs["version"],
|
162
|
+
db=db,
|
163
|
+
)
|
164
|
+
await _verify_non_duplication_group_constraint(
|
165
|
+
user_group_id=task_group_attrs["user_group_id"],
|
166
|
+
pkg_name=task_group_attrs["pkg_name"],
|
167
|
+
version=task_group_attrs["version"],
|
168
|
+
db=db,
|
169
|
+
)
|
170
|
+
|
171
|
+
# FIXME: to be removed with issue #2634
|
172
|
+
stm = select(TaskGroupV2).where(TaskGroupV2.path == task_group_path)
|
173
|
+
res = await db.execute(stm)
|
174
|
+
for conflicting_task_group in res.scalars().all():
|
175
|
+
raise HTTPException(
|
176
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
177
|
+
detail=(
|
178
|
+
f"Another task-group already has path={task_group_path}.\n"
|
179
|
+
f"{conflicting_task_group=}"
|
180
|
+
),
|
181
|
+
)
|
182
|
+
|
183
|
+
if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
|
184
|
+
if Path(task_group_path).exists():
|
185
|
+
raise HTTPException(
|
186
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
187
|
+
detail=f"{task_group_path} already exists.",
|
188
|
+
)
|
189
|
+
|
190
|
+
task_group = TaskGroupV2(**task_group_attrs)
|
191
|
+
db.add(task_group)
|
192
|
+
await db.commit()
|
193
|
+
await db.refresh(task_group)
|
194
|
+
db.expunge(task_group)
|
195
|
+
|
196
|
+
task_group_activity = TaskGroupActivityV2(
|
197
|
+
user_id=task_group.user_id,
|
198
|
+
taskgroupv2_id=task_group.id,
|
199
|
+
status=TaskGroupActivityStatusV2.PENDING,
|
200
|
+
action=TaskGroupActivityActionV2.COLLECT,
|
201
|
+
pkg_name=task_group.pkg_name,
|
202
|
+
version=task_group.version,
|
203
|
+
)
|
204
|
+
db.add(task_group_activity)
|
205
|
+
await db.commit()
|
206
|
+
await db.refresh(task_group_activity)
|
207
|
+
|
208
|
+
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
209
|
+
ssh_config = SSHConfig(
|
210
|
+
user=user_settings.ssh_username,
|
211
|
+
host=user_settings.ssh_host,
|
212
|
+
key_path=user_settings.ssh_private_key_path,
|
213
|
+
)
|
214
|
+
|
215
|
+
background_tasks.add_task(
|
216
|
+
collect_ssh_pixi,
|
217
|
+
task_group_id=task_group.id,
|
218
|
+
task_group_activity_id=task_group_activity.id,
|
219
|
+
ssh_config=ssh_config,
|
220
|
+
tasks_base_dir=user_settings.ssh_tasks_dir,
|
221
|
+
tar_gz_file=tar_gz_file,
|
222
|
+
)
|
223
|
+
else:
|
224
|
+
background_tasks.add_task(
|
225
|
+
collect_local_pixi,
|
226
|
+
task_group_id=task_group.id,
|
227
|
+
task_group_activity_id=task_group_activity.id,
|
228
|
+
tar_gz_file=tar_gz_file,
|
229
|
+
)
|
230
|
+
logger.info(
|
231
|
+
"Task-collection endpoint: start background collection "
|
232
|
+
"and return task_group_activity. "
|
233
|
+
f"Current pid is {os.getpid()}. "
|
234
|
+
)
|
235
|
+
response.status_code = status.HTTP_202_ACCEPTED
|
236
|
+
return task_group_activity
|
@@ -25,9 +25,13 @@ from fractal_server.logger import set_logger
|
|
25
25
|
from fractal_server.ssh._fabric import SSHConfig
|
26
26
|
from fractal_server.syringe import Inject
|
27
27
|
from fractal_server.tasks.v2.local import deactivate_local
|
28
|
+
from fractal_server.tasks.v2.local import deactivate_local_pixi
|
28
29
|
from fractal_server.tasks.v2.local import reactivate_local
|
30
|
+
from fractal_server.tasks.v2.local import reactivate_local_pixi
|
29
31
|
from fractal_server.tasks.v2.ssh import deactivate_ssh
|
32
|
+
from fractal_server.tasks.v2.ssh import deactivate_ssh_pixi
|
30
33
|
from fractal_server.tasks.v2.ssh import reactivate_ssh
|
34
|
+
from fractal_server.tasks.v2.ssh import reactivate_ssh_pixi
|
31
35
|
from fractal_server.utils import get_timestamp
|
32
36
|
|
33
37
|
router = APIRouter()
|
@@ -125,9 +129,12 @@ async def deactivate_task_group(
|
|
125
129
|
host=user_settings.ssh_host,
|
126
130
|
key_path=user_settings.ssh_private_key_path,
|
127
131
|
)
|
128
|
-
|
132
|
+
if task_group.origin == TaskGroupV2OriginEnum.PIXI:
|
133
|
+
deactivate_function = deactivate_ssh_pixi
|
134
|
+
else:
|
135
|
+
deactivate_function = deactivate_ssh
|
129
136
|
background_tasks.add_task(
|
130
|
-
|
137
|
+
deactivate_function,
|
131
138
|
task_group_id=task_group.id,
|
132
139
|
task_group_activity_id=task_group_activity.id,
|
133
140
|
ssh_config=ssh_config,
|
@@ -135,8 +142,12 @@ async def deactivate_task_group(
|
|
135
142
|
)
|
136
143
|
|
137
144
|
else:
|
145
|
+
if task_group.origin == TaskGroupV2OriginEnum.PIXI:
|
146
|
+
deactivate_function = deactivate_local_pixi
|
147
|
+
else:
|
148
|
+
deactivate_function = deactivate_local
|
138
149
|
background_tasks.add_task(
|
139
|
-
|
150
|
+
deactivate_function,
|
140
151
|
task_group_id=task_group.id,
|
141
152
|
task_group_activity_id=task_group_activity.id,
|
142
153
|
)
|
@@ -210,12 +221,12 @@ async def reactivate_task_group(
|
|
210
221
|
response.status_code = status.HTTP_202_ACCEPTED
|
211
222
|
return task_group_activity
|
212
223
|
|
213
|
-
if task_group.
|
224
|
+
if task_group.env_info is None:
|
214
225
|
raise HTTPException(
|
215
226
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
216
227
|
detail=(
|
217
228
|
"Cannot reactivate a task group with "
|
218
|
-
f"{task_group.
|
229
|
+
f"{task_group.env_info=}."
|
219
230
|
),
|
220
231
|
)
|
221
232
|
|
@@ -247,8 +258,12 @@ async def reactivate_task_group(
|
|
247
258
|
key_path=user_settings.ssh_private_key_path,
|
248
259
|
)
|
249
260
|
|
261
|
+
if task_group.origin == TaskGroupV2OriginEnum.PIXI:
|
262
|
+
reactivate_function = reactivate_ssh_pixi
|
263
|
+
else:
|
264
|
+
reactivate_function = reactivate_ssh
|
250
265
|
background_tasks.add_task(
|
251
|
-
|
266
|
+
reactivate_function,
|
252
267
|
task_group_id=task_group.id,
|
253
268
|
task_group_activity_id=task_group_activity.id,
|
254
269
|
ssh_config=ssh_config,
|
@@ -256,8 +271,12 @@ async def reactivate_task_group(
|
|
256
271
|
)
|
257
272
|
|
258
273
|
else:
|
274
|
+
if task_group.origin == TaskGroupV2OriginEnum.PIXI:
|
275
|
+
reactivate_function = reactivate_local_pixi
|
276
|
+
else:
|
277
|
+
reactivate_function = reactivate_local
|
259
278
|
background_tasks.add_task(
|
260
|
-
|
279
|
+
reactivate_function,
|
261
280
|
task_group_id=task_group.id,
|
262
281
|
task_group_activity_id=task_group_activity.id,
|
263
282
|
)
|
@@ -33,14 +33,15 @@ from .task import TaskImportV2Legacy # noqa F401
|
|
33
33
|
from .task import TaskReadV2 # noqa F401
|
34
34
|
from .task import TaskType # noqa F401
|
35
35
|
from .task import TaskUpdateV2 # noqa F401
|
36
|
+
from .task_collection import FractalUploadedFile # noqa F401
|
36
37
|
from .task_collection import TaskCollectCustomV2 # noqa F401
|
37
38
|
from .task_collection import TaskCollectPipV2 # noqa F401
|
38
|
-
from .task_collection import WheelFile # noqa F401
|
39
39
|
from .task_group import TaskGroupActivityActionV2 # noqa F401
|
40
40
|
from .task_group import TaskGroupActivityStatusV2 # noqa F401
|
41
41
|
from .task_group import TaskGroupActivityV2Read # noqa F401
|
42
42
|
from .task_group import TaskGroupCreateV2 # noqa F401
|
43
43
|
from .task_group import TaskGroupCreateV2Strict # noqa F401
|
44
|
+
from .task_group import TaskGroupCreateV2StrictPixi # noqa F401
|
44
45
|
from .task_group import TaskGroupReadV2 # noqa F401
|
45
46
|
from .task_group import TaskGroupUpdateV2 # noqa F401
|
46
47
|
from .task_group import TaskGroupV2OriginEnum # noqa F401
|
@@ -16,6 +16,7 @@ from fractal_server.types import NonEmptyStr
|
|
16
16
|
class TaskGroupV2OriginEnum(StrEnum):
|
17
17
|
PYPI = "pypi"
|
18
18
|
WHEELFILE = "wheel-file"
|
19
|
+
PIXI = "pixi"
|
19
20
|
OTHER = "other"
|
20
21
|
|
21
22
|
|
@@ -41,11 +42,12 @@ class TaskGroupCreateV2(BaseModel):
|
|
41
42
|
pkg_name: str
|
42
43
|
version: str | None = None
|
43
44
|
python_version: NonEmptyStr = None
|
45
|
+
pixi_version: NonEmptyStr = None
|
44
46
|
path: AbsolutePathStr = None
|
45
47
|
venv_path: AbsolutePathStr = None
|
46
|
-
|
48
|
+
archive_path: AbsolutePathStr = None
|
47
49
|
pip_extras: NonEmptyStr = None
|
48
|
-
|
50
|
+
env_info: str | None = None
|
49
51
|
pinned_package_versions: DictStrStr = Field(default_factory=dict)
|
50
52
|
|
51
53
|
|
@@ -55,11 +57,20 @@ class TaskGroupCreateV2Strict(TaskGroupCreateV2):
|
|
55
57
|
"""
|
56
58
|
|
57
59
|
path: AbsolutePathStr
|
58
|
-
venv_path: AbsolutePathStr
|
59
60
|
version: NonEmptyStr
|
61
|
+
venv_path: AbsolutePathStr
|
60
62
|
python_version: NonEmptyStr
|
61
63
|
|
62
64
|
|
65
|
+
class TaskGroupCreateV2StrictPixi(TaskGroupCreateV2):
|
66
|
+
"""
|
67
|
+
A strict version of TaskGroupCreateV2, to be used for pixi task collection.
|
68
|
+
"""
|
69
|
+
|
70
|
+
path: AbsolutePathStr
|
71
|
+
pixi_version: NonEmptyStr
|
72
|
+
|
73
|
+
|
63
74
|
class TaskGroupReadV2(BaseModel):
|
64
75
|
id: int
|
65
76
|
task_list: list[TaskReadV2]
|
@@ -71,10 +82,10 @@ class TaskGroupReadV2(BaseModel):
|
|
71
82
|
pkg_name: str
|
72
83
|
version: str | None = None
|
73
84
|
python_version: str | None = None
|
85
|
+
pixi_version: str | None = None
|
74
86
|
path: str | None = None
|
75
87
|
venv_path: str | None = None
|
76
|
-
|
77
|
-
pip_freeze: str | None = None
|
88
|
+
archive_path: str | None = None
|
78
89
|
pip_extras: str | None = None
|
79
90
|
pinned_package_versions: dict[str, str] = Field(default_factory=dict)
|
80
91
|
|
fractal_server/config.py
CHANGED
@@ -11,6 +11,7 @@
|
|
11
11
|
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
12
12
|
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
13
13
|
# Zurich.
|
14
|
+
import json
|
14
15
|
import logging
|
15
16
|
import shutil
|
16
17
|
import sys
|
@@ -34,6 +35,7 @@ from sqlalchemy.engine import URL
|
|
34
35
|
|
35
36
|
import fractal_server
|
36
37
|
from fractal_server.types import AbsolutePathStr
|
38
|
+
from fractal_server.types import DictStrStr
|
37
39
|
|
38
40
|
|
39
41
|
class MailSettings(BaseModel):
|
@@ -62,6 +64,35 @@ class MailSettings(BaseModel):
|
|
62
64
|
use_login: bool
|
63
65
|
|
64
66
|
|
67
|
+
class PixiSettings(BaseModel):
|
68
|
+
default_version: str
|
69
|
+
versions: DictStrStr
|
70
|
+
|
71
|
+
@model_validator(mode="after")
|
72
|
+
def check_pixi_settings(self):
|
73
|
+
|
74
|
+
if self.default_version not in self.versions:
|
75
|
+
raise ValueError(
|
76
|
+
f"Default version '{self.default_version}' not in "
|
77
|
+
f"available version {list(self.versions.keys())}."
|
78
|
+
)
|
79
|
+
|
80
|
+
pixi_base_dir = Path(self.versions[self.default_version]).parent
|
81
|
+
|
82
|
+
for key, value in self.versions.items():
|
83
|
+
|
84
|
+
pixi_path = Path(value)
|
85
|
+
|
86
|
+
if pixi_path.parent != pixi_base_dir:
|
87
|
+
raise ValueError(
|
88
|
+
f"{pixi_path=} is not located within the {pixi_base_dir=}."
|
89
|
+
)
|
90
|
+
if pixi_path.name != key:
|
91
|
+
raise ValueError(f"{pixi_path.name=} is not equal to {key=}")
|
92
|
+
|
93
|
+
return self
|
94
|
+
|
95
|
+
|
65
96
|
class FractalConfigurationError(RuntimeError):
|
66
97
|
pass
|
67
98
|
|
@@ -513,6 +544,17 @@ class Settings(BaseSettings):
|
|
513
544
|
FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders".
|
514
545
|
"""
|
515
546
|
|
547
|
+
FRACTAL_PIXI_CONFIG_FILE: Path | None = None
|
548
|
+
|
549
|
+
pixi: PixiSettings | None = None
|
550
|
+
|
551
|
+
@model_validator(mode="after")
|
552
|
+
def populate_pixi_settings(self):
|
553
|
+
if self.FRACTAL_PIXI_CONFIG_FILE is not None:
|
554
|
+
with self.FRACTAL_PIXI_CONFIG_FILE.open("r") as f:
|
555
|
+
self.pixi = PixiSettings(**json.load(f))
|
556
|
+
return self
|
557
|
+
|
516
558
|
###########################################################################
|
517
559
|
# SMTP SERVICE
|
518
560
|
###########################################################################
|
@@ -0,0 +1,53 @@
|
|
1
|
+
"""Task group for pixi
|
2
|
+
|
3
|
+
Revision ID: b1e7f7a1ff71
|
4
|
+
Revises: 791ce783d3d8
|
5
|
+
Create Date: 2025-05-29 16:31:17.565973
|
6
|
+
|
7
|
+
"""
|
8
|
+
import sqlalchemy as sa
|
9
|
+
import sqlmodel
|
10
|
+
from alembic import op
|
11
|
+
|
12
|
+
|
13
|
+
# revision identifiers, used by Alembic.
|
14
|
+
revision = "b1e7f7a1ff71"
|
15
|
+
down_revision = "791ce783d3d8"
|
16
|
+
branch_labels = None
|
17
|
+
depends_on = None
|
18
|
+
|
19
|
+
|
20
|
+
def upgrade() -> None:
|
21
|
+
with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
|
22
|
+
batch_op.add_column(
|
23
|
+
sa.Column(
|
24
|
+
"pixi_version",
|
25
|
+
sqlmodel.sql.sqltypes.AutoString(),
|
26
|
+
nullable=True,
|
27
|
+
)
|
28
|
+
)
|
29
|
+
batch_op.alter_column(
|
30
|
+
"wheel_path",
|
31
|
+
nullable=True,
|
32
|
+
new_column_name="archive_path",
|
33
|
+
)
|
34
|
+
batch_op.alter_column(
|
35
|
+
"pip_freeze",
|
36
|
+
nullable=True,
|
37
|
+
new_column_name="env_info",
|
38
|
+
)
|
39
|
+
|
40
|
+
|
41
|
+
def downgrade() -> None:
|
42
|
+
with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
|
43
|
+
batch_op.alter_column(
|
44
|
+
"archive_path",
|
45
|
+
nullable=True,
|
46
|
+
new_column_name="wheel_path",
|
47
|
+
)
|
48
|
+
batch_op.alter_column(
|
49
|
+
"env_info",
|
50
|
+
nullable=True,
|
51
|
+
new_column_name="pip_freeze",
|
52
|
+
)
|
53
|
+
batch_op.drop_column("pixi_version")
|