fractal-server 2.9.2__py3-none-any.whl → 2.10.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/routes/api/v2/task_collection.py +110 -21
- fractal_server/app/schemas/v2/__init__.py +1 -0
- fractal_server/app/schemas/v2/task_collection.py +31 -35
- fractal_server/app/schemas/v2/task_group.py +0 -16
- fractal_server/tasks/v2/local/collect.py +17 -22
- fractal_server/tasks/v2/ssh/collect.py +26 -15
- {fractal_server-2.9.2.dist-info → fractal_server-2.10.0a0.dist-info}/METADATA +1 -1
- {fractal_server-2.9.2.dist-info → fractal_server-2.10.0a0.dist-info}/RECORD +12 -12
- {fractal_server-2.9.2.dist-info → fractal_server-2.10.0a0.dist-info}/LICENSE +0 -0
- {fractal_server-2.9.2.dist-info → fractal_server-2.10.0a0.dist-info}/WHEEL +0 -0
- {fractal_server-2.9.2.dist-info → fractal_server-2.10.0a0.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.
|
1
|
+
__VERSION__ = "2.10.0a0"
|
@@ -1,13 +1,19 @@
|
|
1
|
+
import json
|
1
2
|
from pathlib import Path
|
2
3
|
from typing import Optional
|
3
4
|
|
4
5
|
from fastapi import APIRouter
|
5
6
|
from fastapi import BackgroundTasks
|
6
7
|
from fastapi import Depends
|
8
|
+
from fastapi import File
|
9
|
+
from fastapi import Form
|
7
10
|
from fastapi import HTTPException
|
8
11
|
from fastapi import Request
|
9
12
|
from fastapi import Response
|
10
13
|
from fastapi import status
|
14
|
+
from fastapi import UploadFile
|
15
|
+
from pydantic import BaseModel
|
16
|
+
from pydantic import root_validator
|
11
17
|
from pydantic import ValidationError
|
12
18
|
from sqlmodel import select
|
13
19
|
|
@@ -22,6 +28,7 @@ from ....schemas.v2 import TaskCollectPipV2
|
|
22
28
|
from ....schemas.v2 import TaskGroupActivityStatusV2
|
23
29
|
from ....schemas.v2 import TaskGroupActivityV2Read
|
24
30
|
from ....schemas.v2 import TaskGroupCreateV2Strict
|
31
|
+
from ....schemas.v2 import WheelFile
|
25
32
|
from ...aux.validate_user_settings import validate_user_settings
|
26
33
|
from ._aux_functions_task_lifecycle import get_package_version_from_pypi
|
27
34
|
from ._aux_functions_tasks import _get_valid_user_group_id
|
@@ -44,20 +51,99 @@ from fractal_server.tasks.v2.utils_python_interpreter import (
|
|
44
51
|
get_python_interpreter_v2,
|
45
52
|
)
|
46
53
|
|
54
|
+
|
47
55
|
router = APIRouter()
|
48
56
|
|
49
57
|
logger = set_logger(__name__)
|
50
58
|
|
51
59
|
|
60
|
+
class CollectionRequestData(BaseModel):
|
61
|
+
"""
|
62
|
+
Validate form data _and_ wheel file.
|
63
|
+
"""
|
64
|
+
|
65
|
+
task_collect: TaskCollectPipV2
|
66
|
+
file: Optional[UploadFile] = None
|
67
|
+
origin: TaskGroupV2OriginEnum
|
68
|
+
|
69
|
+
@root_validator(pre=True)
|
70
|
+
def validate_data(cls, values):
|
71
|
+
file = values.get("file")
|
72
|
+
package = values.get("task_collect").package
|
73
|
+
package_version = values.get("task_collect").package_version
|
74
|
+
|
75
|
+
if file is None:
|
76
|
+
if package is None:
|
77
|
+
raise ValueError(
|
78
|
+
"When no `file` is provided, `package` is required."
|
79
|
+
)
|
80
|
+
values["origin"] = TaskGroupV2OriginEnum.PYPI
|
81
|
+
else:
|
82
|
+
if package is not None:
|
83
|
+
raise ValueError(
|
84
|
+
"Cannot set `package` when `file` is provided "
|
85
|
+
f"(given package='{package}')."
|
86
|
+
)
|
87
|
+
if package_version is not None:
|
88
|
+
raise ValueError(
|
89
|
+
"Cannot set `package_version` when `file` is "
|
90
|
+
f"provided (given package_version='{package_version}')."
|
91
|
+
)
|
92
|
+
values["origin"] = TaskGroupV2OriginEnum.WHEELFILE
|
93
|
+
return values
|
94
|
+
|
95
|
+
|
96
|
+
def parse_request_data(
|
97
|
+
package: Optional[str] = Form(None),
|
98
|
+
package_version: Optional[str] = Form(None),
|
99
|
+
package_extras: Optional[str] = Form(None),
|
100
|
+
python_version: Optional[str] = Form(None),
|
101
|
+
pinned_package_versions: Optional[str] = Form(None),
|
102
|
+
file: Optional[UploadFile] = File(None),
|
103
|
+
) -> CollectionRequestData:
|
104
|
+
"""
|
105
|
+
Expand the parsing/validation of `parse_form_data`, based on `file`.
|
106
|
+
"""
|
107
|
+
|
108
|
+
try:
|
109
|
+
# Convert dict_pinned_pkg from a JSON string into a Python dictionary
|
110
|
+
dict_pinned_pkg = (
|
111
|
+
json.loads(pinned_package_versions)
|
112
|
+
if pinned_package_versions
|
113
|
+
else None
|
114
|
+
)
|
115
|
+
# Validate and coerce form data
|
116
|
+
task_collect_pip = TaskCollectPipV2(
|
117
|
+
package=package,
|
118
|
+
package_version=package_version,
|
119
|
+
package_extras=package_extras,
|
120
|
+
python_version=python_version,
|
121
|
+
pinned_package_versions=dict_pinned_pkg,
|
122
|
+
)
|
123
|
+
|
124
|
+
data = CollectionRequestData(
|
125
|
+
task_collect=task_collect_pip,
|
126
|
+
file=file,
|
127
|
+
)
|
128
|
+
|
129
|
+
except (ValidationError, json.JSONDecodeError) as e:
|
130
|
+
raise HTTPException(
|
131
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
132
|
+
detail=f"Invalid request-body\n{str(e)}",
|
133
|
+
)
|
134
|
+
|
135
|
+
return data
|
136
|
+
|
137
|
+
|
52
138
|
@router.post(
|
53
139
|
"/collect/pip/",
|
54
140
|
response_model=TaskGroupActivityV2Read,
|
55
141
|
)
|
56
142
|
async def collect_tasks_pip(
|
57
|
-
task_collect: TaskCollectPipV2,
|
58
|
-
background_tasks: BackgroundTasks,
|
59
|
-
response: Response,
|
60
143
|
request: Request,
|
144
|
+
response: Response,
|
145
|
+
background_tasks: BackgroundTasks,
|
146
|
+
request_data: CollectionRequestData = Depends(parse_request_data),
|
61
147
|
private: bool = False,
|
62
148
|
user_group_id: Optional[int] = None,
|
63
149
|
user: UserOAuth = Depends(current_active_verified_user),
|
@@ -66,12 +152,17 @@ async def collect_tasks_pip(
|
|
66
152
|
"""
|
67
153
|
Task-collection endpoint
|
68
154
|
"""
|
69
|
-
|
70
155
|
# Get settings
|
71
156
|
settings = Inject(get_settings)
|
72
157
|
|
158
|
+
# Get some validated request data
|
159
|
+
task_collect = request_data.task_collect
|
160
|
+
|
73
161
|
# Initialize task-group attributes
|
74
|
-
task_group_attrs = dict(
|
162
|
+
task_group_attrs = dict(
|
163
|
+
user_id=user.id,
|
164
|
+
origin=request_data.origin,
|
165
|
+
)
|
75
166
|
|
76
167
|
# Set/check python version
|
77
168
|
if task_collect.python_version is None:
|
@@ -103,12 +194,19 @@ async def collect_tasks_pip(
|
|
103
194
|
"pinned_package_versions"
|
104
195
|
] = task_collect.pinned_package_versions
|
105
196
|
|
197
|
+
# Initialize wheel_file_content as None
|
198
|
+
wheel_file = None
|
199
|
+
|
106
200
|
# Set pkg_name, version, origin and wheel_path
|
107
|
-
if
|
201
|
+
if request_data.origin == TaskGroupV2OriginEnum.WHEELFILE:
|
108
202
|
try:
|
109
|
-
|
110
|
-
wheel_filename = Path(task_group_attrs["wheel_path"]).name
|
203
|
+
wheel_filename = request_data.file.filename
|
111
204
|
wheel_info = _parse_wheel_filename(wheel_filename)
|
205
|
+
wheel_file_content = await request_data.file.read()
|
206
|
+
wheel_file = WheelFile(
|
207
|
+
filename=wheel_filename,
|
208
|
+
contents=wheel_file_content,
|
209
|
+
)
|
112
210
|
except ValueError as e:
|
113
211
|
raise HTTPException(
|
114
212
|
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
@@ -121,11 +219,9 @@ async def collect_tasks_pip(
|
|
121
219
|
wheel_info["distribution"]
|
122
220
|
)
|
123
221
|
task_group_attrs["version"] = wheel_info["version"]
|
124
|
-
|
125
|
-
else:
|
222
|
+
elif request_data.origin == TaskGroupV2OriginEnum.PYPI:
|
126
223
|
pkg_name = task_collect.package
|
127
224
|
task_group_attrs["pkg_name"] = normalize_package_name(pkg_name)
|
128
|
-
task_group_attrs["origin"] = TaskGroupV2OriginEnum.PYPI
|
129
225
|
latest_version = await get_package_version_from_pypi(
|
130
226
|
task_collect.package,
|
131
227
|
task_collect.package_version,
|
@@ -202,7 +298,6 @@ async def collect_tasks_pip(
|
|
202
298
|
# On-disk checks
|
203
299
|
|
204
300
|
if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
|
205
|
-
|
206
301
|
# Verify that folder does not exist (for local collection)
|
207
302
|
if Path(task_group_path).exists():
|
208
303
|
raise HTTPException(
|
@@ -210,15 +305,6 @@ async def collect_tasks_pip(
|
|
210
305
|
detail=f"{task_group_path} already exists.",
|
211
306
|
)
|
212
307
|
|
213
|
-
# Verify that wheel file exists
|
214
|
-
wheel_path = task_group_attrs.get("wheel_path", None)
|
215
|
-
if wheel_path is not None:
|
216
|
-
if not Path(wheel_path).exists():
|
217
|
-
raise HTTPException(
|
218
|
-
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
219
|
-
detail=f"No such file: {wheel_path}.",
|
220
|
-
)
|
221
|
-
|
222
308
|
# Create TaskGroupV2 object
|
223
309
|
task_group = TaskGroupV2(**task_group_attrs)
|
224
310
|
db.add(task_group)
|
@@ -259,14 +345,17 @@ async def collect_tasks_pip(
|
|
259
345
|
task_group_activity_id=task_group_activity.id,
|
260
346
|
fractal_ssh=fractal_ssh,
|
261
347
|
tasks_base_dir=user_settings.ssh_tasks_dir,
|
348
|
+
wheel_file=wheel_file,
|
262
349
|
)
|
263
350
|
|
264
351
|
else:
|
265
352
|
# Local task collection
|
353
|
+
|
266
354
|
background_tasks.add_task(
|
267
355
|
collect_local,
|
268
356
|
task_group_id=task_group.id,
|
269
357
|
task_group_activity_id=task_group_activity.id,
|
358
|
+
wheel_file=wheel_file,
|
270
359
|
)
|
271
360
|
logger.debug(
|
272
361
|
"Task-collection endpoint: start background collection "
|
@@ -25,6 +25,7 @@ from .task import TaskReadV2 # noqa F401
|
|
25
25
|
from .task import TaskUpdateV2 # noqa F401
|
26
26
|
from .task_collection import TaskCollectCustomV2 # noqa F401
|
27
27
|
from .task_collection import TaskCollectPipV2 # noqa F401
|
28
|
+
from .task_collection import WheelFile # noqa F401
|
28
29
|
from .task_group import TaskGroupActivityActionV2 # noqa F401
|
29
30
|
from .task_group import TaskGroupActivityStatusV2 # noqa F401
|
30
31
|
from .task_group import TaskGroupActivityV2Read # noqa F401
|
@@ -12,6 +12,15 @@ from fractal_server.app.schemas.v2 import ManifestV2
|
|
12
12
|
from fractal_server.string_tools import validate_cmd
|
13
13
|
|
14
14
|
|
15
|
+
class WheelFile(BaseModel):
|
16
|
+
"""
|
17
|
+
Model for data sent from the endpoint to the background task.
|
18
|
+
"""
|
19
|
+
|
20
|
+
filename: str
|
21
|
+
contents: bytes
|
22
|
+
|
23
|
+
|
15
24
|
class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
|
16
25
|
"""
|
17
26
|
TaskCollectPipV2 class
|
@@ -22,14 +31,11 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
|
|
22
31
|
|
23
32
|
Two cases are supported:
|
24
33
|
|
25
|
-
1. `package` is the
|
26
|
-
|
27
|
-
|
34
|
+
1. `package` is the name of a package that can be installed via `pip`.
|
35
|
+
1. `package=None`, and a wheel file is uploaded within the API request.
|
28
36
|
|
29
37
|
Attributes:
|
30
|
-
package:
|
31
|
-
The name of a `pip`-installable package, or the path to a local
|
32
|
-
wheel file.
|
38
|
+
package: The name of a `pip`-installable package, or `None`.
|
33
39
|
package_version: Version of the package
|
34
40
|
package_extras: Package extras to include in the `pip install` command
|
35
41
|
python_version: Python version to install and run the package tasks
|
@@ -39,12 +45,28 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
|
|
39
45
|
|
40
46
|
"""
|
41
47
|
|
42
|
-
package: str
|
48
|
+
package: Optional[str] = None
|
43
49
|
package_version: Optional[str] = None
|
44
50
|
package_extras: Optional[str] = None
|
45
51
|
python_version: Optional[Literal["3.9", "3.10", "3.11", "3.12"]] = None
|
46
52
|
pinned_package_versions: Optional[dict[str, str]] = None
|
47
53
|
|
54
|
+
@validator("package")
|
55
|
+
def package_validator(cls, value: Optional[str]) -> Optional[str]:
|
56
|
+
if value is None:
|
57
|
+
return value
|
58
|
+
value = valstr("package")(value)
|
59
|
+
validate_cmd(value, attribute_name="package")
|
60
|
+
return value
|
61
|
+
|
62
|
+
@validator("package_version")
|
63
|
+
def package_version_validator(cls, value: Optional[str]) -> Optional[str]:
|
64
|
+
if value is None:
|
65
|
+
return value
|
66
|
+
value = valstr("package_version")(value)
|
67
|
+
validate_cmd(value, attribute_name="package_version")
|
68
|
+
return value
|
69
|
+
|
48
70
|
@validator("pinned_package_versions")
|
49
71
|
def pinned_package_versions_validator(cls, value):
|
50
72
|
if value is None:
|
@@ -65,36 +87,10 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
|
|
65
87
|
validate_cmd(version)
|
66
88
|
return value
|
67
89
|
|
68
|
-
@validator("package")
|
69
|
-
def package_validator(cls, value):
|
70
|
-
value = valstr("package")(value)
|
71
|
-
if "/" in value or value.endswith(".whl"):
|
72
|
-
if not value.endswith(".whl"):
|
73
|
-
raise ValueError(
|
74
|
-
"Local-package path must be a wheel file "
|
75
|
-
f"(given {value})."
|
76
|
-
)
|
77
|
-
if not Path(value).is_absolute():
|
78
|
-
raise ValueError(
|
79
|
-
f"Local-package path must be absolute: (given {value})."
|
80
|
-
)
|
81
|
-
validate_cmd(value, attribute_name="package")
|
82
|
-
return value
|
83
|
-
|
84
|
-
@validator("package_version")
|
85
|
-
def package_version_validator(
|
86
|
-
cls, v: Optional[str], values
|
87
|
-
) -> Optional[str]:
|
88
|
-
v = valstr("package_version")(v)
|
89
|
-
if values["package"].endswith(".whl"):
|
90
|
-
raise ValueError(
|
91
|
-
"Cannot provide package version when package is a wheel file."
|
92
|
-
)
|
93
|
-
validate_cmd(v, attribute_name="package_version")
|
94
|
-
return v
|
95
|
-
|
96
90
|
@validator("package_extras")
|
97
91
|
def package_extras_validator(cls, value: Optional[str]) -> Optional[str]:
|
92
|
+
if value is None:
|
93
|
+
return value
|
98
94
|
value = valstr("package_extras")(value)
|
99
95
|
validate_cmd(value, attribute_name="package_extras")
|
100
96
|
return value
|
@@ -5,7 +5,6 @@ from typing import Optional
|
|
5
5
|
from pydantic import BaseModel
|
6
6
|
from pydantic import Extra
|
7
7
|
from pydantic import Field
|
8
|
-
from pydantic import root_validator
|
9
8
|
from pydantic import validator
|
10
9
|
|
11
10
|
from .._validators import val_absolute_path
|
@@ -77,21 +76,6 @@ class TaskGroupCreateV2Strict(TaskGroupCreateV2):
|
|
77
76
|
version: str
|
78
77
|
python_version: str
|
79
78
|
|
80
|
-
@root_validator
|
81
|
-
def check_wheel_file(cls, values):
|
82
|
-
origin = values.get("origin")
|
83
|
-
wheel_path = values.get("wheel_path")
|
84
|
-
bad_condition_1 = (
|
85
|
-
origin == TaskGroupV2OriginEnum.WHEELFILE and wheel_path is None
|
86
|
-
)
|
87
|
-
bad_condition_2 = (
|
88
|
-
origin != TaskGroupV2OriginEnum.WHEELFILE
|
89
|
-
and wheel_path is not None
|
90
|
-
)
|
91
|
-
if bad_condition_1 or bad_condition_2:
|
92
|
-
raise ValueError(f"Cannot have {origin=} and {wheel_path=}.")
|
93
|
-
return values
|
94
|
-
|
95
79
|
|
96
80
|
class TaskGroupReadV2(BaseModel):
|
97
81
|
id: int
|
@@ -4,6 +4,7 @@ import shutil
|
|
4
4
|
import time
|
5
5
|
from pathlib import Path
|
6
6
|
from tempfile import TemporaryDirectory
|
7
|
+
from typing import Optional
|
7
8
|
|
8
9
|
from ..utils_database import create_db_tasks_and_update_task_group
|
9
10
|
from ._utils import _customize_and_run_template
|
@@ -12,8 +13,8 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
12
13
|
from fractal_server.app.models.v2 import TaskGroupV2
|
13
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
14
15
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
16
|
+
from fractal_server.app.schemas.v2 import WheelFile
|
15
17
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
16
|
-
from fractal_server.logger import get_logger
|
17
18
|
from fractal_server.logger import set_logger
|
18
19
|
from fractal_server.tasks.utils import get_log_path
|
19
20
|
from fractal_server.tasks.v2.local._utils import check_task_files_exist
|
@@ -35,30 +36,19 @@ from fractal_server.utils import get_timestamp
|
|
35
36
|
LOGGER_NAME = __name__
|
36
37
|
|
37
38
|
|
38
|
-
def _copy_wheel_file_local(task_group: TaskGroupV2) -> str:
|
39
|
-
logger = get_logger(LOGGER_NAME)
|
40
|
-
source = task_group.wheel_path
|
41
|
-
dest = (
|
42
|
-
Path(task_group.path) / Path(task_group.wheel_path).name
|
43
|
-
).as_posix()
|
44
|
-
logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")
|
45
|
-
shutil.copy(task_group.wheel_path, task_group.path)
|
46
|
-
logger.debug(f"[_copy_wheel_file] END {source=} {dest=}")
|
47
|
-
return dest
|
48
|
-
|
49
|
-
|
50
39
|
def collect_local(
|
51
40
|
*,
|
52
41
|
task_group_activity_id: int,
|
53
42
|
task_group_id: int,
|
43
|
+
wheel_file: Optional[WheelFile] = None,
|
54
44
|
) -> None:
|
55
45
|
"""
|
56
46
|
Collect a task package.
|
57
47
|
|
58
|
-
This function
|
48
|
+
This function runs as a background task, therefore exceptions must be
|
59
49
|
handled.
|
60
50
|
|
61
|
-
NOTE:
|
51
|
+
NOTE: since this function is sync, it runs within a thread - due to
|
62
52
|
starlette/fastapi handling of background tasks (see
|
63
53
|
https://github.com/encode/starlette/blob/master/starlette/background.py).
|
64
54
|
|
@@ -66,6 +56,7 @@ def collect_local(
|
|
66
56
|
Arguments:
|
67
57
|
task_group_id:
|
68
58
|
task_group_activity_id:
|
59
|
+
wheel_file:
|
69
60
|
"""
|
70
61
|
|
71
62
|
with TemporaryDirectory() as tmpdir:
|
@@ -76,7 +67,6 @@ def collect_local(
|
|
76
67
|
)
|
77
68
|
|
78
69
|
with next(get_sync_db()) as db:
|
79
|
-
|
80
70
|
# Get main objects from db
|
81
71
|
activity = db.get(TaskGroupActivityV2, task_group_activity_id)
|
82
72
|
task_group = db.get(TaskGroupV2, task_group_id)
|
@@ -109,17 +99,22 @@ def collect_local(
|
|
109
99
|
return
|
110
100
|
|
111
101
|
try:
|
112
|
-
|
113
102
|
# Create task_group.path folder
|
114
103
|
Path(task_group.path).mkdir(parents=True)
|
115
104
|
logger.debug(f"Created {task_group.path}")
|
116
105
|
|
117
|
-
#
|
118
|
-
if
|
119
|
-
|
120
|
-
|
106
|
+
# Write wheel file and set task_group.wheel_path
|
107
|
+
if wheel_file is not None:
|
108
|
+
|
109
|
+
wheel_path = (
|
110
|
+
Path(task_group.path) / wheel_file.filename
|
111
|
+
).as_posix()
|
112
|
+
logger.debug(
|
113
|
+
f"Write wheel-file contents into {wheel_path}"
|
121
114
|
)
|
122
|
-
|
115
|
+
with open(wheel_path, "wb") as f:
|
116
|
+
f.write(wheel_file.contents)
|
117
|
+
task_group.wheel_path = wheel_path
|
123
118
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
124
119
|
|
125
120
|
# Prepare replacements for templates
|
@@ -2,6 +2,7 @@ import logging
|
|
2
2
|
import time
|
3
3
|
from pathlib import Path
|
4
4
|
from tempfile import TemporaryDirectory
|
5
|
+
from typing import Optional
|
5
6
|
|
6
7
|
from ..utils_background import _prepare_tasks_metadata
|
7
8
|
from ..utils_background import fail_and_cleanup
|
@@ -11,10 +12,10 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
11
12
|
from fractal_server.app.models.v2 import TaskGroupV2
|
12
13
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
13
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
15
|
+
from fractal_server.app.schemas.v2 import WheelFile
|
14
16
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
15
17
|
from fractal_server.logger import set_logger
|
16
18
|
from fractal_server.ssh._fabric import FractalSSH
|
17
|
-
from fractal_server.tasks.v2.ssh._utils import _copy_wheel_file_ssh
|
18
19
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
|
19
20
|
from fractal_server.tasks.v2.utils_background import add_commit_refresh
|
20
21
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
@@ -38,14 +39,15 @@ def collect_ssh(
|
|
38
39
|
task_group_activity_id: int,
|
39
40
|
fractal_ssh: FractalSSH,
|
40
41
|
tasks_base_dir: str,
|
42
|
+
wheel_file: Optional[WheelFile] = None,
|
41
43
|
) -> None:
|
42
44
|
"""
|
43
45
|
Collect a task package over SSH
|
44
46
|
|
45
|
-
This function
|
47
|
+
This function runs as a background task, therefore exceptions must be
|
46
48
|
handled.
|
47
49
|
|
48
|
-
NOTE:
|
50
|
+
NOTE: since this function is sync, it runs within a thread - due to
|
49
51
|
starlette/fastapi handling of background tasks (see
|
50
52
|
https://github.com/encode/starlette/blob/master/starlette/background.py).
|
51
53
|
|
@@ -57,6 +59,7 @@ def collect_ssh(
|
|
57
59
|
tasks_base_dir:
|
58
60
|
Only used as a `safe_root` in `remove_dir`, and typically set to
|
59
61
|
`user_settings.ssh_tasks_dir`.
|
62
|
+
wheel_file:
|
60
63
|
"""
|
61
64
|
|
62
65
|
# Work within a temporary folder, where also logs will be placed
|
@@ -116,27 +119,36 @@ def collect_ssh(
|
|
116
119
|
return
|
117
120
|
|
118
121
|
try:
|
119
|
-
script_dir_remote = (
|
120
|
-
Path(task_group.path) / SCRIPTS_SUBFOLDER
|
121
|
-
).as_posix()
|
122
122
|
# Create remote `task_group.path` and `script_dir_remote`
|
123
123
|
# folders (note that because of `parents=True` we are in
|
124
124
|
# the `no error if existing, make parent directories as
|
125
125
|
# needed` scenario for `mkdir`)
|
126
|
+
script_dir_remote = (
|
127
|
+
Path(task_group.path) / SCRIPTS_SUBFOLDER
|
128
|
+
).as_posix()
|
126
129
|
fractal_ssh.mkdir(folder=task_group.path, parents=True)
|
127
130
|
fractal_ssh.mkdir(folder=script_dir_remote, parents=True)
|
128
131
|
|
129
|
-
#
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
132
|
+
# Write wheel file locally and send it to remote path,
|
133
|
+
# and set task_group.wheel_path
|
134
|
+
if wheel_file is not None:
|
135
|
+
wheel_filename = wheel_file.filename
|
136
|
+
wheel_path = (
|
137
|
+
Path(task_group.path) / wheel_filename
|
138
|
+
).as_posix()
|
139
|
+
tmp_wheel_path = (Path(tmpdir) / wheel_filename).as_posix()
|
140
|
+
logger.debug(
|
141
|
+
f"Write wheel-file contents into {tmp_wheel_path}"
|
135
142
|
)
|
136
|
-
|
143
|
+
with open(tmp_wheel_path, "wb") as f:
|
144
|
+
f.write(wheel_file.contents)
|
145
|
+
fractal_ssh.send_file(
|
146
|
+
local=tmp_wheel_path,
|
147
|
+
remote=wheel_path,
|
148
|
+
)
|
149
|
+
task_group.wheel_path = wheel_path
|
137
150
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
138
151
|
|
139
|
-
# Prepare replacements for templates
|
140
152
|
replacements = get_collection_replacements(
|
141
153
|
task_group=task_group,
|
142
154
|
python_bin=get_python_interpreter_v2(
|
@@ -173,7 +185,6 @@ def collect_ssh(
|
|
173
185
|
)
|
174
186
|
activity.log = get_current_log(log_file_path)
|
175
187
|
activity = add_commit_refresh(obj=activity, db=db)
|
176
|
-
|
177
188
|
# Run script 2
|
178
189
|
stdout = _customize_and_run_template(
|
179
190
|
template_filename="2_pip_install.sh",
|
@@ -1,4 +1,4 @@
|
|
1
|
-
fractal_server/__init__.py,sha256=
|
1
|
+
fractal_server/__init__.py,sha256=Qb19IXoi0fWBhP9l0Xx5EOFNybthKtzgo46qgF8V_s0,25
|
2
2
|
fractal_server/__main__.py,sha256=dEkCfzLLQrIlxsGC-HBfoR-RBMWnJDgNrxYTyzmE9c0,6146
|
3
3
|
fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
|
4
4
|
fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -53,7 +53,7 @@ fractal_server/app/routes/api/v2/project.py,sha256=eWYFJ7F2ZYQcpi-_n-rhPF-Q4gJhz
|
|
53
53
|
fractal_server/app/routes/api/v2/status.py,sha256=6N9DSZ4iFqbZImorWfEAPoyoFUgEruo4Hweqo0x0xXU,6435
|
54
54
|
fractal_server/app/routes/api/v2/submit.py,sha256=cQwt0oK8xjHMGA_bQrw4Um8jd_aCvgmWfoqSQDh12hQ,8246
|
55
55
|
fractal_server/app/routes/api/v2/task.py,sha256=K0ik33t7vL8BAK5S7fqyJDNdRK4stGqb_73bSa8tvPE,7159
|
56
|
-
fractal_server/app/routes/api/v2/task_collection.py,sha256=
|
56
|
+
fractal_server/app/routes/api/v2/task_collection.py,sha256=snX_E3OSBsgjbVwQMgKvV7pLmfNGD0OyqgAsxSGtB5E,12359
|
57
57
|
fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=cctW61-C2QYF2KXluS15lLhZJS_kt30Ca6UGLFO32z0,6207
|
58
58
|
fractal_server/app/routes/api/v2/task_group.py,sha256=4o2N0z7jK7VUVlJZMM4GveCCc4JKxYJx9-PMmsYIlJQ,8256
|
59
59
|
fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=3o9bCC8ubMwffQPPaxQZy-CjH9IB2RkIReIecI6L2_w,9300
|
@@ -149,7 +149,7 @@ fractal_server/app/schemas/v1/state.py,sha256=tBXzp_qW2TNNNPBo-AWEaffEU-1GkMBtUo
|
|
149
149
|
fractal_server/app/schemas/v1/task.py,sha256=7BxOZ_qoRQ8n3YbQpDvB7VMcxB5fSYQmR5RLIWhuJ5U,3704
|
150
150
|
fractal_server/app/schemas/v1/task_collection.py,sha256=uvq9bcMaGD_qHsh7YtcpoSAkVAbw12eY4DocIO3MKOg,3057
|
151
151
|
fractal_server/app/schemas/v1/workflow.py,sha256=oRKamLSuAgrTcv3gMMxGcotDloLL2c3NNgPA39UEmmM,4467
|
152
|
-
fractal_server/app/schemas/v2/__init__.py,sha256=
|
152
|
+
fractal_server/app/schemas/v2/__init__.py,sha256=ILWYZu_PfVVuZ8-qMRAuRuzBhne6nhS6sUzsigLcl-E,2557
|
153
153
|
fractal_server/app/schemas/v2/dataset.py,sha256=zRlcO0wDZahTW1PINdVEuARZ7GZUuEqqop7UdE3-5do,2470
|
154
154
|
fractal_server/app/schemas/v2/dumps.py,sha256=s6dg-pHZFui6t2Ktm0SMxjKDN-v-ZqBHz9iTsBQF3eU,1712
|
155
155
|
fractal_server/app/schemas/v2/job.py,sha256=42V-bFfMvysRplwTKGsL_WshAVsWSM6yjFqypxwrY3k,3020
|
@@ -157,8 +157,8 @@ fractal_server/app/schemas/v2/manifest.py,sha256=Uqtd7DbyOkf9bxBOKkU7Sv7nToBIFGU
|
|
157
157
|
fractal_server/app/schemas/v2/project.py,sha256=ABv9LSLVCq1QYthEhBvZOTn_4DFEC-7cH28tFGFdM7I,589
|
158
158
|
fractal_server/app/schemas/v2/status.py,sha256=SQaUpQkjFq5c5k5J4rOjNhuQaDOEg8lksPhkKmPU5VU,332
|
159
159
|
fractal_server/app/schemas/v2/task.py,sha256=FFAbYwDlqowB8gVMdjFVPVHvAM0T89PYLixUth49xfQ,6870
|
160
|
-
fractal_server/app/schemas/v2/task_collection.py,sha256=
|
161
|
-
fractal_server/app/schemas/v2/task_group.py,sha256=
|
160
|
+
fractal_server/app/schemas/v2/task_collection.py,sha256=9c_yyFcVBXdAZpQQniy1bROhYnQT7G1BflOpMY1joPE,6250
|
161
|
+
fractal_server/app/schemas/v2/task_group.py,sha256=EPQ1WHjIA8WDrpsTfvfRESjwUVzu6jKiaKZx45b36N4,3215
|
162
162
|
fractal_server/app/schemas/v2/workflow.py,sha256=-KWvXnbHBFA3pj5n7mfSyLKJQSqkJmoziIEe7mpLl3M,1875
|
163
163
|
fractal_server/app/schemas/v2/workflowtask.py,sha256=vDdMktYbHeYBgB5OuWSv6wRPRXWqvetkeqQ7IC5YtfA,5751
|
164
164
|
fractal_server/app/security/__init__.py,sha256=MlWVrLFPj9M2Gug-k8yATM-Cw066RugVU4KK6kMRbnQ,13019
|
@@ -218,12 +218,12 @@ fractal_server/tasks/v1/utils.py,sha256=HYFyNAyZofmf--mVgdwGC5TJpGShIWIDaS01yRr4
|
|
218
218
|
fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
219
219
|
fractal_server/tasks/v2/local/__init__.py,sha256=9RVItnS7OyLsJOuJjWMCicaky4ASUPQEYD4SzDs0hOE,141
|
220
220
|
fractal_server/tasks/v2/local/_utils.py,sha256=EvhmVwYjqaNyDCUMEsTWYOUXLgEwR1xr6bu32apCEI8,2491
|
221
|
-
fractal_server/tasks/v2/local/collect.py,sha256=
|
221
|
+
fractal_server/tasks/v2/local/collect.py,sha256=JuMplfREqrPvVEGlT5kJhcmZXC_iYlwvNlkgFrCaCC0,12107
|
222
222
|
fractal_server/tasks/v2/local/deactivate.py,sha256=XR1nvJY3mKCRqwPwV79rVaQmtb3J83KdmJKjTOHD-cU,9250
|
223
223
|
fractal_server/tasks/v2/local/reactivate.py,sha256=R3rArAzUpMGf6xa3dGVwwXHW9WVDi5ia28AFisZsqNc,6112
|
224
224
|
fractal_server/tasks/v2/ssh/__init__.py,sha256=aSQbVi6Ummt9QzcSLWNmSqYjfdxrn9ROmqgH6bDpI7k,135
|
225
225
|
fractal_server/tasks/v2/ssh/_utils.py,sha256=LjaEYVUJDChilu3YuhxuGWYRNnVJ_zqNE9SDHdRTIHY,2824
|
226
|
-
fractal_server/tasks/v2/ssh/collect.py,sha256=
|
226
|
+
fractal_server/tasks/v2/ssh/collect.py,sha256=2XXEPpl4LS22A75v_k4Bd46k46tmnLNZfceHyPi3kXo,13457
|
227
227
|
fractal_server/tasks/v2/ssh/deactivate.py,sha256=Ffk_UuQSBUBNBCiviuKNhEUGyZPQa4_erJKFdwgMcE8,10616
|
228
228
|
fractal_server/tasks/v2/ssh/reactivate.py,sha256=jdO8iyzavzSVPcOpIZrYSEkGPYTvz5XJ5h_5-nz9yzA,7896
|
229
229
|
fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
|
@@ -240,8 +240,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=MS8zu24qimJSktZaHruPxkwIl81ZoU
|
|
240
240
|
fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
|
241
241
|
fractal_server/utils.py,sha256=utvmBx8K9I8hRWFquxna2pBaOqe0JifDL_NVPmihEJI,3525
|
242
242
|
fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
|
243
|
-
fractal_server-2.
|
244
|
-
fractal_server-2.
|
245
|
-
fractal_server-2.
|
246
|
-
fractal_server-2.
|
247
|
-
fractal_server-2.
|
243
|
+
fractal_server-2.10.0a0.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
|
244
|
+
fractal_server-2.10.0a0.dist-info/METADATA,sha256=G-j04Q-B4uv3h77NOrY7lh-IEVFO_7AzmV3Lp3zqw8w,4546
|
245
|
+
fractal_server-2.10.0a0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
246
|
+
fractal_server-2.10.0a0.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
|
247
|
+
fractal_server-2.10.0a0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|