fractal-server 1.3.0a2__py3-none-any.whl → 1.3.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/api/v1/dataset.py +21 -0
- fractal_server/app/api/v1/task.py +78 -16
- fractal_server/app/api/v1/workflow.py +36 -25
- fractal_server/app/models/job.py +3 -2
- fractal_server/app/models/project.py +4 -3
- fractal_server/app/models/security.py +2 -0
- fractal_server/app/models/state.py +2 -1
- fractal_server/app/models/task.py +20 -9
- fractal_server/app/models/workflow.py +34 -29
- fractal_server/app/runner/_common.py +13 -12
- fractal_server/app/runner/_slurm/executor.py +2 -1
- fractal_server/common/requirements.txt +1 -1
- fractal_server/common/schemas/__init__.py +2 -0
- fractal_server/common/schemas/applyworkflow.py +6 -7
- fractal_server/common/schemas/manifest.py +32 -15
- fractal_server/common/schemas/project.py +8 -10
- fractal_server/common/schemas/state.py +3 -4
- fractal_server/common/schemas/task.py +28 -97
- fractal_server/common/schemas/task_collection.py +101 -0
- fractal_server/common/schemas/user.py +5 -0
- fractal_server/common/schemas/workflow.py +9 -11
- fractal_server/common/tests/test_manifest.py +36 -4
- fractal_server/common/tests/test_task.py +16 -0
- fractal_server/common/tests/test_task_collection.py +24 -0
- fractal_server/common/tests/test_user.py +12 -0
- fractal_server/main.py +3 -0
- fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +38 -0
- fractal_server/migrations/versions/{e8f4051440be_new_initial_schema.py → 50a13d6138fd_initial_schema.py} +18 -10
- fractal_server/migrations/versions/{fda995215ae9_drop_applyworkflow_overwrite_input.py → f384e1c0cf5d_drop_task_default_args_columns.py} +9 -10
- fractal_server/tasks/collection.py +180 -115
- {fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/METADATA +2 -1
- {fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/RECORD +36 -34
- {fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/WHEEL +1 -1
- fractal_server/migrations/versions/bb1cca2acc40_add_applyworkflow_end_timestamp.py +0 -31
- {fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/LICENSE +0 -0
- {fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/entry_points.txt +0 -0
@@ -12,6 +12,20 @@ def test_task_update():
|
|
12
12
|
debug(t)
|
13
13
|
assert list(t.dict(exclude_none=True).keys()) == ["name"]
|
14
14
|
assert list(t.dict(exclude_unset=True).keys()) == ["name"]
|
15
|
+
# Some failures
|
16
|
+
with pytest.raises(ValidationError):
|
17
|
+
TaskUpdate(name="task", version="")
|
18
|
+
with pytest.raises(ValidationError):
|
19
|
+
TaskUpdate(name="task", version=None)
|
20
|
+
# Successful cretion, with mutliple fields set
|
21
|
+
t = TaskUpdate(
|
22
|
+
name="task",
|
23
|
+
version="1.2.3",
|
24
|
+
owner="someone",
|
25
|
+
)
|
26
|
+
debug(t)
|
27
|
+
assert t.name
|
28
|
+
assert t.version
|
15
29
|
|
16
30
|
|
17
31
|
def test_task_create():
|
@@ -22,6 +36,8 @@ def test_task_create():
|
|
22
36
|
command="command",
|
23
37
|
input_type="input_type",
|
24
38
|
output_type="output_type",
|
39
|
+
version="1.2.3",
|
40
|
+
owner="someone",
|
25
41
|
)
|
26
42
|
debug(t)
|
27
43
|
# Missing arguments
|
@@ -0,0 +1,24 @@
|
|
1
|
+
import pytest
|
2
|
+
from devtools import debug
|
3
|
+
from pydantic.error_wrappers import ValidationError
|
4
|
+
|
5
|
+
from schemas import TaskCollectPip
|
6
|
+
|
7
|
+
|
8
|
+
def test_TaskCollectPip():
|
9
|
+
# Successful creation
|
10
|
+
c = TaskCollectPip(package="some-package")
|
11
|
+
debug(c)
|
12
|
+
assert c
|
13
|
+
c = TaskCollectPip(package="/some/package.whl")
|
14
|
+
debug(c)
|
15
|
+
assert c
|
16
|
+
# Failed creation
|
17
|
+
with pytest.raises(ValidationError):
|
18
|
+
c = TaskCollectPip(package="some/package")
|
19
|
+
with pytest.raises(ValidationError):
|
20
|
+
c = TaskCollectPip(package="/some/package.tar.gz")
|
21
|
+
with pytest.raises(ValidationError):
|
22
|
+
c = TaskCollectPip(package="some-package", package_extras="")
|
23
|
+
with pytest.raises(ValidationError):
|
24
|
+
c = TaskCollectPip(package="some-package", package_extras=None)
|
@@ -31,3 +31,15 @@ def test_user_create():
|
|
31
31
|
u = UserCreate(email="a@b.c", password="asd", cache_dir="xxx")
|
32
32
|
debug(e.value)
|
33
33
|
assert "must be an absolute path" in e.value.errors()[0]["msg"]
|
34
|
+
# With all attributes
|
35
|
+
u = UserCreate(
|
36
|
+
email="a@b.c",
|
37
|
+
password="pwd",
|
38
|
+
slurm_user="slurm_user",
|
39
|
+
username="username",
|
40
|
+
cache_dir="/some/path",
|
41
|
+
)
|
42
|
+
debug(u)
|
43
|
+
assert u.slurm_user
|
44
|
+
assert u.cache_dir
|
45
|
+
assert u.username
|
fractal_server/main.py
CHANGED
@@ -85,6 +85,7 @@ async def _create_first_user(
|
|
85
85
|
is_superuser: bool = False,
|
86
86
|
slurm_user: Optional[str] = None,
|
87
87
|
cache_dir: Optional[str] = None,
|
88
|
+
username: Optional[str] = None,
|
88
89
|
) -> None:
|
89
90
|
"""
|
90
91
|
Private method to create the first fractal-server user
|
@@ -121,6 +122,8 @@ async def _create_first_user(
|
|
121
122
|
kwargs["slurm_user"] = slurm_user
|
122
123
|
if cache_dir:
|
123
124
|
kwargs["cache_dir"] = cache_dir
|
125
|
+
if username:
|
126
|
+
kwargs["username"] = username
|
124
127
|
user = await user_manager.create(UserCreate(**kwargs))
|
125
128
|
logger.info(f"User {user.email} created")
|
126
129
|
|
@@ -0,0 +1,38 @@
|
|
1
|
+
"""Add task.args_schema and task.args_schema_version
|
2
|
+
|
3
|
+
Revision ID: 4c308bcaea2b
|
4
|
+
Revises: 50a13d6138fd
|
5
|
+
Create Date: 2023-05-29 17:09:02.492639
|
6
|
+
|
7
|
+
"""
|
8
|
+
import sqlalchemy as sa
|
9
|
+
import sqlmodel
|
10
|
+
from alembic import op
|
11
|
+
|
12
|
+
|
13
|
+
# revision identifiers, used by Alembic.
|
14
|
+
revision = "4c308bcaea2b"
|
15
|
+
down_revision = "50a13d6138fd"
|
16
|
+
branch_labels = None
|
17
|
+
depends_on = None
|
18
|
+
|
19
|
+
|
20
|
+
def upgrade() -> None:
|
21
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
22
|
+
op.add_column("task", sa.Column("args_schema", sa.JSON(), nullable=True))
|
23
|
+
op.add_column(
|
24
|
+
"task",
|
25
|
+
sa.Column(
|
26
|
+
"args_schema_version",
|
27
|
+
sqlmodel.sql.sqltypes.AutoString(),
|
28
|
+
nullable=True,
|
29
|
+
),
|
30
|
+
)
|
31
|
+
# ### end Alembic commands ###
|
32
|
+
|
33
|
+
|
34
|
+
def downgrade() -> None:
|
35
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
36
|
+
op.drop_column("task", "args_schema_version")
|
37
|
+
op.drop_column("task", "args_schema")
|
38
|
+
# ### end Alembic commands ###
|
@@ -1,8 +1,8 @@
|
|
1
|
-
"""
|
1
|
+
"""Initial schema
|
2
2
|
|
3
|
-
Revision ID:
|
3
|
+
Revision ID: 50a13d6138fd
|
4
4
|
Revises:
|
5
|
-
Create Date: 2023-05-
|
5
|
+
Create Date: 2023-05-29 12:14:56.670243
|
6
6
|
|
7
7
|
"""
|
8
8
|
import sqlalchemy as sa
|
@@ -11,7 +11,7 @@ from alembic import op
|
|
11
11
|
|
12
12
|
|
13
13
|
# revision identifiers, used by Alembic.
|
14
|
-
revision = "
|
14
|
+
revision = "50a13d6138fd"
|
15
15
|
down_revision = None
|
16
16
|
branch_labels = None
|
17
17
|
depends_on = None
|
@@ -37,11 +37,11 @@ def upgrade() -> None:
|
|
37
37
|
"task",
|
38
38
|
sa.Column("default_args", sa.JSON(), nullable=True),
|
39
39
|
sa.Column("meta", sa.JSON(), nullable=True),
|
40
|
-
sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
41
40
|
sa.Column(
|
42
41
|
"source", sqlmodel.sql.sqltypes.AutoString(), nullable=False
|
43
42
|
),
|
44
43
|
sa.Column("id", sa.Integer(), nullable=False),
|
44
|
+
sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
45
45
|
sa.Column(
|
46
46
|
"command", sqlmodel.sql.sqltypes.AutoString(), nullable=False
|
47
47
|
),
|
@@ -51,7 +51,12 @@ def upgrade() -> None:
|
|
51
51
|
sa.Column(
|
52
52
|
"output_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
|
53
53
|
),
|
54
|
+
sa.Column("owner", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
55
|
+
sa.Column(
|
56
|
+
"version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
|
57
|
+
),
|
54
58
|
sa.PrimaryKeyConstraint("id"),
|
59
|
+
sa.UniqueConstraint("source"),
|
55
60
|
)
|
56
61
|
op.create_table(
|
57
62
|
"user_oauth",
|
@@ -71,6 +76,9 @@ def upgrade() -> None:
|
|
71
76
|
sa.Column(
|
72
77
|
"cache_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
|
73
78
|
),
|
79
|
+
sa.Column(
|
80
|
+
"username", sqlmodel.sql.sqltypes.AutoString(), nullable=True
|
81
|
+
),
|
74
82
|
sa.PrimaryKeyConstraint("id"),
|
75
83
|
)
|
76
84
|
op.create_index(
|
@@ -158,15 +166,15 @@ def upgrade() -> None:
|
|
158
166
|
sa.Column(
|
159
167
|
"start_timestamp", sa.DateTime(timezone=True), nullable=True
|
160
168
|
),
|
161
|
-
sa.Column("
|
162
|
-
sa.Column("input_dataset_id", sa.Integer(), nullable=False),
|
163
|
-
sa.Column("output_dataset_id", sa.Integer(), nullable=False),
|
164
|
-
sa.Column("workflow_id", sa.Integer(), nullable=False),
|
165
|
-
sa.Column("overwrite_input", sa.Boolean(), nullable=False),
|
169
|
+
sa.Column("end_timestamp", sa.DateTime(timezone=True), nullable=True),
|
166
170
|
sa.Column(
|
167
171
|
"worker_init", sqlmodel.sql.sqltypes.AutoString(), nullable=True
|
168
172
|
),
|
169
173
|
sa.Column("id", sa.Integer(), nullable=False),
|
174
|
+
sa.Column("project_id", sa.Integer(), nullable=False),
|
175
|
+
sa.Column("input_dataset_id", sa.Integer(), nullable=False),
|
176
|
+
sa.Column("output_dataset_id", sa.Integer(), nullable=False),
|
177
|
+
sa.Column("workflow_id", sa.Integer(), nullable=False),
|
170
178
|
sa.Column(
|
171
179
|
"working_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
|
172
180
|
),
|
@@ -1,31 +1,30 @@
|
|
1
|
-
"""Drop
|
1
|
+
"""Drop task.default_args columns
|
2
2
|
|
3
|
-
Revision ID:
|
4
|
-
Revises:
|
5
|
-
Create Date: 2023-
|
3
|
+
Revision ID: f384e1c0cf5d
|
4
|
+
Revises: 4c308bcaea2b
|
5
|
+
Create Date: 2023-06-06 15:10:51.838607
|
6
6
|
|
7
7
|
"""
|
8
8
|
import sqlalchemy as sa
|
9
9
|
from alembic import op
|
10
|
-
|
10
|
+
from sqlalchemy.dialects import sqlite
|
11
11
|
|
12
12
|
# revision identifiers, used by Alembic.
|
13
|
-
revision = "
|
14
|
-
down_revision = "
|
13
|
+
revision = "f384e1c0cf5d"
|
14
|
+
down_revision = "4c308bcaea2b"
|
15
15
|
branch_labels = None
|
16
16
|
depends_on = None
|
17
17
|
|
18
18
|
|
19
19
|
def upgrade() -> None:
|
20
20
|
# ### commands auto generated by Alembic - please adjust! ###
|
21
|
-
op.drop_column("
|
21
|
+
op.drop_column("task", "default_args")
|
22
22
|
# ### end Alembic commands ###
|
23
23
|
|
24
24
|
|
25
25
|
def downgrade() -> None:
|
26
26
|
# ### commands auto generated by Alembic - please adjust! ###
|
27
27
|
op.add_column(
|
28
|
-
"
|
29
|
-
sa.Column("overwrite_input", sa.BOOLEAN(), nullable=False),
|
28
|
+
"task", sa.Column("default_args", sqlite.JSON(), nullable=True)
|
30
29
|
)
|
31
30
|
# ### end Alembic commands ###
|
@@ -11,14 +11,12 @@
|
|
11
11
|
"""
|
12
12
|
This module takes care of installing tasks so that fractal can execute them
|
13
13
|
|
14
|
-
Tasks
|
15
|
-
`
|
16
|
-
.fractal`.
|
14
|
+
Tasks are installed under `Settings.FRACTAL_TASKS_DIR/{username}`, with
|
15
|
+
`username = ".fractal"`.
|
17
16
|
"""
|
18
17
|
import json
|
19
18
|
import shutil
|
20
19
|
import sys
|
21
|
-
from io import IOBase
|
22
20
|
from pathlib import Path
|
23
21
|
from typing import Optional
|
24
22
|
from typing import Union
|
@@ -110,11 +108,19 @@ class _TaskCollectPip(TaskCollectPip):
|
|
110
108
|
"""
|
111
109
|
Internal TaskCollectPip schema
|
112
110
|
|
113
|
-
|
114
|
-
|
111
|
+
Differences with its parent class (`TaskCollectPip`):
|
112
|
+
|
113
|
+
1. We check if the package corresponds to a path in the filesystem, and
|
114
|
+
whether it exists (via new validator `check_local_package`, new
|
115
|
+
method `is_local_package` and new attribute `package_path`).
|
116
|
+
2. We include an additional `package_manifest` attribute.
|
117
|
+
3. We expose an additional attribute `package_name`, which is filled
|
118
|
+
during task collection.
|
115
119
|
"""
|
116
120
|
|
121
|
+
package_name: Optional[str] = None
|
117
122
|
package_path: Optional[Path] = None
|
123
|
+
package_manifest: Optional[ManifestV1] = None
|
118
124
|
|
119
125
|
@property
|
120
126
|
def is_local_package(self) -> bool:
|
@@ -145,47 +151,64 @@ class _TaskCollectPip(TaskCollectPip):
|
|
145
151
|
return values
|
146
152
|
|
147
153
|
@property
|
148
|
-
def
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
@property
|
156
|
-
def source(self):
|
154
|
+
def package_source(self):
|
155
|
+
if not self.package_name or not self.package_version:
|
156
|
+
raise ValueError(
|
157
|
+
"Cannot construct `package_source` property with "
|
158
|
+
f"{self.package_name=} and {self.package_version=}."
|
159
|
+
)
|
157
160
|
if self.is_local_package:
|
158
|
-
|
161
|
+
collection_type = "pip_local"
|
159
162
|
else:
|
160
|
-
|
161
|
-
|
162
|
-
def check(self):
|
163
|
-
if not self.version:
|
164
|
-
raise ValueError("Version is not set or cannot be determined")
|
163
|
+
collection_type = "pip_remote"
|
165
164
|
|
165
|
+
package_extras = self.package_extras or ""
|
166
|
+
if self.python_version:
|
167
|
+
python_version = f"py{self.python_version}"
|
168
|
+
else:
|
169
|
+
python_version = "" # FIXME: can we allow this?
|
170
|
+
|
171
|
+
source = ":".join(
|
172
|
+
(
|
173
|
+
collection_type,
|
174
|
+
self.package_name,
|
175
|
+
self.package_version,
|
176
|
+
package_extras,
|
177
|
+
python_version,
|
178
|
+
)
|
179
|
+
)
|
180
|
+
return source
|
166
181
|
|
167
|
-
def
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
182
|
+
def check(self):
|
183
|
+
"""
|
184
|
+
Verify that the package has all attributes that are needed to continue
|
185
|
+
with task collection
|
186
|
+
"""
|
187
|
+
if not self.package_name:
|
188
|
+
raise ValueError("`package_name` attribute is not set")
|
189
|
+
if not self.package_version:
|
190
|
+
raise ValueError("`package_version` attribute is not set")
|
191
|
+
if not self.package_manifest:
|
192
|
+
raise ValueError("`package_manifest` attribute is not set")
|
174
193
|
|
175
194
|
|
176
195
|
def create_package_dir_pip(
|
177
196
|
*,
|
178
197
|
task_pkg: _TaskCollectPip,
|
179
|
-
user: Optional[str] = None,
|
180
198
|
create: bool = True,
|
181
|
-
**_, # FIXME remove this catch-all argument
|
182
199
|
) -> Path:
|
200
|
+
"""
|
201
|
+
Create venv folder for a task package and return corresponding Path object
|
202
|
+
"""
|
183
203
|
settings = Inject(get_settings)
|
184
|
-
user =
|
185
|
-
|
186
|
-
|
204
|
+
user = FRACTAL_PUBLIC_TASK_SUBDIR
|
205
|
+
if task_pkg.package_version is None:
|
206
|
+
raise ValueError(
|
207
|
+
f"Cannot create venv folder for package `{task_pkg.package}` "
|
208
|
+
"with `version=None`."
|
209
|
+
)
|
210
|
+
package_dir = f"{task_pkg.package}{task_pkg.package_version}"
|
187
211
|
venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir # type: ignore
|
188
|
-
# TODO check the access right of the venv_path and subdirs
|
189
212
|
if create:
|
190
213
|
venv_path.mkdir(exist_ok=False, parents=True)
|
191
214
|
return venv_path
|
@@ -201,10 +224,11 @@ async def download_package(
|
|
201
224
|
"""
|
202
225
|
interpreter = get_python_interpreter(version=task_pkg.python_version)
|
203
226
|
pip = f"{interpreter} -m pip"
|
204
|
-
|
205
|
-
f"{
|
206
|
-
f"-d {dest}"
|
227
|
+
version = (
|
228
|
+
f"=={task_pkg.package_version}" if task_pkg.package_version else ""
|
207
229
|
)
|
230
|
+
package_and_version = f"{task_pkg.package}{version}"
|
231
|
+
cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
|
208
232
|
stdout = await execute_command(command=cmd, cwd=Path("."))
|
209
233
|
pkg_file = next(
|
210
234
|
line.split()[-1] for line in stdout.split("\n") if "Saved" in line
|
@@ -212,9 +236,40 @@ async def download_package(
|
|
212
236
|
return Path(pkg_file)
|
213
237
|
|
214
238
|
|
215
|
-
def
|
239
|
+
def _load_manifest_from_wheel(
|
240
|
+
path: Path, wheel: ZipFile, logger_name: Optional[str] = None
|
241
|
+
) -> ManifestV1:
|
242
|
+
logger = get_logger(logger_name)
|
243
|
+
namelist = wheel.namelist()
|
244
|
+
try:
|
245
|
+
manifest = next(
|
246
|
+
name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
|
247
|
+
)
|
248
|
+
except StopIteration:
|
249
|
+
msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
|
250
|
+
logger.error(msg)
|
251
|
+
raise ValueError(msg)
|
252
|
+
with wheel.open(manifest) as manifest_fd:
|
253
|
+
manifest_dict = json.load(manifest_fd)
|
254
|
+
manifest_version = str(manifest_dict["manifest_version"])
|
255
|
+
if manifest_version == "1":
|
256
|
+
pkg_manifest = ManifestV1(**manifest_dict)
|
257
|
+
return pkg_manifest
|
258
|
+
else:
|
259
|
+
msg = f"Manifest version {manifest_version=} not supported"
|
260
|
+
logger.error(msg)
|
261
|
+
raise ValueError(msg)
|
262
|
+
|
263
|
+
|
264
|
+
def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
|
216
265
|
"""
|
217
|
-
Inspect task package
|
266
|
+
Inspect task package to extract version, name and manifest
|
267
|
+
|
268
|
+
Note that this only works with wheel files, which have a well-defined
|
269
|
+
dist-info section. If we need to generalize to to tar.gz archives, we would
|
270
|
+
need to go and look for `PKG-INFO`.
|
271
|
+
|
272
|
+
Note: package name is normalized by replacing `{-,.}` with `_`.
|
218
273
|
|
219
274
|
Args:
|
220
275
|
path: Path
|
@@ -225,32 +280,54 @@ def inspect_package(path: Path) -> dict:
|
|
225
280
|
pacakge, and `manifest`, the Fractal manifest object relative to the
|
226
281
|
tasks.
|
227
282
|
"""
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
283
|
+
|
284
|
+
logger = get_logger(logger_name)
|
285
|
+
|
286
|
+
if not path.as_posix().endswith(".whl"):
|
287
|
+
raise ValueError(
|
288
|
+
f"Only wheel packages are supported, given {path.as_posix()}."
|
289
|
+
)
|
290
|
+
|
291
|
+
with ZipFile(path) as wheel:
|
292
|
+
namelist = wheel.namelist()
|
293
|
+
|
294
|
+
# Read and validate task manifest
|
295
|
+
logger.debug(f"Now reading manifest for {path.as_posix()}")
|
296
|
+
pkg_manifest = _load_manifest_from_wheel(
|
297
|
+
path, wheel, logger_name=logger_name
|
298
|
+
)
|
299
|
+
logger.debug("Manifest read correctly.")
|
300
|
+
|
301
|
+
# Read package name and version from *.dist-info/METADATA
|
302
|
+
logger.debug(
|
303
|
+
f"Now reading package name and version for {path.as_posix()}"
|
304
|
+
)
|
305
|
+
metadata = next(
|
306
|
+
name for name in namelist if "dist-info/METADATA" in name
|
307
|
+
)
|
308
|
+
with wheel.open(metadata) as metadata_fd:
|
309
|
+
meta = metadata_fd.read().decode("utf-8")
|
310
|
+
pkg_name = next(
|
311
|
+
line.split()[-1]
|
312
|
+
for line in meta.splitlines()
|
313
|
+
if line.startswith("Name")
|
236
314
|
)
|
237
|
-
|
238
|
-
|
315
|
+
pkg_version = next(
|
316
|
+
line.split()[-1]
|
317
|
+
for line in meta.splitlines()
|
318
|
+
if line.startswith("Version")
|
239
319
|
)
|
320
|
+
logger.debug("Package name and version read correctly.")
|
240
321
|
|
241
|
-
|
242
|
-
|
243
|
-
version = next(
|
244
|
-
line.split()[-1]
|
245
|
-
for line in meta.splitlines()
|
246
|
-
if line.startswith("Version")
|
247
|
-
)
|
248
|
-
|
249
|
-
with wheel.open(manifest) as manifest_fd:
|
250
|
-
manifest_obj = read_manifest(manifest_fd) # type: ignore
|
322
|
+
# Normalize package name:
|
323
|
+
pkg_name = pkg_name.replace("-", "_").replace(".", "_")
|
251
324
|
|
252
|
-
|
253
|
-
|
325
|
+
info = dict(
|
326
|
+
pkg_name=pkg_name,
|
327
|
+
pkg_version=pkg_version,
|
328
|
+
pkg_manifest=pkg_manifest,
|
329
|
+
)
|
330
|
+
return info
|
254
331
|
|
255
332
|
|
256
333
|
async def create_package_environment_pip(
|
@@ -260,74 +337,60 @@ async def create_package_environment_pip(
|
|
260
337
|
logger_name: str,
|
261
338
|
) -> list[TaskCreate]:
|
262
339
|
"""
|
263
|
-
Create environment and
|
340
|
+
Create environment, install package, and prepare task list
|
264
341
|
"""
|
342
|
+
|
265
343
|
logger = get_logger(logger_name)
|
344
|
+
|
345
|
+
# Only proceed if package, version and manifest attributes are set
|
346
|
+
task_pkg.check()
|
347
|
+
|
266
348
|
try:
|
267
|
-
logger.debug("Creating venv and installing package")
|
268
349
|
|
350
|
+
logger.debug("Creating venv and installing package")
|
269
351
|
python_bin, package_root = await _create_venv_install_package(
|
270
352
|
path=venv_path,
|
271
353
|
task_pkg=task_pkg,
|
272
354
|
logger_name=logger_name,
|
273
355
|
)
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
356
|
+
logger.debug("Venv creation and package installation ended correctly.")
|
357
|
+
|
358
|
+
# Prepare task_list with appropriate metadata
|
359
|
+
logger.debug("Creating task list from manifest")
|
360
|
+
task_list = []
|
361
|
+
for t in task_pkg.package_manifest.task_list:
|
362
|
+
# Fill in attributes for TaskCreate
|
363
|
+
task_executable = package_root / t.executable
|
364
|
+
cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
|
365
|
+
task_name_slug = t.name.replace(" ", "_").lower()
|
366
|
+
task_source = f"{task_pkg.package_source}:{task_name_slug}"
|
367
|
+
if not task_executable.exists():
|
368
|
+
raise FileNotFoundError(
|
369
|
+
f"Cannot find executable `{task_executable}` "
|
370
|
+
f"for task `{t.name}`"
|
371
|
+
)
|
372
|
+
manifest = task_pkg.package_manifest
|
373
|
+
if manifest.has_args_schemas:
|
374
|
+
additional_attrs = dict(
|
375
|
+
args_schema_version=manifest.args_schema_version
|
376
|
+
)
|
377
|
+
else:
|
378
|
+
additional_attrs = {}
|
379
|
+
this_task = TaskCreate(
|
380
|
+
**t.dict(),
|
381
|
+
command=cmd,
|
382
|
+
version=task_pkg.package_version,
|
383
|
+
**additional_attrs,
|
384
|
+
source=task_source,
|
385
|
+
)
|
386
|
+
task_list.append(this_task)
|
387
|
+
logger.debug("Task list created correctly")
|
283
388
|
except Exception as e:
|
284
389
|
logger.error("Task manifest loading failed")
|
285
390
|
raise e
|
286
391
|
return task_list
|
287
392
|
|
288
393
|
|
289
|
-
def read_manifest(file: Union[Path, IOBase]) -> ManifestV1:
|
290
|
-
"""
|
291
|
-
Read and parse manifest file
|
292
|
-
"""
|
293
|
-
if isinstance(file, IOBase):
|
294
|
-
manifest_dict = json.load(file)
|
295
|
-
else:
|
296
|
-
with file.open("r") as f:
|
297
|
-
manifest_dict = json.load(f)
|
298
|
-
|
299
|
-
manifest_version = str(manifest_dict["manifest_version"])
|
300
|
-
if manifest_version == "1":
|
301
|
-
manifest = ManifestV1(**manifest_dict)
|
302
|
-
else:
|
303
|
-
raise ValueError("Manifest version {manifest_version=} not supported")
|
304
|
-
|
305
|
-
return manifest
|
306
|
-
|
307
|
-
|
308
|
-
def load_manifest(
|
309
|
-
package_root: Path,
|
310
|
-
python_bin: Path,
|
311
|
-
source: str,
|
312
|
-
) -> list[TaskCreate]:
|
313
|
-
|
314
|
-
manifest_file = package_root / "__FRACTAL_MANIFEST__.json"
|
315
|
-
manifest = read_manifest(manifest_file)
|
316
|
-
|
317
|
-
task_list = []
|
318
|
-
for t in manifest.task_list:
|
319
|
-
task_executable = package_root / t.executable
|
320
|
-
if not task_executable.exists():
|
321
|
-
raise FileNotFoundError(
|
322
|
-
f"Cannot find executable `{task_executable}` "
|
323
|
-
f"for task `{t.name}`"
|
324
|
-
)
|
325
|
-
cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
|
326
|
-
this_task = TaskCreate(**t.dict(), command=cmd, source=source)
|
327
|
-
task_list.append(this_task)
|
328
|
-
return task_list
|
329
|
-
|
330
|
-
|
331
394
|
async def _create_venv_install_package(
|
332
395
|
*,
|
333
396
|
task_pkg: _TaskCollectPip,
|
@@ -403,7 +466,9 @@ async def _pip_install(
|
|
403
466
|
if task_pkg.is_local_package:
|
404
467
|
pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
|
405
468
|
else:
|
406
|
-
version_string =
|
469
|
+
version_string = (
|
470
|
+
f"=={task_pkg.package_version}" if task_pkg.package_version else ""
|
471
|
+
)
|
407
472
|
pip_install_str = f"{task_pkg.package}{extras}{version_string}"
|
408
473
|
|
409
474
|
cmd_install = f"{pip} install {pip_install_str}"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fractal-server
|
3
|
-
Version: 1.3.
|
3
|
+
Version: 1.3.0a3
|
4
4
|
Summary: Server component of the Fractal analytics platform
|
5
5
|
Home-page: https://github.com/fractal-analytics-platform/fractal-server
|
6
6
|
License: BSD-3-Clause
|
@@ -25,6 +25,7 @@ Requires-Dist: fastapi (>=0.95.0,<0.96.0)
|
|
25
25
|
Requires-Dist: fastapi-users[oauth] (>=10.1,<11.0)
|
26
26
|
Requires-Dist: gunicorn (>=20.1.0,<21.0.0) ; extra == "gunicorn"
|
27
27
|
Requires-Dist: psycopg2 (>=2.9.5,<3.0.0) ; extra == "postgres"
|
28
|
+
Requires-Dist: pydantic (>=1.10.8)
|
28
29
|
Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
|
29
30
|
Requires-Dist: sqlalchemy (>=1.4,<2.0)
|
30
31
|
Requires-Dist: sqlmodel (>=0.0.8,<0.0.9)
|