fractal-server 2.7.0a11__py3-none-any.whl → 2.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/models/user_settings.py +1 -0
- fractal_server/app/models/v2/task.py +16 -2
- fractal_server/app/routes/admin/v2/task_group.py +7 -0
- fractal_server/app/routes/api/v2/dataset.py +39 -6
- fractal_server/app/routes/api/v2/task.py +4 -6
- fractal_server/app/routes/api/v2/task_collection.py +17 -44
- fractal_server/app/routes/api/v2/task_collection_custom.py +5 -4
- fractal_server/app/schemas/user_settings.py +18 -0
- fractal_server/app/schemas/v2/__init__.py +1 -0
- fractal_server/app/schemas/v2/dataset.py +5 -3
- fractal_server/app/schemas/v2/task_collection.py +20 -4
- fractal_server/app/schemas/v2/task_group.py +8 -1
- fractal_server/app/security/__init__.py +8 -1
- fractal_server/config.py +8 -28
- fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +39 -0
- fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +42 -0
- fractal_server/tasks/utils.py +0 -31
- fractal_server/tasks/v1/background_operations.py +11 -11
- fractal_server/tasks/v1/endpoint_operations.py +5 -5
- fractal_server/tasks/v1/utils.py +2 -2
- fractal_server/tasks/v2/collection_local.py +357 -0
- fractal_server/tasks/v2/{background_operations_ssh.py → collection_ssh.py} +108 -102
- fractal_server/tasks/v2/templates/_1_create_venv.sh +0 -8
- fractal_server/tasks/v2/templates/{_2_upgrade_pip.sh → _2_preliminary_pip_operations.sh} +2 -1
- fractal_server/tasks/v2/templates/_3_pip_install.sh +22 -1
- fractal_server/tasks/v2/templates/_5_pip_show.sh +5 -5
- fractal_server/tasks/v2/utils_background.py +209 -0
- fractal_server/tasks/v2/utils_package_names.py +77 -0
- fractal_server/tasks/v2/{utils.py → utils_python_interpreter.py} +0 -26
- fractal_server/tasks/v2/utils_templates.py +59 -0
- fractal_server/utils.py +48 -3
- {fractal_server-2.7.0a11.dist-info → fractal_server-2.8.0.dist-info}/METADATA +14 -17
- {fractal_server-2.7.0a11.dist-info → fractal_server-2.8.0.dist-info}/RECORD +38 -35
- fractal_server/data_migrations/2_7_0.py +0 -323
- fractal_server/tasks/v2/_venv_pip.py +0 -193
- fractal_server/tasks/v2/background_operations.py +0 -456
- /fractal_server/{tasks/v2/endpoint_operations.py → app/routes/api/v2/_aux_functions_task_collection.py} +0 -0
- {fractal_server-2.7.0a11.dist-info → fractal_server-2.8.0.dist-info}/LICENSE +0 -0
- {fractal_server-2.7.0a11.dist-info → fractal_server-2.8.0.dist-info}/WHEEL +0 -0
- {fractal_server-2.7.0a11.dist-info → fractal_server-2.8.0.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.
|
1
|
+
__VERSION__ = "2.8.0"
|
@@ -29,7 +29,6 @@ class TaskV2(SQLModel, table=True):
|
|
29
29
|
sa_column=Column(JSON, server_default="{}", default={}, nullable=False)
|
30
30
|
)
|
31
31
|
|
32
|
-
owner: Optional[str] = None
|
33
32
|
version: Optional[str] = None
|
34
33
|
args_schema_non_parallel: Optional[dict[str, Any]] = Field(
|
35
34
|
sa_column=Column(JSON), default=None
|
@@ -44,7 +43,7 @@ class TaskV2(SQLModel, table=True):
|
|
44
43
|
input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
|
45
44
|
output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
|
46
45
|
|
47
|
-
taskgroupv2_id:
|
46
|
+
taskgroupv2_id: int = Field(foreign_key="taskgroupv2.id")
|
48
47
|
|
49
48
|
category: Optional[str] = None
|
50
49
|
modality: Optional[str] = None
|
@@ -104,3 +103,18 @@ class TaskGroupV2(SQLModel, table=True):
|
|
104
103
|
f"{self.pkg_name=}, {self.wheel_path=}, {self.version=}."
|
105
104
|
)
|
106
105
|
return f"{self.pkg_name}{extras}=={self.version}"
|
106
|
+
|
107
|
+
@property
|
108
|
+
def pinned_package_versions_string(self) -> str:
|
109
|
+
"""
|
110
|
+
Prepare string to be used in `python -m pip install`.
|
111
|
+
"""
|
112
|
+
if self.pinned_package_versions is None:
|
113
|
+
return ""
|
114
|
+
output = " ".join(
|
115
|
+
[
|
116
|
+
f"{key}=={value}"
|
117
|
+
for key, value in self.pinned_package_versions.items()
|
118
|
+
]
|
119
|
+
)
|
120
|
+
return output
|
@@ -21,6 +21,7 @@ from fractal_server.app.routes.auth._aux_auth import (
|
|
21
21
|
)
|
22
22
|
from fractal_server.app.schemas.v2 import TaskGroupReadV2
|
23
23
|
from fractal_server.app.schemas.v2 import TaskGroupUpdateV2
|
24
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
24
25
|
from fractal_server.logger import set_logger
|
25
26
|
|
26
27
|
router = APIRouter()
|
@@ -50,6 +51,8 @@ async def query_task_group_list(
|
|
50
51
|
user_group_id: Optional[int] = None,
|
51
52
|
private: Optional[bool] = None,
|
52
53
|
active: Optional[bool] = None,
|
54
|
+
pkg_name: Optional[str] = None,
|
55
|
+
origin: Optional[TaskGroupV2OriginEnum] = None,
|
53
56
|
user: UserOAuth = Depends(current_active_superuser),
|
54
57
|
db: AsyncSession = Depends(get_async_db),
|
55
58
|
) -> list[TaskGroupReadV2]:
|
@@ -75,6 +78,10 @@ async def query_task_group_list(
|
|
75
78
|
stm = stm.where(is_(TaskGroupV2.active, True))
|
76
79
|
else:
|
77
80
|
stm = stm.where(is_(TaskGroupV2.active, False))
|
81
|
+
if origin is not None:
|
82
|
+
stm = stm.where(TaskGroupV2.origin == origin)
|
83
|
+
if pkg_name is not None:
|
84
|
+
stm = stm.where(TaskGroupV2.pkg_name.icontains(pkg_name))
|
78
85
|
|
79
86
|
res = await db.execute(stm)
|
80
87
|
task_groups_list = res.scalars().all()
|
@@ -22,6 +22,8 @@ from ._aux_functions import _get_project_check_owner
|
|
22
22
|
from ._aux_functions import _get_submitted_jobs_statement
|
23
23
|
from fractal_server.app.models import UserOAuth
|
24
24
|
from fractal_server.app.routes.auth import current_active_user
|
25
|
+
from fractal_server.string_tools import sanitize_string
|
26
|
+
from fractal_server.urls import normalize_url
|
25
27
|
|
26
28
|
router = APIRouter()
|
27
29
|
|
@@ -40,14 +42,45 @@ async def create_dataset(
|
|
40
42
|
"""
|
41
43
|
Add new dataset to current project
|
42
44
|
"""
|
43
|
-
await _get_project_check_owner(
|
45
|
+
project = await _get_project_check_owner(
|
44
46
|
project_id=project_id, user_id=user.id, db=db
|
45
47
|
)
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
|
49
|
+
if dataset.zarr_dir is None:
|
50
|
+
|
51
|
+
if user.settings.project_dir is None:
|
52
|
+
raise HTTPException(
|
53
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
54
|
+
detail=(
|
55
|
+
"Both 'dataset.zarr_dir' and 'user.settings.project_dir' "
|
56
|
+
"are null"
|
57
|
+
),
|
58
|
+
)
|
59
|
+
|
60
|
+
db_dataset = DatasetV2(
|
61
|
+
project_id=project_id,
|
62
|
+
zarr_dir="__PLACEHOLDER__",
|
63
|
+
**dataset.dict(exclude={"zarr_dir"}),
|
64
|
+
)
|
65
|
+
db.add(db_dataset)
|
66
|
+
await db.commit()
|
67
|
+
await db.refresh(db_dataset)
|
68
|
+
path = (
|
69
|
+
f"{user.settings.project_dir}/fractal/"
|
70
|
+
f"{project_id}_{sanitize_string(project.name)}/"
|
71
|
+
f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
|
72
|
+
)
|
73
|
+
normalized_path = normalize_url(path)
|
74
|
+
db_dataset.zarr_dir = normalized_path
|
75
|
+
|
76
|
+
db.add(db_dataset)
|
77
|
+
await db.commit()
|
78
|
+
await db.refresh(db_dataset)
|
79
|
+
else:
|
80
|
+
db_dataset = DatasetV2(project_id=project_id, **dataset.dict())
|
81
|
+
db.add(db_dataset)
|
82
|
+
await db.commit()
|
83
|
+
await db.refresh(db_dataset)
|
51
84
|
|
52
85
|
return db_dataset
|
53
86
|
|
@@ -24,6 +24,7 @@ from fractal_server.app.models.v2 import TaskV2
|
|
24
24
|
from fractal_server.app.routes.auth import current_active_user
|
25
25
|
from fractal_server.app.routes.auth import current_active_verified_user
|
26
26
|
from fractal_server.app.schemas.v2 import TaskCreateV2
|
27
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
27
28
|
from fractal_server.app.schemas.v2 import TaskReadV2
|
28
29
|
from fractal_server.app.schemas.v2 import TaskUpdateV2
|
29
30
|
from fractal_server.logger import set_logger
|
@@ -35,8 +36,7 @@ logger = set_logger(__name__)
|
|
35
36
|
|
36
37
|
@router.get("/", response_model=list[TaskReadV2])
|
37
38
|
async def get_list_task(
|
38
|
-
|
39
|
-
args_schema_non_parallel: bool = True,
|
39
|
+
args_schema: bool = True,
|
40
40
|
category: Optional[str] = None,
|
41
41
|
modality: Optional[str] = None,
|
42
42
|
author: Optional[str] = None,
|
@@ -71,11 +71,9 @@ async def get_list_task(
|
|
71
71
|
res = await db.execute(stm)
|
72
72
|
task_list = res.scalars().all()
|
73
73
|
await db.close()
|
74
|
-
if
|
74
|
+
if args_schema is False:
|
75
75
|
for task in task_list:
|
76
76
|
setattr(task, "args_schema_parallel", None)
|
77
|
-
if args_schema_non_parallel is False:
|
78
|
-
for task in task_list:
|
79
77
|
setattr(task, "args_schema_non_parallel", None)
|
80
78
|
|
81
79
|
return task_list
|
@@ -200,7 +198,7 @@ async def create_task(
|
|
200
198
|
user_group_id=user_group_id,
|
201
199
|
active=True,
|
202
200
|
task_list=[db_task],
|
203
|
-
origin=
|
201
|
+
origin=TaskGroupV2OriginEnum.OTHER,
|
204
202
|
version=db_task.version,
|
205
203
|
pkg_name=pkg_name,
|
206
204
|
)
|
@@ -24,22 +24,22 @@ from ....schemas.v2 import CollectionStatusV2
|
|
24
24
|
from ....schemas.v2 import TaskCollectPipV2
|
25
25
|
from ....schemas.v2 import TaskGroupCreateV2
|
26
26
|
from ...aux.validate_user_settings import validate_user_settings
|
27
|
+
from ._aux_functions_task_collection import get_package_version_from_pypi
|
27
28
|
from ._aux_functions_tasks import _get_valid_user_group_id
|
28
29
|
from ._aux_functions_tasks import _verify_non_duplication_group_constraint
|
29
30
|
from ._aux_functions_tasks import _verify_non_duplication_user_constraint
|
30
31
|
from fractal_server.app.models import UserOAuth
|
31
32
|
from fractal_server.app.routes.auth import current_active_user
|
32
33
|
from fractal_server.app.routes.auth import current_active_verified_user
|
33
|
-
from fractal_server.
|
34
|
-
from fractal_server.tasks.
|
35
|
-
|
36
|
-
background_collect_pip,
|
34
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
35
|
+
from fractal_server.tasks.v2.collection_local import (
|
36
|
+
collect_package_local,
|
37
37
|
)
|
38
|
-
from fractal_server.tasks.v2.
|
39
|
-
|
38
|
+
from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
|
39
|
+
from fractal_server.tasks.v2.utils_package_names import normalize_package_name
|
40
|
+
from fractal_server.tasks.v2.utils_python_interpreter import (
|
41
|
+
get_python_interpreter_v2,
|
40
42
|
)
|
41
|
-
from fractal_server.tasks.v2.utils import _parse_wheel_filename
|
42
|
-
from fractal_server.tasks.v2.utils import get_python_interpreter_v2
|
43
43
|
|
44
44
|
router = APIRouter()
|
45
45
|
|
@@ -117,15 +117,15 @@ async def collect_tasks_pip(
|
|
117
117
|
f"Original error: {str(e)}",
|
118
118
|
),
|
119
119
|
)
|
120
|
-
task_group_attrs["pkg_name"] =
|
120
|
+
task_group_attrs["pkg_name"] = normalize_package_name(
|
121
121
|
wheel_info["distribution"]
|
122
122
|
)
|
123
123
|
task_group_attrs["version"] = wheel_info["version"]
|
124
|
-
task_group_attrs["origin"] =
|
124
|
+
task_group_attrs["origin"] = TaskGroupV2OriginEnum.WHEELFILE
|
125
125
|
else:
|
126
126
|
pkg_name = task_collect.package
|
127
|
-
task_group_attrs["pkg_name"] =
|
128
|
-
task_group_attrs["origin"] =
|
127
|
+
task_group_attrs["pkg_name"] = normalize_package_name(pkg_name)
|
128
|
+
task_group_attrs["origin"] = TaskGroupV2OriginEnum.PYPI
|
129
129
|
latest_version = await get_package_version_from_pypi(
|
130
130
|
task_collect.package,
|
131
131
|
task_collect.package_version,
|
@@ -248,8 +248,8 @@ async def collect_tasks_pip(
|
|
248
248
|
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
249
249
|
# SSH task collection
|
250
250
|
|
251
|
-
from fractal_server.tasks.v2.
|
252
|
-
|
251
|
+
from fractal_server.tasks.v2.collection_ssh import (
|
252
|
+
collect_package_ssh,
|
253
253
|
)
|
254
254
|
|
255
255
|
# User appropriate FractalSSH object
|
@@ -262,7 +262,7 @@ async def collect_tasks_pip(
|
|
262
262
|
fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
|
263
263
|
|
264
264
|
background_tasks.add_task(
|
265
|
-
|
265
|
+
collect_package_ssh,
|
266
266
|
state_id=state.id,
|
267
267
|
task_group=task_group,
|
268
268
|
fractal_ssh=fractal_ssh,
|
@@ -272,7 +272,7 @@ async def collect_tasks_pip(
|
|
272
272
|
else:
|
273
273
|
# Local task collection
|
274
274
|
background_tasks.add_task(
|
275
|
-
|
275
|
+
collect_package_local,
|
276
276
|
state_id=state.id,
|
277
277
|
task_group=task_group,
|
278
278
|
)
|
@@ -295,42 +295,15 @@ async def collect_tasks_pip(
|
|
295
295
|
async def check_collection_status(
|
296
296
|
state_id: int,
|
297
297
|
user: UserOAuth = Depends(current_active_user),
|
298
|
-
verbose: bool = False,
|
299
298
|
db: AsyncSession = Depends(get_async_db),
|
300
299
|
) -> CollectionStateReadV2: # State[TaskCollectStatus]
|
301
300
|
"""
|
302
301
|
Check status of background task collection
|
303
302
|
"""
|
304
|
-
|
305
|
-
logger = set_logger(logger_name="check_collection_status")
|
306
|
-
logger.debug(f"Querying state for state.id={state_id}")
|
307
303
|
state = await db.get(CollectionStateV2, state_id)
|
308
|
-
if
|
309
|
-
await db.close()
|
304
|
+
if state is None:
|
310
305
|
raise HTTPException(
|
311
306
|
status_code=status.HTTP_404_NOT_FOUND,
|
312
307
|
detail=f"No task collection info with id={state_id}",
|
313
308
|
)
|
314
|
-
|
315
|
-
settings = Inject(get_settings)
|
316
|
-
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
317
|
-
# FIXME SSH: add logic for when data.state["log"] is empty
|
318
|
-
pass
|
319
|
-
else:
|
320
|
-
# Non-SSH mode
|
321
|
-
# In some cases (i.e. a successful or ongoing task collection),
|
322
|
-
# state.data["log"] is not set; if so, we collect the current logs.
|
323
|
-
if verbose and not state.data.get("log"):
|
324
|
-
if "path" not in state.data.keys():
|
325
|
-
await db.close()
|
326
|
-
raise HTTPException(
|
327
|
-
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
328
|
-
detail=(
|
329
|
-
f"No 'path' in CollectionStateV2[{state_id}].data"
|
330
|
-
),
|
331
|
-
)
|
332
|
-
state.data["log"] = get_collection_log_v2(Path(state.data["path"]))
|
333
|
-
|
334
|
-
reset_logger_handlers(logger)
|
335
|
-
await db.close()
|
336
309
|
return state
|
@@ -21,17 +21,18 @@ from fractal_server.app.routes.auth import current_active_verified_user
|
|
21
21
|
from fractal_server.app.schemas.v2 import TaskCollectCustomV2
|
22
22
|
from fractal_server.app.schemas.v2 import TaskCreateV2
|
23
23
|
from fractal_server.app.schemas.v2 import TaskGroupCreateV2
|
24
|
+
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
24
25
|
from fractal_server.app.schemas.v2 import TaskReadV2
|
25
26
|
from fractal_server.config import get_settings
|
26
27
|
from fractal_server.logger import set_logger
|
27
28
|
from fractal_server.string_tools import validate_cmd
|
28
29
|
from fractal_server.syringe import Inject
|
29
|
-
from fractal_server.tasks.v2.background_operations import (
|
30
|
-
_prepare_tasks_metadata,
|
31
|
-
)
|
32
30
|
from fractal_server.tasks.v2.database_operations import (
|
33
31
|
create_db_tasks_and_update_task_group,
|
34
32
|
)
|
33
|
+
from fractal_server.tasks.v2.utils_background import (
|
34
|
+
_prepare_tasks_metadata,
|
35
|
+
)
|
35
36
|
|
36
37
|
router = APIRouter()
|
37
38
|
|
@@ -140,7 +141,7 @@ async def collect_task_custom(
|
|
140
141
|
|
141
142
|
# Prepare task-group attributes
|
142
143
|
task_group_attrs = dict(
|
143
|
-
origin=
|
144
|
+
origin=TaskGroupV2OriginEnum.OTHER,
|
144
145
|
pkg_name=task_collect.label,
|
145
146
|
user_id=user.id,
|
146
147
|
user_group_id=user_group_id,
|
@@ -19,6 +19,10 @@ __all__ = (
|
|
19
19
|
|
20
20
|
|
21
21
|
class UserSettingsRead(BaseModel):
|
22
|
+
"""
|
23
|
+
Schema reserved for superusers
|
24
|
+
"""
|
25
|
+
|
22
26
|
id: int
|
23
27
|
ssh_host: Optional[str] = None
|
24
28
|
ssh_username: Optional[str] = None
|
@@ -28,6 +32,7 @@ class UserSettingsRead(BaseModel):
|
|
28
32
|
slurm_user: Optional[str] = None
|
29
33
|
slurm_accounts: list[str]
|
30
34
|
cache_dir: Optional[str] = None
|
35
|
+
project_dir: Optional[str] = None
|
31
36
|
|
32
37
|
|
33
38
|
class UserSettingsReadStrict(BaseModel):
|
@@ -35,9 +40,14 @@ class UserSettingsReadStrict(BaseModel):
|
|
35
40
|
slurm_accounts: list[str]
|
36
41
|
cache_dir: Optional[str] = None
|
37
42
|
ssh_username: Optional[str] = None
|
43
|
+
project_dir: Optional[str] = None
|
38
44
|
|
39
45
|
|
40
46
|
class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
|
47
|
+
"""
|
48
|
+
Schema reserved for superusers
|
49
|
+
"""
|
50
|
+
|
41
51
|
ssh_host: Optional[str] = None
|
42
52
|
ssh_username: Optional[str] = None
|
43
53
|
ssh_private_key_path: Optional[str] = None
|
@@ -46,6 +56,7 @@ class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
|
|
46
56
|
slurm_user: Optional[str] = None
|
47
57
|
slurm_accounts: Optional[list[StrictStr]] = None
|
48
58
|
cache_dir: Optional[str] = None
|
59
|
+
project_dir: Optional[str] = None
|
49
60
|
|
50
61
|
_ssh_host = validator("ssh_host", allow_reuse=True)(
|
51
62
|
valstr("ssh_host", accept_none=True)
|
@@ -83,6 +94,13 @@ class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
|
|
83
94
|
validate_cmd(value)
|
84
95
|
return val_absolute_path("cache_dir")(value)
|
85
96
|
|
97
|
+
@validator("project_dir")
|
98
|
+
def project_dir_validator(cls, value):
|
99
|
+
if value is None:
|
100
|
+
return None
|
101
|
+
validate_cmd(value)
|
102
|
+
return val_absolute_path("project_dir")(value)
|
103
|
+
|
86
104
|
|
87
105
|
class UserSettingsUpdateStrict(BaseModel, extra=Extra.forbid):
|
88
106
|
slurm_accounts: Optional[list[StrictStr]] = None
|
@@ -30,6 +30,7 @@ from .task_collection import TaskCollectPipV2 # noqa F401
|
|
30
30
|
from .task_group import TaskGroupCreateV2 # noqa F401
|
31
31
|
from .task_group import TaskGroupReadV2 # noqa F401
|
32
32
|
from .task_group import TaskGroupUpdateV2 # noqa F401
|
33
|
+
from .task_group import TaskGroupV2OriginEnum # noqa F401
|
33
34
|
from .workflow import WorkflowCreateV2 # noqa F401
|
34
35
|
from .workflow import WorkflowExportV2 # noqa F401
|
35
36
|
from .workflow import WorkflowImportV2 # noqa F401
|
@@ -33,14 +33,16 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
|
|
33
33
|
|
34
34
|
name: str
|
35
35
|
|
36
|
-
zarr_dir: str
|
36
|
+
zarr_dir: Optional[str] = None
|
37
37
|
|
38
38
|
filters: Filters = Field(default_factory=Filters)
|
39
39
|
|
40
40
|
# Validators
|
41
41
|
@validator("zarr_dir")
|
42
42
|
def normalize_zarr_dir(cls, v: str) -> str:
|
43
|
-
|
43
|
+
if v is not None:
|
44
|
+
return normalize_url(v)
|
45
|
+
return v
|
44
46
|
|
45
47
|
_name = validator("name", allow_reuse=True)(valstr("name"))
|
46
48
|
|
@@ -95,7 +97,7 @@ class DatasetImportV2(BaseModel, extra=Extra.forbid):
|
|
95
97
|
|
96
98
|
name: str
|
97
99
|
zarr_dir: str
|
98
|
-
images: list[SingleImage] = Field(default_factory=
|
100
|
+
images: list[SingleImage] = Field(default_factory=list)
|
99
101
|
filters: Filters = Field(default_factory=Filters)
|
100
102
|
|
101
103
|
# Validators
|
@@ -10,7 +10,6 @@ from pydantic import Extra
|
|
10
10
|
from pydantic import root_validator
|
11
11
|
from pydantic import validator
|
12
12
|
|
13
|
-
from .._validators import valdictkeys
|
14
13
|
from .._validators import valstr
|
15
14
|
from fractal_server.app.schemas._validators import valutc
|
16
15
|
from fractal_server.app.schemas.v2 import ManifestV2
|
@@ -62,13 +61,30 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
|
|
62
61
|
_package_version = validator("package_version", allow_reuse=True)(
|
63
62
|
valstr("package_version")
|
64
63
|
)
|
65
|
-
_pinned_package_versions = validator(
|
66
|
-
"pinned_package_versions", allow_reuse=True
|
67
|
-
)(valdictkeys("pinned_package_versions"))
|
68
64
|
_package_extras = validator("package_extras", allow_reuse=True)(
|
69
65
|
valstr("package_extras")
|
70
66
|
)
|
71
67
|
|
68
|
+
@validator("pinned_package_versions")
|
69
|
+
def pinned_package_validator(cls, value):
|
70
|
+
if value is None:
|
71
|
+
return value
|
72
|
+
old_keys = list(value.keys())
|
73
|
+
new_keys = [
|
74
|
+
valstr(f"pinned_package_versions[{key}]")(key) for key in old_keys
|
75
|
+
]
|
76
|
+
if len(new_keys) != len(set(new_keys)):
|
77
|
+
raise ValueError(
|
78
|
+
f"Dictionary contains multiple identical keys: {value}."
|
79
|
+
)
|
80
|
+
for old_key, new_key in zip(old_keys, new_keys):
|
81
|
+
if new_key != old_key:
|
82
|
+
value[new_key] = value.pop(old_key)
|
83
|
+
for pkg, version in value.items():
|
84
|
+
validate_cmd(pkg)
|
85
|
+
validate_cmd(version)
|
86
|
+
return value
|
87
|
+
|
72
88
|
@validator("package")
|
73
89
|
def package_validator(cls, value):
|
74
90
|
if "/" in value or value.endswith(".whl"):
|
@@ -1,4 +1,5 @@
|
|
1
1
|
from datetime import datetime
|
2
|
+
from enum import Enum
|
2
3
|
from typing import Literal
|
3
4
|
from typing import Optional
|
4
5
|
|
@@ -13,11 +14,17 @@ from .._validators import valstr
|
|
13
14
|
from .task import TaskReadV2
|
14
15
|
|
15
16
|
|
17
|
+
class TaskGroupV2OriginEnum(str, Enum):
|
18
|
+
PYPI = "pypi"
|
19
|
+
WHEELFILE = "wheel-file"
|
20
|
+
OTHER = "other"
|
21
|
+
|
22
|
+
|
16
23
|
class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
|
17
24
|
user_id: int
|
18
25
|
user_group_id: Optional[int] = None
|
19
26
|
active: bool = True
|
20
|
-
origin:
|
27
|
+
origin: TaskGroupV2OriginEnum
|
21
28
|
pkg_name: str
|
22
29
|
version: Optional[str] = None
|
23
30
|
python_version: Optional[str] = None
|
@@ -315,13 +315,20 @@ async def _create_first_user(
|
|
315
315
|
|
316
316
|
|
317
317
|
def _create_first_group():
|
318
|
+
"""
|
319
|
+
Create a `UserGroup` with `name=FRACTAL_DEFAULT_GROUP_NAME`, if missing.
|
320
|
+
"""
|
318
321
|
function_logger = set_logger("fractal_server.create_first_group")
|
319
322
|
|
320
323
|
function_logger.info(
|
321
324
|
f"START _create_first_group, with name '{FRACTAL_DEFAULT_GROUP_NAME}'"
|
322
325
|
)
|
323
326
|
with next(get_sync_db()) as db:
|
324
|
-
group_all = db.execute(
|
327
|
+
group_all = db.execute(
|
328
|
+
select(UserGroup).where(
|
329
|
+
UserGroup.name == FRACTAL_DEFAULT_GROUP_NAME
|
330
|
+
)
|
331
|
+
)
|
325
332
|
if group_all.scalars().one_or_none() is None:
|
326
333
|
first_group = UserGroup(name=FRACTAL_DEFAULT_GROUP_NAME)
|
327
334
|
db.add(first_group)
|
fractal_server/config.py
CHANGED
@@ -167,9 +167,9 @@ class Settings(BaseSettings):
|
|
167
167
|
###########################################################################
|
168
168
|
# DATABASE
|
169
169
|
###########################################################################
|
170
|
-
DB_ENGINE: Literal["sqlite", "postgres
|
170
|
+
DB_ENGINE: Literal["sqlite", "postgres-psycopg"] = "sqlite"
|
171
171
|
"""
|
172
|
-
|
172
|
+
Database engine to use (supported: `sqlite`, `postgres-psycopg`).
|
173
173
|
"""
|
174
174
|
DB_ECHO: bool = False
|
175
175
|
"""
|
@@ -203,16 +203,7 @@ class Settings(BaseSettings):
|
|
203
203
|
|
204
204
|
@property
|
205
205
|
def DATABASE_ASYNC_URL(self) -> URL:
|
206
|
-
if self.DB_ENGINE == "postgres":
|
207
|
-
url = URL.create(
|
208
|
-
drivername="postgresql+asyncpg",
|
209
|
-
username=self.POSTGRES_USER,
|
210
|
-
password=self.POSTGRES_PASSWORD,
|
211
|
-
host=self.POSTGRES_HOST,
|
212
|
-
port=self.POSTGRES_PORT,
|
213
|
-
database=self.POSTGRES_DB,
|
214
|
-
)
|
215
|
-
elif self.DB_ENGINE == "postgres-psycopg":
|
206
|
+
if self.DB_ENGINE == "postgres-psycopg":
|
216
207
|
url = URL.create(
|
217
208
|
drivername="postgresql+psycopg",
|
218
209
|
username=self.POSTGRES_USER,
|
@@ -235,11 +226,7 @@ class Settings(BaseSettings):
|
|
235
226
|
|
236
227
|
@property
|
237
228
|
def DATABASE_SYNC_URL(self):
|
238
|
-
if self.DB_ENGINE == "postgres":
|
239
|
-
return self.DATABASE_ASYNC_URL.set(
|
240
|
-
drivername="postgresql+psycopg2"
|
241
|
-
)
|
242
|
-
elif self.DB_ENGINE == "postgres-psycopg":
|
229
|
+
if self.DB_ENGINE == "postgres-psycopg":
|
243
230
|
return self.DATABASE_ASYNC_URL.set(drivername="postgresql+psycopg")
|
244
231
|
else:
|
245
232
|
if not self.SQLITE_PATH:
|
@@ -546,20 +533,13 @@ class Settings(BaseSettings):
|
|
546
533
|
"""
|
547
534
|
Checks that db environment variables are properly set.
|
548
535
|
"""
|
549
|
-
if self.DB_ENGINE == "postgres":
|
536
|
+
if self.DB_ENGINE == "postgres-psycopg":
|
550
537
|
if not self.POSTGRES_DB:
|
551
538
|
raise FractalConfigurationError(
|
552
|
-
"POSTGRES_DB cannot be None when DB_ENGINE=
|
553
|
-
|
554
|
-
try:
|
555
|
-
import psycopg2 # noqa: F401
|
556
|
-
import asyncpg # noqa: F401
|
557
|
-
except ModuleNotFoundError:
|
558
|
-
raise FractalConfigurationError(
|
559
|
-
"DB engine is `postgres` but `psycopg2` or `asyncpg` "
|
560
|
-
"are not available"
|
539
|
+
"POSTGRES_DB cannot be None when DB_ENGINE="
|
540
|
+
"postgres-psycopg."
|
561
541
|
)
|
562
|
-
|
542
|
+
|
563
543
|
try:
|
564
544
|
import psycopg # noqa: F401
|
565
545
|
except ModuleNotFoundError:
|
@@ -0,0 +1,39 @@
|
|
1
|
+
"""user settings project dir
|
2
|
+
|
3
|
+
Revision ID: 19eca0dd47a9
|
4
|
+
Revises: 8e8f227a3e36
|
5
|
+
Create Date: 2024-10-30 14:34:28.219355
|
6
|
+
|
7
|
+
"""
|
8
|
+
import sqlalchemy as sa
|
9
|
+
import sqlmodel
|
10
|
+
from alembic import op
|
11
|
+
|
12
|
+
|
13
|
+
# revision identifiers, used by Alembic.
|
14
|
+
revision = "19eca0dd47a9"
|
15
|
+
down_revision = "8e8f227a3e36"
|
16
|
+
branch_labels = None
|
17
|
+
depends_on = None
|
18
|
+
|
19
|
+
|
20
|
+
def upgrade() -> None:
|
21
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
22
|
+
with op.batch_alter_table("user_settings", schema=None) as batch_op:
|
23
|
+
batch_op.add_column(
|
24
|
+
sa.Column(
|
25
|
+
"project_dir",
|
26
|
+
sqlmodel.sql.sqltypes.AutoString(),
|
27
|
+
nullable=True,
|
28
|
+
)
|
29
|
+
)
|
30
|
+
|
31
|
+
# ### end Alembic commands ###
|
32
|
+
|
33
|
+
|
34
|
+
def downgrade() -> None:
|
35
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
36
|
+
with op.batch_alter_table("user_settings", schema=None) as batch_op:
|
37
|
+
batch_op.drop_column("project_dir")
|
38
|
+
|
39
|
+
# ### end Alembic commands ###
|
@@ -0,0 +1,42 @@
|
|
1
|
+
"""Update TaskV2 post 2.7.0
|
2
|
+
|
3
|
+
Revision ID: 8e8f227a3e36
|
4
|
+
Revises: 034a469ec2eb
|
5
|
+
Create Date: 2024-10-29 09:01:33.075251
|
6
|
+
|
7
|
+
"""
|
8
|
+
import sqlalchemy as sa
|
9
|
+
from alembic import op
|
10
|
+
|
11
|
+
|
12
|
+
# revision identifiers, used by Alembic.
|
13
|
+
revision = "8e8f227a3e36"
|
14
|
+
down_revision = "034a469ec2eb"
|
15
|
+
branch_labels = None
|
16
|
+
depends_on = None
|
17
|
+
|
18
|
+
|
19
|
+
def upgrade() -> None:
|
20
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
21
|
+
with op.batch_alter_table("taskv2", schema=None) as batch_op:
|
22
|
+
batch_op.alter_column(
|
23
|
+
"taskgroupv2_id", existing_type=sa.INTEGER(), nullable=False
|
24
|
+
)
|
25
|
+
batch_op.drop_column("owner")
|
26
|
+
|
27
|
+
# ### end Alembic commands ###
|
28
|
+
|
29
|
+
|
30
|
+
def downgrade() -> None:
|
31
|
+
# ### commands auto generated by Alembic - please adjust! ###
|
32
|
+
with op.batch_alter_table("taskv2", schema=None) as batch_op:
|
33
|
+
batch_op.add_column(
|
34
|
+
sa.Column(
|
35
|
+
"owner", sa.VARCHAR(), autoincrement=False, nullable=True
|
36
|
+
)
|
37
|
+
)
|
38
|
+
batch_op.alter_column(
|
39
|
+
"taskgroupv2_id", existing_type=sa.INTEGER(), nullable=True
|
40
|
+
)
|
41
|
+
|
42
|
+
# ### end Alembic commands ###
|