fractal-server 2.17.1a1__py3-none-any.whl → 2.17.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +19 -18
- fractal_server/app/db/__init__.py +3 -3
- fractal_server/app/models/__init__.py +1 -0
- fractal_server/app/models/linkuserproject.py +3 -1
- fractal_server/app/models/security.py +21 -3
- fractal_server/app/models/v2/__init__.py +3 -1
- fractal_server/app/models/v2/accounting.py +9 -1
- fractal_server/app/models/v2/dataset.py +5 -1
- fractal_server/app/models/v2/history.py +15 -1
- fractal_server/app/models/v2/job.py +4 -0
- fractal_server/app/models/v2/profile.py +29 -0
- fractal_server/app/models/v2/project.py +4 -10
- fractal_server/app/models/v2/resource.py +4 -0
- fractal_server/app/models/v2/task_group.py +4 -3
- fractal_server/app/models/v2/workflow.py +2 -1
- fractal_server/app/routes/admin/v2/__init__.py +1 -2
- fractal_server/app/routes/admin/v2/accounting.py +1 -1
- fractal_server/app/routes/admin/v2/job.py +9 -9
- fractal_server/app/routes/admin/v2/profile.py +3 -2
- fractal_server/app/routes/admin/v2/resource.py +5 -5
- fractal_server/app/routes/admin/v2/task.py +28 -18
- fractal_server/app/routes/admin/v2/task_group.py +0 -1
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
- fractal_server/app/routes/api/__init__.py +1 -0
- fractal_server/app/routes/api/v2/__init__.py +5 -6
- fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
- fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
- fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
- fractal_server/app/routes/api/v2/dataset.py +13 -32
- fractal_server/app/routes/api/v2/history.py +35 -21
- fractal_server/app/routes/api/v2/images.py +3 -2
- fractal_server/app/routes/api/v2/job.py +17 -14
- fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
- fractal_server/app/routes/api/v2/project.py +22 -17
- fractal_server/app/routes/api/v2/status_legacy.py +12 -11
- fractal_server/app/routes/api/v2/submit.py +11 -12
- fractal_server/app/routes/api/v2/task.py +4 -3
- fractal_server/app/routes/api/v2/task_collection.py +28 -30
- fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
- fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
- fractal_server/app/routes/api/v2/task_group.py +7 -6
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
- fractal_server/app/routes/api/v2/task_version_update.py +13 -12
- fractal_server/app/routes/api/v2/workflow.py +14 -31
- fractal_server/app/routes/api/v2/workflow_import.py +17 -19
- fractal_server/app/routes/api/v2/workflowtask.py +10 -12
- fractal_server/app/routes/auth/__init__.py +1 -3
- fractal_server/app/routes/auth/_aux_auth.py +1 -2
- fractal_server/app/routes/auth/current_user.py +4 -5
- fractal_server/app/routes/auth/group.py +7 -5
- fractal_server/app/routes/auth/login.py +1 -0
- fractal_server/app/routes/auth/oauth.py +4 -3
- fractal_server/app/routes/auth/register.py +4 -2
- fractal_server/app/routes/auth/users.py +10 -10
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/routes/aux/_runner.py +2 -2
- fractal_server/app/routes/pagination.py +1 -1
- fractal_server/app/schemas/user.py +3 -3
- fractal_server/app/schemas/v2/accounting.py +11 -0
- fractal_server/app/schemas/v2/dataset.py +28 -4
- fractal_server/app/schemas/v2/dumps.py +1 -0
- fractal_server/app/schemas/v2/manifest.py +4 -3
- fractal_server/app/schemas/v2/profile.py +53 -2
- fractal_server/app/schemas/v2/resource.py +109 -13
- fractal_server/app/schemas/v2/task.py +0 -1
- fractal_server/app/schemas/v2/task_collection.py +1 -1
- fractal_server/app/schemas/v2/workflowtask.py +4 -3
- fractal_server/app/security/__init__.py +4 -7
- fractal_server/app/security/signup_email.py +4 -5
- fractal_server/config/_data.py +36 -25
- fractal_server/config/_database.py +19 -20
- fractal_server/config/_email.py +30 -38
- fractal_server/config/_main.py +33 -52
- fractal_server/config/_oauth.py +17 -21
- fractal_server/exceptions.py +4 -0
- fractal_server/images/models.py +3 -3
- fractal_server/images/status_tools.py +4 -2
- fractal_server/logger.py +1 -1
- fractal_server/main.py +4 -3
- fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
- fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
- fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
- fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
- fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
- fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
- fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
- fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
- fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
- fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
- fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
- fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
- fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
- fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
- fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
- fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
- fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
- fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
- fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
- fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
- fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
- fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
- fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
- fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
- fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
- fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
- fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
- fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
- fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
- fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
- fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
- fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
- fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
- fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
- fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
- fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
- fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
- fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
- fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
- fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
- fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
- fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
- fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
- fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
- fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
- fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
- fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
- fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
- fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
- fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
- fractal_server/runner/config/_local.py +8 -5
- fractal_server/runner/config/_slurm.py +37 -33
- fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
- fractal_server/runner/executors/base_runner.py +29 -4
- fractal_server/runner/executors/local/get_local_config.py +1 -0
- fractal_server/runner/executors/local/runner.py +14 -13
- fractal_server/runner/executors/slurm_common/_batching.py +5 -10
- fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
- fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
- fractal_server/runner/executors/slurm_common/remote.py +3 -1
- fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
- fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
- fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
- fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
- fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
- fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
- fractal_server/runner/task_files.py +9 -3
- fractal_server/runner/v2/_local.py +9 -4
- fractal_server/runner/v2/_slurm_ssh.py +11 -5
- fractal_server/runner/v2/_slurm_sudo.py +11 -5
- fractal_server/runner/v2/db_tools.py +0 -1
- fractal_server/runner/v2/deduplicate_list.py +2 -1
- fractal_server/runner/v2/runner.py +11 -14
- fractal_server/runner/v2/runner_functions.py +11 -14
- fractal_server/runner/v2/submit_workflow.py +7 -6
- fractal_server/ssh/_fabric.py +6 -13
- fractal_server/string_tools.py +0 -1
- fractal_server/syringe.py +1 -1
- fractal_server/tasks/config/_pixi.py +1 -1
- fractal_server/tasks/config/_python.py +16 -9
- fractal_server/tasks/utils.py +0 -1
- fractal_server/tasks/v2/local/_utils.py +1 -1
- fractal_server/tasks/v2/local/collect.py +10 -12
- fractal_server/tasks/v2/local/collect_pixi.py +9 -10
- fractal_server/tasks/v2/local/deactivate.py +7 -8
- fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
- fractal_server/tasks/v2/local/delete.py +1 -3
- fractal_server/tasks/v2/local/reactivate.py +7 -7
- fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
- fractal_server/tasks/v2/ssh/_utils.py +3 -3
- fractal_server/tasks/v2/ssh/collect.py +14 -19
- fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
- fractal_server/tasks/v2/ssh/deactivate.py +10 -8
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
- fractal_server/tasks/v2/ssh/delete.py +7 -5
- fractal_server/tasks/v2/ssh/reactivate.py +11 -11
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
- fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
- fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
- fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
- fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
- fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
- fractal_server/tasks/v2/utils_background.py +3 -3
- fractal_server/tasks/v2/utils_package_names.py +1 -2
- fractal_server/tasks/v2/utils_pixi.py +1 -3
- fractal_server/types/__init__.py +76 -1
- fractal_server/types/validators/_common_validators.py +1 -3
- fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
- fractal_server/utils.py +1 -0
- fractal_server/zip_tools.py +34 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
- fractal_server-2.17.2.dist-info/RECORD +265 -0
- fractal_server/app/routes/admin/v2/project.py +0 -41
- fractal_server-2.17.1a1.dist-info/RECORD +0 -264
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -2,12 +2,6 @@ import time
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from tempfile import TemporaryDirectory
|
|
4
4
|
|
|
5
|
-
from ..utils_background import fail_and_cleanup
|
|
6
|
-
from ..utils_background import get_activity_and_task_group
|
|
7
|
-
from ..utils_pixi import SOURCE_DIR_NAME
|
|
8
|
-
from ._pixi_slurm_ssh import run_script_on_remote_slurm
|
|
9
|
-
from ._utils import check_ssh_or_fail_and_cleanup
|
|
10
|
-
from ._utils import edit_pyproject_toml_in_place_ssh
|
|
11
5
|
from fractal_server.app.db import get_sync_db
|
|
12
6
|
from fractal_server.app.models import Profile
|
|
13
7
|
from fractal_server.app.models import Resource
|
|
@@ -21,10 +15,17 @@ from fractal_server.tasks.utils import get_log_path
|
|
|
21
15
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
|
|
22
16
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_send_template
|
|
23
17
|
from fractal_server.tasks.v2.utils_background import add_commit_refresh
|
|
18
|
+
from fractal_server.tasks.v2.utils_background import fail_and_cleanup
|
|
19
|
+
from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
|
|
24
20
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
|
21
|
+
from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
|
|
25
22
|
from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
|
|
26
23
|
from fractal_server.utils import get_timestamp
|
|
27
24
|
|
|
25
|
+
from ._pixi_slurm_ssh import run_script_on_remote_slurm
|
|
26
|
+
from ._utils import check_ssh_or_fail_and_cleanup
|
|
27
|
+
from ._utils import edit_pyproject_toml_in_place_ssh
|
|
28
|
+
|
|
28
29
|
|
|
29
30
|
def reactivate_ssh_pixi(
|
|
30
31
|
*,
|
|
@@ -233,9 +234,7 @@ def reactivate_ssh_pixi(
|
|
|
233
234
|
remote_script3_path,
|
|
234
235
|
f"chmod -R 755 {source_dir}",
|
|
235
236
|
],
|
|
236
|
-
slurm_config=resource.tasks_pixi_config[
|
|
237
|
-
"SLURM_CONFIG"
|
|
238
|
-
],
|
|
237
|
+
slurm_config=resource.tasks_pixi_config["SLURM_CONFIG"],
|
|
239
238
|
fractal_ssh=fractal_ssh,
|
|
240
239
|
logger_name=LOGGER_NAME,
|
|
241
240
|
prefix=common_args["prefix"],
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
1
3
|
set -e
|
|
2
4
|
|
|
3
5
|
# Variables to be filled within fractal-server
|
|
@@ -7,4 +9,4 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
|
|
|
7
9
|
ENV_DISK_USAGE=$(du -sk "${PACKAGE_ENV_DIR}" | cut -f1)
|
|
8
10
|
ENV_FILE_NUMBER=$(find "${PACKAGE_ENV_DIR}" -type f | wc -l)
|
|
9
11
|
|
|
10
|
-
echo $ENV_DISK_USAGE $ENV_FILE_NUMBER
|
|
12
|
+
echo "$ENV_DISK_USAGE" "$ENV_FILE_NUMBER"
|
|
@@ -112,9 +112,9 @@ def prepare_tasks_metadata(
|
|
|
112
112
|
if package_version is not None:
|
|
113
113
|
task_attributes["version"] = package_version
|
|
114
114
|
if package_manifest.has_args_schemas:
|
|
115
|
-
task_attributes[
|
|
116
|
-
|
|
117
|
-
|
|
115
|
+
task_attributes["args_schema_version"] = (
|
|
116
|
+
package_manifest.args_schema_version
|
|
117
|
+
)
|
|
118
118
|
# Set command attributes
|
|
119
119
|
if _task.executable_non_parallel is not None:
|
|
120
120
|
non_parallel_path = package_root / _task.executable_non_parallel
|
|
@@ -62,8 +62,7 @@ def compare_package_names(
|
|
|
62
62
|
return
|
|
63
63
|
|
|
64
64
|
logger.warning(
|
|
65
|
-
f"Package name mismatch: "
|
|
66
|
-
f"{pkg_name_task_group=}, {pkg_name_pip_show=}."
|
|
65
|
+
f"Package name mismatch: {pkg_name_task_group=}, {pkg_name_pip_show=}."
|
|
67
66
|
)
|
|
68
67
|
normalized_pkg_name_pip = normalize_package_name(pkg_name_pip_show)
|
|
69
68
|
normalized_pkg_name_taskgroup = normalize_package_name(pkg_name_task_group)
|
|
@@ -95,9 +95,7 @@ def simplify_pyproject_toml(
|
|
|
95
95
|
if key == pixi_environment
|
|
96
96
|
}
|
|
97
97
|
if pixi_data["environments"] == {}:
|
|
98
|
-
raise ValueError(
|
|
99
|
-
f"No '{pixi_environment}' pixi environment found."
|
|
100
|
-
)
|
|
98
|
+
raise ValueError(f"No '{pixi_environment}' pixi environment found.")
|
|
101
99
|
except KeyError:
|
|
102
100
|
logger.info("KeyError for workspace/platforms - skip.")
|
|
103
101
|
|
fractal_server/types/__init__.py
CHANGED
|
@@ -6,7 +6,8 @@ from pydantic import AfterValidator
|
|
|
6
6
|
from pydantic.types import NonNegativeInt
|
|
7
7
|
from pydantic.types import StringConstraints
|
|
8
8
|
|
|
9
|
-
from
|
|
9
|
+
from fractal_server.urls import normalize_url
|
|
10
|
+
|
|
10
11
|
from .validators import val_absolute_path
|
|
11
12
|
from .validators import val_http_url
|
|
12
13
|
from .validators import val_unique_list
|
|
@@ -18,69 +19,143 @@ NonEmptyStr = Annotated[
|
|
|
18
19
|
str,
|
|
19
20
|
StringConstraints(min_length=1, strip_whitespace=True),
|
|
20
21
|
]
|
|
22
|
+
"""
|
|
23
|
+
A non-empty string, with no leading/trailing whitespaces.
|
|
24
|
+
"""
|
|
25
|
+
|
|
21
26
|
|
|
22
27
|
AbsolutePathStr = Annotated[
|
|
23
28
|
NonEmptyStr,
|
|
24
29
|
AfterValidator(val_absolute_path),
|
|
25
30
|
]
|
|
31
|
+
"""
|
|
32
|
+
String representing an absolute path.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
|
|
26
36
|
HttpUrlStr = Annotated[
|
|
27
37
|
NonEmptyStr,
|
|
28
38
|
AfterValidator(val_http_url),
|
|
29
39
|
]
|
|
40
|
+
"""
|
|
41
|
+
String representing an URL.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
|
|
30
45
|
ZarrUrlStr = Annotated[
|
|
31
46
|
NonEmptyStr,
|
|
32
47
|
AfterValidator(normalize_url),
|
|
33
48
|
]
|
|
49
|
+
"""
|
|
50
|
+
String representing a zarr URL/path.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
34
54
|
ZarrDirStr = Annotated[
|
|
35
55
|
NonEmptyStr,
|
|
36
56
|
AfterValidator(normalize_url),
|
|
37
57
|
]
|
|
58
|
+
"""
|
|
59
|
+
String representing a `zarr_dir` path.
|
|
60
|
+
"""
|
|
38
61
|
|
|
39
62
|
DictStrAny = Annotated[
|
|
40
63
|
dict[str, Any],
|
|
41
64
|
AfterValidator(valdict_keys),
|
|
42
65
|
]
|
|
66
|
+
"""
|
|
67
|
+
Dictionary where keys are strings with no leading/trailing whitespaces.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
|
|
43
71
|
DictStrStr = Annotated[
|
|
44
72
|
dict[str, NonEmptyStr],
|
|
45
73
|
AfterValidator(valdict_keys),
|
|
46
74
|
]
|
|
75
|
+
"""
|
|
76
|
+
Dictionary where keys are strings with no leading/trailing whitespaces and
|
|
77
|
+
values are non-empty strings.
|
|
78
|
+
"""
|
|
47
79
|
|
|
48
80
|
ListUniqueNonEmptyString = Annotated[
|
|
49
81
|
list[NonEmptyStr],
|
|
50
82
|
AfterValidator(val_unique_list),
|
|
51
83
|
]
|
|
84
|
+
"""
|
|
85
|
+
List of unique non-empty-string items.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
|
|
52
89
|
ListUniqueNonNegativeInt = Annotated[
|
|
53
90
|
list[NonNegativeInt],
|
|
54
91
|
AfterValidator(val_unique_list),
|
|
55
92
|
]
|
|
93
|
+
"""
|
|
94
|
+
List of unique non-negative-integer items.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
|
|
56
98
|
ListUniqueAbsolutePathStr = Annotated[
|
|
57
99
|
list[AbsolutePathStr],
|
|
58
100
|
AfterValidator(val_unique_list),
|
|
59
101
|
]
|
|
102
|
+
"""
|
|
103
|
+
List of unique absolute-path-string items.
|
|
104
|
+
"""
|
|
60
105
|
|
|
61
106
|
WorkflowTaskArgument = Annotated[
|
|
62
107
|
DictStrAny,
|
|
63
108
|
AfterValidator(validate_wft_args),
|
|
64
109
|
]
|
|
110
|
+
"""
|
|
111
|
+
Dictionary with no keys from a given forbid-list.
|
|
112
|
+
"""
|
|
65
113
|
|
|
66
114
|
ImageAttributeValue = Union[int, float, str, bool]
|
|
115
|
+
"""
|
|
116
|
+
Possible values for image attributes.
|
|
117
|
+
"""
|
|
118
|
+
|
|
67
119
|
ImageAttributes = Annotated[
|
|
68
120
|
dict[str, ImageAttributeValue],
|
|
69
121
|
AfterValidator(valdict_keys),
|
|
70
122
|
]
|
|
123
|
+
"""
|
|
124
|
+
Image-attributes dictionary.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
|
|
71
128
|
ImageAttributesWithNone = Annotated[
|
|
72
129
|
dict[str, ImageAttributeValue | None],
|
|
73
130
|
AfterValidator(valdict_keys),
|
|
74
131
|
]
|
|
132
|
+
"""
|
|
133
|
+
Image-attributes dictionary, including `None` attributes.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
|
|
75
137
|
AttributeFilters = Annotated[
|
|
76
138
|
dict[str, list[ImageAttributeValue]],
|
|
77
139
|
AfterValidator(validate_attribute_filters),
|
|
78
140
|
]
|
|
141
|
+
"""
|
|
142
|
+
Image-attributes filters.
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
|
|
79
146
|
TypeFilters = Annotated[
|
|
80
147
|
dict[str, bool],
|
|
81
148
|
AfterValidator(valdict_keys),
|
|
82
149
|
]
|
|
150
|
+
"""
|
|
151
|
+
Image-type filters.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
|
|
83
155
|
ImageTypes = Annotated[
|
|
84
156
|
dict[str, bool],
|
|
85
157
|
AfterValidator(valdict_keys),
|
|
86
158
|
]
|
|
159
|
+
"""
|
|
160
|
+
Image types.
|
|
161
|
+
"""
|
|
@@ -13,9 +13,7 @@ def valdict_keys(d: dict[str, Any]) -> dict[str, Any]:
|
|
|
13
13
|
if any(k == "" for k in new_keys):
|
|
14
14
|
raise ValueError(f"Empty string in {new_keys}.")
|
|
15
15
|
if len(new_keys) != len(set(new_keys)):
|
|
16
|
-
raise ValueError(
|
|
17
|
-
f"Dictionary contains multiple identical keys: '{d}'."
|
|
18
|
-
)
|
|
16
|
+
raise ValueError(f"Dictionary contains multiple identical keys: '{d}'.")
|
|
19
17
|
for old_key, new_key in zip(old_keys, new_keys):
|
|
20
18
|
if new_key != old_key:
|
|
21
19
|
d[new_key] = d.pop(old_key)
|
|
@@ -4,7 +4,6 @@ def validate_wft_args(value: dict) -> dict:
|
|
|
4
4
|
intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
|
|
5
5
|
if intersect_keys:
|
|
6
6
|
raise ValueError(
|
|
7
|
-
"`args` contains the following forbidden keys: "
|
|
8
|
-
f"{intersect_keys}"
|
|
7
|
+
f"`args` contains the following forbidden keys: {intersect_keys}"
|
|
9
8
|
)
|
|
10
9
|
return value
|
fractal_server/utils.py
CHANGED
fractal_server/zip_tools.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import shutil
|
|
3
|
+
import subprocess # nosec
|
|
3
4
|
from collections.abc import Iterator
|
|
4
5
|
from io import BytesIO
|
|
5
6
|
from pathlib import Path
|
|
@@ -144,3 +145,36 @@ def _zip_folder_to_file_and_remove(folder: str) -> None:
|
|
|
144
145
|
logger.info(f"Removing folder '{folder}'.")
|
|
145
146
|
shutil.rmtree(folder)
|
|
146
147
|
logger.info("Folder removed.")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _read_single_file_from_zip(*, file_path: str, archive_path: str) -> str:
|
|
151
|
+
"""
|
|
152
|
+
Reads and returns the contents of a single file from a ZIP archive using
|
|
153
|
+
`unzip -p`.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
file_path:
|
|
157
|
+
relative to the archive
|
|
158
|
+
archive_path:
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
The file content
|
|
162
|
+
|
|
163
|
+
Raises:
|
|
164
|
+
FileNotFoundError:
|
|
165
|
+
if the file is not inside the archive
|
|
166
|
+
"""
|
|
167
|
+
result = subprocess.run( # nosec
|
|
168
|
+
["unzip", "-p", archive_path, file_path],
|
|
169
|
+
capture_output=True,
|
|
170
|
+
encoding="utf-8",
|
|
171
|
+
check=False,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
if result.returncode != 0:
|
|
175
|
+
# The caller function should handle this error
|
|
176
|
+
raise FileNotFoundError(
|
|
177
|
+
f"File '{file_path}' not found inside archive '{archive_path}'."
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
return result.stdout
|