fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl

fractal_server/tasks/v2/ssh/reactivate.py

@@ -2,29 +2,30 @@ import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-from ..utils_background import add_commit_refresh
-from ..utils_background import fail_and_cleanup
-from ..utils_background import get_activity_and_task_group
-from ..utils_templates import get_collection_replacements
-from ._utils import _customize_and_run_template
-from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models import Profile
 from fractal_server.app.models import Resource
-from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
-from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityAction
+from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatus
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import SingleUseFractalSSH
 from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import fail_and_cleanup
+from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
 from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_python_interpreter import (
     get_python_interpreter,
 )
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.tasks.v2.utils_templates import get_collection_replacements
 from fractal_server.utils import get_timestamp
 
+from ._utils import _customize_and_run_template
+from ._utils import check_ssh_or_fail_and_cleanup
+
 
 def reactivate_ssh(
     *,
@@ -40,9 +41,10 @@ def reactivate_ssh(
     handled.
 
     Args:
-        task_group_id:
         task_group_activity_id:
-        ssh_config:
+        task_group_id:
+        resource:
+        profile:
     """
 
     LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
@@ -101,7 +103,7 @@ def reactivate_ssh(
                         )
                         return
 
-                    activity.status = TaskGroupActivityStatusV2.ONGOING
+                    activity.status = TaskGroupActivityStatus.ONGOING
                     activity = add_commit_refresh(obj=activity, db=db)
 
                     # Prepare replacements for templates
@@ -145,7 +147,7 @@ def reactivate_ssh(
                         script_dir_remote=script_dir_remote,
                         prefix=(
                             f"{int(time.time())}_"
-                            f"{TaskGroupActivityActionV2.REACTIVATE}"
+                            f"{TaskGroupActivityAction.REACTIVATE}"
                         ),
                         fractal_ssh=fractal_ssh,
                         logger_name=LOGGER_NAME,
@@ -170,7 +172,7 @@ def reactivate_ssh(
                     )
                     logger.info("end - install from pip freeze")
                     activity.log = get_current_log(log_file_path)
-                    activity.status = TaskGroupActivityStatusV2.OK
+                    activity.status = TaskGroupActivityStatus.OK
                     activity.timestamp_ended = get_timestamp()
                     activity = add_commit_refresh(obj=activity, db=db)
                     task_group.active = True
@@ -182,9 +184,7 @@ def reactivate_ssh(
                 except Exception as reactivate_e:
                     # Delete corrupted venv_path
                     try:
-                        logger.info(
-                            f"Now delete folder {task_group.venv_path}"
-                        )
+                        logger.info(f"Now delete folder {task_group.venv_path}")
                         fractal_ssh.remove_folder(
                             folder=task_group.venv_path,
                             safe_root=profile.tasks_remote_dir,

fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -2,17 +2,11 @@ import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-from ..utils_background import fail_and_cleanup
-from ..utils_background import get_activity_and_task_group
-from ..utils_pixi import SOURCE_DIR_NAME
-from ._pixi_slurm_ssh import run_script_on_remote_slurm
-from ._utils import check_ssh_or_fail_and_cleanup
-from ._utils import edit_pyproject_toml_in_place_ssh
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models import Profile
 from fractal_server.app.models import Resource
-from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
-from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityAction
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import SingleUseFractalSSH
@@ -21,10 +15,17 @@ from fractal_server.tasks.utils import get_log_path
 from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
 from fractal_server.tasks.v2.ssh._utils import _customize_and_send_template
 from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import fail_and_cleanup
+from fractal_server.tasks.v2.utils_background import get_activity_and_task_group
 from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_pixi import SOURCE_DIR_NAME
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
 from fractal_server.utils import get_timestamp
 
+from ._pixi_slurm_ssh import run_script_on_remote_slurm
+from ._utils import check_ssh_or_fail_and_cleanup
+from ._utils import edit_pyproject_toml_in_place_ssh
+
 
 def reactivate_ssh_pixi(
     *,
@@ -149,7 +150,7 @@ def reactivate_ssh_pixi(
                     logger.info("installing - START")
 
                     # Set status to ONGOING and refresh logs
-                    activity.status = TaskGroupActivityStatusV2.ONGOING
+                    activity.status = TaskGroupActivityStatus.ONGOING
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
@@ -163,7 +164,7 @@ def reactivate_ssh_pixi(
                         script_dir_remote=script_dir_remote,
                         prefix=(
                             f"{int(time.time())}_"
-                            f"{TaskGroupActivityActionV2.REACTIVATE}"
+                            f"{TaskGroupActivityAction.REACTIVATE}"
                         ),
                         logger_name=LOGGER_NAME,
                         fractal_ssh=fractal_ssh,
@@ -233,9 +234,7 @@ def reactivate_ssh_pixi(
                             remote_script3_path,
                             f"chmod -R 755 {source_dir}",
                         ],
-                        slurm_config=resource.tasks_pixi_config[
-                            "SLURM_CONFIG"
-                        ],
+                        slurm_config=resource.tasks_pixi_config["SLURM_CONFIG"],
                         fractal_ssh=fractal_ssh,
                         logger_name=LOGGER_NAME,
                         prefix=common_args["prefix"],
@@ -248,7 +247,7 @@ def reactivate_ssh_pixi(
                     activity = add_commit_refresh(obj=activity, db=db)
 
                     # Finalize (write metadata to DB)
-                    activity.status = TaskGroupActivityStatusV2.OK
+                    activity.status = TaskGroupActivityStatus.OK
                     activity.timestamp_ended = get_timestamp()
                     activity = add_commit_refresh(obj=activity, db=db)
                     task_group.active = True

fractal_server/tasks/v2/templates/1_create_venv.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/2_pip_install.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/3_pip_freeze.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 # Variables to be filled within fractal-server

fractal_server/tasks/v2/templates/4_pip_show.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 # Variables to be filled within fractal-server
@@ -7,4 +9,4 @@ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
 ENV_DISK_USAGE=$(du -sk "${PACKAGE_ENV_DIR}" | cut -f1)
 ENV_FILE_NUMBER=$(find "${PACKAGE_ENV_DIR}" -type f | wc -l)
 
-echo $ENV_DISK_USAGE $ENV_FILE_NUMBER
+echo "$ENV_DISK_USAGE" "$ENV_FILE_NUMBER"

fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/pixi_1_extract.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/pixi_2_install.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/templates/pixi_3_post_install.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 write_log(){

fractal_server/tasks/v2/utils_background.py

@@ -6,10 +6,10 @@ from sqlalchemy.orm import Session as DBSyncSession
 
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
-from fractal_server.app.schemas.v2 import TaskCreateV2
-from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import TaskCreate
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatus
 from fractal_server.app.schemas.v2.manifest import ManifestV2
-from fractal_server.app.schemas.v2.task_group import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2.task_group import TaskGroupActivityAction
 from fractal_server.exceptions import UnreachableBranchError
 from fractal_server.logger import get_logger
 from fractal_server.logger import reset_logger_handlers
@@ -66,11 +66,11 @@ def fail_and_cleanup(
         f"Original error: {str(exception)}"
     )
 
-    task_group_activity.status = TaskGroupActivityStatusV2.FAILED
+    task_group_activity.status = TaskGroupActivityStatus.FAILED
     task_group_activity.timestamp_ended = get_timestamp()
     task_group_activity.log = get_current_log(log_file_path)
     task_group_activity = add_commit_refresh(obj=task_group_activity, db=db)
-    if task_group_activity.action == TaskGroupActivityActionV2.COLLECT:
+    if task_group_activity.action == TaskGroupActivityAction.COLLECT:
         db.delete(task_group)
     db.commit()
     reset_logger_handlers(logger)
@@ -83,7 +83,7 @@ def prepare_tasks_metadata(
     python_bin: Path | None = None,
     project_python_wrapper: Path | None = None,
     package_version: str | None = None,
-) -> list[TaskCreateV2]:
+) -> list[TaskCreate]:
     """
     Based on the package manifest and additional info, prepare the task list.
 
@@ -112,9 +112,9 @@ def prepare_tasks_metadata(
         if package_version is not None:
             task_attributes["version"] = package_version
         if package_manifest.has_args_schemas:
-            task_attributes[
-                "args_schema_version"
-            ] = package_manifest.args_schema_version
+            task_attributes["args_schema_version"] = (
+                package_manifest.args_schema_version
+            )
         # Set command attributes
         if _task.executable_non_parallel is not None:
             non_parallel_path = package_root / _task.executable_non_parallel
@@ -129,7 +129,7 @@ def prepare_tasks_metadata(
             )
             task_attributes["command_parallel"] = cmd_parallel
         # Create object
-        task_obj = TaskCreateV2(
+        task_obj = TaskCreate(
             **_task.model_dump(
                 exclude={
                     "executable_non_parallel",

fractal_server/tasks/v2/utils_database.py

@@ -3,13 +3,13 @@ from sqlalchemy.orm import Session as DBSyncSession
 
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import TaskV2
-from fractal_server.app.schemas.v2 import TaskCreateV2
+from fractal_server.app.schemas.v2 import TaskCreate
 
 
 def create_db_tasks_and_update_task_group_sync(
     *,
     task_group_id: int,
-    task_list: list[TaskCreateV2],
+    task_list: list[TaskCreate],
     db: DBSyncSession,
 ) -> TaskGroupV2:
     """
@@ -17,7 +17,7 @@ def create_db_tasks_and_update_task_group_sync(
 
     Args:
         task_group_id: ID of an existing `TaskGroupV2` object.
-        task_list: List of `TaskCreateV2` objects to be inserted into the db.
+        task_list: List of `TaskCreate` objects to be inserted into the db.
         db: Synchronous database session
 
     Returns:
@@ -36,7 +36,7 @@ def create_db_tasks_and_update_task_group_sync(
 async def create_db_tasks_and_update_task_group_async(
     *,
     task_group_id: int,
-    task_list: list[TaskCreateV2],
+    task_list: list[TaskCreate],
     db: AsyncSession,
 ) -> TaskGroupV2:
     """
@@ -44,7 +44,7 @@ async def create_db_tasks_and_update_task_group_async(
 
     Args:
         task_group_id: ID of an existing `TaskGroupV2` object.
-        task_list: List of `TaskCreateV2` objects to be inserted into the db.
+        task_list: List of `TaskCreate` objects to be inserted into the db.
         db: Synchronous database session
 
     Returns:

fractal_server/tasks/v2/utils_package_names.py

@@ -62,8 +62,7 @@ def compare_package_names(
         return
 
     logger.warning(
-        f"Package name mismatch: "
-        f"{pkg_name_task_group=}, {pkg_name_pip_show=}."
+        f"Package name mismatch: {pkg_name_task_group=}, {pkg_name_pip_show=}."
     )
     normalized_pkg_name_pip = normalize_package_name(pkg_name_pip_show)
     normalized_pkg_name_taskgroup = normalize_package_name(pkg_name_task_group)

fractal_server/tasks/v2/utils_pixi.py

@@ -95,9 +95,7 @@ def simplify_pyproject_toml(
             if key == pixi_environment
         }
         if pixi_data["environments"] == {}:
-            raise ValueError(
-                f"No '{pixi_environment}' pixi environment found."
-            )
+            raise ValueError(f"No '{pixi_environment}' pixi environment found.")
     except KeyError:
         logger.info("KeyError for workspace/platforms - skip.")
 

fractal_server/types/__init__.py

@@ -6,9 +6,13 @@ from pydantic import AfterValidator
 from pydantic.types import NonNegativeInt
 from pydantic.types import StringConstraints
 
-from ..urls import normalize_url
+from fractal_server.urls import normalize_url
+
 from .validators import val_absolute_path
 from .validators import val_http_url
+from .validators import val_no_dotdot_in_path
+from .validators import val_non_absolute_path
+from .validators import val_os_path_normpath
 from .validators import val_unique_list
 from .validators import valdict_keys
 from .validators import validate_attribute_filters
@@ -18,69 +22,162 @@ NonEmptyStr = Annotated[
     str,
     StringConstraints(min_length=1, strip_whitespace=True),
 ]
+"""
+A non-empty string, with no leading/trailing whitespaces.
+"""
+
 
 AbsolutePathStr = Annotated[
     NonEmptyStr,
     AfterValidator(val_absolute_path),
+    AfterValidator(val_no_dotdot_in_path),
+    AfterValidator(val_os_path_normpath),
 ]
+"""
+String representing an absolute path.
+
+Validation fails if the path is not absolute or if it contains a
+parent-directory reference "/../".
+"""
+
+RelativePathStr = Annotated[
+    NonEmptyStr,
+    AfterValidator(val_no_dotdot_in_path),
+    AfterValidator(val_os_path_normpath),
+    AfterValidator(val_non_absolute_path),
+]
+
 HttpUrlStr = Annotated[
     NonEmptyStr,
     AfterValidator(val_http_url),
 ]
+"""
+String representing an URL.
+"""
+
+
 ZarrUrlStr = Annotated[
     NonEmptyStr,
+    AfterValidator(val_no_dotdot_in_path),
     AfterValidator(normalize_url),
 ]
+"""
+String representing a zarr URL/path.
+
+Validation fails if the path is not absolute or if it contains a
+parent-directory reference "/../".
+"""
+
+
 ZarrDirStr = Annotated[
     NonEmptyStr,
+    AfterValidator(val_no_dotdot_in_path),
     AfterValidator(normalize_url),
 ]
+"""
+String representing a `zarr_dir` path.
+
+Validation fails if the path is not absolute or if it contains a
+parent-directory reference "/../".
+"""
 
 DictStrAny = Annotated[
     dict[str, Any],
     AfterValidator(valdict_keys),
 ]
+"""
+Dictionary where keys are strings with no leading/trailing whitespaces.
+"""
+
+
 DictStrStr = Annotated[
     dict[str, NonEmptyStr],
     AfterValidator(valdict_keys),
 ]
+"""
+Dictionary where keys are strings with no leading/trailing whitespaces and
+values are non-empty strings.
+"""
 
 ListUniqueNonEmptyString = Annotated[
     list[NonEmptyStr],
     AfterValidator(val_unique_list),
 ]
+"""
+List of unique non-empty-string items.
+"""
+
+
 ListUniqueNonNegativeInt = Annotated[
     list[NonNegativeInt],
     AfterValidator(val_unique_list),
 ]
+"""
+List of unique non-negative-integer items.
+"""
+
+
 ListUniqueAbsolutePathStr = Annotated[
     list[AbsolutePathStr],
     AfterValidator(val_unique_list),
 ]
+"""
+List of unique absolute-path-string items.
+"""
 
 WorkflowTaskArgument = Annotated[
     DictStrAny,
     AfterValidator(validate_wft_args),
 ]
+"""
+Dictionary with no keys from a given forbid-list.
+"""
 
 ImageAttributeValue = Union[int, float, str, bool]
+"""
+Possible values for image attributes.
+"""
+
 ImageAttributes = Annotated[
     dict[str, ImageAttributeValue],
     AfterValidator(valdict_keys),
 ]
+"""
+Image-attributes dictionary.
+"""
+
+
 ImageAttributesWithNone = Annotated[
     dict[str, ImageAttributeValue | None],
     AfterValidator(valdict_keys),
 ]
+"""
+Image-attributes dictionary, including `None` attributes.
+"""
+
+
 AttributeFilters = Annotated[
     dict[str, list[ImageAttributeValue]],
     AfterValidator(validate_attribute_filters),
 ]
+"""
+Image-attributes filters.
+"""
+
+
 TypeFilters = Annotated[
     dict[str, bool],
     AfterValidator(valdict_keys),
 ]
+"""
+Image-type filters.
+"""
+
+
 ImageTypes = Annotated[
     dict[str, bool],
     AfterValidator(valdict_keys),
 ]
+"""
+Image types.
+"""

fractal_server/types/validators/__init__.py

@@ -1,6 +1,9 @@
 from ._common_validators import val_absolute_path  # noqa F401
+from ._common_validators import val_no_dotdot_in_path  # noqa F401
+from ._common_validators import val_os_path_normpath  # noqa F401
 from ._common_validators import val_http_url  # noqa F401
 from ._common_validators import val_unique_list  # noqa F401
 from ._common_validators import valdict_keys  # noqa F401
+from ._common_validators import val_non_absolute_path  # noqa F401
 from ._filter_validators import validate_attribute_filters  # noqa F401
 from ._workflow_task_arguments_validators import validate_wft_args  # noqa F401

fractal_server/types/validators/_common_validators.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 from typing import Any
 
 from pydantic import HttpUrl
@@ -13,9 +14,7 @@ def valdict_keys(d: dict[str, Any]) -> dict[str, Any]:
     if any(k == "" for k in new_keys):
         raise ValueError(f"Empty string in {new_keys}.")
     if len(new_keys) != len(set(new_keys)):
-        raise ValueError(
-            f"Dictionary contains multiple identical keys: '{d}'."
-        )
+        raise ValueError(f"Dictionary contains multiple identical keys: '{d}'.")
     for old_key, new_key in zip(old_keys, new_keys):
         if new_key != old_key:
             d[new_key] = d.pop(old_key)
@@ -31,6 +30,37 @@ def val_absolute_path(path: str) -> str:
     return path
 
 
+def val_non_absolute_path(path: str) -> str:
+    """
+    Check that a string attribute is not an absolute path
+    """
+    if os.path.isabs(path):
+        raise ValueError(
+            f"String must not be an absolute path (given '{path}')."
+        )
+    return path
+
+
+def val_no_dotdot_in_path(path: str) -> str:
+    """
+    Check that a string attribute has no '/../' in it
+    """
+    if ".." in Path(path).parts:
+        raise ValueError("String must not contain '/../'.")
+    return path
+
+
+def val_os_path_normpath(path: str) -> str:
+    """
+    Apply `os.path.normpath` to `path`.
+
+    Note: we keep this separate from `fractal_server.urls.normalize_url`,
+    because this function only applies to on-disk paths, while `normalize_url`
+    may apply to s3 URLs as well.
+    """
+    return os.path.normpath(path)
+
+
 def val_unique_list(must_be_unique: list) -> list:
     if len(set(must_be_unique)) != len(must_be_unique):
         raise ValueError("List has repetitions")

fractal_server/types/validators/_workflow_task_arguments_validators.py

@@ -4,7 +4,6 @@ def validate_wft_args(value: dict) -> dict:
     intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
     if intersect_keys:
         raise ValueError(
-            "`args` contains the following forbidden keys: "
-            f"{intersect_keys}"
+            f"`args` contains the following forbidden keys: {intersect_keys}"
         )
     return value

fractal_server/utils.py

@@ -13,6 +13,7 @@
 This module provides general purpose utilities that are not specific to any
 subsystem.
 """
+
 import shlex
 import subprocess  # nosec
 from datetime import datetime

fractal_server/zip_tools.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import subprocess  # nosec
 from collections.abc import Iterator
 from io import BytesIO
 from pathlib import Path
@@ -144,3 +145,36 @@ def _zip_folder_to_file_and_remove(folder: str) -> None:
         logger.info(f"Removing folder '{folder}'.")
         shutil.rmtree(folder)
         logger.info("Folder removed.")
+
+
+def _read_single_file_from_zip(*, file_path: str, archive_path: str) -> str:
+    """
+    Reads and returns the contents of a single file from a ZIP archive using
+    `unzip -p`.
+
+    Args:
+        file_path:
+            relative to the archive
+        archive_path:
+
+    Returns:
+        The file content
+
+    Raises:
+        FileNotFoundError:
+            if the file is not inside the archive
+    """
+    result = subprocess.run(  # nosec
+        ["unzip", "-p", archive_path, file_path],
+        capture_output=True,
+        encoding="utf-8",
+        check=False,
+    )
+
+    if result.returncode != 0:
+        # The caller function should handle this error
+        raise FileNotFoundError(
+            f"File '{file_path}' not found inside archive '{archive_path}'."
+        )
+
+    return result.stdout