fractal-server 2.15.0a4__tar.gz → 2.15.0a5__tar.gz

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.0a4
+Version: 2.15.0a5
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.15.0a5/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.15.0a5"

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -333,6 +333,28 @@ async def _verify_non_duplication_group_constraint(
         )
 
 
+async def _verify_non_duplication_group_path(
+    path: str | None,
+    db: AsyncSession,
+) -> None:
+    """
+    Verify uniqueness of non-`None` `TaskGroupV2.path`
+    """
+    if path is None:
+        return
+    stm = select(TaskGroupV2.id).where(TaskGroupV2.path == path)
+    res = await db.execute(stm)
+    duplicate_ids = res.scalars().all()
+    if duplicate_ids:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Other TaskGroups already have {path=}: "
+                f"{sorted(duplicate_ids)}."
+            ),
+        )
+
+
 async def _add_warnings_to_workflow_tasks(
     wftask_list: list[WorkflowTaskV2], user_id: int, db: AsyncSession
 ) -> list[dict[str, Any]]:

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/app/routes/api/v2/task_collection.py

@@ -13,7 +13,6 @@ from fastapi import UploadFile
 from pydantic import BaseModel
 from pydantic import model_validator
 from pydantic import ValidationError
-from sqlmodel import select
 
 from .....config import get_settings
 from .....logger import reset_logger_handlers
@@ -31,6 +30,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions_task_lifecycle import get_package_version_from_pypi
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
+from ._aux_functions_tasks import _verify_non_duplication_group_path
 from ._aux_functions_tasks import _verify_non_duplication_user_constraint
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupActivityV2
@@ -291,18 +291,10 @@ async def collect_tasks_pip(
         version=task_group_attrs["version"],
         db=db,
     )
-
-    # Verify that task-group path is unique
-    stm = select(TaskGroupV2).where(TaskGroupV2.path == task_group_path)
-    res = await db.execute(stm)
-    for conflicting_task_group in res.scalars().all():
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Another task-group already has path={task_group_path}.\n"
-                f"{conflicting_task_group=}"
-            ),
-        )
+    await _verify_non_duplication_group_path(
+        path=task_group_attrs["path"],
+        db=db,
+    )
 
     # On-disk checks
 

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/app/routes/api/v2/task_collection_pixi.py

@@ -9,7 +9,6 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from fastapi import UploadFile
-from sqlmodel import select
 
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
@@ -22,6 +21,9 @@ from fractal_server.app.routes.api.v2._aux_functions_tasks import (
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _verify_non_duplication_group_constraint,
 )
+from fractal_server.app.routes.api.v2._aux_functions_tasks import (
+    _verify_non_duplication_group_path,
+)
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _verify_non_duplication_user_constraint,
 )
@@ -156,18 +158,10 @@ async def collect_task_pixi(
         version=task_group_attrs["version"],
         db=db,
     )
-
-    # NOTE: to be removed with issue #2634
-    stm = select(TaskGroupV2).where(TaskGroupV2.path == task_group_path)
-    res = await db.execute(stm)
-    for conflicting_task_group in res.scalars().all():
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Another task-group already has path={task_group_path}.\n"
-                f"{conflicting_task_group=}"
-            ),
-        )
+    await _verify_non_duplication_group_path(
+        path=task_group_attrs["path"],
+        db=db,
+    )
 
     if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
         if Path(task_group_path).exists():

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/app/routes/api/v2/task_group.py

@@ -12,6 +12,7 @@ from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
 
+from ._aux_functions_task_lifecycle import check_no_ongoing_activity
 from ._aux_functions_tasks import _get_task_group_full_access
 from ._aux_functions_tasks import _get_task_group_read_access
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
@@ -216,6 +217,8 @@ async def delete_task_group(
         db=db,
     )
 
+    await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
+
     stm = select(WorkflowTaskV2).where(
         WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
     )

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/config.py

@@ -65,8 +65,36 @@ class MailSettings(BaseModel):
 
 
 class PixiSettings(BaseModel):
-    default_version: str
+    """
+    Configuration for Pixi task collection
+
+    See https://pixi.sh/latest/reference/cli/pixi/install/#config-options for
+    `pixi install` concurrency options.
+    See https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
+    for `tokio` configuration.
+
+    versions:
+        Available `pixi` versions and their `PIXI_HOME` folders.
+    default_version:
+        Default `pixi` version to use for task collection - must be one
+        of `versions` keys.
+    PIXI_CONCURRENT_SOLVES:
+        Value of `--concurrent-solves` for `pixi install`.
+    PIXI_CONCURRENT_DOWNLOADS:
+        Value of `--concurrent-downloads for `pixi install`.
+    TOKIO_WORKER_THREADS:
+        From tokio docs, "The core threads are where all asynchronous code
+        runs, and Tokio will by default spawn one for each CPU core. You can
+        use the environment variable TOKIO_WORKER_THREADS to override the
+        default value."
+    """
+
     versions: DictStrStr
+    default_version: str
+
+    PIXI_CONCURRENT_SOLVES: int = 4
+    PIXI_CONCURRENT_DOWNLOADS: int = 4
+    TOKIO_WORKER_THREADS: int = 2
 
     @model_validator(mode="after")
     def check_pixi_settings(self):

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/local/collect_pixi.py

@@ -27,6 +27,7 @@ from fractal_server.tasks.v2.utils_background import (
 from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import execute_command_sync
 from fractal_server.utils import get_timestamp
 
 
@@ -98,6 +99,18 @@ def collect_local_pixi(
                         ),
                         ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
                         ("__FROZEN_OPTION__", ""),
+                        (
+                            "__TOKIO_WORKER_THREADS__",
+                            str(settings.pixi.TOKIO_WORKER_THREADS),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_SOLVES__",
+                            str(settings.pixi.PIXI_CONCURRENT_SOLVES),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_DOWNLOADS__",
+                            str(settings.pixi.PIXI_CONCURRENT_DOWNLOADS),
+                        ),
                     },
                     script_dir=Path(
                         task_group.path, SCRIPTS_SUBFOLDER
@@ -146,6 +159,14 @@ def collect_local_pixi(
                     "project_python_wrapper"
                 ]
 
+                # Make task folder 755
+                source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
+                command = f"chmod 755 {source_dir} -R"
+                execute_command_sync(
+                    command=command,
+                    logger_name=LOGGER_NAME,
+                )
+
                 # Read and validate manifest
                 # NOTE: we are only supporting the manifest path being relative
                 # to the top-level folder

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/local/reactivate_pixi.py

@@ -18,6 +18,7 @@ from fractal_server.tasks.utils import get_log_path
 from fractal_server.tasks.v2.local._utils import _customize_and_run_template
 from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import execute_command_sync
 from fractal_server.utils import get_timestamp
 
 
@@ -89,6 +90,18 @@ def reactivate_local_pixi(
                         ),
                         ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
                         ("__FROZEN_OPTION__", "--frozen"),
+                        (
+                            "__TOKIO_WORKER_THREADS__",
+                            str(settings.pixi.TOKIO_WORKER_THREADS),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_SOLVES__",
+                            str(settings.pixi.PIXI_CONCURRENT_SOLVES),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_DOWNLOADS__",
+                            str(settings.pixi.PIXI_CONCURRENT_DOWNLOADS),
+                        ),
                     },
                     script_dir=Path(
                         task_group.path, SCRIPTS_SUBFOLDER
@@ -130,6 +143,14 @@ def reactivate_local_pixi(
                 activity.log = get_current_log(log_file_path)
                 activity = add_commit_refresh(obj=activity, db=db)
 
+                # Make task folder 755
+                source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
+                command = f"chmod 755 {source_dir} -R"
+                execute_command_sync(
+                    command=command,
+                    logger_name=LOGGER_NAME,
+                )
+
                 activity.log = get_current_log(log_file_path)
                 activity.status = TaskGroupActivityStatusV2.OK
                 activity.timestamp_ended = get_timestamp()

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -147,6 +147,18 @@ def collect_ssh_pixi(
                         ),
                         ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
                         ("__FROZEN_OPTION__", ""),
+                        (
+                            "__TOKIO_WORKER_THREADS__",
+                            str(settings.pixi.TOKIO_WORKER_THREADS),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_SOLVES__",
+                            str(settings.pixi.PIXI_CONCURRENT_SOLVES),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_DOWNLOADS__",
+                            str(settings.pixi.PIXI_CONCURRENT_DOWNLOADS),
+                        ),
                     }
 
                     logger.info("installing - START")
@@ -203,6 +215,11 @@ def collect_ssh_pixi(
                         "project_python_wrapper"
                     ]
 
+                    source_dir = Path(
+                        task_group.path, SOURCE_DIR_NAME
+                    ).as_posix()
+                    fractal_ssh.run_command(cmd=f"chmod 755 {source_dir} -R")
+
                     # Read and validate remote manifest file
                     manifest_path_remote = (
                         f"{package_root_remote}/__FRACTAL_MANIFEST__.json"

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -114,6 +114,18 @@ def reactivate_ssh_pixi(
                         ),
                         ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
                         ("__FROZEN_OPTION__", "--frozen"),
+                        (
+                            "__TOKIO_WORKER_THREADS__",
+                            str(settings.pixi.TOKIO_WORKER_THREADS),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_SOLVES__",
+                            str(settings.pixi.PIXI_CONCURRENT_SOLVES),
+                        ),
+                        (
+                            "__PIXI_CONCURRENT_DOWNLOADS__",
+                            str(settings.pixi.PIXI_CONCURRENT_DOWNLOADS),
+                        ),
                     }
 
                     logger.info("installing - START")
@@ -181,6 +193,8 @@ def reactivate_ssh_pixi(
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
+                    fractal_ssh.run_command(cmd=f"chmod 755 {source_dir} -R")
+
                     # Finalize (write metadata to DB)
                     activity.status = TaskGroupActivityStatusV2.OK
                     activity.timestamp_ended = get_timestamp()

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/templates/pixi_2_install.sh

@@ -10,6 +10,9 @@ PIXI_HOME="__PIXI_HOME__"
 PACKAGE_DIR="__PACKAGE_DIR__"
 SOURCE_DIR_NAME="__SOURCE_DIR_NAME__"
 FROZEN_OPTION="__FROZEN_OPTION__"
+TOKIO_WORKER_THREADS="__TOKIO_WORKER_THREADS__"
+PIXI_CONCURRENT_SOLVES="__PIXI_CONCURRENT_SOLVES__"
+PIXI_CONCURRENT_DOWNLOADS="__PIXI_CONCURRENT_DOWNLOADS__"
 
 # Strip trailing `/` from `PACKAGE_DIR`
 PIXI_HOME=${PIXI_HOME%/}
@@ -24,16 +27,22 @@ PYPROJECT_TOML="${SOURCE_DIR}/pyproject.toml"
 export PIXI_HOME="${PIXI_HOME}"
 export PIXI_CACHE_DIR="${PIXI_HOME}/cache"
 export RATTLER_AUTH_FILE="${PIXI_HOME}/credentials.json"
+export TOKIO_WORKER_THREADS="${TOKIO_WORKER_THREADS}"
 
 TIME_START=$(date +%s)
 
+echo "Hostname: $(hostname)"
+
 cd "${PACKAGE_DIR}"
 write_log "Changed working directory to ${PACKAGE_DIR}"
 
 # -----------------------------------------------------------------------------
 
 write_log "START '${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path ${PYPROJECT_TOML}'"
-${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path "${PYPROJECT_TOML}"
+${PIXI_EXECUTABLE} install \
+    --concurrent-solves "${PIXI_CONCURRENT_SOLVES}" \
+    --concurrent-downloads "${PIXI_CONCURRENT_DOWNLOADS}" \
+    ${FROZEN_OPTION} --manifest-path "${PYPROJECT_TOML}"
 write_log "END   '${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path ${PYPROJECT_TOML}'"
 echo
 

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/fractal_server/tasks/v2/templates/pixi_3_post_install.sh

@@ -70,10 +70,6 @@ write_log "Disk usage: ${ENV_DISK_USAGE}"
 write_log "Number of files: ${ENV_FILE_NUMBER}"
 echo
 
-write_log "START chmod 755 ${SOURCE_DIR} -R"
-chmod 755 "${SOURCE_DIR}" -R
-write_log "END chmod 755 ${SOURCE_DIR} -R"
-
 TIME_END=$(date +%s)
 write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
 write_log "All ok, exit."

{fractal_server-2.15.0a4 → fractal_server-2.15.0a5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.15.0a4"
+version = "2.15.0a5"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -94,7 +94,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.15.0a4"
+current_version = "2.15.0a5"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.15.0a4/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.15.0a4"