fractal-server 2.16.5__py3-none-any.whl → 2.17.0__py3-none-any.whl

fractal_server/data_migrations/2_17_0.py

@@ -0,0 +1,339 @@
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+from dotenv.main import DotEnv
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.operators import is_
+from sqlalchemy.sql.operators import is_not
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import Profile
+from fractal_server.app.models import ProjectV2
+from fractal_server.app.models import Resource
+from fractal_server.app.models import TaskGroupV2
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
+from fractal_server.app.schemas.v2.profile import cast_serialize_profile
+from fractal_server.app.schemas.v2.resource import cast_serialize_resource
+from fractal_server.config import get_settings
+from fractal_server.runner.config import JobRunnerConfigLocal
+from fractal_server.runner.config import JobRunnerConfigSLURM
+from fractal_server.tasks.config import TasksPixiSettings
+from fractal_server.tasks.config import TasksPythonSettings
+from fractal_server.types import AbsolutePathStr
+from fractal_server.types import ListUniqueNonEmptyString
+from fractal_server.urls import normalize_url
+
+logging.basicConfig(level=logging.INFO)
+
+
+class UserUpdateInfo(BaseModel):
+    user_id: int
+    project_dir: AbsolutePathStr
+    slurm_accounts: ListUniqueNonEmptyString
+
+
+class ProfileUsersUpdateInfo(BaseModel):
+    data: dict[str, Any]
+    user_updates: list[UserUpdateInfo]
+
+
+def _get_user_settings(user: UserOAuth, db: Session) -> UserSettings:
+    if user.user_settings_id is None:
+        sys.exit(f"User {user.email} is active but {user.user_settings_id=}.")
+    user_settings = db.get(UserSettings, user.user_settings_id)
+    return user_settings
+
+
+def assert_user_setting_key(
+    user: UserOAuth,
+    user_settings: UserSettings,
+    keys: list[str],
+) -> None:
+    for key in keys:
+        if getattr(user_settings, key) is None:
+            sys.exit(
+                f"User {user.email} is active and verified but their "
+                f"user settings have {key}=None."
+            )
+
+
+def prepare_profile_and_user_updates() -> dict[str, ProfileUsersUpdateInfo]:
+    settings = get_settings()
+    profiles_and_users: dict[str, ProfileUsersUpdateInfo] = {}
+    with next(get_sync_db()) as db:
+        # Get active&verified users
+        res = db.execute(
+            select(UserOAuth)
+            .where(is_(UserOAuth.is_active, True))
+            .where(is_(UserOAuth.is_verified, True))
+            .order_by(UserOAuth.id)
+        )
+        for user in res.unique().scalars().all():
+            # Get user settings
+            user_settings = _get_user_settings(user=user, db=db)
+            assert_user_setting_key(user, user_settings, ["project_dir"])
+
+            # Prepare profile data and user update
+            new_profile_data = dict()
+            if settings.FRACTAL_RUNNER_BACKEND == "local":
+                username = None
+            if settings.FRACTAL_RUNNER_BACKEND == "slurm_sudo":
+                assert_user_setting_key(user, user_settings, ["slurm_user"])
+                username = user_settings.slurm_user
+            elif settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+                assert_user_setting_key(
+                    user,
+                    user_settings,
+                    [
+                        "ssh_username",
+                        "ssh_private_key_path",
+                        "ssh_tasks_dir",
+                        "ssh_jobs_dir",
+                    ],
+                )
+                username = user_settings.ssh_username
+                new_profile_data.update(
+                    ssh_key_path=user_settings.ssh_private_key_path,
+                    tasks_remote_dir=normalize_url(
+                        user_settings.ssh_tasks_dir
+                    ),
+                    jobs_remote_dir=normalize_url(user_settings.ssh_jobs_dir),
+                )
+
+            new_profile_data.update(
+                name=f"Profile {username}",
+                username=username,
+                resource_type=settings.FRACTAL_RUNNER_BACKEND,
+            )
+            cast_serialize_profile(new_profile_data)
+
+            user_update_info = UserUpdateInfo(
+                user_id=user.id,
+                project_dir=normalize_url(user_settings.project_dir),
+                slurm_accounts=user_settings.slurm_accounts or [],
+            )
+
+            if username in profiles_and_users.keys():
+                if profiles_and_users[username].data != new_profile_data:
+                    error_msg = (
+                        "Profile data mismatch.\n"
+                        f"{profiles_and_users[username].data=}\n"
+                        f"{new_profile_data=}"
+                    )
+                    logging.error(error_msg)
+                    sys.exit(error_msg)
+                profiles_and_users[username].user_updates.append(
+                    user_update_info
+                )
+            else:
+                profiles_and_users[username] = ProfileUsersUpdateInfo(
+                    data=new_profile_data,
+                    user_updates=[user_update_info],
+                )
+
+    return profiles_and_users
+
+
+def get_old_dotenv_variables() -> dict[str, str | None]:
+    """
+    See
+    https://github.com/fractal-analytics-platform/fractal-server/blob/2.16.x/fractal_server/config.py
+    """
+    OLD_DOTENV_FILE = ".fractal_server.env.old"
+    return dict(
+        **DotEnv(
+            dotenv_path=OLD_DOTENV_FILE,
+            override=False,
+        ).dict()
+    )
+
+
+def get_TasksPythonSettings(
+    old_config: dict[str, str | None]
+) -> dict[str, Any]:
+    versions = {}
+    for version_underscore in ["3_9", "3_10", "3_11", "3_12"]:
+        key = f"FRACTAL_TASKS_PYTHON_{version_underscore}"
+        version_dot = version_underscore.replace("_", ".")
+        value = old_config.get(key, None)
+        if value is not None:
+            versions[version_dot] = value
+    obj = TasksPythonSettings(
+        default_version=old_config["FRACTAL_TASKS_PYTHON_DEFAULT_VERSION"],
+        versions=versions,
+        pip_cache_dir=old_config.get("FRACTAL_PIP_CACHE_DIR", None),
+    )
+    return obj.model_dump()
+
+
+def get_TasksPixiSettings(old_config: dict[str, str | None]) -> dict[str, Any]:
+    pixi_file = old_config.get("FRACTAL_PIXI_CONFIG_FILE", None)
+    if pixi_file is None:
+        return {}
+    with open(pixi_file) as f:
+        old_pixi_config = json.load(f)
+    TasksPixiSettings(**old_pixi_config)
+    return old_pixi_config
+
+
+def get_JobRunnerConfigSLURM(
+    old_config: dict[str, str | None]
+) -> dict[str, Any]:
+    slurm_file = old_config["FRACTAL_SLURM_CONFIG_FILE"]
+    with open(slurm_file) as f:
+        old_slurm_config = json.load(f)
+    JobRunnerConfigSLURM(**old_slurm_config)
+    return old_slurm_config
+
+
+def get_JobRunnerConfigLocal(
+    old_config: dict[str, str | None]
+) -> dict[str, Any]:
+    local_file = old_config.get("FRACTAL_LOCAL_CONFIG_FILE", None)
+    if local_file is None or not Path(local_file).exists():
+        return JobRunnerConfigLocal().model_dump()
+    else:
+        with open(local_file) as f:
+            old_local_config = json.load(f)
+        JobRunnerConfigLocal(**old_local_config)
+        return old_local_config
+
+
+def get_ssh_host() -> str:
+    with next(get_sync_db()) as db:
+        res = db.execute(
+            select(UserSettings.ssh_host).where(
+                is_not(UserSettings.ssh_host, None)
+            )
+        )
+        hosts = res.scalars().all()
+    if len(set(hosts)) > 1:
+        host = max(set(hosts), key=hosts.count)
+        print(f"MOST FREQUENT HOST: {host}")
+    else:
+        host = hosts[0]
+    return host
+
+
+def prepare_resource_data(old_config: dict[str, str | None]) -> dict[str, Any]:
+    settings = get_settings()
+
+    resource_data = dict(
+        type=settings.FRACTAL_RUNNER_BACKEND,
+        name="Resource Name",
+        tasks_python_config=get_TasksPythonSettings(old_config),
+        tasks_pixi_config=get_TasksPixiSettings(old_config),
+        tasks_local_dir=old_config["FRACTAL_TASKS_DIR"],
+        jobs_local_dir=old_config["FRACTAL_RUNNER_WORKING_BASE_DIR"],
+        jobs_poll_interval=int(
+            old_config.get("FRACTAL_SLURM_POLL_INTERVAL", 15)
+        ),
+    )
+    if settings.FRACTAL_RUNNER_BACKEND == "local":
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigLocal(
+            old_config
+        )
+    elif settings.FRACTAL_RUNNER_BACKEND == "slurm_sudo":
+        resource_data["jobs_slurm_python_worker"] = old_config[
+            "FRACTAL_SLURM_WORKER_PYTHON"
+        ]
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigSLURM(
+            old_config
+        )
+    else:
+        resource_data["jobs_slurm_python_worker"] = old_config[
+            "FRACTAL_SLURM_WORKER_PYTHON"
+        ]
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigSLURM(
+            old_config
+        )
+        resource_data["host"] = get_ssh_host()
+
+    resource_data = cast_serialize_resource(resource_data)
+
+    return resource_data
+
+
+def fix_db():
+    logging.info("START preliminary checks.")
+
+    # Read old env file
+    old_config = get_old_dotenv_variables()
+
+    # Prepare resource data
+    logging.info("START prepare_resource_data")
+    resource_data = prepare_resource_data(old_config)
+    logging.info("END prepare_resource_data")
+
+    # Prepare profile/users data
+    logging.info("START prepare_profile_and_user_updates")
+    profile_and_user_updates = prepare_profile_and_user_updates()
+    logging.info("END prepare_profile_and_user_updates")
+
+    logging.info("END preliminary checks.")
+    print()
+
+    with next(get_sync_db()) as db:
+        # Create new resource
+        resource = Resource(**resource_data)
+        db.add(resource)
+        db.commit()
+        db.refresh(resource)
+        db.expunge(resource)
+        resource_id = resource.id
+        logging.info(f"Created resource with {resource_id=}.")
+
+        # Update task groups
+        res = db.execute(select(TaskGroupV2).order_by(TaskGroupV2.id))
+        for taskgroup in res.scalars().all():
+            taskgroup.resource_id = resource_id
+            db.add(taskgroup)
+        db.commit()
+        logging.info(f"Set {resource_id=} foreign key for all task groups.")
+
+        # Update projects
+        res = db.execute(select(ProjectV2).order_by(ProjectV2.id))
+        for project in res.scalars().all():
+            project.resource_id = resource_id
+            db.add(project)
+        db.commit()
+        logging.info(f"Set {resource_id=} foreign key for all projects.")
+        print()
+
+        db.expunge_all()
+
+        for _, info in profile_and_user_updates.items():
+            # Create profile
+            profile_data = info.data
+            profile_data["resource_id"] = resource_id
+            profile = Profile(**profile_data)
+            db.add(profile)
+            db.commit()
+            db.refresh(profile)
+            db.expunge(profile)
+            profile_id = profile.id
+            logging.info(
+                f"Created profile '{profile.name}', with {profile.id=}."
+            )
+
+            # Update users
+            for user_update in info.user_updates:
+                user = db.get(UserOAuth, user_update.user_id)
+                user.profile_id = profile_id
+                user.project_dir = user_update.project_dir
+                user.slurm_accounts = user_update.slurm_accounts
+                db.add(user)
+                logging.info(f"Updated {user.email} with {user.project_dir=}.")
+                logging.info(
+                    f"Associated {user.email} to profile {profile.name}."
+                )
+            print()
+            db.commit()
+
+    logging.info("END - all ok.")

fractal_server/images/tools.py

@@ -16,7 +16,7 @@ def find_image_by_zarr_url(
     """
     Return a copy of the image with a given zarr_url, and its positional index.
 
-    Arguments:
+    Args:
         images: List of images.
         zarr_url: Path that the returned image must have.
 
@@ -40,7 +40,7 @@ def match_filter(
     """
     Find whether an image matches a filter set.
 
-    Arguments:
+    Args:
         image: A single image.
         type_filters:
         attribute_filters:
@@ -70,7 +70,7 @@ def filter_image_list(
     """
     Compute a sublist with images that match a filter set.
 
-    Arguments:
+    Args:
         images: A list of images.
         type_filters:
         attribute_filters:

fractal_server/logger.py

@@ -44,7 +44,7 @@ def get_logger(logger_name: str | None = None) -> logging.Logger:
         close_logger(logger)
     ```
 
-    Arguments:
+    Args:
         logger_name: Name of logger
     Returns:
         Logger with name `logger_name`
@@ -124,7 +124,7 @@ def close_logger(logger: logging.Logger) -> None:
     """
     Close all handlers associated to a `logging.Logger` object
 
-    Arguments:
+    Args:
         logger: The actual logger
     """
     for handle in logger.handlers:
@@ -135,7 +135,7 @@ def reset_logger_handlers(logger: logging.Logger) -> None:
     """
     Close and remove all handlers associated to a `logging.Logger` object
 
-    Arguments:
+    Args:
         logger: The actual logger
     """
     close_logger(logger)

fractal_server/main.py

@@ -1,34 +1,22 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-# Tommaso Comaprin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-# Application factory
-
-This module sets up the FastAPI application that serves the Fractal Server.
-"""
 import os
 from contextlib import asynccontextmanager
+from itertools import chain
 
 from fastapi import FastAPI
 
 from .app.routes.aux._runner import _backend_supports_shutdown
+from .app.shutdown import cleanup_after_shutdown
+from .config import get_data_settings
+from .config import get_db_settings
+from .config import get_email_settings
 from .config import get_settings
 from .logger import config_uvicorn_loggers
 from .logger import get_logger
 from .logger import reset_logger_handlers
 from .logger import set_logger
-from .runner.shutdown import cleanup_after_shutdown
 from .syringe import Inject
 from fractal_server import __VERSION__
+from fractal_server.app.schemas.v2 import ResourceType
 
 
 def collect_routers(app: FastAPI) -> None:
@@ -63,11 +51,17 @@ def check_settings() -> None:
         ValidationError: If the configuration is invalid.
     """
     settings = Inject(get_settings)
-    settings.check()
-
+    db_settings = Inject(get_db_settings)
+    email_settings = Inject(get_email_settings)
+    data_settings = Inject(get_data_settings)
     logger = set_logger("fractal_server_settings")
     logger.debug("Fractal Settings:")
-    for key, value in settings.model_dump().items():
+    for key, value in chain(
+        db_settings.model_dump().items(),
+        settings.model_dump().items(),
+        email_settings.model_dump().items(),
+        data_settings.model_dump().items(),
+    ):
         if any(s in key.upper() for s in ["PASSWORD", "SECRET", "KEY"]):
             value = "*****"
         logger.debug(f"  {key}: {value}")
@@ -82,7 +76,7 @@ async def lifespan(app: FastAPI):
     check_settings()
     settings = Inject(get_settings)
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if settings.FRACTAL_RUNNER_BACKEND == ResourceType.SLURM_SSH:
         from fractal_server.ssh._fabric import FractalSSHList
 
         app.state.fractal_ssh_list = FractalSSHList()
@@ -103,7 +97,7 @@ async def lifespan(app: FastAPI):
     logger = get_logger("fractal_server.lifespan")
     logger.info("[teardown] START")
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if settings.FRACTAL_RUNNER_BACKEND == ResourceType.SLURM_SSH:
         logger.info(
             "[teardown] Close FractalSSH connections "
             f"(current size: {app.state.fractal_ssh_list.size})."

fractal_server/migrations/naming_convention.py

@@ -1,7 +1,7 @@
 NAMING_CONVENTION = {
     "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
-    "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
     "pk": "pk_%(table_name)s",
 }

fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py

@@ -0,0 +1,195 @@
+"""2.17.0
+
+Revision ID: 83bc2ad3ffcc
+Revises: 981d588fe248
+Create Date: 2025-10-30 14:16:53.639006
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "83bc2ad3ffcc"
+down_revision = "981d588fe248"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "resource",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column("host", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "jobs_local_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column(
+            "jobs_runner_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column(
+            "jobs_slurm_python_worker",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column("jobs_poll_interval", sa.Integer(), nullable=False),
+        sa.Column(
+            "tasks_local_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column(
+            "tasks_python_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column(
+            "tasks_pixi_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.CheckConstraint(
+            "(type = 'local') OR (jobs_slurm_python_worker IS NOT NULL)",
+            name=op.f("ck_resource_jobs_slurm_python_worker_set"),
+        ),
+        sa.CheckConstraint(
+            "type IN ('local', 'slurm_sudo', 'slurm_ssh')",
+            name=op.f("ck_resource_correct_type"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_resource")),
+        sa.UniqueConstraint("name", name=op.f("uq_resource_name")),
+    )
+    op.create_table(
+        "profile",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("resource_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "resource_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column(
+            "username", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "ssh_key_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "jobs_remote_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column(
+            "tasks_remote_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["resource_id"],
+            ["resource.id"],
+            name=op.f("fk_profile_resource_id_resource"),
+            ondelete="RESTRICT",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_profile")),
+        sa.UniqueConstraint("name", name=op.f("uq_profile_name")),
+    )
+    with op.batch_alter_table("projectv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("resource_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_projectv2_resource_id_resource"),
+            "resource",
+            ["resource_id"],
+            ["id"],
+            ondelete="RESTRICT",
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("resource_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupv2_resource_id_resource"),
+            "resource",
+            ["resource_id"],
+            ["id"],
+            ondelete="RESTRICT",
+        )
+
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("profile_id", sa.Integer(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column(
+                "project_dir",
+                sa.String(),
+                server_default="/PLACEHOLDER",
+                nullable=False,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "slurm_accounts",
+                postgresql.ARRAY(sa.String()),
+                server_default="{}",
+                nullable=True,
+            )
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_user_oauth_profile_id_profile"),
+            "profile",
+            ["profile_id"],
+            ["id"],
+            ondelete="RESTRICT",
+        )
+        batch_op.drop_column("username")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "username", sa.VARCHAR(), autoincrement=False, nullable=True
+            )
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_user_oauth_profile_id_profile"), type_="foreignkey"
+        )
+        batch_op.drop_column("slurm_accounts")
+        batch_op.drop_column("project_dir")
+        batch_op.drop_column("profile_id")
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupv2_resource_id_resource"),
+            type_="foreignkey",
+        )
+        batch_op.drop_column("resource_id")
+
+    with op.batch_alter_table("projectv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_projectv2_resource_id_resource"), type_="foreignkey"
+        )
+        batch_op.drop_column("resource_id")
+
+    op.drop_table("profile")
+    op.drop_table("resource")
+    # ### end Alembic commands ###

fractal_server/runner/config/__init__.py

@@ -0,0 +1,2 @@
+from ._local import JobRunnerConfigLocal  # noqa F401
+from ._slurm import JobRunnerConfigSLURM  # noqa F401