fractal-server 2.16.5__py3-none-any.whl → 2.17.0a0__py3-none-any.whl

fractal_server/config/_main.py

@@ -0,0 +1,216 @@
+import logging
+from os import environ
+from os import getenv
+from typing import Literal
+from typing import TypeVar
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import model_validator
+from pydantic import SecretStr
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from ._settings_config import SETTINGS_CONFIG_DICT
+from fractal_server.types import AbsolutePathStr
+
+
+class FractalConfigurationError(ValueError):
+    pass
+
+
+T = TypeVar("T")
+
+
+class OAuthClientConfig(BaseModel):
+    """
+    OAuth Client Config Model
+
+    This model wraps the variables that define a client against an Identity
+    Provider. As some providers are supported by the libraries used within the
+    server, some attributes are optional.
+
+    Attributes:
+        CLIENT_NAME:
+            The name of the client
+        CLIENT_ID:
+            ID of client
+        CLIENT_SECRET:
+            Secret to authorise against the identity provider
+        OIDC_CONFIGURATION_ENDPOINT:
+            OpenID configuration endpoint,
+            allowing to discover the required endpoints automatically
+        REDIRECT_URL:
+            String to be used as `redirect_url` argument for
+            `fastapi_users.get_oauth_router`, and then in
+            `httpx_oauth.integrations.fastapi.OAuth2AuthorizeCallback`.
+    """
+
+    CLIENT_NAME: str
+    CLIENT_ID: str
+    CLIENT_SECRET: SecretStr
+    OIDC_CONFIGURATION_ENDPOINT: str | None = None
+    REDIRECT_URL: str | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def check_configuration(cls, values):
+        if values.get("CLIENT_NAME") not in ["GOOGLE", "GITHUB"]:
+            if not values.get("OIDC_CONFIGURATION_ENDPOINT"):
+                raise FractalConfigurationError(
+                    f"Missing OAUTH_{values.get('CLIENT_NAME')}"
+                    "_OIDC_CONFIGURATION_ENDPOINT"
+                )
+        return values
+
+
+class Settings(BaseSettings):
+    """
+    Contains all the configuration variables for Fractal Server
+
+    The attributes of this class are set from the environment.
+    """
+
+    model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT)
+
+    OAUTH_CLIENTS_CONFIG: list[OAuthClientConfig] = Field(default_factory=list)
+
+    # JWT TOKEN
+    JWT_EXPIRE_SECONDS: int = 180
+    """
+    JWT token lifetime, in seconds.
+    """
+
+    JWT_SECRET_KEY: SecretStr
+    """
+    JWT secret
+
+    ⚠️ **IMPORTANT**: set this variable to a secure string, and do not disclose
+    it.
+    """
+
+    # COOKIE TOKEN
+    COOKIE_EXPIRE_SECONDS: int = 86400
+    """
+    Cookie token lifetime, in seconds.
+    """
+
+    @model_validator(mode="before")
+    @classmethod
+    def collect_oauth_clients(cls, values):
+        """
+        Automatic collection of OAuth Clients
+
+        This method collects the environment variables relative to a single
+        OAuth client and saves them within the `Settings` object in the form
+        of an `OAuthClientConfig` instance.
+
+        Fractal can support an arbitrary number of OAuth providers, which are
+        automatically detected by parsing the environment variable names. In
+        particular, to set the provider `FOO`, one must specify the variables
+
+            OAUTH_FOO_CLIENT_ID
+            OAUTH_FOO_CLIENT_SECRET
+            ...
+
+        etc (cf. OAuthClientConfig).
+        """
+        oauth_env_variable_keys = [
+            key for key in environ.keys() if key.startswith("OAUTH_")
+        ]
+        clients_available = {
+            var.split("_")[1] for var in oauth_env_variable_keys
+        }
+
+        values["OAUTH_CLIENTS_CONFIG"] = []
+        for client in clients_available:
+            prefix = f"OAUTH_{client}"
+            oauth_client_config = OAuthClientConfig(
+                CLIENT_NAME=client,
+                CLIENT_ID=getenv(f"{prefix}_CLIENT_ID", None),
+                CLIENT_SECRET=getenv(f"{prefix}_CLIENT_SECRET", None),
+                OIDC_CONFIGURATION_ENDPOINT=getenv(
+                    f"{prefix}_OIDC_CONFIGURATION_ENDPOINT", None
+                ),
+                REDIRECT_URL=getenv(f"{prefix}_REDIRECT_URL", None),
+            )
+            values["OAUTH_CLIENTS_CONFIG"].append(oauth_client_config)
+        return values
+
+    ###########################################################################
+    # FRACTAL SPECIFIC
+    ###########################################################################
+
+    # Note: we do not use ResourceType here to avoid circular imports
+    FRACTAL_RUNNER_BACKEND: Literal[
+        "local", "slurm_ssh", "slurm_sudo"
+    ] = "local"
+    """
+    Select which runner backend to use.
+    """
+
+    FRACTAL_LOGGING_LEVEL: int = logging.INFO
+    """
+    Logging-level threshold for logging
+
+    Only logs of with this level (or higher) will appear in the console logs.
+    """
+
+    FRACTAL_API_MAX_JOB_LIST_LENGTH: int = 50
+    """
+    Number of ids that can be stored in the `jobsV2` attribute of
+    `app.state`.
+    """
+
+    FRACTAL_GRACEFUL_SHUTDOWN_TIME: int = 30
+    """
+    Waiting time for the shutdown phase of executors
+    """
+
+    FRACTAL_VIEWER_AUTHORIZATION_SCHEME: Literal[
+        "viewer-paths", "users-folders", "none"
+    ] = "none"
+    """
+    Defines how the list of allowed viewer paths is built.
+
+    This variable affects the `GET /auth/current-user/allowed-viewer-paths/`
+    response, which is then consumed by
+    [fractal-vizarr-viewer](https://github.com/fractal-analytics-platform/fractal-vizarr-viewer).
+
+    Options:
+
+    - "viewer-paths": The list of allowed viewer paths will include the user's
+      `project_dir` along with any path defined in user groups' `viewer_paths`
+      attributes.
+    - "users-folders": The list will consist of the user's `project_dir` and a
+       user-specific folder. The user folder is constructed by concatenating
+       the base folder `FRACTAL_VIEWER_BASE_FOLDER` with the user's
+       `slurm_user`.
+    - "none": An empty list will be returned, indicating no access to
+       viewer paths. Useful when vizarr viewer is not used.
+    """
+
+    FRACTAL_VIEWER_BASE_FOLDER: AbsolutePathStr | None = None
+    """
+    Base path to Zarr files that will be served by fractal-vizarr-viewer;
+    This variable is required and used only when
+    FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders".
+    """
+
+    def check(self):
+        """
+        Make sure that required variables are set
+
+        This method must be called before the server starts
+        """
+        # FRACTAL_VIEWER_BASE_FOLDER is required when
+        # FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders"
+        # and it must be an absolute path
+        if self.FRACTAL_VIEWER_AUTHORIZATION_SCHEME == "users-folders":
+            viewer_base_folder = self.FRACTAL_VIEWER_BASE_FOLDER
+            if viewer_base_folder is None:
+                raise FractalConfigurationError(
+                    "FRACTAL_VIEWER_BASE_FOLDER is required when "
+                    "FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "
+                    "users-folders"
+                )

fractal_server/config/_settings_config.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+SETTINGS_CONFIG_DICT = dict(
+    case_sensitive=True,
+    env_file=Path(".fractal_server.env"),
+    extra="ignore",
+)

fractal_server/images/tools.py

@@ -16,7 +16,7 @@ def find_image_by_zarr_url(
     """
     Return a copy of the image with a given zarr_url, and its positional index.
 
-    Arguments:
+    Args:
         images: List of images.
         zarr_url: Path that the returned image must have.
 
@@ -40,7 +40,7 @@ def match_filter(
     """
     Find whether an image matches a filter set.
 
-    Arguments:
+    Args:
         image: A single image.
         type_filters:
         attribute_filters:
@@ -70,7 +70,7 @@ def filter_image_list(
     """
     Compute a sublist with images that match a filter set.
 
-    Arguments:
+    Args:
         images: A list of images.
         type_filters:
         attribute_filters:

fractal_server/logger.py

@@ -44,7 +44,7 @@ def get_logger(logger_name: str | None = None) -> logging.Logger:
         close_logger(logger)
     ```
 
-    Arguments:
+    Args:
         logger_name: Name of logger
     Returns:
         Logger with name `logger_name`
@@ -124,7 +124,7 @@ def close_logger(logger: logging.Logger) -> None:
     """
     Close all handlers associated to a `logging.Logger` object
 
-    Arguments:
+    Args:
         logger: The actual logger
     """
     for handle in logger.handlers:
@@ -135,7 +135,7 @@ def reset_logger_handlers(logger: logging.Logger) -> None:
     """
     Close and remove all handlers associated to a `logging.Logger` object
 
-    Arguments:
+    Args:
         logger: The actual logger
     """
     close_logger(logger)

fractal_server/main.py

@@ -1,34 +1,21 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-# Tommaso Comaprin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-# Application factory
-
-This module sets up the FastAPI application that serves the Fractal Server.
-"""
 import os
 from contextlib import asynccontextmanager
+from itertools import chain
 
 from fastapi import FastAPI
 
 from .app.routes.aux._runner import _backend_supports_shutdown
+from .app.shutdown import cleanup_after_shutdown
+from .config import get_db_settings
+from .config import get_email_settings
 from .config import get_settings
 from .logger import config_uvicorn_loggers
 from .logger import get_logger
 from .logger import reset_logger_handlers
 from .logger import set_logger
-from .runner.shutdown import cleanup_after_shutdown
 from .syringe import Inject
 from fractal_server import __VERSION__
+from fractal_server.app.schemas.v2 import ResourceType
 
 
 def collect_routers(app: FastAPI) -> None:
@@ -64,10 +51,16 @@ def check_settings() -> None:
     """
     settings = Inject(get_settings)
     settings.check()
+    db_settings = Inject(get_db_settings)
+    email_settings = Inject(get_email_settings)
 
     logger = set_logger("fractal_server_settings")
     logger.debug("Fractal Settings:")
-    for key, value in settings.model_dump().items():
+    for key, value in chain(
+        db_settings.model_dump().items(),
+        settings.model_dump().items(),
+        email_settings.model_dump().items(),
+    ):
         if any(s in key.upper() for s in ["PASSWORD", "SECRET", "KEY"]):
             value = "*****"
         logger.debug(f"  {key}: {value}")
@@ -82,7 +75,7 @@ async def lifespan(app: FastAPI):
     check_settings()
     settings = Inject(get_settings)
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if settings.FRACTAL_RUNNER_BACKEND == ResourceType.SLURM_SSH:
         from fractal_server.ssh._fabric import FractalSSHList
 
         app.state.fractal_ssh_list = FractalSSHList()
@@ -103,7 +96,7 @@ async def lifespan(app: FastAPI):
     logger = get_logger("fractal_server.lifespan")
     logger.info("[teardown] START")
 
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if settings.FRACTAL_RUNNER_BACKEND == ResourceType.SLURM_SSH:
         logger.info(
             "[teardown] Close FractalSSH connections "
             f"(current size: {app.state.fractal_ssh_list.size})."

fractal_server/migrations/versions/90f6508c6379_drop_useroauth_username.py

@@ -0,0 +1,36 @@
+"""drop useroauth.username
+
+Revision ID: 90f6508c6379
+Revises: a80ac5a352bf
+Create Date: 2025-10-15 16:09:40.922407
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "90f6508c6379"
+down_revision = "a80ac5a352bf"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.drop_column("username")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "username", sa.VARCHAR(), autoincrement=False, nullable=True
+            )
+        )
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/a80ac5a352bf_resource_profile.py

@@ -0,0 +1,195 @@
+"""resource-profile
+
+Revision ID: a80ac5a352bf
+Revises: 981d588fe248
+Create Date: 2025-10-15 15:53:34.823398
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "a80ac5a352bf"
+down_revision = "981d588fe248"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "resource",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column("host", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "jobs_local_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column(
+            "jobs_runner_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column(
+            "jobs_slurm_python_worker",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column("jobs_poll_interval", sa.Integer(), nullable=False),
+        sa.Column(
+            "tasks_local_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=False,
+        ),
+        sa.Column(
+            "tasks_python_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column(
+            "tasks_pixi_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.CheckConstraint(
+            "(type = 'local') OR (jobs_slurm_python_worker IS NOT NULL)",
+            name=op.f(
+                "ck_resource_`ck_resource_jobs_slurm_python_worker_set`"
+            ),
+        ),
+        sa.CheckConstraint(
+            "type IN ('local', 'slurm_sudo', 'slurm_ssh')",
+            name=op.f("ck_resource_`ck_resource_correct_type`"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_resource")),
+        sa.UniqueConstraint("name", name=op.f("uq_resource_name")),
+    )
+    op.create_table(
+        "profile",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("resource_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "resource_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column(
+            "username", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "ssh_key_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "jobs_remote_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column(
+            "tasks_remote_dir",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["resource_id"],
+            ["resource.id"],
+            name=op.f("fk_profile_resource_id_resource"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_profile")),
+        sa.UniqueConstraint("name", name=op.f("uq_profile_name")),
+    )
+    with op.batch_alter_table("projectv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("resource_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_projectv2_resource_id_resource"),
+            "resource",
+            ["resource_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("resource_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupv2_resource_id_resource"),
+            "resource",
+            ["resource_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("profile_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_user_oauth_profile_id_profile"),
+            "profile",
+            ["profile_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.drop_column("ssh_jobs_dir")
+        batch_op.drop_column("ssh_tasks_dir")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "ssh_tasks_dir",
+                sa.VARCHAR(),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "ssh_jobs_dir",
+                sa.VARCHAR(),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_user_oauth_profile_id_profile"), type_="foreignkey"
+        )
+        batch_op.drop_column("profile_id")
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupv2_resource_id_resource"),
+            type_="foreignkey",
+        )
+        batch_op.drop_column("resource_id")
+
+    with op.batch_alter_table("projectv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_projectv2_resource_id_resource"), type_="foreignkey"
+        )
+        batch_op.drop_column("resource_id")
+
+    op.drop_table("profile")
+    op.drop_table("resource")
+    # ### end Alembic commands ###

fractal_server/runner/config/__init__.py

@@ -0,0 +1,2 @@
+from ._local import JobRunnerConfigLocal  # noqa F401
+from ._slurm import JobRunnerConfigSLURM  # noqa F401

fractal_server/runner/config/_local.py

@@ -0,0 +1,21 @@
+from pydantic import BaseModel
+from pydantic import ConfigDict
+
+
+class JobRunnerConfigLocal(BaseModel):
+    """
+    Specifications of the local-backend configuration
+
+    Attributes:
+        parallel_tasks_per_job:
+            Maximum number of tasks to be run in parallel as part of a call to
+            `FractalThreadPoolExecutor.map`; if `None`, then all tasks will
+            start at the same time.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    parallel_tasks_per_job: int | None = None
+
+    @property
+    def batch_size(self) -> int:
+        return self.parallel_tasks_per_job or 1