fractal-server 2.16.5__py3-none-any.whl → 2.17.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.16.5"
+__VERSION__ = "2.17.0"

fractal_server/__main__.py

@@ -2,8 +2,10 @@ import argparse as ap
 import asyncio
 import json
 import sys
+from pathlib import Path
 
 import uvicorn
+from pydantic import ValidationError
 
 
 parser = ap.ArgumentParser(description="fractal-server commands")
@@ -50,11 +52,42 @@ set_db_parser = subparsers.add_parser(
         "Initialise/upgrade database schemas and create first group&user."
     ),
 )
-set_db_parser.add_argument(
-    "--skip-init-data",
-    action="store_true",
-    help="If set, do not try creating first group and user.",
-    default=False,
+
+# fractalctl init-db-data
+init_db_data_parser = subparsers.add_parser(
+    "init-db-data",
+    description="Populate database with initial data.",
+)
+init_db_data_parser.add_argument(
+    "--resource",
+    type=str,
+    help="Either `default` or path to the JSON file of the first resource.",
+    required=False,
+)
+init_db_data_parser.add_argument(
+    "--profile",
+    type=str,
+    help="Either `default` or path to the JSON file of the first profile.",
+    required=False,
+)
+init_db_data_parser.add_argument(
+    "--admin-email",
+    type=str,
+    help="Email of the first admin user.",
+    required=False,
+)
+init_db_data_parser.add_argument(
+    "--admin-pwd",
+    type=str,
+    help="Password for the first admin user.",
+    required=False,
+)
+
+init_db_data_parser.add_argument(
+    "--admin-project-dir",
+    type=str,
+    help="Project_dir for the first admin user.",
+    required=False,
 )
 
 # fractalctl update-db-data
@@ -63,15 +96,6 @@ update_db_data_parser = subparsers.add_parser(
     description="Apply data-migration script to an existing database.",
 )
 
-# fractalctl encrypt-email-password
-encrypt_email_password_parser = subparsers.add_parser(
-    "encrypt-email-password",
-    description=(
-        "Generate valid values for environment variables "
-        "FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY."
-    ),
-)
-
 
 def save_openapi(dest="openapi.json"):
     from fractal_server.main import start_application
@@ -83,28 +107,22 @@ def save_openapi(dest="openapi.json"):
         json.dump(openapi_schema, f)
 
 
-def set_db(skip_init_data: bool = False):
+def set_db():
     """
-    Upgrade database schema *and* create first group/user
+    Upgrade database schemas.
 
     Call alembic to upgrade to the latest migration.
     Ref: https://stackoverflow.com/a/56683030/283972
-
-    Arguments:
-        skip_init_data: If `True`, skip creation of first group and user.
     """
-    from fractal_server.app.security import _create_first_user
-    from fractal_server.app.security import _create_first_group
     from fractal_server.syringe import Inject
-    from fractal_server.config import get_settings
+    from fractal_server.config import get_db_settings
 
     import alembic.config
     from pathlib import Path
     import fractal_server
 
-    # Check settings
-    settings = Inject(get_settings)
-    settings.check_db()
+    # Validate DB settings
+    Inject(get_db_settings)
 
     # Perform migrations
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
@@ -113,25 +131,138 @@ def set_db(skip_init_data: bool = False):
     alembic.config.main(argv=alembic_args)
     print("END: alembic.config")
 
-    if skip_init_data:
-        return
+
+def init_db_data(
+    *,
+    resource: str | None = None,
+    profile: str | None = None,
+    admin_email: str | None = None,
+    admin_password: str | None = None,
+    admin_project_dir: str | None = None,
+) -> None:
+    from fractal_server.app.security import _create_first_user
+    from fractal_server.app.security import _create_first_group
+    from fractal_server.app.db import get_sync_db
+    from sqlalchemy import select, func
+    from fractal_server.app.models.security import UserOAuth
+    from fractal_server.app.models import Resource, Profile
+    from fractal_server.app.schemas.v2.resource import cast_serialize_resource
+    from fractal_server.app.schemas.v2.profile import cast_serialize_profile
+    from fractal_server.app.schemas.v2 import ResourceType
 
     # Create default group and user
     print()
     _create_first_group()
     print()
-    asyncio.run(
-        _create_first_user(
-            email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
-            password=(
-                settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()
-            ),
-            username=settings.FRACTAL_DEFAULT_ADMIN_USERNAME,
-            is_superuser=True,
-            is_verified=True,
+
+    # Create admin user if requested
+    if not (
+        (admin_email is None)
+        == (admin_password is None)
+        == (admin_project_dir is None)
+    ):
+        print(
+            "You must provide either or or none of `--admin-email`, "
+            "`--admin-pwd` and `--admin-project-dir`. Exit."
         )
-    )
-    print()
+        sys.exit(1)
+    if admin_password and admin_email:
+        asyncio.run(
+            _create_first_user(
+                email=admin_email,
+                password=admin_password,
+                project_dir=admin_project_dir,
+                is_superuser=True,
+                is_verified=True,
+            )
+        )
+        print()
+
+    # Create resource and profile if requested
+    if (resource is None) != (profile is None):
+        print("You must provide both --resource and --profile. Exit.")
+        sys.exit(1)
+    if resource and profile:
+        with next(get_sync_db()) as db:
+            # Preliminary check
+            num_resources = db.execute(
+                select(func.count(Resource.id))
+            ).scalar()
+            if num_resources != 0:
+                print(f"There exist already {num_resources=} resources. Exit.")
+                sys.exit(1)
+
+            # Get resource/profile data
+            if resource == "default":
+                _python_version = (
+                    f"{sys.version_info.major}.{sys.version_info.minor}"
+                )
+                resource_data = {
+                    "name": "Local resource",
+                    "type": ResourceType.LOCAL,
+                    "jobs_local_dir": (Path.cwd() / "data-jobs").as_posix(),
+                    "tasks_local_dir": (Path.cwd() / "data-tasks").as_posix(),
+                    "tasks_python_config": {
+                        "default_version": _python_version,
+                        "versions": {
+                            _python_version: sys.executable,
+                        },
+                    },
+                    "jobs_poll_interval": 0,
+                    "jobs_runner_config": {},
+                    "tasks_pixi_config": {},
+                }
+                print("Prepared default resource data.")
+            else:
+                with open(resource) as f:
+                    resource_data = json.load(f)
+                print(f"Read resource data from {resource}.")
+            if profile == "default":
+                profile_data = {
+                    "resource_type": "local",
+                    "name": "Local profile",
+                }
+                print("Prepared default profile data.")
+            else:
+                with open(profile) as f:
+                    profile_data = json.load(f)
+                print(f"Read profile data from {profile}.")
+
+            # Validate resource/profile data
+            try:
+                resource_data = cast_serialize_resource(resource_data)
+            except ValidationError as e:
+                sys.exit(
+                    f"ERROR: Invalid resource data.\nOriginal error:\n{str(e)}"
+                )
+            try:
+                profile_data = cast_serialize_profile(profile_data)
+            except ValidationError as e:
+                sys.exit(
+                    f"ERROR: Invalid profile data.\nOriginal error:\n{str(e)}"
+                )
+
+            # Create resource/profile db objects
+            resource_obj = Resource(**resource_data)
+            db.add(resource_obj)
+            db.commit()
+            db.refresh(resource_obj)
+            profile_data["resource_id"] = resource_obj.id
+            profile_obj = Profile(**profile_data)
+            db.add(profile_obj)
+            db.commit()
+            db.refresh(profile_obj)
+
+            # Associate profile to users
+            res = db.execute(select(UserOAuth))
+            users = res.unique().scalars().all()
+            for user in users:
+                print(f"Now set profile_id={profile_obj.id} for {user.email}.")
+                user.profile_id = profile_obj.id
+                db.add(user)
+            db.commit()
+            db.expunge_all()
+            print()
 
 
 def update_db_data():
@@ -202,24 +333,21 @@ def update_db_data():
     current_update_db_data_module.fix_db()
 
 
-def print_encrypted_password():
-    from cryptography.fernet import Fernet
-
-    password = input("Insert email password: ").encode("utf-8")
-    key = Fernet.generate_key().decode("utf-8")
-    encrypted_password = Fernet(key).encrypt(password).decode("utf-8")
-
-    print(f"\nFRACTAL_EMAIL_PASSWORD={encrypted_password}")
-    print(f"FRACTAL_EMAIL_PASSWORD_KEY={key}")
-
-
 def run():
     args = parser.parse_args(sys.argv[1:])
 
     if args.cmd == "openapi":
         save_openapi(dest=args.openapi_file)
     elif args.cmd == "set-db":
-        set_db(skip_init_data=args.skip_init_data)
+        set_db()
+    elif args.cmd == "init-db-data":
+        init_db_data(
+            resource=args.resource,
+            profile=args.profile,
+            admin_email=args.admin_email,
+            admin_password=args.admin_pwd,
+            admin_project_dir=args.admin_project_dir,
+        )
     elif args.cmd == "update-db-data":
         update_db_data()
     elif args.cmd == "start":
@@ -229,8 +357,6 @@ def run():
             port=args.port,
             reload=args.reload,
         )
-    elif args.cmd == "encrypt-email-password":
-        print_encrypted_password()
     else:
         sys.exit(f"Error: invalid command '{args.cmd}'.")
 

fractal_server/app/db/__init__.py

@@ -11,9 +11,9 @@ from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm import sessionmaker
 
-from ...config import get_settings
-from ...logger import set_logger
-from ...syringe import Inject
+from fractal_server.config import get_db_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 
 logger = set_logger(__name__)
@@ -42,12 +42,11 @@ class DB:
 
     @classmethod
     def set_async_db(cls):
-        settings = Inject(get_settings)
-        settings.check_db()
+        db_settings = Inject(get_db_settings)
 
         cls._engine_async = create_async_engine(
-            settings.DATABASE_ASYNC_URL,
-            echo=settings.DB_ECHO,
+            db_settings.DATABASE_URL,
+            echo=db_settings.DB_ECHO,
             future=True,
             pool_pre_ping=True,
         )
@@ -60,12 +59,11 @@ class DB:
 
     @classmethod
     def set_sync_db(cls):
-        settings = Inject(get_settings)
-        settings.check_db()
+        db_settings = Inject(get_db_settings)
 
         cls._engine_sync = create_engine(
-            settings.DATABASE_SYNC_URL,
-            echo=settings.DB_ECHO,
+            db_settings.DATABASE_URL,
+            echo=db_settings.DB_ECHO,
             future=True,
             pool_pre_ping=True,
         )

fractal_server/app/models/security.py

@@ -1,27 +1,17 @@
-# This is based on fastapi_users_db_sqlmodel
-# <https://github.com/fastapi-users/fastapi-users-db-sqlmodel>
-# Original Copyright
-# Copyright 2022 François Voron
-# License: MIT
-#
-# Modified by:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
 from datetime import datetime
 from typing import Optional
 
 from pydantic import ConfigDict
 from pydantic import EmailStr
 from sqlalchemy import Column
+from sqlalchemy import String
+from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from .user_settings import UserSettings
 from fractal_server.utils import get_timestamp
 
 
@@ -73,37 +63,55 @@ class UserOAuth(SQLModel, table=True):
         is_active:
         is_superuser:
         is_verified:
-        username:
         oauth_accounts:
-        settings:
+        profile_id:
+        project_dir:
+        slurm_accounts:
     """
 
+    model_config = ConfigDict(from_attributes=True)
+
     __tablename__ = "user_oauth"
 
     id: int | None = Field(default=None, primary_key=True)
 
     email: EmailStr = Field(
-        sa_column_kwargs={"unique": True, "index": True}, nullable=False
+        sa_column_kwargs={"unique": True, "index": True},
+        nullable=False,
     )
     hashed_password: str
     is_active: bool = Field(default=True, nullable=False)
     is_superuser: bool = Field(default=False, nullable=False)
     is_verified: bool = Field(default=False, nullable=False)
 
-    username: str | None = None
-
     oauth_accounts: list["OAuthAccount"] = Relationship(
         back_populates="user",
         sa_relationship_kwargs={"lazy": "joined", "cascade": "all, delete"},
     )
 
-    user_settings_id: int | None = Field(
-        foreign_key="user_settings.id", default=None
+    profile_id: int | None = Field(
+        foreign_key="profile.id",
+        default=None,
+        ondelete="RESTRICT",
+    )
+
+    # TODO-2.17.1: update to `project_dir: str`
+    project_dir: str = Field(
+        sa_column=Column(
+            String,
+            server_default="/PLACEHOLDER",
+            nullable=False,
+        )
     )
-    settings: UserSettings | None = Relationship(
-        sa_relationship_kwargs=dict(lazy="selectin", cascade="all, delete")
+    slurm_accounts: list[str] = Field(
+        sa_column=Column(ARRAY(String), server_default="{}"),
+    )
+
+    # TODO-2.17.1: remove
+    user_settings_id: int | None = Field(
+        foreign_key="user_settings.id",
+        default=None,
     )
-    model_config = ConfigDict(from_attributes=True)
 
 
 class UserGroup(SQLModel, table=True):

fractal_server/app/models/user_settings.py

@@ -4,6 +4,7 @@ from sqlmodel import Field
 from sqlmodel import SQLModel
 
 
+# TODO-2.17.1: Drop `UserSettings`
 class UserSettings(SQLModel, table=True):
     """
     Comprehensive list of user settings.
@@ -15,8 +16,6 @@ class UserSettings(SQLModel, table=True):
         ssh_host: SSH-reachable host where a SLURM client is available.
         ssh_username: User on `ssh_host`.
         ssh_private_key_path: Path of private SSH key for `ssh_username`.
-        ssh_tasks_dir: Task-venvs base folder on `ssh_host`.
-        ssh_jobs_dir: Jobs base folder on `ssh_host`.
         slurm_user: Local user, to be impersonated via `sudo -u`
         project_dir: Folder where `slurm_user` can write.
     """
@@ -30,7 +29,9 @@ class UserSettings(SQLModel, table=True):
     ssh_host: str | None = None
     ssh_username: str | None = None
     ssh_private_key_path: str | None = None
-    ssh_tasks_dir: str | None = None
-    ssh_jobs_dir: str | None = None
+
     slurm_user: str | None = None
     project_dir: str | None = None
+
+    ssh_tasks_dir: str | None = None
+    ssh_jobs_dir: str | None = None

fractal_server/app/models/v2/__init__.py

@@ -9,7 +9,9 @@ from .history import HistoryImageCache
 from .history import HistoryRun
 from .history import HistoryUnit
 from .job import JobV2
+from .profile import Profile
 from .project import ProjectV2
+from .resource import Resource
 from .task import TaskV2
 from .task_group import TaskGroupActivityV2
 from .task_group import TaskGroupV2
@@ -31,4 +33,6 @@ __all__ = [
     "HistoryRun",
     "HistoryUnit",
     "HistoryImageCache",
+    "Resource",
+    "Profile",
 ]

fractal_server/app/models/v2/job.py

@@ -8,9 +8,8 @@ from sqlalchemy.types import DateTime
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
-from ....utils import get_timestamp
-from ...schemas.v2 import JobStatusTypeV2
-from fractal_server.types import AttributeFilters
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.utils import get_timestamp
 
 
 class JobV2(SQLModel, table=True):
@@ -57,7 +56,7 @@ class JobV2(SQLModel, table=True):
     log: str | None = None
     executor_error_log: str | None = None
 
-    attribute_filters: AttributeFilters = Field(
+    attribute_filters: dict[str, list[int | float | str | bool]] = Field(
         sa_column=Column(JSONB, nullable=False, server_default="{}")
     )
     type_filters: dict[str, bool] = Field(

fractal_server/app/models/v2/profile.py

@@ -0,0 +1,16 @@
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+
+class Profile(SQLModel, table=True):
+    id: int | None = Field(default=None, primary_key=True)
+    resource_id: int = Field(foreign_key="resource.id", ondelete="RESTRICT")
+    resource_type: str
+
+    name: str = Field(unique=True)
+
+    username: str | None = None
+    ssh_key_path: str | None = None
+
+    jobs_remote_dir: str | None = None
+    tasks_remote_dir: str | None = None

fractal_server/app/models/v2/project.py

@@ -14,6 +14,11 @@ from fractal_server.utils import get_timestamp
 class ProjectV2(SQLModel, table=True):
     id: int | None = Field(default=None, primary_key=True)
     name: str
+
+    # TODO-2.17.1: make `resource_id` not nullable
+    resource_id: int | None = Field(
+        foreign_key="resource.id", default=None, ondelete="RESTRICT"
+    )
     timestamp_created: datetime = Field(
         default_factory=get_timestamp,
         sa_column=Column(DateTime(timezone=True), nullable=False),

fractal_server/app/models/v2/resource.py

@@ -0,0 +1,130 @@
+from datetime import datetime
+from typing import Any
+from typing import Self
+
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import DateTime
+from sqlmodel import CheckConstraint
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from fractal_server.utils import get_timestamp
+
+
+class Resource(SQLModel, table=True):
+    id: int | None = Field(default=None, primary_key=True)
+
+    type: str
+    """
+    One of `local`, `slurm_sudo` or `slurm_ssh` - matching with
+    `settings.FRACTAL_RUNNER_BACKEND`.
+    """
+
+    name: str = Field(unique=True)
+    """
+    Resource name.
+    """
+
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    """
+    Creation timestamp (autogenerated).
+    """
+
+    host: str | None = None
+    """
+    Address for ssh connections, when `type="slurm_ssh"`.
+    """
+
+    jobs_local_dir: str
+    """
+    Base local folder for job subfolders (containing artifacts and logs).
+    """
+
+    jobs_runner_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Runner configuration, matching one of `JobRunnerConfigLocal` or
+    `JobRunnerConfigSLURM` schemas.
+    """
+
+    jobs_slurm_python_worker: str | None = None
+    """
+    On SLURM deloyments, this is the Python interpreter that runs the
+    `fractal-server` worker from within the SLURM jobs.
+    """
+
+    jobs_poll_interval: int
+    """
+    On SLURM resources: the interval to wait before new `squeue` calls.
+    On local resources: ignored.
+    """
+
+    # task_settings
+    tasks_local_dir: str
+    """
+    Base local folder for task-package subfolders.
+    """
+
+    tasks_python_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Python configuration for task collection. Example:
+    ```json
+    {
+      "default_version": "3.10",
+      "versions:{
+        "3.10": "/xxx/venv-3.10/bin/python",
+        "3.11": "/xxx/venv-3.11/bin/python",
+        "3.12": "/xxx/venv-3.12/bin/python"
+       }
+    }
+    ```
+    """
+
+    tasks_pixi_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Pixi configuration for task collection. Basic example:
+    ```json
+    {
+        "default_version": "0.41.0",
+        "versions": {
+            "0.40.0": "/xxx/pixi/0.40.0/",
+            "0.41.0": "/xxx/pixi/0.41.0/"
+        },
+    }
+    ```
+    """
+
+    @property
+    def pip_cache_dir_arg(self: Self) -> str:
+        """
+        If `pip_cache_dir` is set (in `self.tasks_python_config`), then
+        return `--cache_dir /something`; else return `--no-cache-dir`.
+        """
+        _pip_cache_dir = self.tasks_python_config.get("pip_cache_dir", None)
+        if _pip_cache_dir is not None:
+            return f"--cache-dir {_pip_cache_dir}"
+        else:
+            return "--no-cache-dir"
+
+    # Check constraints
+    __table_args__ = (
+        # `type` column must be one of "local", "slurm_sudo" or "slurm_ssh"
+        CheckConstraint(
+            "type IN ('local', 'slurm_sudo', 'slurm_ssh')",
+            name="correct_type",
+        ),
+        # If `type` is not "local", `jobs_slurm_python_worker` must be set
+        CheckConstraint(
+            "(type = 'local') OR (jobs_slurm_python_worker IS NOT NULL)",
+            name="jobs_slurm_python_worker_set",
+        ),
+    )

fractal_server/app/models/v2/task_group.py

@@ -42,6 +42,10 @@ class TaskGroupV2(SQLModel, table=True):
     user_group_id: int | None = Field(
         foreign_key="usergroup.id", default=None, ondelete="SET NULL"
     )
+    # TODO-2.17.1: make `resource_id` not nullable
+    resource_id: int | None = Field(
+        foreign_key="resource.id", default=None, ondelete="RESTRICT"
+    )
 
     origin: str
     pkg_name: str