fractal-server 2.16.6__tar.gz → 2.17.0a1__tar.gz

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fractal-server
-Version: 2.16.6
+Version: 2.17.0a1
 Summary: Backend component of the Fractal analytics platform
 License-Expression: BSD-3-Clause
 License-File: LICENSE

fractal_server-2.17.0a1/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.17.0a1"

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/__main__.py

@@ -2,9 +2,12 @@ import argparse as ap
 import asyncio
 import json
 import sys
+from pathlib import Path
 
 import uvicorn
+from pydantic import ValidationError
 
+from fractal_server.app.schemas.v2 import ResourceType
 
 parser = ap.ArgumentParser(description="fractal-server commands")
 
@@ -50,11 +53,23 @@ set_db_parser = subparsers.add_parser(
         "Initialise/upgrade database schemas and create first group&user."
     ),
 )
-set_db_parser.add_argument(
-    "--skip-init-data",
-    action="store_true",
-    help="If set, do not try creating first group and user.",
-    default=False,
+
+# fractalctl init-db-data
+init_db_data_parser = subparsers.add_parser(
+    "init-db-data",
+    description="Populate database with initial data.",
+)
+init_db_data_parser.add_argument(
+    "--resource",
+    type=str,
+    help=("Either `default` or path to the JSON file of the first resource."),
+    required=True,
+)
+init_db_data_parser.add_argument(
+    "--profile",
+    type=str,
+    help=("Either `default` or path to the JSON file of the first profile."),
+    required=True,
 )
 
 # fractalctl update-db-data
@@ -83,28 +98,22 @@ def save_openapi(dest="openapi.json"):
         json.dump(openapi_schema, f)
 
 
-def set_db(skip_init_data: bool = False):
+def set_db():
     """
-    Upgrade database schema *and* create first group/user
+    Upgrade database schemas.
 
     Call alembic to upgrade to the latest migration.
     Ref: https://stackoverflow.com/a/56683030/283972
-
-    Arguments:
-        skip_init_data: If `True`, skip creation of first group and user.
     """
-    from fractal_server.app.security import _create_first_user
-    from fractal_server.app.security import _create_first_group
     from fractal_server.syringe import Inject
-    from fractal_server.config import get_settings
+    from fractal_server.config import get_db_settings
 
     import alembic.config
     from pathlib import Path
     import fractal_server
 
-    # Check settings
-    settings = Inject(get_settings)
-    settings.check_db()
+    # Validate DB settings
+    Inject(get_db_settings)
 
     # Perform migrations
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
@@ -113,8 +122,24 @@ def set_db(skip_init_data: bool = False):
     alembic.config.main(argv=alembic_args)
     print("END: alembic.config")
 
-    if skip_init_data:
-        return
+
+def init_db_data(
+    *,
+    resource: str,
+    profile: str,
+) -> None:
+    from fractal_server.app.security import _create_first_user
+    from fractal_server.app.security import _create_first_group
+    from fractal_server.syringe import Inject
+    from fractal_server.config import get_init_data_settings
+    from fractal_server.app.db import get_sync_db
+    from sqlalchemy import select, func
+    from fractal_server.app.models.security import UserOAuth
+    from fractal_server.app.models import Resource, Profile
+    from fractal_server.app.schemas.v2.resource import cast_serialize_resource
+    from fractal_server.app.schemas.v2.profile import cast_serialize_profile
+
+    init_data_settings = Inject(get_init_data_settings)
 
     # Create default group and user
     print()
@@ -122,17 +147,96 @@ def set_db(skip_init_data: bool = False):
     print()
     asyncio.run(
         _create_first_user(
-            email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
+            email=init_data_settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
             password=(
-                settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()
+                init_data_settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()  # noqa E501
             ),
-            username=settings.FRACTAL_DEFAULT_ADMIN_USERNAME,
             is_superuser=True,
             is_verified=True,
         )
     )
     print()
 
+    # Create resource and profile
+    with next(get_sync_db()) as db:
+        # Preliminary check
+        num_resources = db.execute(select(func.count(Resource.id))).scalar()
+        if num_resources != 0:
+            print(f"There exist already {num_resources=} resources. Exit.")
+            sys.exit(1)
+
+        # Get resource/profile data
+        if resource == "default":
+            _python_version = (
+                f"{sys.version_info.major}.{sys.version_info.minor}"
+            )
+            resource_data = {
+                "name": "Local resource",
+                "type": ResourceType.LOCAL,
+                "jobs_local_dir": (Path.cwd() / "data-jobs").as_posix(),
+                "tasks_local_dir": (Path.cwd() / "data-tasks").as_posix(),
+                "tasks_python_config": {
+                    "default_version": _python_version,
+                    "versions": {
+                        _python_version: sys.executable,
+                    },
+                },
+                "jobs_poll_interval": 0,
+                "jobs_runner_config": {},
+                "tasks_pixi_config": {},
+            }
+            print("Prepared default resource data.")
+        else:
+            with open(resource) as f:
+                resource_data = json.load(f)
+            print(f"Read resource data from {resource}.")
+        if profile == "default":
+            profile_data = {
+                "resource_type": "local",
+                "name": "Local profile",
+            }
+            print("Prepared default profile data.")
+        else:
+            with open(profile) as f:
+                profile_data = json.load(f)
+            print(f"Read profile data from {profile}.")
+
+        # Validate resource/profile data
+        try:
+            resource_data = cast_serialize_resource(resource_data)
+        except ValidationError as e:
+            sys.exit(
+                f"ERROR: Invalid resource data.\nOriginal error:\n{str(e)}"
+            )
+        try:
+            profile_data = cast_serialize_profile(profile_data)
+        except ValidationError as e:
+            sys.exit(
+                f"ERROR: Invalid profile data.\nOriginal error:\n{str(e)}"
+            )
+
+        # Create resource/profile db objects
+        resource_obj = Resource(**resource_data)
+        db.add(resource_obj)
+        db.commit()
+        db.refresh(resource_obj)
+        profile_data["resource_id"] = resource_obj.id
+        profile_obj = Profile(**profile_data)
+        db.add(profile_obj)
+        db.commit()
+        db.refresh(profile_obj)
+
+        # Associate profile to users
+        res = db.execute(select(UserOAuth))
+        users = res.unique().scalars().all()
+        for user in users:
+            print(f"Now set profile_id={profile_obj.id} for {user.email}.")
+            user.profile_id = profile_obj.id
+            db.add(user)
+        db.commit()
+        db.expunge_all()
+        print()
+
 
 def update_db_data():
     """
@@ -219,7 +323,12 @@ def run():
     if args.cmd == "openapi":
         save_openapi(dest=args.openapi_file)
     elif args.cmd == "set-db":
-        set_db(skip_init_data=args.skip_init_data)
+        set_db()
+    elif args.cmd == "init-db-data":
+        init_db_data(
+            resource=args.resource,
+            profile=args.profile,
+        )
     elif args.cmd == "update-db-data":
         update_db_data()
     elif args.cmd == "start":

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/db/__init__.py

@@ -11,9 +11,9 @@ from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm import sessionmaker
 
-from ...config import get_settings
-from ...logger import set_logger
-from ...syringe import Inject
+from fractal_server.config import get_db_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 
 logger = set_logger(__name__)
@@ -42,12 +42,11 @@ class DB:
 
     @classmethod
     def set_async_db(cls):
-        settings = Inject(get_settings)
-        settings.check_db()
+        db_settings = Inject(get_db_settings)
 
         cls._engine_async = create_async_engine(
-            settings.DATABASE_ASYNC_URL,
-            echo=settings.DB_ECHO,
+            db_settings.DATABASE_URL,
+            echo=db_settings.DB_ECHO,
             future=True,
             pool_pre_ping=True,
         )
@@ -60,12 +59,11 @@ class DB:
 
     @classmethod
     def set_sync_db(cls):
-        settings = Inject(get_settings)
-        settings.check_db()
+        db_settings = Inject(get_db_settings)
 
         cls._engine_sync = create_engine(
-            settings.DATABASE_SYNC_URL,
-            echo=settings.DB_ECHO,
+            db_settings.DATABASE_URL,
+            echo=db_settings.DB_ECHO,
             future=True,
             pool_pre_ping=True,
         )

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/models/security.py

@@ -73,8 +73,9 @@ class UserOAuth(SQLModel, table=True):
         is_active:
         is_superuser:
         is_verified:
-        username:
         oauth_accounts:
+        user_settings_id:
+        profile_id:
         settings:
     """
 
@@ -90,8 +91,6 @@ class UserOAuth(SQLModel, table=True):
     is_superuser: bool = Field(default=False, nullable=False)
     is_verified: bool = Field(default=False, nullable=False)
 
-    username: str | None = None
-
     oauth_accounts: list["OAuthAccount"] = Relationship(
         back_populates="user",
         sa_relationship_kwargs={"lazy": "joined", "cascade": "all, delete"},
@@ -100,6 +99,11 @@ class UserOAuth(SQLModel, table=True):
     user_settings_id: int | None = Field(
         foreign_key="user_settings.id", default=None
     )
+    profile_id: int | None = Field(
+        foreign_key="profile.id",
+        default=None,
+        ondelete="SET NULL",
+    )
     settings: UserSettings | None = Relationship(
         sa_relationship_kwargs=dict(lazy="selectin", cascade="all, delete")
     )

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/models/user_settings.py

@@ -15,8 +15,6 @@ class UserSettings(SQLModel, table=True):
         ssh_host: SSH-reachable host where a SLURM client is available.
         ssh_username: User on `ssh_host`.
         ssh_private_key_path: Path of private SSH key for `ssh_username`.
-        ssh_tasks_dir: Task-venvs base folder on `ssh_host`.
-        ssh_jobs_dir: Jobs base folder on `ssh_host`.
         slurm_user: Local user, to be impersonated via `sudo -u`
         project_dir: Folder where `slurm_user` can write.
     """
@@ -30,7 +28,5 @@ class UserSettings(SQLModel, table=True):
     ssh_host: str | None = None
     ssh_username: str | None = None
     ssh_private_key_path: str | None = None
-    ssh_tasks_dir: str | None = None
-    ssh_jobs_dir: str | None = None
     slurm_user: str | None = None
     project_dir: str | None = None

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/models/v2/__init__.py

@@ -9,7 +9,9 @@ from .history import HistoryImageCache
 from .history import HistoryRun
 from .history import HistoryUnit
 from .job import JobV2
+from .profile import Profile
 from .project import ProjectV2
+from .resource import Resource
 from .task import TaskV2
 from .task_group import TaskGroupActivityV2
 from .task_group import TaskGroupV2
@@ -31,4 +33,6 @@ __all__ = [
     "HistoryRun",
     "HistoryUnit",
     "HistoryImageCache",
+    "Resource",
+    "Profile",
 ]

fractal_server-2.17.0a1/fractal_server/app/models/v2/profile.py

@@ -0,0 +1,16 @@
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+
+class Profile(SQLModel, table=True):
+    id: int | None = Field(default=None, primary_key=True)
+    resource_id: int = Field(foreign_key="resource.id", ondelete="CASCADE")
+    resource_type: str
+
+    name: str = Field(unique=True)
+
+    username: str | None = None
+    ssh_key_path: str | None = None
+
+    jobs_remote_dir: str | None = None
+    tasks_remote_dir: str | None = None

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/models/v2/project.py

@@ -14,6 +14,9 @@ from fractal_server.utils import get_timestamp
 class ProjectV2(SQLModel, table=True):
     id: int | None = Field(default=None, primary_key=True)
     name: str
+    resource_id: int | None = Field(
+        foreign_key="resource.id", default=None, ondelete="SET NULL"
+    )
     timestamp_created: datetime = Field(
         default_factory=get_timestamp,
         sa_column=Column(DateTime(timezone=True), nullable=False),

fractal_server-2.17.0a1/fractal_server/app/models/v2/resource.py

@@ -0,0 +1,130 @@
+from datetime import datetime
+from typing import Any
+from typing import Self
+
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import DateTime
+from sqlmodel import CheckConstraint
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from fractal_server.utils import get_timestamp
+
+
+class Resource(SQLModel, table=True):
+    id: int | None = Field(default=None, primary_key=True)
+
+    type: str
+    """
+    One of `local`, `slurm_sudo` or `slurm_ssh` - matching with
+    `settings.FRACTAL_RUNNER_BACKEND`.
+    """
+
+    name: str = Field(unique=True)
+    """
+    Resource name.
+    """
+
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    """
+    Creation timestamp (autogenerated).
+    """
+
+    host: str | None = None
+    """
+    Address for ssh connections, when `type="slurm_ssh"`.
+    """
+
+    jobs_local_dir: str
+    """
+    Base local folder for job subfolders (containing artifacts and logs).
+    """
+
+    jobs_runner_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Runner configuration, matching one of `JobRunnerConfigLocal` or
+    `JobRunnerConfigSLURM` schemas.
+    """
+
+    jobs_slurm_python_worker: str | None = None
+    """
+    On SLURM deloyments, this is the Python interpreter that runs the
+    `fractal-server` worker from within the SLURM jobs.
+    """
+
+    jobs_poll_interval: int
+    """
+    On SLURM resources: the interval to wait before new `squeue` calls.
+    On local resources: ignored.
+    """
+
+    # task_settings
+    tasks_local_dir: str
+    """
+    Base local folder for task-package subfolders.
+    """
+
+    tasks_python_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Python configuration for task collection. Example:
+    ```json
+    {
+      "default_version": "3.10",
+      "versions:{
+        "3.10": "/xxx/venv-3.10/bin/python",
+        "3.11": "/xxx/venv-3.11/bin/python",
+        "3.12": "/xxx/venv-3.12/bin/python"
+       }
+    }
+    ```
+    """
+
+    tasks_pixi_config: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
+    )
+    """
+    Pixi configuration for task collection. Basic example:
+    ```json
+    {
+        "default_version": "0.41.0",
+        "versions": {
+            "0.40.0": "/xxx/pixi/0.40.0/",
+            "0.41.0": "/xxx/pixi/0.41.0/"
+        },
+    }
+    ```
+    """
+
+    @property
+    def pip_cache_dir_arg(self: Self) -> str:
+        """
+        If `pip_cache_dir` is set (in `self.tasks_python_config`), then
+        return `--cache_dir /something`; else return `--no-cache-dir`.
+        """
+        _pip_cache_dir = self.tasks_python_config.get("pip_cache_dir", None)
+        if _pip_cache_dir is not None:
+            return f"--cache-dir {_pip_cache_dir}"
+        else:
+            return "--no-cache-dir"
+
+    # Check constraints
+    __table_args__ = (
+        # `type` column must be one of "local", "slurm_sudo" or "slurm_ssh"
+        CheckConstraint(
+            "type IN ('local', 'slurm_sudo', 'slurm_ssh')",
+            name="ck_resource_correct_type",
+        ),
+        # If `type` is not "local", `jobs_slurm_python_worker` must be set
+        CheckConstraint(
+            "(type = 'local') OR (jobs_slurm_python_worker IS NOT NULL)",
+            name="ck_resource_jobs_slurm_python_worker_set",
+        ),
+    )

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/models/v2/task_group.py

@@ -42,6 +42,9 @@ class TaskGroupV2(SQLModel, table=True):
     user_group_id: int | None = Field(
         foreign_key="usergroup.id", default=None, ondelete="SET NULL"
     )
+    resource_id: int | None = Field(
+        foreign_key="resource.id", default=None, ondelete="SET NULL"
+    )
 
     origin: str
     pkg_name: str

{fractal_server-2.16.6 → fractal_server-2.17.0a1}/fractal_server/app/routes/admin/v2/__init__.py

@@ -6,7 +6,9 @@ from fastapi import APIRouter
 from .accounting import router as accounting_router
 from .impersonate import router as impersonate_router
 from .job import router as job_router
+from .profile import router as profile_router
 from .project import router as project_router
+from .resource import router as resource_router
 from .task import router as task_router
 from .task_group import router as task_group_router
 from .task_group_lifecycle import router as task_group_lifecycle_router
@@ -22,3 +24,5 @@ router_admin_v2.include_router(
     task_group_lifecycle_router, prefix="/task-group"
 )
 router_admin_v2.include_router(impersonate_router, prefix="/impersonate")
+router_admin_v2.include_router(resource_router, prefix="/resource")
+router_admin_v2.include_router(profile_router, prefix="/profile")

fractal_server-2.17.0a1/fractal_server/app/routes/admin/v2/_aux_functions.py

@@ -0,0 +1,55 @@
+from fastapi import HTTPException
+from fastapi import status
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models.v2 import Profile
+from fractal_server.app.models.v2 import Resource
+
+
+async def _get_resource_or_404(
+    *,
+    resource_id: int,
+    db: AsyncSession,
+) -> Resource:
+    resource = await db.get(Resource, resource_id)
+    if resource is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Resource {resource_id} not found",
+        )
+    return resource
+
+
+async def _get_profile_or_404(
+    *,
+    profile_id: int,
+    db: AsyncSession,
+) -> Profile:
+    profile = await db.get(Profile, profile_id)
+    if profile is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Profile {profile_id} not found",
+        )
+    return profile
+
+
+async def _check_profile_name(*, name: str, db: AsyncSession) -> None:
+    res = await db.execute(select(Profile).where(Profile.name == name))
+    namesake = res.scalars().one_or_none()
+    if namesake is not None:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=f"Profile with name '{name}' already exists.",
+        )
+
+
+async def _check_resource_name(*, name: str, db: AsyncSession) -> None:
+    res = await db.execute(select(Resource).where(Resource.name == name))
+    namesake = res.scalars().one_or_none()
+    if namesake is not None:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=f"Resource with name '{name}' already exists.",
+        )

fractal_server-2.17.0a1/fractal_server/app/routes/admin/v2/profile.py

@@ -0,0 +1,86 @@
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import func
+from sqlmodel import select
+
+from ._aux_functions import _check_profile_name
+from ._aux_functions import _get_profile_or_404
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.schemas.v2 import ProfileCreate
+from fractal_server.app.schemas.v2 import ProfileRead
+
+router = APIRouter()
+
+
+@router.get("/{profile_id}/", response_model=ProfileRead, status_code=200)
+async def get_single_profile(
+    profile_id: int,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> ProfileRead:
+    """
+    Query single `Profile`.
+    """
+    profile = await _get_profile_or_404(profile_id=profile_id, db=db)
+    return profile
+
+
+@router.put("/{profile_id}/", response_model=ProfileRead, status_code=200)
+async def put_profile(
+    profile_id: int,
+    profile_update: ProfileCreate,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> ProfileRead:
+    """
+    Override single `Profile`.
+    """
+    profile = await _get_profile_or_404(profile_id=profile_id, db=db)
+
+    if profile_update.name and profile_update.name != profile.name:
+        await _check_profile_name(name=profile_update.name, db=db)
+
+    for key, value in profile_update.model_dump().items():
+        setattr(profile, key, value)
+    await db.commit()
+    await db.refresh(profile)
+    return profile
+
+
+@router.delete("/{profile_id}/", status_code=204)
+async def delete_profile(
+    profile_id: int,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+):
+    """
+    Delete single `Profile`.
+    """
+    profile = await _get_profile_or_404(profile_id=profile_id, db=db)
+
+    # Fail if at least one UserOAuth is associated with the Profile.
+    res = await db.execute(
+        select(func.count(UserOAuth.id)).where(
+            UserOAuth.profile_id == profile.id
+        )
+    )
+    associated_users_count = res.scalar()
+    if associated_users_count > 0:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=(
+                f"Cannot delete Profile {profile_id} because it's associated"
+                f" with {associated_users_count} UserOAuths."
+            ),
+        )
+
+    # Delete
+    await db.delete(profile)
+    await db.commit()
+    return Response(status_code=status.HTTP_204_NO_CONTENT)