fractal-server 2.8.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

fractal_server/app/schemas/v2/task_collection.py

@@ -1,7 +1,4 @@
-from datetime import datetime
-from enum import Enum
 from pathlib import Path
-from typing import Any
 from typing import Literal
 from typing import Optional
 
@@ -11,19 +8,10 @@ from pydantic import root_validator
 from pydantic import validator
 
 from .._validators import valstr
-from fractal_server.app.schemas._validators import valutc
 from fractal_server.app.schemas.v2 import ManifestV2
 from fractal_server.string_tools import validate_cmd
 
 
-class CollectionStatusV2(str, Enum):
-    PENDING = "pending"
-    INSTALLING = "installing"
-    COLLECTING = "collecting"
-    FAIL = "fail"
-    OK = "OK"
-
-
 class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
     """
     TaskCollectPipV2 class
@@ -57,16 +45,8 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
     python_version: Optional[Literal["3.9", "3.10", "3.11", "3.12"]] = None
     pinned_package_versions: Optional[dict[str, str]] = None
 
-    _package = validator("package", allow_reuse=True)(valstr("package"))
-    _package_version = validator("package_version", allow_reuse=True)(
-        valstr("package_version")
-    )
-    _package_extras = validator("package_extras", allow_reuse=True)(
-        valstr("package_extras")
-    )
-
     @validator("pinned_package_versions")
-    def pinned_package_validator(cls, value):
+    def pinned_package_versions_validator(cls, value):
         if value is None:
             return value
         old_keys = list(value.keys())
@@ -87,6 +67,7 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
 
     @validator("package")
     def package_validator(cls, value):
+        value = valstr("package")(value)
         if "/" in value or value.endswith(".whl"):
             if not value.endswith(".whl"):
                 raise ValueError(
@@ -97,17 +78,27 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
                 raise ValueError(
                     f"Local-package path must be absolute: (given {value})."
                 )
+        validate_cmd(value, attribute_name="package")
         return value
 
     @validator("package_version")
-    def package_version_validator(cls, v, values):
+    def package_version_validator(
+        cls, v: Optional[str], values
+    ) -> Optional[str]:
         v = valstr("package_version")(v)
         if values["package"].endswith(".whl"):
             raise ValueError(
                 "Cannot provide package version when package is a wheel file."
             )
+        validate_cmd(v, attribute_name="package_version")
         return v
 
+    @validator("package_extras")
+    def package_extras_validator(cls, value: Optional[str]) -> Optional[str]:
+        value = valstr("package_extras")(value)
+        validate_cmd(value, attribute_name="package_extras")
+        return value
+
 
 class TaskCollectCustomV2(BaseModel, extra=Extra.forbid):
     """
@@ -188,12 +179,3 @@ class TaskCollectCustomV2(BaseModel, extra=Extra.forbid):
                 f"Python interpreter path must be absolute: (given {value})."
             )
         return value
-
-
-class CollectionStateReadV2(BaseModel):
-
-    id: Optional[int]
-    data: dict[str, Any]
-    timestamp: datetime
-
-    _timestamp = validator("timestamp", allow_reuse=True)(valutc("timestamp"))

fractal_server/app/schemas/v2/task_group.py

@@ -1,11 +1,11 @@
 from datetime import datetime
 from enum import Enum
-from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
 from .._validators import val_absolute_path
@@ -20,6 +20,19 @@ class TaskGroupV2OriginEnum(str, Enum):
     OTHER = "other"
 
 
+class TaskGroupActivityStatusV2(str, Enum):
+    PENDING = "pending"
+    ONGOING = "ongoing"
+    FAILED = "failed"
+    OK = "OK"
+
+
+class TaskGroupActivityActionV2(str, Enum):
+    COLLECT = "collect"
+    DEACTIVATE = "deactivate"
+    REACTIVATE = "reactivate"
+
+
 class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
     user_id: int
     user_group_id: Optional[int] = None
@@ -32,6 +45,7 @@ class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
     venv_path: Optional[str] = None
     wheel_path: Optional[str] = None
     pip_extras: Optional[str] = None
+    pip_freeze: Optional[str] = None
     pinned_package_versions: dict[str, str] = Field(default_factory=dict)
 
     # Validators
@@ -53,6 +67,32 @@ class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
     )
 
 
+class TaskGroupCreateV2Strict(TaskGroupCreateV2):
+    """
+    A strict version of TaskGroupCreateV2, to be used for task collection.
+    """
+
+    path: str
+    venv_path: str
+    version: str
+    python_version: str
+
+    @root_validator
+    def check_wheel_file(cls, values):
+        origin = values.get("origin")
+        wheel_path = values.get("wheel_path")
+        bad_condition_1 = (
+            origin == TaskGroupV2OriginEnum.WHEELFILE and wheel_path is None
+        )
+        bad_condition_2 = (
+            origin != TaskGroupV2OriginEnum.WHEELFILE
+            and wheel_path is not None
+        )
+        if bad_condition_1 or bad_condition_2:
+            raise ValueError(f"Cannot have {origin=} and {wheel_path=}.")
+        return values
+
+
 class TaskGroupReadV2(BaseModel):
     id: int
     task_list: list[TaskReadV2]
@@ -60,26 +100,37 @@ class TaskGroupReadV2(BaseModel):
     user_id: int
     user_group_id: Optional[int] = None
 
-    origin: Literal["pypi", "wheel-file", "other"]
+    origin: TaskGroupV2OriginEnum
     pkg_name: str
     version: Optional[str] = None
     python_version: Optional[str] = None
     path: Optional[str] = None
     venv_path: Optional[str] = None
     wheel_path: Optional[str] = None
+    pip_freeze: Optional[str] = None
     pip_extras: Optional[str] = None
     pinned_package_versions: dict[str, str] = Field(default_factory=dict)
 
+    venv_size_in_kB: Optional[int] = None
+    venv_file_number: Optional[int] = None
+
     active: bool
     timestamp_created: datetime
+    timestamp_last_used: datetime
 
 
 class TaskGroupUpdateV2(BaseModel, extra=Extra.forbid):
     user_group_id: Optional[int] = None
-    active: Optional[bool] = None
 
-    @validator("active")
-    def active_cannot_be_None(cls, value):
-        if value is None:
-            raise ValueError("`active` cannot be set to None")
-        return value
+
+class TaskGroupActivityV2Read(BaseModel):
+    id: int
+    user_id: int
+    taskgroupv2_id: Optional[int] = None
+    timestamp_started: datetime
+    timestamp_ended: Optional[datetime] = None
+    pkg_name: str
+    version: str
+    status: TaskGroupActivityStatusV2
+    action: TaskGroupActivityActionV2
+    log: Optional[str] = None

fractal_server/app/schemas/v2/workflow.py

@@ -6,7 +6,6 @@ from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
-from .._validators import valutc
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskExportV2
 from .workflowtask import WorkflowTaskImportV2
@@ -31,10 +30,6 @@ class WorkflowReadV2(BaseModel):
     project: ProjectReadV2
     timestamp_created: datetime
 
-    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
-        valutc("timestamp_created")
-    )
-
 
 class WorkflowReadV2WithWarnings(WorkflowReadV2):
     task_list: list[WorkflowTaskReadV2WithWarning]

fractal_server/app/security/__init__.py

@@ -43,6 +43,9 @@ from fastapi_users.exceptions import UserAlreadyExists
 from fastapi_users.models import ID
 from fastapi_users.models import OAP
 from fastapi_users.models import UP
+from fastapi_users.password import PasswordHelper
+from pwdlib import PasswordHash
+from pwdlib.hashers.bcrypt import BcryptHasher
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 from sqlmodel import func
@@ -177,7 +180,21 @@ async def get_user_db(
     yield SQLModelUserDatabaseAsync(session, UserOAuth, OAuthAccount)
 
 
+password_hash = PasswordHash(hashers=(BcryptHasher(),))
+password_helper = PasswordHelper(password_hash=password_hash)
+
+
 class UserManager(IntegerIDMixin, BaseUserManager[UserOAuth, int]):
+    def __init__(self, user_db):
+        """
+        Override `__init__` of `BaseUserManager` to define custom
+        `password_helper`.
+        """
+        super().__init__(
+            user_db=user_db,
+            password_helper=password_helper,
+        )
+
     async def validate_password(self, password: str, user: UserOAuth) -> None:
         # check password length
         min_length = 4

fractal_server/config.py

@@ -16,7 +16,6 @@ import shutil
 import sys
 from os import environ
 from os import getenv
-from os.path import abspath
 from pathlib import Path
 from typing import Literal
 from typing import Optional
@@ -88,7 +87,7 @@ class Settings(BaseSettings):
     """
     Contains all the configuration variables for Fractal Server
 
-    The attributes of this class are set from the environtment.
+    The attributes of this class are set from the environment.
     """
 
     class Config:
@@ -167,10 +166,6 @@ class Settings(BaseSettings):
     ###########################################################################
     # DATABASE
     ###########################################################################
-    DB_ENGINE: Literal["sqlite", "postgres-psycopg"] = "sqlite"
-    """
-    Database engine to use (supported: `sqlite`, `postgres-psycopg`).
-    """
     DB_ECHO: bool = False
     """
     If `True`, make database operations verbose.
@@ -196,44 +191,21 @@ class Settings(BaseSettings):
     Name of the PostgreSQL database to connect to.
     """
 
-    SQLITE_PATH: Optional[str]
-    """
-    File path where the SQLite database is located (or will be located).
-    """
-
     @property
     def DATABASE_ASYNC_URL(self) -> URL:
-        if self.DB_ENGINE == "postgres-psycopg":
-            url = URL.create(
-                drivername="postgresql+psycopg",
-                username=self.POSTGRES_USER,
-                password=self.POSTGRES_PASSWORD,
-                host=self.POSTGRES_HOST,
-                port=self.POSTGRES_PORT,
-                database=self.POSTGRES_DB,
-            )
-        else:
-            if not self.SQLITE_PATH:
-                raise FractalConfigurationError(
-                    "SQLITE_PATH path cannot be None"
-                )
-            sqlite_path = abspath(self.SQLITE_PATH)
-            url = URL.create(
-                drivername="sqlite+aiosqlite",
-                database=sqlite_path,
-            )
+        url = URL.create(
+            drivername="postgresql+psycopg",
+            username=self.POSTGRES_USER,
+            password=self.POSTGRES_PASSWORD,
+            host=self.POSTGRES_HOST,
+            port=self.POSTGRES_PORT,
+            database=self.POSTGRES_DB,
+        )
         return url
 
     @property
     def DATABASE_SYNC_URL(self):
-        if self.DB_ENGINE == "postgres-psycopg":
-            return self.DATABASE_ASYNC_URL.set(drivername="postgresql+psycopg")
-        else:
-            if not self.SQLITE_PATH:
-                raise FractalConfigurationError(
-                    "SQLITE_PATH path cannot be None"
-                )
-            return self.DATABASE_ASYNC_URL.set(drivername="sqlite")
+        return self.DATABASE_ASYNC_URL.set(drivername="postgresql+psycopg")
 
     ###########################################################################
     # FRACTAL SPECIFIC
@@ -411,7 +383,7 @@ class Settings(BaseSettings):
     @root_validator(pre=True)
     def check_tasks_python(cls, values) -> None:
         """
-        Perform multiple checks of the Python-intepreter variables.
+        Perform multiple checks of the Python-interpreter variables.
 
         1. Each `FRACTAL_TASKS_PYTHON_X_Y` variable must be an absolute path,
             if set.
@@ -460,7 +432,7 @@ class Settings(BaseSettings):
                 f"{current_version_dot}"
             )
 
-            # Unset all existing intepreters variable
+            # Unset all existing interpreters variable
             for _version in ["3_9", "3_10", "3_11", "3_12"]:
                 key = f"FRACTAL_TASKS_PYTHON_{_version}"
                 if _version == current_version:
@@ -526,6 +498,36 @@ class Settings(BaseSettings):
     Maximum value at which to update `pip` before performing task collection.
     """
 
+    FRACTAL_VIEWER_AUTHORIZATION_SCHEME: Literal[
+        "viewer-paths", "users-folders", "none"
+    ] = "none"
+    """
+    Defines how the list of allowed viewer paths is built.
+
+    This variable affects the `GET /auth/current-user/allowed-viewer-paths/`
+    response, which is then consumed by
+    [fractal-vizarr-viewer](https://github.com/fractal-analytics-platform/fractal-vizarr-viewer).
+
+    Options:
+
+    - "viewer-paths": The list of allowed viewer paths will include the user's
+      `project_dir` along with any path defined in user groups' `viewer_paths`
+      attributes.
+    - "users-folders": The list will consist of the user's `project_dir` and a
+       user-specific folder. The user folder is constructed by concatenating
+       the base folder `FRACTAL_VIEWER_BASE_FOLDER` with the user's
+       `slurm_user`.
+    - "none": An empty list will be returned, indicating no access to
+       viewer paths. Useful when vizarr viewer is not used.
+    """
+
+    FRACTAL_VIEWER_BASE_FOLDER: Optional[str] = None
+    """
+    Base path to Zarr files that will be served by fractal-vizarr-viewer;
+    This variable is required and used only when
+    FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders".
+    """
+
     ###########################################################################
     # BUSINESS LOGIC
     ###########################################################################
@@ -533,25 +535,8 @@ class Settings(BaseSettings):
         """
         Checks that db environment variables are properly set.
         """
-        if self.DB_ENGINE == "postgres-psycopg":
-            if not self.POSTGRES_DB:
-                raise FractalConfigurationError(
-                    "POSTGRES_DB cannot be None when DB_ENGINE="
-                    "postgres-psycopg."
-                )
-
-            try:
-                import psycopg  # noqa: F401
-            except ModuleNotFoundError:
-                raise FractalConfigurationError(
-                    "DB engine is `postgres-psycopg` but `psycopg` is not "
-                    "available"
-                )
-        else:
-            if not self.SQLITE_PATH:
-                raise FractalConfigurationError(
-                    "SQLITE_PATH cannot be None when DB_ENGINE=sqlite."
-                )
+        if not self.POSTGRES_DB:
+            raise FractalConfigurationError("POSTGRES_DB cannot be None.")
 
     def check_runner(self) -> None:
 
@@ -634,6 +619,23 @@ class Settings(BaseSettings):
         if not self.FRACTAL_TASKS_DIR:
             raise FractalConfigurationError("FRACTAL_TASKS_DIR cannot be None")
 
+        # FRACTAL_VIEWER_BASE_FOLDER is required when
+        # FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders"
+        # and it must be an absolute path
+        if self.FRACTAL_VIEWER_AUTHORIZATION_SCHEME == "users-folders":
+            viewer_base_folder = self.FRACTAL_VIEWER_BASE_FOLDER
+            if viewer_base_folder is None:
+                raise FractalConfigurationError(
+                    "FRACTAL_VIEWER_BASE_FOLDER is required when "
+                    "FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "
+                    "users-folders"
+                )
+            if not Path(viewer_base_folder).is_absolute():
+                raise FractalConfigurationError(
+                    f"Non-absolute value for "
+                    f"FRACTAL_VIEWER_BASE_FOLDER={viewer_base_folder}"
+                )
+
         self.check_db()
         self.check_runner()
 

fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py

@@ -0,0 +1,117 @@
+"""TaskGroup Activity and venv-info to TaskGroup
+
+Revision ID: d256a7379ab8
+Revises: 19eca0dd47a9
+Create Date: 2024-11-20 15:01:52.659832
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "d256a7379ab8"
+down_revision = "19eca0dd47a9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "taskgroupactivityv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("taskgroupv2_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "timestamp_started", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column(
+            "pkg_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "version", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "action", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("log", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "timestamp_ended", sa.DateTime(timezone=True), nullable=True
+        ),
+        sa.ForeignKeyConstraint(
+            ["taskgroupv2_id"],
+            ["taskgroupv2.id"],
+            name=op.f("fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user_oauth.id"],
+            name=op.f("fk_taskgroupactivityv2_user_id_user_oauth"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_taskgroupactivityv2")),
+    )
+    op.drop_table("collectionstatev2")
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "pip_freeze", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column("venv_size_in_kB", sa.Integer(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("venv_file_number", sa.Integer(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column(
+                "timestamp_last_used",
+                sa.DateTime(timezone=True),
+                server_default="2024-11-20T00:00:00+00:00",
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_column("timestamp_last_used")
+        batch_op.drop_column("venv_file_number")
+        batch_op.drop_column("venv_size_in_kB")
+        batch_op.drop_column("pip_freeze")
+
+    op.create_table(
+        "collectionstatev2",
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column(
+            "data",
+            postgresql.JSON(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column(
+            "timestamp",
+            postgresql.TIMESTAMP(timezone=True),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column(
+            "taskgroupv2_id", sa.INTEGER(), autoincrement=False, nullable=True
+        ),
+        sa.ForeignKeyConstraint(
+            ["taskgroupv2_id"],
+            ["taskgroupv2.id"],
+            name="fk_collectionstatev2_taskgroupv2_id_taskgroupv2",
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_collectionstatev2"),
+    )
+    op.drop_table("taskgroupactivityv2")
+    # ### end Alembic commands ###