fractal-server 2.8.1__py3-none-any.whl → 2.9.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.8.1"
+__VERSION__ = "2.9.0"

fractal_server/app/db/__init__.py

@@ -2,17 +2,14 @@
 `db` module, loosely adapted from
 https://testdriven.io/blog/fastapi-sqlmodel/#async-sqlmodel
 """
-import sqlite3
 from typing import AsyncGenerator
 from typing import Generator
 
 from sqlalchemy import create_engine
-from sqlalchemy import event
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import StaticPool
 
 from ...config import get_settings
 from ...logger import set_logger
@@ -21,14 +18,6 @@ from ...syringe import Inject
 
 logger = set_logger(__name__)
 
-SQLITE_WARNING_MESSAGE = (
-    "SQLite is supported (supported version >=3.37, "
-    f"current {sqlite3.sqlite_version=}) "
-    "but discouraged in production. "
-    "Given its partial support for ForeignKey constraints, "
-    "database consistency cannot be guaranteed."
-)
-
 
 class DB:
     """
@@ -56,14 +45,7 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        if settings.DB_ENGINE == "sqlite":
-            logger.warning(SQLITE_WARNING_MESSAGE)
-            # Set some sqlite-specific options
-            engine_kwargs_async = dict(poolclass=StaticPool)
-        else:
-            engine_kwargs_async = {
-                "pool_pre_ping": True,
-            }
+        engine_kwargs_async = {"pool_pre_ping": True}
 
         cls._engine_async = create_async_engine(
             settings.DATABASE_ASYNC_URL,
@@ -83,15 +65,7 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        if settings.DB_ENGINE == "sqlite":
-            logger.warning(SQLITE_WARNING_MESSAGE)
-            # Set some sqlite-specific options
-            engine_kwargs_sync = dict(
-                poolclass=StaticPool,
-                connect_args={"check_same_thread": False},
-            )
-        else:
-            engine_kwargs_sync = {}
+        engine_kwargs_sync = {}
 
         cls._engine_sync = create_engine(
             settings.DATABASE_SYNC_URL,
@@ -107,13 +81,6 @@ class DB:
             future=True,
         )
 
-        @event.listens_for(cls._engine_sync, "connect")
-        def set_sqlite_pragma(dbapi_connection, connection_record):
-            if settings.DB_ENGINE == "sqlite":
-                cursor = dbapi_connection.cursor()
-                cursor.execute("PRAGMA journal_mode=WAL")
-                cursor.close()
-
     @classmethod
     async def get_async_db(cls) -> AsyncGenerator[AsyncSession, None]:
         """

fractal_server/app/models/v2/__init__.py

@@ -2,12 +2,12 @@
 v2 `models` module
 """
 from ..linkuserproject import LinkUserProjectV2
-from .collection_state import CollectionStateV2
 from .dataset import DatasetV2
 from .job import JobV2
 from .project import ProjectV2
-from .task import TaskGroupV2
 from .task import TaskV2
+from .task_group import TaskGroupActivityV2
+from .task_group import TaskGroupV2
 from .workflow import WorkflowV2
 from .workflowtask import WorkflowTaskV2
 
@@ -16,8 +16,8 @@ __all__ = [
     "DatasetV2",
     "JobV2",
     "ProjectV2",
-    "CollectionStateV2",
     "TaskGroupV2",
+    "TaskGroupActivityV2",
     "TaskV2",
     "WorkflowTaskV2",
     "WorkflowV2",

fractal_server/app/models/v2/task.py

@@ -1,17 +1,12 @@
-from datetime import datetime
 from typing import Any
 from typing import Optional
 
 from pydantic import HttpUrl
 from sqlalchemy import Column
-from sqlalchemy.types import DateTime
 from sqlalchemy.types import JSON
 from sqlmodel import Field
-from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from fractal_server.utils import get_timestamp
-
 
 class TaskV2(SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
@@ -51,70 +46,3 @@ class TaskV2(SQLModel, table=True):
     tags: list[str] = Field(
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
-
-
-class TaskGroupV2(SQLModel, table=True):
-    id: Optional[int] = Field(default=None, primary_key=True)
-    task_list: list[TaskV2] = Relationship(
-        sa_relationship_kwargs=dict(
-            lazy="selectin", cascade="all, delete-orphan"
-        ),
-    )
-
-    user_id: int = Field(foreign_key="user_oauth.id")
-    user_group_id: Optional[int] = Field(foreign_key="usergroup.id")
-
-    origin: str
-    pkg_name: str
-    version: Optional[str] = None
-    python_version: Optional[str] = None
-    path: Optional[str] = None
-    venv_path: Optional[str] = None
-    wheel_path: Optional[str] = None
-    pip_extras: Optional[str] = None
-    pinned_package_versions: dict[str, str] = Field(
-        sa_column=Column(
-            JSON,
-            server_default="{}",
-            default={},
-            nullable=True,
-        ),
-    )
-
-    active: bool = True
-    timestamp_created: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(DateTime(timezone=True), nullable=False),
-    )
-
-    @property
-    def pip_install_string(self) -> str:
-        """
-        Prepare string to be used in `python -m pip install`.
-        """
-        extras = f"[{self.pip_extras}]" if self.pip_extras is not None else ""
-
-        if self.wheel_path is not None:
-            return f"{self.wheel_path}{extras}"
-        else:
-            if self.version is None:
-                raise ValueError(
-                    "Cannot run `pip_install_string` with "
-                    f"{self.pkg_name=}, {self.wheel_path=}, {self.version=}."
-                )
-            return f"{self.pkg_name}{extras}=={self.version}"
-
-    @property
-    def pinned_package_versions_string(self) -> str:
-        """
-        Prepare string to be used in `python -m pip install`.
-        """
-        if self.pinned_package_versions is None:
-            return ""
-        output = " ".join(
-            [
-                f"{key}=={value}"
-                for key, value in self.pinned_package_versions.items()
-            ]
-        )
-        return output

fractal_server/app/models/v2/task_group.py

@@ -0,0 +1,113 @@
+from datetime import datetime
+from datetime import timezone
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import DateTime
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+from .task import TaskV2
+from fractal_server.utils import get_timestamp
+
+
+class TaskGroupV2(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    task_list: list[TaskV2] = Relationship(
+        sa_relationship_kwargs=dict(
+            lazy="selectin", cascade="all, delete-orphan"
+        ),
+    )
+
+    user_id: int = Field(foreign_key="user_oauth.id")
+    user_group_id: Optional[int] = Field(foreign_key="usergroup.id")
+
+    origin: str
+    pkg_name: str
+    version: Optional[str] = None
+    python_version: Optional[str] = None
+    path: Optional[str] = None
+    wheel_path: Optional[str] = None
+    pip_extras: Optional[str] = None
+    pinned_package_versions: dict[str, str] = Field(
+        sa_column=Column(
+            JSON,
+            server_default="{}",
+            default={},
+            nullable=True,
+        ),
+    )
+    pip_freeze: Optional[str] = None
+    venv_path: Optional[str] = None
+    venv_size_in_kB: Optional[int] = None
+    venv_file_number: Optional[int] = None
+
+    active: bool = True
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    timestamp_last_used: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(
+            DateTime(timezone=True),
+            nullable=False,
+            server_default=(
+                datetime(2024, 11, 20, tzinfo=timezone.utc).isoformat()
+            ),
+        ),
+    )
+
+    @property
+    def pip_install_string(self) -> str:
+        """
+        Prepare string to be used in `python -m pip install`.
+        """
+        extras = f"[{self.pip_extras}]" if self.pip_extras is not None else ""
+
+        if self.wheel_path is not None:
+            return f"{self.wheel_path}{extras}"
+        else:
+            if self.version is None:
+                raise ValueError(
+                    "Cannot run `pip_install_string` with "
+                    f"{self.pkg_name=}, {self.wheel_path=}, {self.version=}."
+                )
+            return f"{self.pkg_name}{extras}=={self.version}"
+
+    @property
+    def pinned_package_versions_string(self) -> str:
+        """
+        Prepare string to be used in `python -m pip install`.
+        """
+        if self.pinned_package_versions is None:
+            return ""
+        output = " ".join(
+            [
+                f"{key}=={value}"
+                for key, value in self.pinned_package_versions.items()
+            ]
+        )
+        return output
+
+
+class TaskGroupActivityV2(SQLModel, table=True):
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    user_id: int = Field(foreign_key="user_oauth.id")
+    taskgroupv2_id: Optional[int] = Field(foreign_key="taskgroupv2.id")
+    timestamp_started: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    pkg_name: str
+    version: str
+    status: str
+    action: str
+    log: Optional[str] = None
+    timestamp_ended: Optional[datetime] = Field(
+        default=None,
+        sa_column=Column(DateTime(timezone=True)),
+    )

fractal_server/app/routes/admin/v1.py

@@ -2,7 +2,6 @@
 Definition of `/admin` routes.
 """
 from datetime import datetime
-from datetime import timezone
 from pathlib import Path
 from typing import Optional
 
@@ -15,8 +14,6 @@ from fastapi.responses import StreamingResponse
 from sqlalchemy import func
 from sqlmodel import select
 
-from ....config import get_settings
-from ....syringe import Inject
 from ....utils import get_timestamp
 from ....zip_tools import _zip_folder_to_byte_stream_iterator
 from ...db import AsyncSession
@@ -36,27 +33,11 @@ from ..aux._job import _write_shutdown_file
 from ..aux._runner import _check_shutdown_is_supported
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.aux import _raise_if_naive_datetime
 
 router_admin_v1 = APIRouter()
 
 
-def _convert_to_db_timestamp(dt: datetime) -> datetime:
-    """
-    This function takes a timezone-aware datetime and converts it to UTC.
-    If using SQLite, it also removes the timezone information in order to make
-    the datetime comparable with datetimes in the database.
-    """
-    if dt.tzinfo is None:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"The timestamp provided has no timezone information: {dt}",
-        )
-    _dt = dt.astimezone(timezone.utc)
-    if Inject(get_settings).DB_ENGINE == "sqlite":
-        return _dt.replace(tzinfo=None)
-    return _dt
-
-
 @router_admin_v1.get("/project/", response_model=list[ProjectReadV1])
 async def view_project(
     id: Optional[int] = None,
@@ -73,6 +54,7 @@ async def view_project(
         id: If not `None`, select a given `project.id`.
         user_id: If not `None`, select a given `project.user_id`.
     """
+    _raise_if_naive_datetime(timestamp_created_min, timestamp_created_max)
 
     stm = select(Project)
 
@@ -82,10 +64,8 @@ async def view_project(
     if user_id is not None:
         stm = stm.where(Project.user_list.any(UserOAuth.id == user_id))
     if timestamp_created_min is not None:
-        timestamp_created_min = _convert_to_db_timestamp(timestamp_created_min)
         stm = stm.where(Project.timestamp_created >= timestamp_created_min)
     if timestamp_created_max is not None:
-        timestamp_created_max = _convert_to_db_timestamp(timestamp_created_max)
         stm = stm.where(Project.timestamp_created <= timestamp_created_max)
 
     res = await db.execute(stm)
@@ -115,6 +95,8 @@ async def view_workflow(
         name_contains: If not `None`, select workflows such that their
             `name` attribute contains `name_contains` (case-insensitive).
     """
+    _raise_if_naive_datetime(timestamp_created_min, timestamp_created_max)
+
     stm = select(Workflow)
 
     if user_id is not None:
@@ -131,10 +113,8 @@ async def view_workflow(
             func.lower(Workflow.name).contains(name_contains.lower())
         )
     if timestamp_created_min is not None:
-        timestamp_created_min = _convert_to_db_timestamp(timestamp_created_min)
         stm = stm.where(Workflow.timestamp_created >= timestamp_created_min)
     if timestamp_created_max is not None:
-        timestamp_created_max = _convert_to_db_timestamp(timestamp_created_max)
         stm = stm.where(Workflow.timestamp_created <= timestamp_created_max)
 
     res = await db.execute(stm)
@@ -166,6 +146,8 @@ async def view_dataset(
             `name` attribute contains `name_contains` (case-insensitive).
         type: If not `None`, select a given `dataset.type`.
     """
+    _raise_if_naive_datetime(timestamp_created_min, timestamp_created_max)
+
     stm = select(Dataset)
 
     if user_id is not None:
@@ -184,10 +166,8 @@ async def view_dataset(
     if type is not None:
         stm = stm.where(Dataset.type == type)
     if timestamp_created_min is not None:
-        timestamp_created_min = _convert_to_db_timestamp(timestamp_created_min)
         stm = stm.where(Dataset.timestamp_created >= timestamp_created_min)
     if timestamp_created_max is not None:
-        timestamp_created_max = _convert_to_db_timestamp(timestamp_created_max)
         stm = stm.where(Dataset.timestamp_created <= timestamp_created_max)
 
     res = await db.execute(stm)
@@ -237,6 +217,13 @@ async def view_job(
         log: If `True`, include `job.log`, if `False`
             `job.log` is set to `None`.
     """
+    _raise_if_naive_datetime(
+        start_timestamp_min,
+        start_timestamp_max,
+        end_timestamp_min,
+        end_timestamp_max,
+    )
+
     stm = select(ApplyWorkflow)
 
     if id is not None:
@@ -256,16 +243,12 @@ async def view_job(
     if status is not None:
         stm = stm.where(ApplyWorkflow.status == status)
     if start_timestamp_min is not None:
-        start_timestamp_min = _convert_to_db_timestamp(start_timestamp_min)
         stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp_min)
     if start_timestamp_max is not None:
-        start_timestamp_max = _convert_to_db_timestamp(start_timestamp_max)
         stm = stm.where(ApplyWorkflow.start_timestamp <= start_timestamp_max)
     if end_timestamp_min is not None:
-        end_timestamp_min = _convert_to_db_timestamp(end_timestamp_min)
         stm = stm.where(ApplyWorkflow.end_timestamp >= end_timestamp_min)
     if end_timestamp_max is not None:
-        end_timestamp_max = _convert_to_db_timestamp(end_timestamp_max)
         stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp_max)
 
     res = await db.execute(stm)

fractal_server/app/routes/admin/v2/__init__.py

@@ -7,6 +7,7 @@ from .job import router as job_router
 from .project import router as project_router
 from .task import router as task_router
 from .task_group import router as task_group_router
+from .task_group_lifecycle import router as task_group_lifecycle_router
 
 router_admin_v2 = APIRouter()
 
@@ -14,3 +15,6 @@ router_admin_v2.include_router(job_router, prefix="/job")
 router_admin_v2.include_router(project_router, prefix="/project")
 router_admin_v2.include_router(task_router, prefix="/task")
 router_admin_v2.include_router(task_group_router, prefix="/task-group")
+router_admin_v2.include_router(
+    task_group_lifecycle_router, prefix="/task-group"
+)

fractal_server/app/routes/admin/v2/job.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from datetime import timezone
 from pathlib import Path
 from typing import Optional
 
@@ -17,37 +16,19 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.aux import _raise_if_naive_datetime
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
 from fractal_server.app.runner.filenames import WORKFLOW_LOG_FILENAME
 from fractal_server.app.schemas.v2 import JobReadV2
 from fractal_server.app.schemas.v2 import JobStatusTypeV2
 from fractal_server.app.schemas.v2 import JobUpdateV2
-from fractal_server.config import get_settings
-from fractal_server.syringe import Inject
 from fractal_server.utils import get_timestamp
 from fractal_server.zip_tools import _zip_folder_to_byte_stream_iterator
 
 router = APIRouter()
 
 
-def _convert_to_db_timestamp(dt: datetime) -> datetime:
-    """
-    This function takes a timezone-aware datetime and converts it to UTC.
-    If using SQLite, it also removes the timezone information in order to make
-    the datetime comparable with datetimes in the database.
-    """
-    if dt.tzinfo is None:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"The timestamp provided has no timezone information: {dt}",
-        )
-    _dt = dt.astimezone(timezone.utc)
-    if Inject(get_settings).DB_ENGINE == "sqlite":
-        return _dt.replace(tzinfo=None)
-    return _dt
-
-
 @router.get("/", response_model=list[JobReadV2])
 async def view_job(
     id: Optional[int] = None,
@@ -85,6 +66,14 @@ async def view_job(
         log: If `True`, include `job.log`, if `False`
             `job.log` is set to `None`.
     """
+
+    _raise_if_naive_datetime(
+        start_timestamp_min,
+        start_timestamp_max,
+        end_timestamp_min,
+        end_timestamp_max,
+    )
+
     stm = select(JobV2)
 
     if id is not None:
@@ -102,16 +91,16 @@ async def view_job(
     if status is not None:
         stm = stm.where(JobV2.status == status)
     if start_timestamp_min is not None:
-        start_timestamp_min = _convert_to_db_timestamp(start_timestamp_min)
+        start_timestamp_min = start_timestamp_min
         stm = stm.where(JobV2.start_timestamp >= start_timestamp_min)
     if start_timestamp_max is not None:
-        start_timestamp_max = _convert_to_db_timestamp(start_timestamp_max)
+        start_timestamp_max = start_timestamp_max
         stm = stm.where(JobV2.start_timestamp <= start_timestamp_max)
     if end_timestamp_min is not None:
-        end_timestamp_min = _convert_to_db_timestamp(end_timestamp_min)
+        end_timestamp_min = end_timestamp_min
         stm = stm.where(JobV2.end_timestamp >= end_timestamp_min)
     if end_timestamp_max is not None:
-        end_timestamp_max = _convert_to_db_timestamp(end_timestamp_max)
+        end_timestamp_max = end_timestamp_max
         stm = stm.where(JobV2.end_timestamp <= end_timestamp_max)
 
     res = await db.execute(stm)

fractal_server/app/routes/admin/v2/task.py

@@ -7,6 +7,7 @@ from fastapi import status
 from pydantic import BaseModel
 from pydantic import EmailStr
 from pydantic import Field
+from sqlmodel import func
 from sqlmodel import select
 
 from fractal_server.app.db import AsyncSession
@@ -60,6 +61,9 @@ async def query_tasks(
     version: Optional[str] = None,
     name: Optional[str] = None,
     max_number_of_results: int = 25,
+    category: Optional[str] = None,
+    modality: Optional[str] = None,
+    author: Optional[str] = None,
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskV2Info]:
@@ -74,6 +78,9 @@ async def query_tasks(
         version: If not `None`, query for matching `task.version`.
         name: If not `None`, query for contained case insensitive `task.name`.
         max_number_of_results: The maximum length of the response.
+        category:
+        modality:
+        author:
     """
 
     stm = select(TaskV2)
@@ -86,6 +93,12 @@ async def query_tasks(
         stm = stm.where(TaskV2.version == version)
     if name is not None:
         stm = stm.where(TaskV2.name.icontains(name))
+    if category is not None:
+        stm = stm.where(func.lower(TaskV2.category) == category.lower())
+    if modality is not None:
+        stm = stm.where(func.lower(TaskV2.modality) == modality.lower())
+    if author is not None:
+        stm = stm.where(TaskV2.authors.icontains(author))
 
     res = await db.execute(stm)
     task_list = res.scalars().all()