fractal-server 1.4.3a0__py3-none-any.whl → 1.4.3a1__py3-none-any.whl

fractal_server/app/routes/api/v1/task_collection.py

@@ -1,7 +1,5 @@
-import json
 from pathlib import Path
 from shutil import copy as shell_copy
-from shutil import rmtree as shell_rmtree
 from tempfile import TemporaryDirectory
 
 from fastapi import APIRouter
@@ -17,27 +15,21 @@ from .....config import get_settings
 from .....logger import close_logger
 from .....logger import set_logger
 from .....syringe import Inject
-from .....tasks.collection import _TaskCollectPip
-from .....tasks.collection import create_package_dir_pip
-from .....tasks.collection import create_package_environment_pip
-from .....tasks.collection import download_package
-from .....tasks.collection import get_collection_data
-from .....tasks.collection import get_collection_log
-from .....tasks.collection import get_collection_path
-from .....tasks.collection import get_log_path
-from .....tasks.collection import inspect_package
-from .....tasks.collection import slugify_task_name
+from .....tasks._TaskCollectPip import _TaskCollectPip
+from .....tasks.background_operations import background_collect_pip
+from .....tasks.endpoint_operations import create_package_dir_pip
+from .....tasks.endpoint_operations import download_package
+from .....tasks.endpoint_operations import get_collection_data
+from .....tasks.endpoint_operations import inspect_package
+from .....tasks.utils import get_collection_log
+from .....tasks.utils import slugify_task_name
 from ....db import AsyncSession
-from ....db import DBSyncSession
-from ....db import get_db
-from ....db import get_sync_db
+from ....db import get_async_db
 from ....models import State
 from ....models import Task
 from ....schemas import StateRead
 from ....schemas import TaskCollectPip
 from ....schemas import TaskCollectStatus
-from ....schemas import TaskCreate
-from ....schemas import TaskRead
 from ....security import current_active_user
 from ....security import current_active_verified_user
 from ....security import User
@@ -47,112 +39,6 @@ router = APIRouter()
 logger = set_logger(__name__)
 
 
-async def _background_collect_pip(
-    state_id: int,
-    venv_path: Path,
-    task_pkg: _TaskCollectPip,
-) -> None:
-    """
-    Install package and collect tasks
-
-    Install a python package and collect the tasks it provides according to
-    the manifest.
-
-    In case of error, copy the log into the state and delete the package
-    directory.
-    """
-    logger_name = task_pkg.package.replace("/", "_")
-    logger = set_logger(
-        logger_name=logger_name,
-        log_file_path=get_log_path(venv_path),
-    )
-    logger.debug("Start background task collection")
-    for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
-        logger.debug(f"{key}: {value}")
-
-    with next(get_sync_db()) as db:
-        state: State = db.get(State, state_id)
-        data = TaskCollectStatus(**state.data)
-        data.info = None
-
-        try:
-            # install
-            logger.debug("Task-collection status: installing")
-            data.status = "installing"
-
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            task_list = await create_package_environment_pip(
-                venv_path=venv_path,
-                task_pkg=task_pkg,
-                logger_name=logger_name,
-            )
-
-            # collect
-            logger.debug("Task-collection status: collecting")
-            data.status = "collecting"
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            tasks = await _insert_tasks(task_list=task_list, db=db)
-
-            # finalise
-            logger.debug("Task-collection status: finalising")
-            collection_path = get_collection_path(venv_path)
-            data.task_list = [TaskRead(**task.model_dump()) for task in tasks]
-            with collection_path.open("w") as f:
-                json.dump(data.sanitised_dict(), f)
-
-            # Update DB
-            data.status = "OK"
-            data.log = get_collection_log(venv_path)
-            state.data = data.sanitised_dict()
-            db.add(state)
-            db.merge(state)
-            db.commit()
-
-            # Write last logs to file
-            logger.debug("Task-collection status: OK")
-            logger.info("Background task collection completed successfully")
-            close_logger(logger)
-            db.close()
-
-        except Exception as e:
-            # Write last logs to file
-            logger.debug("Task-collection status: fail")
-            logger.info(f"Background collection failed. Original error: {e}")
-            close_logger(logger)
-
-            # Update db
-            data.status = "fail"
-            data.info = f"Original error: {e}"
-            data.log = get_collection_log(venv_path)
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            db.close()
-
-            # Delete corrupted package dir
-            shell_rmtree(venv_path)
-
-
-async def _insert_tasks(
-    task_list: list[TaskCreate],
-    db: DBSyncSession,
-) -> list[Task]:
-    """
-    Insert tasks into database
-    """
-    task_db_list = [Task(**t.dict()) for t in task_list]
-    db.add_all(task_db_list)
-    db.commit()
-    for t in task_db_list:
-        db.refresh(t)
-    db.close()
-    return task_db_list
-
-
 @router.post(
     "/collect/pip/",
     response_model=StateRead,
@@ -175,7 +61,7 @@ async def collect_tasks_pip(
     background_tasks: BackgroundTasks,
     response: Response,
     user: User = Depends(current_active_verified_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> StateRead:  # State[TaskCollectStatus]
     """
     Task collection endpoint
@@ -289,7 +175,7 @@ async def collect_tasks_pip(
     await db.refresh(state)
 
     background_tasks.add_task(
-        _background_collect_pip,
+        background_collect_pip,
         state_id=state.id,
         venv_path=venv_path,
         task_pkg=task_pkg,
@@ -314,7 +200,7 @@ async def check_collection_status(
     state_id: int,
     user: User = Depends(current_active_user),
     verbose: bool = False,
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> StateRead:  # State[TaskCollectStatus]
     """
     Check status of background task collection

fractal_server/app/routes/api/v1/workflow.py

@@ -22,7 +22,7 @@ from sqlmodel import select
 from .....logger import close_logger
 from .....logger import set_logger
 from ....db import AsyncSession
-from ....db import get_db
+from ....db import get_async_db
 from ....models import ApplyWorkflow
 from ....models import Project
 from ....models import Task
@@ -36,8 +36,8 @@ from ....schemas import WorkflowUpdate
 from ....security import current_active_user
 from ....security import User
 from ._aux_functions import _check_workflow_exists
-from ._aux_functions import _get_active_jobs_statement
 from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _workflow_insert_task
 
@@ -52,7 +52,7 @@ router = APIRouter()
 async def get_workflow_list(
     project_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[WorkflowRead]]:
     """
     Get workflow list for given project
@@ -79,7 +79,7 @@ async def create_workflow(
     project_id: int,
     workflow: WorkflowCreate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowRead]:
     """
     Create a workflow, associate to a project
@@ -109,7 +109,7 @@ async def read_workflow(
     project_id: int,
     workflow_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowRead]:
     """
     Get info on an existing workflow
@@ -131,7 +131,7 @@ async def update_workflow(
     workflow_id: int,
     patch: WorkflowUpdate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowRead]:
     """
     Edit a workflow
@@ -182,7 +182,7 @@ async def delete_workflow(
     project_id: int,
     workflow_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Delete a workflow
@@ -192,9 +192,9 @@ async def delete_workflow(
         project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
     )
 
-    # Fail if there exist jobs that are active (that is, pending or running)
-    # and in relation with the current workflow.
-    stm = _get_active_jobs_statement().where(
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current workflow.
+    stm = _get_submitted_jobs_statement().where(
         ApplyWorkflow.workflow_id == workflow.id
     )
     res = await db.execute(stm)
@@ -234,7 +234,7 @@ async def export_worfklow(
     project_id: int,
     workflow_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowExport]:
     """
     Export an existing workflow, after stripping all IDs
@@ -267,7 +267,7 @@ async def import_workflow(
     project_id: int,
     workflow: WorkflowImport,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowRead]:
     """
     Import an existing workflow into a project
@@ -338,7 +338,7 @@ async def import_workflow(
 @router.get("/workflow/", response_model=list[WorkflowRead])
 async def get_user_workflows(
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[WorkflowRead]:
     """
     Returns all the workflows of the current user

fractal_server/app/routes/api/v1/workflowtask.py

@@ -21,7 +21,7 @@ from fastapi import Response
 from fastapi import status
 
 from ....db import AsyncSession
-from ....db import get_db
+from ....db import get_async_db
 from ....models import Task
 from ....schemas import WorkflowTaskCreate
 from ....schemas import WorkflowTaskRead
@@ -46,7 +46,7 @@ async def create_workflowtask(
     task_id: int,
     new_task: WorkflowTaskCreate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowTaskRead]:
     """
     Add a WorkflowTask to a Workflow
@@ -85,7 +85,7 @@ async def read_workflowtask(
     workflow_id: int,
     workflow_task_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ):
     workflow_task, _ = await _get_workflow_task_check_owner(
         project_id=project_id,
@@ -107,7 +107,7 @@ async def update_workflowtask(
     workflow_task_id: int,
     workflow_task_update: WorkflowTaskUpdate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowTaskRead]:
     """
     Edit a WorkflowTask of a Workflow
@@ -162,7 +162,7 @@ async def delete_workflowtask(
     workflow_id: int,
     workflow_task_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Delete a WorkflowTask of a Workflow

fractal_server/app/routes/auth.py

@@ -13,7 +13,7 @@ from sqlmodel import select
 
 from ...config import get_settings
 from ...syringe import Inject
-from ..db import get_db
+from ..db import get_async_db
 from ..models.security import UserOAuth as User
 from ..schemas.user import UserCreate
 from ..schemas.user import UserRead
@@ -97,7 +97,7 @@ async def get_current_user(user: User = Depends(current_active_user)):
 @router_auth.get("/users/", response_model=list[UserRead])
 async def list_users(
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ):
     """
     Return list of all users

fractal_server/app/runner/__init__.py

@@ -184,7 +184,6 @@ async def submit_workflow(
         # Update db
         job.working_dir = WORKFLOW_DIR.as_posix()
         job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
-        job.status = JobStatusType.RUNNING
         db_sync.merge(job)
         db_sync.commit()
 

fractal_server/app/schemas/__init__.py

@@ -26,6 +26,7 @@ from .task import TaskUpdate  # noqa: F401
 from .task_collection import TaskCollectPip  # noqa: F401
 from .task_collection import TaskCollectStatus  # noqa: F401
 from .user import UserCreate  # noqa: F401
+from .user import UserRead  # noqa: F401
 from .user import UserUpdate  # noqa: F401
 from .user import UserUpdateStrict  # noqa: F401
 from .workflow import WorkflowCreate  # noqa: F401

fractal_server/app/schemas/applyworkflow.py

@@ -25,21 +25,17 @@ class JobStatusType(str, Enum):
 
     Attributes:
         SUBMITTED:
-            The workflow has been applied but not yet scheduled with an
-            executor. In this phase, due diligence takes place, such as
-            creating working directory, assemblying arguments, etc.
-        RUNNING:
-            The workflow was scheduled with an executor. Note that it might not
-            yet be running within the executor, e.g., jobs could still be
-            pending within a SLURM executor.
+            The job was created. This does not guarantee that it was also
+            submitted to an executor (e.g. other errors could have prevented
+            this), nor that it is actually running (e.g. SLURM jobs could be
+            still in the queue).
         DONE:
-            The workflow was applied successfully
+            The job successfully reached its end.
         FAILED:
             The workflow terminated with an error.
     """
 
     SUBMITTED = "submitted"
-    RUNNING = "running"
     DONE = "done"
     FAILED = "failed"
 

fractal_server/app/schemas/task_collection.py

@@ -15,15 +15,7 @@ __all__ = (
 )
 
 
-class _TaskCollectBase(BaseModel):
-    """
-    Base class for `TaskCollectPip`.
-    """
-
-    pass
-
-
-class TaskCollectPip(_TaskCollectBase):
+class TaskCollectPip(BaseModel):
     """
     TaskCollectPip class
 
@@ -89,7 +81,7 @@ class TaskCollectPip(_TaskCollectBase):
         return v
 
 
-class TaskCollectStatus(_TaskCollectBase):
+class TaskCollectStatus(BaseModel):
     """
     TaskCollectStatus class
 

fractal_server/app/security/__init__.py

@@ -56,7 +56,7 @@ from sqlmodel import select
 
 from ...config import get_settings
 from ...syringe import Inject
-from ..db import get_db
+from ..db import get_async_db
 from ..models.security import OAuthAccount
 from ..models.security import UserOAuth as User
 from fractal_server.app.models.security import UserOAuth
@@ -175,7 +175,7 @@ class SQLModelUserDatabaseAsync(Generic[UP, ID], BaseUserDatabase[UP, ID]):
 
 
 async def get_user_db(
-    session: AsyncSession = Depends(get_db),
+    session: AsyncSession = Depends(get_async_db),
 ) -> AsyncGenerator[SQLModelUserDatabaseAsync, None]:
     yield SQLModelUserDatabaseAsync(session, User, OAuthAccount)
 
@@ -247,7 +247,7 @@ current_active_superuser = fastapi_users.current_user(
     active=True, superuser=True
 )
 
-get_async_session_context = contextlib.asynccontextmanager(get_db)
+get_async_session_context = contextlib.asynccontextmanager(get_async_db)
 get_user_db_context = contextlib.asynccontextmanager(get_user_db)
 get_user_manager_context = contextlib.asynccontextmanager(get_user_manager)
 

fractal_server/config.py

@@ -360,6 +360,14 @@ class Settings(BaseSettings):
                 raise FractalConfigurationError(
                     "POSTGRES_DB cannot be None when DB_ENGINE=postgres."
                 )
+            try:
+                import psycopg2  # noqa: F401
+                import asyncpg  # noqa: F401
+            except ModuleNotFoundError:
+                raise FractalConfigurationError(
+                    "DB engine is `postgres` but `psycopg2` or `asyncpg` "
+                    "are not available"
+                )
         else:
             if not self.SQLITE_PATH:
                 raise FractalConfigurationError(
@@ -375,6 +383,12 @@ class Settings(BaseSettings):
 
         info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
         if self.FRACTAL_RUNNER_BACKEND == "slurm":
+            try:
+                import cfut  # noqa: F401
+            except ModuleNotFoundError:
+                raise FractalConfigurationError(
+                    f"{info} but `clusterfutures` is not available"
+                )
             if not self.FRACTAL_SLURM_CONFIG_FILE:
                 raise FractalConfigurationError(
                     f"Must set FRACTAL_SLURM_CONFIG_FILE when {info}"

fractal_server/tasks/_TaskCollectPip.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+from typing import Optional
+
+from pydantic import root_validator
+
+from fractal_server.app.schemas import ManifestV1
+from fractal_server.app.schemas import TaskCollectPip
+
+
+class _TaskCollectPip(TaskCollectPip):
+    """
+    Internal TaskCollectPip schema
+
+    Differences with its parent class (`TaskCollectPip`):
+
+        1. We check if the package corresponds to a path in the filesystem, and
+           whether it exists (via new validator `check_local_package`, new
+           method `is_local_package` and new attribute `package_path`).
+        2. We include an additional `package_manifest` attribute.
+        3. We expose an additional attribute `package_name`, which is filled
+           during task collection.
+    """
+
+    package_name: Optional[str] = None
+    package_path: Optional[Path] = None
+    package_manifest: Optional[ManifestV1] = None
+
+    @property
+    def is_local_package(self) -> bool:
+        return bool(self.package_path)
+
+    @root_validator(pre=True)
+    def check_local_package(cls, values):
+        """
+        Checks if package corresponds to an existing path on the filesystem
+
+        In this case, the user is providing directly a package file, rather
+        than a remote one from PyPI. We set the `package_path` attribute and
+        get the actual package name and version from the package file name.
+        """
+        if "/" in values["package"]:
+            package_path = Path(values["package"])
+            if not package_path.is_absolute():
+                raise ValueError("Package path must be absolute")
+            if package_path.exists():
+                values["package_path"] = package_path
+                (
+                    values["package"],
+                    values["version"],
+                    *_,
+                ) = package_path.name.split("-")
+            else:
+                raise ValueError(f"Package {package_path} does not exist.")
+        return values
+
+    @property
+    def package_source(self) -> str:
+        """
+        NOTE: As of PR #1188 in `fractal-server`, the attribute
+        `self.package_name` is normalized; this means e.g. that `_` is
+        replaced by `-`. To guarantee backwards compatibility with
+        `Task.source` attributes created before this change, we still replace
+        `-` with `_` upon generation of the `source` attribute, in this
+        method.
+        """
+        if not self.package_name or not self.package_version:
+            raise ValueError(
+                "Cannot construct `package_source` property with "
+                f"{self.package_name=} and {self.package_version=}."
+            )
+        if self.is_local_package:
+            collection_type = "pip_local"
+        else:
+            collection_type = "pip_remote"
+
+        package_extras = self.package_extras or ""
+        if self.python_version:
+            python_version = f"py{self.python_version}"
+        else:
+            python_version = ""  # FIXME: can we allow this?
+
+        source = ":".join(
+            (
+                collection_type,
+                self.package_name.replace("-", "_"),  # see method docstring
+                self.package_version,
+                package_extras,
+                python_version,
+            )
+        )
+        return source
+
+    def check(self):
+        """
+        Verify that the package has all attributes that are needed to continue
+        with task collection
+        """
+        if not self.package_name:
+            raise ValueError("`package_name` attribute is not set")
+        if not self.package_version:
+            raise ValueError("`package_version` attribute is not set")
+        if not self.package_manifest:
+            raise ValueError("`package_manifest` attribute is not set")

fractal_server/tasks/__init__.py

@@ -2,5 +2,7 @@
 `tasks` module
 
 Submodules:
-    * [task collection](collection)
+    * [Endpoint functions](background_operations)
+    * [Background-task functions](endpoint_operations)
+    * [Auxiliary functions](utils)
 """