fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/routes/api/v1/dataset.py

@@ -1,4 +1,5 @@
 import json
+from json.decoder import JSONDecodeError
 from pathlib import Path
 from typing import Optional
 
@@ -12,20 +13,20 @@ from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import ApplyWorkflow
-from ....models import Dataset
-from ....models import Project
-from ....models import Resource
-from ....runner._common import HISTORY_FILENAME
-from ....schemas import DatasetCreate
-from ....schemas import DatasetRead
-from ....schemas import DatasetStatusRead
-from ....schemas import DatasetUpdate
-from ....schemas import ResourceCreate
-from ....schemas import ResourceRead
-from ....schemas import ResourceUpdate
-from ....schemas import WorkflowExport
-from ....schemas import WorkflowTaskExport
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import Dataset
+from ....models.v1 import Project
+from ....models.v1 import Resource
+from ....runner.filenames import HISTORY_FILENAME
+from ....schemas.v1 import DatasetCreateV1
+from ....schemas.v1 import DatasetReadV1
+from ....schemas.v1 import DatasetStatusReadV1
+from ....schemas.v1 import DatasetUpdateV1
+from ....schemas.v1 import ResourceCreateV1
+from ....schemas.v1 import ResourceReadV1
+from ....schemas.v1 import ResourceUpdateV1
+from ....schemas.v1 import WorkflowExportV1
+from ....schemas.v1 import WorkflowTaskExportV1
 from ....security import current_active_user
 from ....security import User
 from ._aux_functions import _get_dataset_check_owner
@@ -39,15 +40,15 @@ router = APIRouter()
 
 @router.post(
     "/project/{project_id}/dataset/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV1,
     status_code=status.HTTP_201_CREATED,
 )
 async def create_dataset(
     project_id: int,
-    dataset: DatasetCreate,
+    dataset: DatasetCreateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[DatasetRead]:
+) -> Optional[DatasetReadV1]:
     """
     Add new dataset to current project
     """
@@ -65,14 +66,14 @@ async def create_dataset(
 
 @router.get(
     "/project/{project_id}/dataset/",
-    response_model=list[DatasetRead],
+    response_model=list[DatasetReadV1],
 )
 async def read_dataset_list(
     project_id: int,
     history: bool = True,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[list[DatasetRead]]:
+) -> Optional[list[DatasetReadV1]]:
     """
     Get dataset list for given project
     """
@@ -96,14 +97,14 @@ async def read_dataset_list(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV1,
 )
 async def read_dataset(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[DatasetRead]:
+) -> Optional[DatasetReadV1]:
     """
     Get info on a dataset associated to the current project
     """
@@ -120,15 +121,15 @@ async def read_dataset(
 
 @router.patch(
     "/project/{project_id}/dataset/{dataset_id}/",
-    response_model=DatasetRead,
+    response_model=DatasetReadV1,
 )
 async def update_dataset(
     project_id: int,
     dataset_id: int,
-    dataset_update: DatasetUpdate,
+    dataset_update: DatasetUpdateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[DatasetRead]:
+) -> Optional[DatasetReadV1]:
     """
     Edit a dataset associated to the current project
     """
@@ -229,16 +230,16 @@ async def delete_dataset(
 
 @router.post(
     "/project/{project_id}/dataset/{dataset_id}/resource/",
-    response_model=ResourceRead,
+    response_model=ResourceReadV1,
     status_code=status.HTTP_201_CREATED,
 )
 async def create_resource(
     project_id: int,
     dataset_id: int,
-    resource: ResourceCreate,
+    resource: ResourceCreateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ResourceRead]:
+) -> Optional[ResourceReadV1]:
     """
     Add resource to an existing dataset
     """
@@ -259,14 +260,14 @@ async def create_resource(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/resource/",
-    response_model=list[ResourceRead],
+    response_model=list[ResourceReadV1],
 )
 async def get_resource_list(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[list[ResourceRead]]:
+) -> Optional[list[ResourceReadV1]]:
     """
     Get resources from a dataset
     """
@@ -285,16 +286,16 @@ async def get_resource_list(
 
 @router.patch(
     "/project/{project_id}/dataset/{dataset_id}/resource/{resource_id}/",
-    response_model=ResourceRead,
+    response_model=ResourceReadV1,
 )
 async def update_resource(
     project_id: int,
     dataset_id: int,
     resource_id: int,
-    resource_update: ResourceUpdate,
+    resource_update: ResourceUpdateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ResourceRead]:
+) -> Optional[ResourceReadV1]:
     """
     Edit a resource of a dataset
     """
@@ -360,14 +361,14 @@ async def delete_resource(
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/export_history/",
-    response_model=WorkflowExport,
+    response_model=WorkflowExportV1,
 )
 async def export_history_as_workflow(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[WorkflowExport]:
+) -> Optional[WorkflowExportV1]:
     """
     Extract a reproducible workflow from the dataset history.
     """
@@ -412,7 +413,7 @@ async def export_history_as_workflow(
         wftask = history_item["workflowtask"]
         wftask_status = history_item["status"]
         if wftask_status == "done":
-            task_list.append(WorkflowTaskExport(**wftask))
+            task_list.append(WorkflowTaskExportV1(**wftask))
 
     def _slugify_dataset_name(_name: str) -> str:
         _new_name = _name
@@ -422,20 +423,20 @@ async def export_history_as_workflow(
 
     name = f"history_{_slugify_dataset_name(dataset.name)}"
 
-    workflow = WorkflowExport(name=name, task_list=task_list)
+    workflow = WorkflowExportV1(name=name, task_list=task_list)
     return workflow
 
 
 @router.get(
     "/project/{project_id}/dataset/{dataset_id}/status/",
-    response_model=DatasetStatusRead,
+    response_model=DatasetStatusReadV1,
 )
 async def get_workflowtask_status(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[DatasetStatusRead]:
+) -> Optional[DatasetStatusReadV1]:
     """
     Extract the status of all `WorkflowTask`s that ran on a given `Dataset`.
     """
@@ -511,21 +512,27 @@ async def get_workflowtask_status(
                 history = json.load(f)
         except FileNotFoundError:
             history = []
+        except JSONDecodeError:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="History file does not include a valid JSON.",
+            )
+
         for history_item in history:
             wftask_id = history_item["workflowtask"]["id"]
             wftask_status = history_item["status"]
             workflow_tasks_status_dict[wftask_id] = wftask_status
 
-    response_body = DatasetStatusRead(status=workflow_tasks_status_dict)
+    response_body = DatasetStatusReadV1(status=workflow_tasks_status_dict)
     return response_body
 
 
-@router.get("/dataset/", response_model=list[DatasetRead])
+@router.get("/dataset/", response_model=list[DatasetReadV1])
 async def get_user_datasets(
     history: bool = True,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> list[DatasetRead]:
+) -> list[DatasetReadV1]:
     """
     Returns all the datasets of the current user
     """

fractal_server/app/routes/api/v1/job.py

@@ -10,11 +10,11 @@ from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import ApplyWorkflow
-from ....models import JobStatusType
-from ....models import Project
-from ....runner._common import WORKFLOW_LOG_FILENAME
-from ....schemas import ApplyWorkflowRead
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import JobStatusTypeV1
+from ....models.v1 import Project
+from ....runner.filenames import WORKFLOW_LOG_FILENAME
+from ....schemas.v1 import ApplyWorkflowReadV1
 from ....security import current_active_user
 from ....security import User
 from ...aux._job import _write_shutdown_file
@@ -27,12 +27,12 @@ from ._aux_functions import _get_workflow_check_owner
 router = APIRouter()
 
 
-@router.get("/job/", response_model=list[ApplyWorkflowRead])
+@router.get("/job/", response_model=list[ApplyWorkflowReadV1])
 async def get_user_jobs(
     user: User = Depends(current_active_user),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[ApplyWorkflowRead]:
+) -> list[ApplyWorkflowReadV1]:
     """
     Returns all the jobs of the current user
     """
@@ -50,14 +50,14 @@ async def get_user_jobs(
 
 @router.get(
     "/project/{project_id}/workflow/{workflow_id}/job/",
-    response_model=list[ApplyWorkflowRead],
+    response_model=list[ApplyWorkflowReadV1],
 )
 async def get_workflow_jobs(
     project_id: int,
     workflow_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[list[ApplyWorkflowRead]]:
+) -> Optional[list[ApplyWorkflowReadV1]]:
     """
     Returns all the jobs related to a specific workflow
     """
@@ -72,7 +72,7 @@ async def get_workflow_jobs(
 
 @router.get(
     "/project/{project_id}/job/{job_id}/",
-    response_model=ApplyWorkflowRead,
+    response_model=ApplyWorkflowReadV1,
 )
 async def read_job(
     project_id: int,
@@ -80,7 +80,7 @@ async def read_job(
     show_tmp_logs: bool = False,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ApplyWorkflowRead]:
+) -> Optional[ApplyWorkflowReadV1]:
     """
     Return info on an existing job
     """
@@ -94,7 +94,7 @@ async def read_job(
     job = output["job"]
     await db.close()
 
-    if show_tmp_logs and (job.status == JobStatusType.SUBMITTED):
+    if show_tmp_logs and (job.status == JobStatusTypeV1.SUBMITTED):
         try:
             with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
                 job.log = f.read()
@@ -140,14 +140,14 @@ async def download_job_logs(
 
 @router.get(
     "/project/{project_id}/job/",
-    response_model=list[ApplyWorkflowRead],
+    response_model=list[ApplyWorkflowReadV1],
 )
 async def get_job_list(
     project_id: int,
     user: User = Depends(current_active_user),
     log: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[list[ApplyWorkflowRead]]:
+) -> Optional[list[ApplyWorkflowReadV1]]:
     """
     Get job list for given project
     """

fractal_server/app/routes/api/v1/project.py

@@ -18,20 +18,22 @@ from .....logger import set_logger
 from .....syringe import Inject
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import ApplyWorkflow
-from ....models import Dataset
 from ....models import LinkUserProject
-from ....models import Project
-from ....models import Workflow
-from ....runner import submit_workflow
-from ....runner import validate_workflow_compatibility
-from ....runner.common import set_start_and_last_task_index
-from ....schemas import ApplyWorkflowCreate
-from ....schemas import ApplyWorkflowRead
-from ....schemas import JobStatusType
-from ....schemas import ProjectCreate
-from ....schemas import ProjectRead
-from ....schemas import ProjectUpdate
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import Dataset
+from ....models.v1 import Project
+from ....models.v1 import Workflow
+from ....runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
+from ....runner.v1 import submit_workflow
+from ....runner.v1 import validate_workflow_compatibility
+from ....schemas.v1 import ApplyWorkflowCreateV1
+from ....schemas.v1 import ApplyWorkflowReadV1
+from ....schemas.v1 import JobStatusTypeV1
+from ....schemas.v1 import ProjectCreateV1
+from ....schemas.v1 import ProjectReadV1
+from ....schemas.v1 import ProjectUpdateV1
 from ....security import current_active_user
 from ....security import current_active_verified_user
 from ....security import User
@@ -48,7 +50,7 @@ def _encode_as_utc(dt: datetime):
     return dt.replace(tzinfo=timezone.utc).isoformat()
 
 
-@router.get("/", response_model=list[ProjectRead])
+@router.get("/", response_model=list[ProjectReadV1])
 async def get_list_project(
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
@@ -67,12 +69,12 @@ async def get_list_project(
     return project_list
 
 
-@router.post("/", response_model=ProjectRead, status_code=201)
+@router.post("/", response_model=ProjectReadV1, status_code=201)
 async def create_project(
-    project: ProjectCreate,
+    project: ProjectCreateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ProjectRead]:
+) -> Optional[ProjectReadV1]:
     """
     Create new poject
     """
@@ -102,12 +104,12 @@ async def create_project(
     return db_project
 
 
-@router.get("/{project_id}/", response_model=ProjectRead)
+@router.get("/{project_id}/", response_model=ProjectReadV1)
 async def read_project(
     project_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ProjectRead]:
+) -> Optional[ProjectReadV1]:
     """
     Return info on an existing project
     """
@@ -118,10 +120,10 @@ async def read_project(
     return project
 
 
-@router.patch("/{project_id}/", response_model=ProjectRead)
+@router.patch("/{project_id}/", response_model=ProjectReadV1)
 async def update_project(
     project_id: int,
-    project_update: ProjectUpdate,
+    project_update: ProjectUpdateV1,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ):
@@ -241,18 +243,18 @@ async def delete_project(
 @router.post(
     "/{project_id}/workflow/{workflow_id}/apply/",
     status_code=status.HTTP_202_ACCEPTED,
-    response_model=ApplyWorkflowRead,
+    response_model=ApplyWorkflowReadV1,
 )
 async def apply_workflow(
     project_id: int,
     workflow_id: int,
-    apply_workflow: ApplyWorkflowCreate,
+    apply_workflow: ApplyWorkflowCreateV1,
     background_tasks: BackgroundTasks,
     input_dataset_id: int,
     output_dataset_id: int,
     user: User = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ApplyWorkflowRead]:
+) -> Optional[ApplyWorkflowReadV1]:
 
     output = await _get_dataset_check_owner(
         project_id=project_id,
@@ -362,7 +364,7 @@ async def apply_workflow(
     stm = (
         select(ApplyWorkflow)
         .where(ApplyWorkflow.output_dataset_id == output_dataset_id)
-        .where(ApplyWorkflow.status == JobStatusType.SUBMITTED)
+        .where(ApplyWorkflow.status == JobStatusTypeV1.SUBMITTED)
     )
     res = await db.execute(stm)
     if res.scalars().all():
@@ -449,8 +451,9 @@ async def apply_workflow(
         raise HTTPException(
             status_code=status.HTTP_429_TOO_MANY_REQUESTS,
             detail=(
-                f"The endpoint 'POST /{project_id}/workflow/{workflow_id}/"
-                "apply/' was called several times with an interval of less "
+                f"The endpoint 'POST /api/v1/project/{project_id}/workflow/"
+                f"{workflow_id}/apply/' "
+                "was called several times within an interval of less "
                 f"than {settings.FRACTAL_API_SUBMIT_RATE_LIMIT} seconds, using"
                 " the same foreign keys. If it was intentional, please wait "
                 "and try again."

fractal_server/app/routes/api/v1/task.py

@@ -11,11 +11,12 @@ from sqlmodel import select
 from .....logger import set_logger
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import Task
-from ....models import WorkflowTask
-from ....schemas import TaskCreate
-from ....schemas import TaskRead
-from ....schemas import TaskUpdate
+from ....models.v1 import Task
+from ....models.v1 import WorkflowTask
+from ....models.v2 import TaskV2
+from ....schemas.v1 import TaskCreateV1
+from ....schemas.v1 import TaskReadV1
+from ....schemas.v1 import TaskUpdateV1
 from ....security import current_active_user
 from ....security import current_active_verified_user
 from ....security import User
@@ -26,12 +27,12 @@ router = APIRouter()
 logger = set_logger(__name__)
 
 
-@router.get("/", response_model=list[TaskRead])
+@router.get("/", response_model=list[TaskReadV1])
 async def get_list_task(
     user: User = Depends(current_active_user),
     args_schema: bool = True,
     db: AsyncSession = Depends(get_async_db),
-) -> list[TaskRead]:
+) -> list[TaskReadV1]:
     """
     Get list of available tasks
     """
@@ -46,12 +47,12 @@ async def get_list_task(
     return task_list
 
 
-@router.get("/{task_id}/", response_model=TaskRead)
+@router.get("/{task_id}/", response_model=TaskReadV1)
 async def get_task(
     task_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> TaskRead:
+) -> TaskReadV1:
     """
     Get info on a specific task
     """
@@ -64,13 +65,13 @@ async def get_task(
     return task
 
 
-@router.patch("/{task_id}/", response_model=TaskRead)
+@router.patch("/{task_id}/", response_model=TaskReadV1)
 async def patch_task(
     task_id: int,
-    task_update: TaskUpdate,
+    task_update: TaskUpdateV1,
     user: User = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[TaskRead]:
+) -> Optional[TaskReadV1]:
     """
     Edit a specific task (restricted to superusers and task owner)
     """
@@ -109,12 +110,14 @@ async def patch_task(
     return db_task
 
 
-@router.post("/", response_model=TaskRead, status_code=status.HTTP_201_CREATED)
+@router.post(
+    "/", response_model=TaskReadV1, status_code=status.HTTP_201_CREATED
+)
 async def create_task(
-    task: TaskCreate,
+    task: TaskCreateV1,
     user: User = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[TaskRead]:
+) -> Optional[TaskReadV1]:
     """
     Create a new task
     """
@@ -143,7 +146,14 @@ async def create_task(
     if res.scalars().all():
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f'Task source "{task.source}" already in use',
+            detail=f"Source '{task.source}' already used by some TaskV1",
+        )
+    stm = select(TaskV2).where(TaskV2.source == task.source)
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Source '{task.source}' already used by some TaskV2",
         )
 
     # Add task

fractal_server/app/routes/api/v1/task_collection.py

@@ -15,24 +15,26 @@ from .....config import get_settings
 from .....logger import close_logger
 from .....logger import set_logger
 from .....syringe import Inject
-from .....tasks._TaskCollectPip import _TaskCollectPip
-from .....tasks.background_operations import background_collect_pip
-from .....tasks.endpoint_operations import create_package_dir_pip
-from .....tasks.endpoint_operations import download_package
-from .....tasks.endpoint_operations import get_collection_data
-from .....tasks.endpoint_operations import inspect_package
-from .....tasks.utils import get_collection_log
-from .....tasks.utils import slugify_task_name
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import State
-from ....models import Task
-from ....schemas import StateRead
-from ....schemas import TaskCollectPip
-from ....schemas import TaskCollectStatus
+from ....models.v1 import State
+from ....models.v1 import Task
+from ....schemas.state import StateRead
+from ....schemas.v1 import TaskCollectPipV1
+from ....schemas.v1 import TaskCollectStatusV1
 from ....security import current_active_user
 from ....security import current_active_verified_user
 from ....security import User
+from fractal_server.tasks.endpoint_operations import create_package_dir_pip
+from fractal_server.tasks.endpoint_operations import download_package
+from fractal_server.tasks.endpoint_operations import inspect_package
+from fractal_server.tasks.utils import get_collection_log
+from fractal_server.tasks.utils import slugify_task_name
+from fractal_server.tasks.v1._TaskCollectPip import _TaskCollectPip
+from fractal_server.tasks.v1.background_operations import (
+    background_collect_pip,
+)
+from fractal_server.tasks.v1.get_collection_data import get_collection_data
 
 router = APIRouter()
 
@@ -57,7 +59,7 @@ logger = set_logger(__name__)
     },
 )
 async def collect_tasks_pip(
-    task_collect: TaskCollectPip,
+    task_collect: TaskCollectPipV1,
     background_tasks: BackgroundTasks,
     response: Response,
     user: User = Depends(current_active_verified_user),
@@ -138,6 +140,16 @@ async def collect_tasks_pip(
                     f"Original error: {e}"
                 ),
             )
+        except ValidationError as e:
+            await db.close()
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Cannot collect package. Possible reason: an old version "
+                    "of the same package has already been collected. "
+                    f"Original error: {e}"
+                ),
+            )
         task_collect_status.info = "Already installed"
         state = State(data=task_collect_status.sanitised_dict())
         response.status_code == status.HTTP_200_OK
@@ -162,7 +174,7 @@ async def collect_tasks_pip(
 
     # All checks are OK, proceed with task collection
     full_venv_path = venv_path.relative_to(settings.FRACTAL_TASKS_DIR)
-    collection_status = TaskCollectStatus(
+    collection_status = TaskCollectStatusV1(
         status="pending", venv_path=full_venv_path, package=task_pkg.package
     )
 
@@ -214,7 +226,7 @@ async def check_collection_status(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=f"No task collection info with id={state_id}",
         )
-    data = TaskCollectStatus(**state.data)
+    data = TaskCollectStatusV1(**state.data)
 
     # In some cases (i.e. a successful or ongoing task collection), data.log is
     # not set; if so, we collect the current logs