fractal-server 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "1.2.2"
+__VERSION__ = "1.2.3"

fractal_server/app/api/v1/job.py

@@ -52,6 +52,7 @@ async def get_job(
     except (KeyError, FileNotFoundError):
         pass
 
+    await db.close()
     return job_read
 
 
@@ -85,6 +86,8 @@ async def download_job_logs(
         for fpath in working_dir_path.glob("*"):
             zipfile.write(filename=str(fpath), arcname=str(fpath.name))
 
+    await db.close()
+
     return StreamingResponse(
         iter([byte_stream.getvalue()]),
         media_type="application/x-zip-compressed",

fractal_server/app/api/v1/project.py

@@ -143,6 +143,7 @@ async def get_list_project(
     )
     res = await db.execute(stm)
     project_list = res.scalars().all()
+    await db.close()
     return project_list
 
 
@@ -177,6 +178,7 @@ async def create_project(
         db.add(db_project)
         await db.commit()
         await db.refresh(db_project)
+        await db.close()
     except IntegrityError as e:
         await db.rollback()
         logger = set_logger("create_project")
@@ -200,7 +202,9 @@ async def apply_workflow(
     background_tasks: BackgroundTasks,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_db),
-    db_sync: DBSyncSession = Depends(get_sync_db),
+    db_sync: DBSyncSession = Depends(
+        get_sync_db
+    ),  # FIXME: why both sync and async?  # noqa
 ) -> Optional[ApplyWorkflowRead]:
     output = await _get_dataset_check_owner(
         project_id=apply_workflow.project_id,
@@ -293,15 +297,18 @@ async def apply_workflow(
 
     background_tasks.add_task(
         submit_workflow,
-        workflow=workflow,
-        input_dataset=input_dataset,
-        output_dataset=output_dataset,
+        workflow_id=workflow.id,
+        input_dataset_id=input_dataset.id,
+        output_dataset_id=output_dataset.id,
         job_id=job.id,
         worker_init=apply_workflow.worker_init,
         slurm_user=user.slurm_user,
         user_cache_dir=user.cache_dir,
     )
 
+    await db.close()
+    db_sync.close()
+
     return job
 
 
@@ -320,6 +327,7 @@ async def get_project(
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user.id, db=db
     )
+    await db.close()
     return project
 
 
@@ -337,6 +345,7 @@ async def delete_project(
     )
     await db.delete(project)
     await db.commit()
+    await db.close()
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
@@ -361,6 +370,8 @@ async def add_dataset(
     db.add(db_dataset)
     await db.commit()
     await db.refresh(db_dataset)
+    await db.close()
+
     return db_dataset
 
 
@@ -379,6 +390,7 @@ async def get_workflow_list(
     stm = select(Workflow).where(Workflow.project_id == project_id)
     res = await db.execute(stm)
     workflow_list = res.scalars().all()
+    await db.close()
     return workflow_list
 
 
@@ -397,6 +409,7 @@ async def get_job_list(
     stm = select(ApplyWorkflow).where(ApplyWorkflow.project_id == project_id)
     res = await db.execute(stm)
     job_list = res.scalars().all()
+    await db.close()
     return job_list
 
 
@@ -415,6 +428,7 @@ async def edit_project(
 
     await db.commit()
     await db.refresh(project)
+    await db.close()
     return project
 
 
@@ -435,6 +449,7 @@ async def get_dataset(
         project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
     )
     dataset = output["dataset"]
+    await db.close()
     return dataset
 
 
@@ -462,6 +477,7 @@ async def patch_dataset(
 
     await db.commit()
     await db.refresh(db_dataset)
+    await db.close()
     return db_dataset
 
 
@@ -488,6 +504,7 @@ async def delete_dataset(
     dataset = res.scalar()
     await db.delete(dataset)
     await db.commit()
+    await db.close()
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
@@ -521,6 +538,7 @@ async def add_resource(
     db.add(db_resource)
     await db.commit()
     await db.refresh(db_resource)
+    await db.close()
     return db_resource
 
 
@@ -543,6 +561,7 @@ async def get_resource(
     stm = select(Resource).where(Resource.dataset_id == dataset_id)
     res = await db.execute(stm)
     resource_list = res.scalars().all()
+    await db.close()
     return resource_list
 
 
@@ -573,6 +592,7 @@ async def delete_resource(
         )
     await db.delete(resource)
     await db.commit()
+    await db.close()
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
@@ -614,6 +634,7 @@ async def edit_resource(
         setattr(orig_resource, key, value)
     await db.commit()
     await db.refresh(orig_resource)
+    await db.close()
     return orig_resource
 
 
@@ -715,4 +736,5 @@ async def import_workflow_into_project(
                 db=db,
             )
 
+    await db.close()
     return db_workflow

fractal_server/app/api/v1/task.py

@@ -35,9 +35,7 @@ from ....tasks.collection import get_collection_path
 from ....tasks.collection import get_log_path
 from ....tasks.collection import inspect_package
 from ...db import AsyncSession
-from ...db import DBSyncSession
 from ...db import get_db
-from ...db import get_sync_db
 from ...models import State
 from ...models import Task
 from ...security import current_active_superuser
@@ -48,7 +46,9 @@ router = APIRouter()
 
 
 async def _background_collect_pip(
-    state: State, venv_path: Path, task_pkg: _TaskCollectPip, db: AsyncSession
+    state_id: int,
+    venv_path: Path,
+    task_pkg: _TaskCollectPip,
 ) -> None:
     """
     Install package and collect tasks
@@ -59,6 +59,12 @@ async def _background_collect_pip(
     In case of error, copy the log into the state and delete the package
     directory.
     """
+
+    # Note: anext(get_db()) is only available for python>=3.10
+    db = await get_db().__anext__()
+
+    state: State = await db.get(State, state_id)
+
     logger_name = task_pkg.package.replace("/", "_")
     logger = set_logger(
         logger_name=logger_name,
@@ -110,6 +116,7 @@ async def _background_collect_pip(
         logger.debug("Task-collection status: OK")
         logger.info("Background task collection completed successfully")
         close_logger(logger)
+        await db.close()
 
     except Exception as e:
         # Write last logs to file
@@ -124,6 +131,7 @@ async def _background_collect_pip(
         state.data = data.sanitised_dict()
         await db.merge(state)
         await db.commit()
+        await db.close()
 
         # Delete corrupted package dir
         shell_rmtree(venv_path)
@@ -140,6 +148,7 @@ async def _insert_tasks(
     db.add_all(task_db_list)
     await db.commit()
     await asyncio.gather(*[db.refresh(t) for t in task_db_list])
+    await db.close()
     return task_db_list
 
 
@@ -208,6 +217,7 @@ async def collect_tasks_pip(
         try:
             task_collect_status = get_collection_data(venv_path)
         except FileNotFoundError as e:
+            await db.close()
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail=(
@@ -219,6 +229,7 @@ async def collect_tasks_pip(
         task_collect_status.info = "Already installed"
         state = State(data=task_collect_status.sanitised_dict())
         response.status_code == status.HTTP_200_OK
+        await db.close()
         return state
     settings = Inject(get_settings)
 
@@ -237,10 +248,9 @@ async def collect_tasks_pip(
 
     background_tasks.add_task(
         _background_collect_pip,
-        state=state,
+        state_id=state.id,
         venv_path=venv_path,
         task_pkg=task_pkg,
-        db=db,
     )
     logger.debug(
         "Task-collection endpoint: start background collection "
@@ -253,6 +263,7 @@ async def collect_tasks_pip(
     )
     state.data["info"] = info
     response.status_code = status.HTTP_201_CREATED
+    await db.close()
     return state
 
 
@@ -270,6 +281,7 @@ async def check_collection_status(
     logger.debug(f"Querying state for state.id={state_id}")
     state = await db.get(State, state_id)
     if not state:
+        await db.close()
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=f"No task collection info with id={state_id}",
@@ -282,6 +294,7 @@ async def check_collection_status(
         data.log = get_collection_log(data.venv_path)
         state.data = data.sanitised_dict()
     close_logger(logger)
+    await db.close()
     return state
 
 
@@ -297,19 +310,21 @@ async def get_list_task(
     res = await db.execute(stm)
     task_list = res.scalars().unique().fetchall()
     await asyncio.gather(*[db.refresh(t) for t in task_list])
+    await db.close()
     return task_list
 
 
 @router.get("/{task_id}", response_model=TaskRead)
-def get_task(
+async def get_task(
     task_id: int,
     user: User = Depends(current_active_user),
-    db_sync: DBSyncSession = Depends(get_sync_db),
+    db: AsyncSession = Depends(get_db),
 ) -> TaskRead:
     """
     Get info on a specific task
     """
-    task = db_sync.get(Task, task_id)
+    task = await db.get(Task, task_id)
+    await db.close()
     return task
 
 
@@ -346,6 +361,7 @@ async def patch_task(
 
     await db.commit()
     await db.refresh(db_task)
+    await db.close()
     return db_task
 
 
@@ -369,4 +385,5 @@ async def create_task(
     db.add(db_task)
     await db.commit()
     await db.refresh(db_task)
+    await db.close()
     return db_task

fractal_server/app/api/v1/workflow.py

@@ -184,6 +184,7 @@ async def create_workflow(
     db.add(db_workflow)
     await db.commit()
     await db.refresh(db_workflow)
+    await db.close()
     return db_workflow
 
 
@@ -247,6 +248,7 @@ async def patch_workflow(
             setattr(workflow, key, value)
     await db.commit()
     await db.refresh(workflow)
+    await db.close()
 
     return workflow
 
@@ -292,6 +294,7 @@ async def add_task_to_workflow(
             db=db,
         )
 
+    await db.close()
     return workflow_task
 
 
@@ -337,6 +340,7 @@ async def patch_workflow_task(
 
     await db.commit()
     await db.refresh(db_workflow_task)
+    await db.close()
 
     return db_workflow_task
 
@@ -387,4 +391,5 @@ async def export_worfklow(
     workflow = await _get_workflow_check_owner(
         workflow_id=workflow_id, user_id=user.id, db=db
     )
+    await db.close()
     return workflow

fractal_server/app/runner/__init__.py

@@ -68,9 +68,9 @@ def get_process_workflow():
 
 async def submit_workflow(
     *,
-    workflow: Workflow,
-    input_dataset: Dataset,
-    output_dataset: Dataset,
+    workflow_id: int,
+    input_dataset_id: int,
+    output_dataset_id: int,
     job_id: int,
     worker_init: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -83,16 +83,14 @@ async def submit_workflow(
     backend (e.g. local or slurm backend).
 
     Args:
-        workflow:
-            Workflow being applied
-        input_dataset:
-            Input dataset
-        output_dataset:
-            the destination dataset of the workflow. If not provided,
+        workflow_id:
+            ID of the workflow being applied
+        input_dataset_id
+            Input dataset ID
+        output_dataset_id:
+            ID of the destination dataset of the workflow. If not provided,
             overwriting of the input dataset is implied and an error is raised
-            if the dataset is in read only mode. If a string is passed and the
-            dataset does not exist, a new dataset with that name is created and
-            within it a new resource with the same name.
+            if the dataset is in read only mode.
         job_id:
             Id of the job record which stores the state for the current
             workflow application.
@@ -107,10 +105,21 @@ async def submit_workflow(
             The username to impersonate for the workflow execution, for the
             slurm backend.
     """
+
     db_sync = next(DB.get_sync_db())
-    job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)  # type: ignore
+
+    job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
     if not job:
         raise ValueError("Cannot fetch job from database")
+    input_dataset: Dataset = db_sync.get(Dataset, input_dataset_id)
+    if not input_dataset:
+        raise ValueError("Cannot fetch input_dataset from database")
+    output_dataset: Dataset = db_sync.get(Dataset, output_dataset_id)
+    if not output_dataset:
+        raise ValueError("Cannot fetch output_dataset from database")
+    workflow: Workflow = db_sync.get(Workflow, workflow_id)
+    if not workflow:
+        raise ValueError("Cannot fetch workflow from database")
 
     # Select backend
     settings = Inject(get_settings)
@@ -178,7 +187,13 @@ async def submit_workflow(
     logger.debug(f"job.workflow_dir_user: {str(WORKFLOW_DIR_USER)}")
     logger.debug(f'START workflow "{workflow.name}"')
 
+    # Note: from the docs, "The Session.close() method does not prevent the
+    # Session from being used again"
+    # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close)
+    db_sync.close()
+
     try:
+
         output_dataset.meta = await process_workflow(
             workflow=workflow,
             input_paths=input_paths,
@@ -192,6 +207,10 @@ async def submit_workflow(
             worker_init=worker_init,
         )
 
+        logger.info(
+            f'End execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
         logger.debug(f'END workflow "{workflow.name}"')
 
         db_sync.merge(output_dataset)
@@ -240,3 +259,4 @@ async def submit_workflow(
     finally:
         close_job_logger(logger)
         db_sync.commit()
+        db_sync.close()

fractal_server/app/security/__init__.py

@@ -173,6 +173,7 @@ async def list_users(
     stm = select(User)
     res = await db.execute(stm)
     user_list = res.scalars().all()
+    await db.close()
     return user_list
 
 

{fractal_server-1.2.2.dist-info → fractal_server-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 1.2.2
+Version: 1.2.3
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-1.2.2.dist-info → fractal_server-1.2.3.dist-info}/RECORD

@@ -1,14 +1,14 @@
 fractal_server/.gitignore,sha256=9swdobOQ_Celjt2OgRUpy9oEpNqTBn-1NRUFfJU4rps,69
-fractal_server/__init__.py,sha256=OOlw46faCY9wzYwazGLg7oNccHXfFeZp3CIso18O8SU,22
+fractal_server/__init__.py,sha256=t3mOgNMHEo0S33akuGM5qsLlIvf6r557V5R6caoFy5w,22
 fractal_server/__main__.py,sha256=znijcImbcEC4P26ICOhEJ9VY3_5vWdMwQcl-WP25sYA,2202
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/api/__init__.py,sha256=A_RS6hUnnFef2dcWbvG0kLmQdNs26g9W47fAbWc_WXo,887
 fractal_server/app/api/v1/__init__.py,sha256=2HMymr1YkUk39V8iof7KENyLnre4ghouOSvNZ_kF1ec,24
-fractal_server/app/api/v1/job.py,sha256=X6jwUL3tNZCR8qXCOBNUEfKd7osFlvT81UqC8zYZeNY,2740
-fractal_server/app/api/v1/project.py,sha256=-sBt9jgwW2DnUuFMs1wV7UBhqBZu68Wb8DjKjV9MpM8,21983
-fractal_server/app/api/v1/task.py,sha256=raoRHkMqJgjfm4ApMX1R-e5sk3a8hC2Sl3_2XlVMw54,11794
-fractal_server/app/api/v1/workflow.py,sha256=elm9mbfcHlis_VUAAnAzxn9FxRMbeSkl-F3Qan3Sdjc,11398
+fractal_server/app/api/v1/job.py,sha256=UldY2tt904rF95JXaMf7yIkEK0BIVI1k7YchJVr847o,2783
+fractal_server/app/api/v1/project.py,sha256=WxSnUaY6Xhn1sHlNVKih7iUp6uEJeHzrucfka4heAl8,22441
+fractal_server/app/api/v1/task.py,sha256=9S62HkQuj8Co1EzBd-vbjxx-fKipbJSwifov0ftNYdo,12138
+fractal_server/app/api/v1/workflow.py,sha256=DPmDgS3Thf1HS46FZCEni9vQEDkA7wPj0BmBa-T2wU0,11503
 fractal_server/app/db/__init__.py,sha256=Ont7FOvNEEx8-_GFsJ5Lv_5nbxySs_bEpLTdKEzOqDM,2724
 fractal_server/app/models/__init__.py,sha256=MyHn6KDfdPvBpfvcR2QrhWuP1qle3St3e_YzNYcu9y8,372
 fractal_server/app/models/job.py,sha256=qBLchAcwUSXwCUfji5OJhsFVU2pDph8CdJu_mqCVtUE,3242
@@ -18,7 +18,7 @@ fractal_server/app/models/state.py,sha256=uVMwkpalARAni7DNbOQ44v9LikRHJqR7MWlsGy
 fractal_server/app/models/task.py,sha256=3lTZMmHcQK0CgWLjcceyOANdqRt-89n1_z1AMEL2L2Q,1080
 fractal_server/app/models/workflow.py,sha256=Tf4sT4kciDMhGRCTUFWDtEfTvNj3mdWhCx6fURJp9Vc,5371
 fractal_server/app/runner/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkddg0xZk6s,16
-fractal_server/app/runner/__init__.py,sha256=AMOa4XqmxSYISl3z_S0DzgACkSADGeY0gg_nB3mjf4Y,8282
+fractal_server/app/runner/__init__.py,sha256=dvv6hAQwMGDuJ1S8AiuiCayIX1BDwQC95ywY8VGTmjc,8991
 fractal_server/app/runner/_common.py,sha256=PoiBIaCclo1DqjqrIY4qbEAvnRgxQ4MgN3x7xfqXUNw,19294
 fractal_server/app/runner/_local/__init__.py,sha256=JuwhEinF-Yu5C7kkXKqs5cl3OYzCWSSl8uz1HK2eIYY,5460
 fractal_server/app/runner/_local/_local_config.py,sha256=tz93dzmG2NtgQiCO3k81QgDzNbC1x38_tOjNKNyqYEY,3273
@@ -34,7 +34,7 @@ fractal_server/app/runner/_slurm/_subprocess_run_as_user.py,sha256=k7JWKJ5oSM-WE
 fractal_server/app/runner/_slurm/executor.py,sha256=AgmDD3qXVyyI8FWU-udQLS3PAuc1V84oZC7HoYM6q3o,40272
 fractal_server/app/runner/_slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
 fractal_server/app/runner/common.py,sha256=JvE0MICApdnuE3b0UeCr0A-GBk8XB1IpZM1tG_lUqko,9749
-fractal_server/app/security/__init__.py,sha256=zK1SyIdJHq6n48cOOsd3TK4wFx09EwioyDNv5HhWGug,7379
+fractal_server/app/security/__init__.py,sha256=MEOl7CN5nMQE-DhFclvSRtpzRVoY-sCqjkaQsIZNJ1c,7400
 fractal_server/common/.git,sha256=Tc99_jnBzoLdeHxTTuFRz3i7bxoCBKM-WemKlGVThi8,49
 fractal_server/common/.github/workflows/ci.yml,sha256=48COJi9_H9cyRV1kWyp54dtS64PoN9Bt9f8IWEYuZTE,717
 fractal_server/common/.github/workflows/project-management.yml,sha256=Qe6bkHmZVGAIL1LP56mL8Aro0vpMElFY85Ozn6DtysY,364
@@ -75,8 +75,8 @@ fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,278
 fractal_server/tasks/__init__.py,sha256=Wzuxf5EoH1v0fYzRpAZHG_S-Z9f6DmbIsuSvllBCGvc,72
 fractal_server/tasks/collection.py,sha256=hvZeOHenJwvjyFgxIxU7OqOwG8SdmCA9MTBuRAt_Onc,12782
 fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
-fractal_server-1.2.2.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-1.2.2.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-fractal_server-1.2.2.dist-info/METADATA,sha256=nLMyfOnpOBc3PyDqmmwllHc2sCuR1tSO_JQMJQJeaac,3589
-fractal_server-1.2.2.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-1.2.2.dist-info/RECORD,,
+fractal_server-1.2.3.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-1.2.3.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+fractal_server-1.2.3.dist-info/METADATA,sha256=OsUYHA4w9Br2-QyFi1ZSg47FgnGFxi-JXiq4rfMeOvE,3589
+fractal_server-1.2.3.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-1.2.3.dist-info/RECORD,,