fractal-server 2.9.0a1__py3-none-any.whl → 2.9.0a2__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.9.0a1"
+__VERSION__ = "2.9.0a2"

fractal_server/app/models/v2/task_group.py

@@ -48,6 +48,10 @@ class TaskGroupV2(SQLModel, table=True):
         default_factory=get_timestamp,
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
+    timestamp_last_used: Optional[datetime] = Field(
+        default=None,
+        sa_column=Column(DateTime(timezone=True), nullable=True),
+    )
 
     @property
     def pip_install_string(self) -> str:

fractal_server/app/routes/api/v2/submit.py

@@ -30,6 +30,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
+from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
@@ -61,6 +62,8 @@ async def apply_workflow(
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[JobReadV2]:
 
+    now = get_timestamp()
+
     # Remove non-submitted V2 jobs from the app state when the list grows
     # beyond a threshold
     settings = Inject(get_settings)
@@ -112,22 +115,23 @@ async def apply_workflow(
         )
 
     # Check that tasks have read-access and are `active`
+    used_task_group_ids = set()
     for wftask in workflow.task_list[
         first_task_index : last_task_index + 1  # noqa: E203
     ]:
-        await _get_task_read_access(
+        task = await _get_task_read_access(
             user_id=user.id,
             task_id=wftask.task_id,
             require_active=True,
             db=db,
         )
+        used_task_group_ids.add(task.taskgroupv2_id)
 
     # Validate user settings
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     user_settings = await validate_user_settings(
         user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
     )
-
     # Check that no other job with the same dataset_id is SUBMITTED
     stm = (
         select(JobV2)
@@ -184,8 +188,18 @@ async def apply_workflow(
     await db.commit()
     await db.refresh(job)
 
+    # Update TaskGroupV2.timestamp_last_used
+    res = await db.execute(
+        select(TaskGroupV2).where(TaskGroupV2.id.in_(used_task_group_ids))
+    )
+    used_task_groups = res.scalars().all()
+    for used_task_group in used_task_groups:
+        used_task_group.timestamp_last_used = now
+        db.add(used_task_group)
+    await db.commit()
+
     # Define server-side job directory
-    timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+    timestamp_string = now.strftime("%Y%m%d_%H%M%S")
     WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
         f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
         f"_{timestamp_string}"

fractal_server/app/schemas/v2/task_group.py

@@ -89,6 +89,7 @@ class TaskGroupReadV2(BaseModel):
 
     active: bool
     timestamp_created: datetime
+    timestamp_last_used: Optional[datetime] = None
 
 
 class TaskGroupUpdateV2(BaseModel, extra=Extra.forbid):

fractal_server/migrations/versions/3082479ac4ea_taskgroup_activity_and_venv_info_to_.py

@@ -67,6 +67,13 @@ def upgrade() -> None:
         batch_op.add_column(
             sa.Column("venv_file_number", sa.Integer(), nullable=True)
         )
+        batch_op.add_column(
+            sa.Column(
+                "timestamp_last_used",
+                sa.DateTime(timezone=True),
+                nullable=True,
+            )
+        )
 
     # ### end Alembic commands ###
 
@@ -74,6 +81,7 @@ def upgrade() -> None:
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_column("timestamp_last_used")
         batch_op.drop_column("venv_file_number")
         batch_op.drop_column("venv_size_in_kB")
         batch_op.drop_column("pip_freeze")

fractal_server/ssh/_fabric.py

@@ -196,23 +196,57 @@ class FractalSSH(object):
         """
         Open the SSH connection and handle exceptions.
 
-        This function can be called from within other functions that use
-        `connection`, so that we can provide a meaningful error in case the
-        SSH connection cannot be opened.
+        This method should always be called at the beginning of background
+        operations that use FractalSSH, so that:
+
+        1. We try to restore unusable connections (e.g. due to closed socket).
+        2. We provide an informative error if connection cannot be established.
         """
-        if not self._connection.is_connected:
+        self.logger.debug(
+            f"[check_connection] {self._connection.is_connected=}"
+        )
+        if self._connection.is_connected:
+            # Even if the connection appears open, it could be broken for
+            # external reasons (e.g. the socket is closed because the SSH
+            # server was restarted). In these cases, we catch the error and
+            # try to re-open the connection.
             try:
-                with _acquire_lock_with_timeout(
-                    lock=self._lock,
-                    label="_connection.open",
-                    timeout=self.default_lock_timeout,
-                ):
-                    self._connection.open()
-            except Exception as e:
-                raise RuntimeError(
-                    f"Cannot open SSH connection. Original error:\n{str(e)}"
+                self.logger.info(
+                    "[check_connection] Run dummy command to check connection."
+                )
+                # Run both an SFTP and an SSH command, as they correspond to
+                # different sockets
+                self.remote_exists("/dummy/path/")
+                self.run_command(cmd="whoami")
+                self.logger.info(
+                    "[check_connection] SSH connection is already OK, exit."
+                )
+                return
+            except (OSError, EOFError) as e:
+                self.logger.warning(
+                    f"[check_connection] Detected error {str(e)}, re-open."
+                )
+        # Try opening the connection (if it was closed) or to re-open it (if
+        # an error happened).
+        try:
+            self.close()
+            with _acquire_lock_with_timeout(
+                lock=self._lock,
+                label="_connection.open",
+                timeout=self.default_lock_timeout,
+                logger_name=self.logger_name,
+            ):
+                self._connection.open()
+                self._connection.client.open_sftp()
+                self.logger.info(
+                    "[check_connection] SSH connection opened, exit."
                 )
 
+        except Exception as e:
+            raise RuntimeError(
+                f"Cannot open SSH connection. Original error:\n{str(e)}"
+            )
+
     def close(self) -> None:
         """
         Aggressively close `self._connection`.
@@ -228,9 +262,8 @@ class FractalSSH(object):
             timeout=self.default_lock_timeout,
         ):
             self._connection.close()
-
-        if self._connection.client is not None:
-            self._connection.client.close()
+            if self._connection.client is not None:
+                self._connection.client.close()
 
     def run_command(
         self,

{fractal_server-2.9.0a1.dist-info → fractal_server-2.9.0a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.9.0a1
+Version: 2.9.0a2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.9.0a1.dist-info → fractal_server-2.9.0a2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=6M5YgTZyA5QmDsgSes9Ql2kogbZ0I5tDiCgSD49xIok,24
+fractal_server/__init__.py,sha256=d0Y48obcx61ZaWA7ZS1o1kdZRZ6sJMhUfuKDOT5xPKU,24
 fractal_server/__main__.py,sha256=dEkCfzLLQrIlxsGC-HBfoR-RBMWnJDgNrxYTyzmE9c0,6146
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,7 +20,7 @@ fractal_server/app/models/v2/dataset.py,sha256=-7sxHEw4IIAvF_uSan7tA3o8hvoakBkQ0
 fractal_server/app/models/v2/job.py,sha256=ypJmN-qspkKBGhBG7Mt-HypSQqcQ2EmB4Bzzb2-y550,1535
 fractal_server/app/models/v2/project.py,sha256=rAHoh5KfYwIaW7rTX0_O0jvWmxEvfo1BafvmcXuSSRk,786
 fractal_server/app/models/v2/task.py,sha256=jebD28Pz8tGcsWCItxj6uKjcD8BMMnnU8dqYhvhEB6c,1520
-fractal_server/app/models/v2/task_group.py,sha256=X2Qlg-at2RcbvlWbpEMT3kZCk5WOVmJ6cOnKF-RDnQ0,3107
+fractal_server/app/models/v2/task_group.py,sha256=Lxtwxk6pfffM8IehqkFvhDuecSXnXqTWeUlf7Y78oss,3254
 fractal_server/app/models/v2/workflow.py,sha256=YBgFGCziUgU0aJ5EM3Svu9W2c46AewZO9VBlFCHiSps,1069
 fractal_server/app/models/v2/workflowtask.py,sha256=iDuJYk8kp4PNqGmbKRtGI7y-QsbjkNd_gDsbMzL4i-g,1274
 fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,7 +50,7 @@ fractal_server/app/routes/api/v2/images.py,sha256=JR1rR6qEs81nacjriOXAOBQjAbCXF4
 fractal_server/app/routes/api/v2/job.py,sha256=Bga2Kz1OjvDIdxZObWaaXVhNIhC_5JKhKRjEH2_ayEE,5157
 fractal_server/app/routes/api/v2/project.py,sha256=eWYFJ7F2ZYQcpi-_n-rhPF-Q4gJhzYBsVGYFhHZZXAE,6653
 fractal_server/app/routes/api/v2/status.py,sha256=6N9DSZ4iFqbZImorWfEAPoyoFUgEruo4Hweqo0x0xXU,6435
-fractal_server/app/routes/api/v2/submit.py,sha256=tq-NGnUlpIcm_MRN47rJRHkRcIJ5HiL4Wj1wItJy3o8,8185
+fractal_server/app/routes/api/v2/submit.py,sha256=jqYix7X6dUJn8w6RWasu1lOFf7T_CcXQlpVY38njE24,8688
 fractal_server/app/routes/api/v2/task.py,sha256=K0ik33t7vL8BAK5S7fqyJDNdRK4stGqb_73bSa8tvPE,7159
 fractal_server/app/routes/api/v2/task_collection.py,sha256=-DVhultvdI3Jh8Jq8W5np6Lnkh5oisjbKCwxFmwddmo,9820
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=cctW61-C2QYF2KXluS15lLhZJS_kt30Ca6UGLFO32z0,6207
@@ -157,7 +157,7 @@ fractal_server/app/schemas/v2/project.py,sha256=UXEA0UUUe0bFFOVLLmVtvDFLBO5vmD1J
 fractal_server/app/schemas/v2/status.py,sha256=SQaUpQkjFq5c5k5J4rOjNhuQaDOEg8lksPhkKmPU5VU,332
 fractal_server/app/schemas/v2/task.py,sha256=FFAbYwDlqowB8gVMdjFVPVHvAM0T89PYLixUth49xfQ,6870
 fractal_server/app/schemas/v2/task_collection.py,sha256=yHpCRxoj6tKqCiQfUjaTj8SfCn1ChD_P6okfEOzyUDE,6518
-fractal_server/app/schemas/v2/task_group.py,sha256=HbwWTabPXaND8cawJnCvOm8S89uYwQUzJJWXWKLQ63A,2963
+fractal_server/app/schemas/v2/task_group.py,sha256=e4NwFuOmiO0afoZLVsI_XHIpD_o_QWDpzlI7ZoNAYwo,3014
 fractal_server/app/schemas/v2/workflow.py,sha256=HSNQSrBRdoBzh8Igr76FUWCAWvVzykrqmUv1vGv-8og,2026
 fractal_server/app/schemas/v2/workflowtask.py,sha256=vDdMktYbHeYBgB5OuWSv6wRPRXWqvetkeqQ7IC5YtfA,5751
 fractal_server/app/security/__init__.py,sha256=8Xd4GxumZgvxEH1Vli3ULehwdesEPiaAbtffJvAEgNo,12509
@@ -178,7 +178,7 @@ fractal_server/migrations/script.py.mako,sha256=oMXw9LC3zRbinWWPPDgeZ4z9FJrV2zhR
 fractal_server/migrations/versions/034a469ec2eb_task_groups.py,sha256=vrPhC8hfFu1c4HmLHNZyCuqEfecFD8-bWc49bXMNes0,6199
 fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py,sha256=-BSS9AFTPcu3gYC-sYbawSy4MWQQx8TfMb5BW5EBKmQ,1450
 fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py,sha256=Q1Gj1cJ0UrdLBJ5AXfFK9QpxTtmcv-4Z3NEGDnxOme4,961
-fractal_server/migrations/versions/3082479ac4ea_taskgroup_activity_and_venv_info_to_.py,sha256=oAqSwW9ilKm5x6Wez3JpJFU2-ls2_O8-j1R6KLEir08,3411
+fractal_server/migrations/versions/3082479ac4ea_taskgroup_activity_and_venv_info_to_.py,sha256=_wy9Yu2YfhpBffUsboAG9junlQ2jRR7qui9wbLvXp7w,3653
 fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py,sha256=-wHe-fOffmYeAm0JXVl_lxZ7hhDkaEVqxgxpHkb_uL8,954
 fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py,sha256=Mob8McGYAcmgvrseyyYOa54E6Gsgr-4SiGdC-r9O4_A,1157
 fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py,sha256=5ROUgcoZOdjf8kMt6cxuvPhzHmV6xaCxvZEbhUEyZM4,3271
@@ -202,7 +202,7 @@ fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py
 fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=udbz8BsbToqf2J_JjSlGUgVQGcP0tY-TjG2plzPoGPM,20180
+fractal_server/ssh/_fabric.py,sha256=Nwsc5uU2BcOE7t1AFFpytKUsZHfbQxB0uKF5HD-dycA,21647
 fractal_server/string_tools.py,sha256=XtMNsr5R7GmgzmFi68zkKMedHs8vjGoVMMCXqWhIk9k,2568
 fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -237,8 +237,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=C5WLuY3uGG2s53OEL-__H35-fmSlgu
 fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
 fractal_server/utils.py,sha256=utvmBx8K9I8hRWFquxna2pBaOqe0JifDL_NVPmihEJI,3525
 fractal_server/zip_tools.py,sha256=xYpzBshysD2nmxkD5WLYqMzPYUcCRM3kYy-7n9bJL-U,4426
-fractal_server-2.9.0a1.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.9.0a1.dist-info/METADATA,sha256=VglByOPkGilAORy-ZhR8noitj8xGEBUBLxnJN8EPil0,4585
-fractal_server-2.9.0a1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fractal_server-2.9.0a1.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.9.0a1.dist-info/RECORD,,
+fractal_server-2.9.0a2.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.9.0a2.dist-info/METADATA,sha256=Qk7nGzAcl_hZRCJp8pdCV2PQACAWwqOzTCH7nFW__pw,4585
+fractal_server-2.9.0a2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fractal_server-2.9.0a2.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.9.0a2.dist-info/RECORD,,