fractal-server 2.9.0a1__tar.gz → 2.9.0a2__tar.gz

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.9.0a1
+Version: 2.9.0a2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.9.0a2/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.9.0a2"

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/fractal_server/app/models/v2/task_group.py

@@ -48,6 +48,10 @@ class TaskGroupV2(SQLModel, table=True):
         default_factory=get_timestamp,
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
+    timestamp_last_used: Optional[datetime] = Field(
+        default=None,
+        sa_column=Column(DateTime(timezone=True), nullable=True),
+    )
 
     @property
     def pip_install_string(self) -> str:

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/fractal_server/app/routes/api/v2/submit.py

@@ -30,6 +30,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
+from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
@@ -61,6 +62,8 @@ async def apply_workflow(
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[JobReadV2]:
 
+    now = get_timestamp()
+
     # Remove non-submitted V2 jobs from the app state when the list grows
     # beyond a threshold
     settings = Inject(get_settings)
@@ -112,22 +115,23 @@ async def apply_workflow(
         )
 
     # Check that tasks have read-access and are `active`
+    used_task_group_ids = set()
     for wftask in workflow.task_list[
         first_task_index : last_task_index + 1  # noqa: E203
     ]:
-        await _get_task_read_access(
+        task = await _get_task_read_access(
             user_id=user.id,
             task_id=wftask.task_id,
             require_active=True,
             db=db,
         )
+        used_task_group_ids.add(task.taskgroupv2_id)
 
     # Validate user settings
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     user_settings = await validate_user_settings(
         user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
     )
-
     # Check that no other job with the same dataset_id is SUBMITTED
     stm = (
         select(JobV2)
@@ -184,8 +188,18 @@ async def apply_workflow(
     await db.commit()
     await db.refresh(job)
 
+    # Update TaskGroupV2.timestamp_last_used
+    res = await db.execute(
+        select(TaskGroupV2).where(TaskGroupV2.id.in_(used_task_group_ids))
+    )
+    used_task_groups = res.scalars().all()
+    for used_task_group in used_task_groups:
+        used_task_group.timestamp_last_used = now
+        db.add(used_task_group)
+    await db.commit()
+
     # Define server-side job directory
-    timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+    timestamp_string = now.strftime("%Y%m%d_%H%M%S")
     WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
         f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
         f"_{timestamp_string}"

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/fractal_server/app/schemas/v2/task_group.py

@@ -89,6 +89,7 @@ class TaskGroupReadV2(BaseModel):
 
     active: bool
     timestamp_created: datetime
+    timestamp_last_used: Optional[datetime] = None
 
 
 class TaskGroupUpdateV2(BaseModel, extra=Extra.forbid):

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/fractal_server/migrations/versions/3082479ac4ea_taskgroup_activity_and_venv_info_to_.py

@@ -67,6 +67,13 @@ def upgrade() -> None:
         batch_op.add_column(
             sa.Column("venv_file_number", sa.Integer(), nullable=True)
         )
+        batch_op.add_column(
+            sa.Column(
+                "timestamp_last_used",
+                sa.DateTime(timezone=True),
+                nullable=True,
+            )
+        )
 
     # ### end Alembic commands ###
 
@@ -74,6 +81,7 @@ def upgrade() -> None:
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_column("timestamp_last_used")
         batch_op.drop_column("venv_file_number")
         batch_op.drop_column("venv_size_in_kB")
         batch_op.drop_column("pip_freeze")

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/fractal_server/ssh/_fabric.py

@@ -196,23 +196,57 @@ class FractalSSH(object):
         """
         Open the SSH connection and handle exceptions.
 
-        This function can be called from within other functions that use
-        `connection`, so that we can provide a meaningful error in case the
-        SSH connection cannot be opened.
+        This method should always be called at the beginning of background
+        operations that use FractalSSH, so that:
+
+        1. We try to restore unusable connections (e.g. due to closed socket).
+        2. We provide an informative error if connection cannot be established.
         """
-        if not self._connection.is_connected:
+        self.logger.debug(
+            f"[check_connection] {self._connection.is_connected=}"
+        )
+        if self._connection.is_connected:
+            # Even if the connection appears open, it could be broken for
+            # external reasons (e.g. the socket is closed because the SSH
+            # server was restarted). In these cases, we catch the error and
+            # try to re-open the connection.
             try:
-                with _acquire_lock_with_timeout(
-                    lock=self._lock,
-                    label="_connection.open",
-                    timeout=self.default_lock_timeout,
-                ):
-                    self._connection.open()
-            except Exception as e:
-                raise RuntimeError(
-                    f"Cannot open SSH connection. Original error:\n{str(e)}"
+                self.logger.info(
+                    "[check_connection] Run dummy command to check connection."
+                )
+                # Run both an SFTP and an SSH command, as they correspond to
+                # different sockets
+                self.remote_exists("/dummy/path/")
+                self.run_command(cmd="whoami")
+                self.logger.info(
+                    "[check_connection] SSH connection is already OK, exit."
+                )
+                return
+            except (OSError, EOFError) as e:
+                self.logger.warning(
+                    f"[check_connection] Detected error {str(e)}, re-open."
+                )
+        # Try opening the connection (if it was closed) or to re-open it (if
+        # an error happened).
+        try:
+            self.close()
+            with _acquire_lock_with_timeout(
+                lock=self._lock,
+                label="_connection.open",
+                timeout=self.default_lock_timeout,
+                logger_name=self.logger_name,
+            ):
+                self._connection.open()
+                self._connection.client.open_sftp()
+                self.logger.info(
+                    "[check_connection] SSH connection opened, exit."
                 )
 
+        except Exception as e:
+            raise RuntimeError(
+                f"Cannot open SSH connection. Original error:\n{str(e)}"
+            )
+
     def close(self) -> None:
         """
         Aggressively close `self._connection`.
@@ -228,9 +262,8 @@ class FractalSSH(object):
             timeout=self.default_lock_timeout,
         ):
             self._connection.close()
-
-        if self._connection.client is not None:
-            self._connection.client.close()
+            if self._connection.client is not None:
+                self._connection.client.close()
 
     def run_command(
         self,

{fractal_server-2.9.0a1 → fractal_server-2.9.0a2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fractal-server"
-version = "2.9.0a1"
+version = "2.9.0a2"
 description = "Server component of the Fractal analytics platform"
 authors = [
     "Tommaso Comparin <tommaso.comparin@exact-lab.it>",
@@ -86,7 +86,7 @@ filterwarnings = [
 ]
 
 [tool.bumpver]
-current_version = "2.9.0a1"
+current_version = "2.9.0a2"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.9.0a1/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.9.0a1"