fractal-server 2.13.1__py3-none-any.whl → 2.14.0__py3-none-any.whl

fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py

@@ -0,0 +1,120 @@
+"""new history items
+
+Revision ID: fbce16ff4e47
+Revises: af1ef1c83c9b
+Create Date: 2025-03-14 15:25:01.083619
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "fbce16ff4e47"
+down_revision = "af1ef1c83c9b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "historyrun",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("dataset_id", sa.Integer(), nullable=False),
+        sa.Column("workflowtask_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "workflowtask_dump",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column(
+            "task_group_dump",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column(
+            "timestamp_started", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column(
+            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("num_available_images", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["datasetv2.id"],
+            name=op.f("fk_historyrun_dataset_id_datasetv2"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflowtask_id"],
+            ["workflowtaskv2.id"],
+            name=op.f("fk_historyrun_workflowtask_id_workflowtaskv2"),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_historyrun")),
+    )
+    op.create_table(
+        "historyunit",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("history_run_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "logfile", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("zarr_urls", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["history_run_id"],
+            ["historyrun.id"],
+            name=op.f("fk_historyunit_history_run_id_historyrun"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_historyunit")),
+    )
+    op.create_table(
+        "historyimagecache",
+        sa.Column(
+            "zarr_url", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("dataset_id", sa.Integer(), nullable=False),
+        sa.Column("workflowtask_id", sa.Integer(), nullable=False),
+        sa.Column("latest_history_unit_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["datasetv2.id"],
+            name=op.f("fk_historyimagecache_dataset_id_datasetv2"),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["latest_history_unit_id"],
+            ["historyunit.id"],
+            name=op.f(
+                "fk_historyimagecache_latest_history_unit_id_historyunit"
+            ),
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflowtask_id"],
+            ["workflowtaskv2.id"],
+            name=op.f("fk_historyimagecache_workflowtask_id_workflowtaskv2"),
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint(
+            "zarr_url",
+            "dataset_id",
+            "workflowtask_id",
+            name=op.f("pk_historyimagecache"),
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("historyimagecache")
+    op.drop_table("historyunit")
+    op.drop_table("historyrun")
+    # ### end Alembic commands ###

fractal_server/ssh/_fabric.py

@@ -23,6 +23,18 @@ class FractalSSHTimeoutError(RuntimeError):
     pass
 
 
+class FractalSSHConnectionError(RuntimeError):
+    pass
+
+
+class FractalSSHCommandError(RuntimeError):
+    pass
+
+
+class FractalSSHUnknownError(RuntimeError):
+    pass
+
+
 logger = set_logger(__name__)
 
 
@@ -170,7 +182,6 @@ class FractalSSH(object):
             label="read_remote_json_file",
             timeout=self.default_lock_timeout,
         ):
-
             try:
                 with self._sftp_unsafe().open(filepath, "r") as f:
                     data = json.load(f)
@@ -263,7 +274,7 @@ class FractalSSH(object):
         cmd: str,
         allow_char: Optional[str] = None,
         max_attempts: Optional[int] = None,
-        base_interval: Optional[int] = None,
+        base_interval: Optional[float] = None,
         lock_timeout: Optional[int] = None,
     ) -> str:
         """
@@ -311,7 +322,7 @@ class FractalSSH(object):
                 t_1 = time.perf_counter()
                 self.logger.info(
                     f"{prefix} END   running '{cmd}' over SSH, "
-                    f"elapsed {t_1-t_0:.3f}"
+                    f"elapsed={t_1 - t_0:.3f}"
                 )
                 self.logger.debug("STDOUT:")
                 self.logger.debug(res.stdout)
@@ -329,12 +340,16 @@ class FractalSSH(object):
                     sleeptime = actual_base_interval**ind_attempt
                     self.logger.warning(
                         f"{prefix} Now sleep {sleeptime:.3f} "
-                        "seconds and continue."
+                        "seconds and retry."
                     )
                     time.sleep(sleeptime)
                 else:
                     self.logger.error(f"{prefix} Reached last attempt")
-                    break
+                    raise FractalSSHConnectionError(
+                        f"Reached last attempt "
+                        f"({max_attempts=}) for running "
+                        f"'{cmd}' over SSH"
+                    )
             except UnexpectedExit as e:
                 # Case 3: Command fails with an actual error
                 error_msg = (
@@ -342,18 +357,15 @@ class FractalSSH(object):
                     f"Original error:\n{str(e)}."
                 )
                 self.logger.error(error_msg)
-                raise RuntimeError(error_msg)
+                raise FractalSSHCommandError(error_msg)
+            except FractalSSHTimeoutError as e:
+                raise e
             except Exception as e:
                 self.logger.error(
                     f"Running command `{cmd}` over SSH failed.\n"
                     f"Original Error:\n{str(e)}."
                 )
-                raise e
-
-        raise RuntimeError(
-            f"Reached last attempt ({max_attempts=}) for running "
-            f"'{cmd}' over SSH"
-        )
+                raise FractalSSHUnknownError(f"{type(e)}: {str(e)}")
 
     def send_file(
         self,
@@ -501,7 +513,8 @@ class FractalSSH(object):
             content: Contents to be written to file.
             lock_timeout: Timeout for lock acquisition (overrides default).
         """
-        self.logger.info(f"START writing to remote file {path}.")
+        t_start = time.perf_counter()
+        self.logger.info(f"[write_remote_file] START ({path}).")
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
@@ -518,7 +531,8 @@ class FractalSSH(object):
                     e=e, message=f"Error in `write_remote_file`, for {path=}."
                 )
 
-        self.logger.info(f"END writing to remote file {path}.")
+        elapsed = time.perf_counter() - t_start
+        self.logger.info(f"[write_remote_file] END, {elapsed=} s ({path}).")
 
     def remote_exists(self, path: str) -> bool:
         """

fractal_server/tasks/v2/local/collect.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Optional
 
-from ..utils_database import create_db_tasks_and_update_task_group
+from ..utils_database import create_db_tasks_and_update_task_group_sync
 from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import TaskGroupActivityV2
@@ -233,7 +233,7 @@ def collect_local(
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 logger.info("create_db_tasks_and_update_task_group - " "start")
-                create_db_tasks_and_update_task_group(
+                create_db_tasks_and_update_task_group_sync(
                     task_list=task_list,
                     task_group_id=task_group.id,
                     db=db,

fractal_server/tasks/v2/ssh/collect.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 from ..utils_background import _prepare_tasks_metadata
 from ..utils_background import fail_and_cleanup
-from ..utils_database import create_db_tasks_and_update_task_group
+from ..utils_database import create_db_tasks_and_update_task_group_sync
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -265,7 +265,7 @@ def collect_ssh(
                 logger.info("_prepare_tasks_metadata - end")
 
                 logger.info("create_db_tasks_and_update_task_group - " "start")
-                create_db_tasks_and_update_task_group(
+                create_db_tasks_and_update_task_group_sync(
                     task_list=task_list,
                     task_group_id=task_group.id,
                     db=db,

fractal_server/tasks/v2/templates/2_pip_install.sh

@@ -25,7 +25,7 @@ echo
 
 # Install package
 write_log "START install ${INSTALL_STRING}"
-"$VENVPYTHON" -m pip install --no-cache-dir "$INSTALL_STRING"
+"$VENVPYTHON" -m pip install ${FRACTAL_PIP_CACHE_DIR_ARG} "$INSTALL_STRING"
 write_log "END   install ${INSTALL_STRING}"
 echo
 

fractal_server/tasks/v2/templates/4_pip_show.sh

@@ -20,7 +20,7 @@ echo
 # FIXME: only run pip-show once!
 
 # Extract information about paths
-# WARNING: this block will fail for paths which inlcude whitespace characters
+# WARNING: this block will fail for paths which include whitespace characters
 write_log "START pip show"
 $VENVPYTHON -m pip show ${PACKAGE_NAME}
 write_log "END   pip show"

fractal_server/tasks/v2/utils_background.py

@@ -3,7 +3,6 @@ from typing import Optional
 from typing import TypeVar
 
 from sqlalchemy.orm import Session as DBSyncSession
-from sqlmodel import select
 
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -44,24 +43,6 @@ def fail_and_cleanup(
     task_group_activity.log = get_current_log(log_file_path)
     task_group_activity = add_commit_refresh(obj=task_group_activity, db=db)
     if task_group_activity.action == TaskGroupActivityActionV2.COLLECT:
-        logger.info(f"Now delete TaskGroupV2 with {task_group.id=}")
-
-        logger.info("Start of TaskGroupActivityV2 cascade operations.")
-        stm = select(TaskGroupActivityV2).where(
-            TaskGroupActivityV2.taskgroupv2_id == task_group.id
-        )
-        res = db.execute(stm)
-        task_group_activity_list = res.scalars().all()
-        for task_group_activity in task_group_activity_list:
-            logger.info(
-                f"Setting TaskGroupActivityV2[{task_group_activity.id}]"
-                ".taskgroupv2_id to None."
-            )
-            task_group_activity.taskgroupv2_id = None
-            db.add(task_group_activity)
-        logger.info("End of TaskGroupActivityV2 cascade operations.")
-        logger.info(f"TaskGroupV2 with {task_group.id=} deleted")
-
         db.delete(task_group)
     db.commit()
     reset_logger_handlers(logger)

fractal_server/tasks/v2/utils_database.py

@@ -1,3 +1,4 @@
+from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session as DBSyncSession
 
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -5,16 +6,7 @@ from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.schemas.v2 import TaskCreateV2
 
 
-def _get_task_type(task: TaskCreateV2) -> str:
-    if task.command_non_parallel is None:
-        return "parallel"
-    elif task.command_parallel is None:
-        return "non_parallel"
-    else:
-        return "compound"
-
-
-def create_db_tasks_and_update_task_group(
+def create_db_tasks_and_update_task_group_sync(
     *,
     task_group_id: int,
     task_list: list[TaskCreateV2],
@@ -31,13 +23,7 @@ def create_db_tasks_and_update_task_group(
     Returns:
         Updated `TaskGroupV2` object.
     """
-    actual_task_list = [
-        TaskV2(
-            **task.model_dump(),
-            type=_get_task_type(task),
-        )
-        for task in task_list
-    ]
+    actual_task_list = [TaskV2(**task.model_dump()) for task in task_list]
     task_group = db.get(TaskGroupV2, task_group_id)
     task_group.task_list = actual_task_list
     db.add(task_group)
@@ -45,3 +31,30 @@ def create_db_tasks_and_update_task_group(
     db.refresh(task_group)
 
     return task_group
+
+
+async def create_db_tasks_and_update_task_group_async(
+    *,
+    task_group_id: int,
+    task_list: list[TaskCreateV2],
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Create a `TaskGroupV2` with N `TaskV2`s, and insert them into the database.
+
+    Arguments:
+        task_group_id: ID of an existing `TaskGroupV2` object.
+        task_list: List of `TaskCreateV2` objects to be inserted into the db.
+        db: Synchronous database session
+
+    Returns:
+        Updated `TaskGroupV2` object.
+    """
+    actual_task_list = [TaskV2(**task.model_dump()) for task in task_list]
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    task_group.task_list = actual_task_list
+    db.add(task_group)
+    await db.commit()
+    await db.refresh(task_group)
+
+    return task_group

fractal_server/tasks/v2/utils_templates.py

@@ -2,12 +2,15 @@ from pathlib import Path
 
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
 TEMPLATES_DIR = Path(__file__).parent / "templates"
 
 SCRIPTS_SUBFOLDER = "scripts"
 
+logger = set_logger(__name__)
+
 
 def customize_template(
     *,
@@ -87,4 +90,7 @@ def get_collection_replacements(
             task_group.pinned_package_versions_string,
         ),
     ]
+    logger.info(
+        f"Cache-dir argument for `pip install`: {settings.PIP_CACHE_DIR_ARG}"
+    )
     return replacements

{fractal_server-2.13.1.dist-info → fractal_server-2.14.0.dist-info}/METADATA

@@ -1,8 +1,7 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.13.1
+Version: 2.14.0
 Summary: Backend component of the Fractal analytics platform
-Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause
 Author: Tommaso Comparin
 Author-email: tommaso.comparin@exact-lab.it
@@ -21,14 +20,15 @@ Requires-Dist: fastapi-users[oauth] (>=14,<15)
 Requires-Dist: gunicorn (>=23.0,<24.0)
 Requires-Dist: packaging (>=24.0.0,<25.0.0)
 Requires-Dist: psycopg[binary] (>=3.1.0,<4.0.0)
-Requires-Dist: pydantic (>=2.10.0,<2.11.0)
+Requires-Dist: pydantic (>=2.11.0,<2.12.0)
 Requires-Dist: pydantic-settings (>=2.7.0)
 Requires-Dist: python-dotenv (>=1.0.0,<1.1.0)
 Requires-Dist: sqlalchemy[asyncio] (>=2.0.23,<2.1)
-Requires-Dist: sqlmodel (==0.0.22)
+Requires-Dist: sqlmodel (==0.0.24)
 Requires-Dist: uvicorn (>=0.29.0,<0.35.0)
 Requires-Dist: uvicorn-worker (==0.3.0)
 Project-URL: Documentation, https://fractal-analytics-platform.github.io/fractal-server
+Project-URL: Homepage, https://github.com/fractal-analytics-platform/fractal-server
 Project-URL: Repository, https://github.com/fractal-analytics-platform/fractal-server
 Project-URL: changelog, https://github.com/fractal-analytics-platform/fractal-server/blob/main/CHANGELOG.md
 Description-Content-Type: text/markdown