fractal-server 2.14.0a2__py3-none-any.whl → 2.14.0a4__py3-none-any.whl

fractal_server/app/runner/executors/slurm_sudo/_executor_wait_thread.py

@@ -1,130 +0,0 @@
-import os
-import threading
-import time
-import traceback
-from itertools import count
-from typing import Optional
-
-from ._check_jobs_status import get_finished_jobs
-from fractal_server.app.runner.exceptions import JobExecutionError
-from fractal_server.logger import set_logger
-
-logger = set_logger(__name__)
-
-
-class FractalSlurmSudoWaitThread(threading.Thread):
-    """
-    Thread that monitors a pool of SLURM jobs
-
-    This class is a custom re-implementation of the waiting thread class from:
-
-    > clusterfutures <https://github.com/sampsyo/clusterfutures>
-    > Original Copyright
-    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
-    > License: MIT
-
-    Attributes:
-        slurm_user:
-        shutdown_file:
-        shutdown_callback:
-        slurm_poll_interval:
-        waiting:
-        shutdown:
-        lock:
-    """
-
-    slurm_user: str
-    shutdown_file: Optional[str] = None
-    shutdown_callback: callable
-    slurm_poll_interval: int = 30
-    waiting: dict[tuple[str, ...], str]
-    shutdown: bool
-    _lock: threading.Lock
-
-    def __init__(self, callback: callable, interval=1):
-        threading.Thread.__init__(self, daemon=True)
-        self.callback = callback
-        self.interval = interval
-        self.waiting = {}
-        self._lock = threading.Lock()  # To protect the .waiting dict
-        self.shutdown = False
-        self.active_job_ids = []
-
-    def wait(
-        self,
-        *,
-        filenames: tuple[str, ...],
-        jobid: str,
-    ):
-        """
-        Add a a new job to the set of jobs being waited for.
-
-        A job consists of a tuple of filenames and a callback value (i.e. a
-        SLURM job ID).
-
-        Note that (with respect to clusterfutures) we replaced `filename` with
-        `filenames`.
-        """
-        if self.shutdown:
-            error_msg = "Cannot call `wait` method after executor shutdown."
-            logger.warning(error_msg)
-            raise JobExecutionError(info=error_msg)
-        with self._lock:
-            self.waiting[filenames] = jobid
-
-    def check_shutdown(self, i):
-        """
-        Do one shutdown-file-existence check.
-
-        Note: the `i` parameter allows subclasses like `SlurmWaitThread` to do
-        something on every Nth check.
-
-        Changed from clusterfutures:
-        * Do not check for output-pickle-file existence (we rather rely on
-          `cfut.slurm.jobs_finished`);
-        * Check for the existence of shutdown-file.
-        """
-        if self.shutdown_file and os.path.exists(self.shutdown_file):
-            logger.info(
-                f"Detected executor-shutdown file {str(self.shutdown_file)}"
-            )
-            self.shutdown = True
-
-    def run(self):
-        """
-        Overrides the original clusterfutures.FileWaitThread.run, adding a call
-        to self.shutdown_callback.
-
-        Changed from clusterfutures:
-        * We do not rely on output-file-existence checks to verify whether a
-          job is complete.
-
-        Note that `shutdown_callback` only takes care of cleaning up the
-        FractalSlurmExecutor variables, and then the `return` here is enough to
-        fully clean up the `FractalFileWaitThread` object.
-        """
-        for i in count():
-            if self.shutdown:
-                self.shutdown_callback()
-                return
-            with self._lock:
-                self.check(i)
-            time.sleep(self.interval)
-
-    def check(self, i):
-        self.check_shutdown(i)
-        if i % (self.slurm_poll_interval // self.interval) == 0:
-            try:
-                finished_jobs = get_finished_jobs(self.waiting.values())
-            except Exception:
-                # Don't abandon completion checking if jobs_finished errors
-                traceback.print_exc()
-                return
-
-            if not finished_jobs:
-                return
-
-            id_to_filenames = {v: k for (k, v) in self.waiting.items()}
-            for finished_id in finished_jobs:
-                self.callback(finished_id)
-                self.waiting.pop(id_to_filenames[finished_id])

fractal_server/app/schemas/v2/history.py

@@ -1,23 +0,0 @@
-from datetime import datetime
-from typing import Any
-
-from pydantic import BaseModel
-from pydantic import field_serializer
-from pydantic.types import AwareDatetime
-
-
-class HistoryItemV2Read(BaseModel):
-    id: int
-    dataset_id: int
-    workflowtask_id: int
-    timestamp_started: AwareDatetime
-    parameters_hash: str
-    num_available_images: int
-    num_current_images: int
-    images: dict[str, str]
-    workflowtask_dump: dict[str, Any]
-    task_group_dump: dict[str, Any]
-
-    @field_serializer("timestamp_started")
-    def serialize_datetime(v: datetime) -> str:
-        return v.isoformat()

fractal_server/migrations/versions/87cd72a537a2_add_historyitem_table.py

@@ -1,68 +0,0 @@
-"""Add HistoryItem table
-
-Revision ID: 87cd72a537a2
-Revises: af1ef1c83c9b
-Create Date: 2025-02-18 10:48:16.401995
-
-"""
-import sqlalchemy as sa
-import sqlmodel
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "87cd72a537a2"
-down_revision = "af1ef1c83c9b"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "historyitemv2",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("dataset_id", sa.Integer(), nullable=False),
-        sa.Column("workflowtask_id", sa.Integer(), nullable=True),
-        sa.Column(
-            "timestamp_started", sa.DateTime(timezone=True), nullable=False
-        ),
-        sa.Column(
-            "workflowtask_dump",
-            postgresql.JSONB(astext_type=sa.Text()),
-            nullable=False,
-        ),
-        sa.Column(
-            "task_group_dump",
-            postgresql.JSONB(astext_type=sa.Text()),
-            nullable=False,
-        ),
-        sa.Column(
-            "parameters_hash",
-            sqlmodel.sql.sqltypes.AutoString(),
-            nullable=False,
-        ),
-        sa.Column("num_available_images", sa.Integer(), nullable=False),
-        sa.Column("num_current_images", sa.Integer(), nullable=False),
-        sa.Column(
-            "images", postgresql.JSONB(astext_type=sa.Text()), nullable=False
-        ),
-        sa.ForeignKeyConstraint(
-            ["dataset_id"],
-            ["datasetv2.id"],
-            name=op.f("fk_historyitemv2_dataset_id_datasetv2"),
-        ),
-        sa.ForeignKeyConstraint(
-            ["workflowtask_id"],
-            ["workflowtaskv2.id"],
-            name=op.f("fk_historyitemv2_workflowtask_id_workflowtaskv2"),
-        ),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_historyitemv2")),
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("historyitemv2")
-    # ### end Alembic commands ###

fractal_server/migrations/versions/954ddc64425a_image_status.py

@@ -1,63 +0,0 @@
-"""image status
-
-Revision ID: 954ddc64425a
-Revises: 87cd72a537a2
-Create Date: 2025-02-28 16:37:38.765883
-
-"""
-import sqlalchemy as sa
-import sqlmodel
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "954ddc64425a"
-down_revision = "87cd72a537a2"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "imagestatus",
-        sa.Column(
-            "zarr_url", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column("workflowtask_id", sa.Integer(), nullable=False),
-        sa.Column("dataset_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "parameters_hash",
-            sqlmodel.sql.sqltypes.AutoString(),
-            nullable=False,
-        ),
-        sa.Column(
-            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
-        ),
-        sa.Column(
-            "logfile", sqlmodel.sql.sqltypes.AutoString(), nullable=True
-        ),
-        sa.ForeignKeyConstraint(
-            ["dataset_id"],
-            ["datasetv2.id"],
-            name=op.f("fk_imagestatus_dataset_id_datasetv2"),
-        ),
-        sa.ForeignKeyConstraint(
-            ["workflowtask_id"],
-            ["workflowtaskv2.id"],
-            name=op.f("fk_imagestatus_workflowtask_id_workflowtaskv2"),
-        ),
-        sa.PrimaryKeyConstraint(
-            "zarr_url",
-            "workflowtask_id",
-            "dataset_id",
-            name=op.f("pk_imagestatus"),
-        ),
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table("imagestatus")
-    # ### end Alembic commands ###