fractal-server 2.11.1__tar.gz → 2.12.0a1__tar.gz

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.11.1
+Version: 2.12.0a1
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.12.0a1/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.12.0a1"

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/models/__init__.py

@@ -4,9 +4,7 @@ thus we should always export all relevant database models from here or they
 will not be picked up by alembic.
 """
 from .linkusergroup import LinkUserGroup  # noqa: F401
-from .linkuserproject import LinkUserProject  # noqa: F401
 from .linkuserproject import LinkUserProjectV2  # noqa: F401
 from .security import *  # noqa
 from .user_settings import UserSettings  # noqa
-from .v1 import *  # noqa
 from .v2 import *  # noqa

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/models/linkuserproject.py

@@ -2,15 +2,6 @@ from sqlmodel import Field
 from sqlmodel import SQLModel
 
 
-class LinkUserProject(SQLModel, table=True):
-    """
-    Crossing table between User and Project
-    """
-
-    project_id: int = Field(foreign_key="project.id", primary_key=True)
-    user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)
-
-
 class LinkUserProjectV2(SQLModel, table=True):
     """
     Crossing table between User and ProjectV2

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/routes/aux/_job.py

@@ -1,12 +1,10 @@
 from pathlib import Path
-from typing import Union
 
-from ...models.v1 import ApplyWorkflow
 from ...models.v2 import JobV2
 from ...runner.filenames import SHUTDOWN_FILENAME
 
 
-def _write_shutdown_file(*, job: Union[ApplyWorkflow, JobV2]):
+def _write_shutdown_file(*, job: JobV2):
     """
     Write job's shutdown file.
 

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -385,9 +385,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         args_batches = []
         batch_size = tasks_per_job
         for ind_chunk in range(0, tot_tasks, batch_size):
-            args_batches.append(
-                list_args[ind_chunk : ind_chunk + batch_size]  # noqa
-            )
+            args_batches.append(list_args[ind_chunk : ind_chunk + batch_size])
         if len(args_batches) != math.ceil(tot_tasks / tasks_per_job):
             raise RuntimeError("Something wrong here while batching tasks")
 
@@ -536,10 +534,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             _prefixes = []
             _subfolder_names = []
             for component in components:
-                if isinstance(component, dict):
+                # In Fractal, `component` is `dict` by construction (e.g.
+                # `component = {"zarr_url": "/something", "param": 1}``). The
+                # try/except covers the case of e.g. `executor.map([1, 2])`,
+                # which is useful for testing.
+                try:
                     actual_component = component.get(_COMPONENT_KEY_, None)
-                else:
-                    actual_component = component
+                except AttributeError:
+                    actual_component = str(component)
+
                 _task_file_paths = get_task_file_paths(
                     workflow_dir_local=task_files.workflow_dir_local,
                     workflow_dir_remote=task_files.workflow_dir_remote,

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -608,11 +608,7 @@ class FractalSlurmExecutor(SlurmExecutor):
             _prefixes = []
             _subfolder_names = []
             for component in components:
-                if isinstance(component, dict):
-                    # This is needed for V2
-                    actual_component = component.get(_COMPONENT_KEY_, None)
-                else:
-                    actual_component = component
+                actual_component = component.get(_COMPONENT_KEY_, None)
                 _task_file_paths = get_task_file_paths(
                     workflow_dir_local=task_files.workflow_dir_local,
                     workflow_dir_remote=task_files.workflow_dir_remote,

fractal_server-2.12.0a1/fractal_server/app/runner/filenames.py

@@ -0,0 +1,2 @@
+SHUTDOWN_FILENAME = "shutdown"
+WORKFLOW_LOG_FILENAME = "workflow.log"

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/runner/shutdown.py

@@ -3,8 +3,6 @@ import time
 from sqlmodel import select
 
 from fractal_server.app.db import get_async_db
-from fractal_server.app.models.v1 import ApplyWorkflow
-from fractal_server.app.models.v1.job import JobStatusTypeV1
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.models.v2.job import JobStatusTypeV2
 from fractal_server.app.routes.aux._job import _write_shutdown_file
@@ -13,9 +11,7 @@ from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
 
 
-async def cleanup_after_shutdown(
-    *, jobsV1: list[int], jobsV2: list[int], logger_name: str
-):
+async def cleanup_after_shutdown(*, jobsV2: list[int], logger_name: str):
     logger = get_logger(logger_name)
     logger.info("Cleanup function after shutdown")
     stm_v2 = (
@@ -24,22 +20,12 @@ async def cleanup_after_shutdown(
         .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
     )
 
-    stm_v1 = (
-        select(ApplyWorkflow)
-        .where(ApplyWorkflow.id.in_(jobsV1))
-        .where(ApplyWorkflow.status == JobStatusTypeV1.SUBMITTED)
-    )
-
     async for session in get_async_db():
         jobsV2_db = (await session.execute(stm_v2)).scalars().all()
-        jobsV1_db = (await session.execute(stm_v1)).scalars().all()
 
         for job in jobsV2_db:
             _write_shutdown_file(job=job)
 
-        for job in jobsV1_db:
-            _write_shutdown_file(job=job)
-
         settings = Inject(get_settings)
 
         t_start = time.perf_counter()
@@ -49,9 +35,8 @@ async def cleanup_after_shutdown(
             logger.info("Waiting 3 seconds before checking")
             time.sleep(3)
             jobsV2_db = (await session.execute(stm_v2)).scalars().all()
-            jobsV1_db = (await session.execute(stm_v1)).scalars().all()
 
-            if len(jobsV2_db) == 0 and len(jobsV1_db) == 0:
+            if len(jobsV2_db) == 0:
                 logger.info(
                     (
                         "All jobs associated to this app are "
@@ -61,10 +46,7 @@ async def cleanup_after_shutdown(
                 return
             else:
                 logger.info(
-                    (
-                        f"Some jobs are still 'submitted' "
-                        f"{jobsV1_db=}, {jobsV2_db=}"
-                    )
+                    (f"Some jobs are still 'submitted' " f"{jobsV2_db=}")
                 )
         logger.info(
             (
@@ -79,10 +61,4 @@ async def cleanup_after_shutdown(
             session.add(job)
         await session.commit()
 
-        for job in jobsV1_db:
-            job.status = "failed"
-            job.log = (job.log or "") + "\nJob stopped due to app shutdown\n"
-            session.add(job)
-        await session.commit()
-
         logger.info("Exit from shutdown logic")

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/app/schemas/_validators.py

@@ -48,25 +48,6 @@ def valdict_keys(attribute: str):
     return val
 
 
-def valint(attribute: str, min_val: int = 1):
-    """
-    Check that an integer attribute (e.g. if it is meant to be the ID of a
-    database entry) is greater or equal to min_val.
-    """
-
-    def val(integer: Optional[int]) -> Optional[int]:
-        if integer is None:
-            raise ValueError(f"Integer attribute '{attribute}' cannot be None")
-        if integer < min_val:
-            raise ValueError(
-                f"Integer attribute '{attribute}' cannot be less than "
-                f"{min_val} (given {integer})"
-            )
-        return integer
-
-    return val
-
-
 def val_absolute_path(attribute: str, accept_none: bool = False):
     """
     Check that a string attribute is an absolute path

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/config.py

@@ -338,7 +338,7 @@ class Settings(BaseSettings):
 
     FRACTAL_API_MAX_JOB_LIST_LENGTH: int = 50
     """
-    Number of ids that can be stored in the `jobsV1` and `jobsV2` attributes of
+    Number of ids that can be stored in the `jobsV2` attribute of
     `app.state`.
     """
 
@@ -496,13 +496,6 @@ class Settings(BaseSettings):
     `JobExecutionError`.
     """
 
-    FRACTAL_API_SUBMIT_RATE_LIMIT: int = 2
-    """
-    Interval to wait (in seconds) to be allowed to call again
-    `POST api/v1/{project_id}/workflow/{workflow_id}/apply/`
-    with the same path and query parameters.
-    """
-
     FRACTAL_RUNNER_TASKS_INCLUDE_IMAGE: str = (
         "Copy OME-Zarr structure;Convert Metadata Components from 2D to 3D"
     )
@@ -511,13 +504,6 @@ class Settings(BaseSettings):
     attribute in their input-arguments JSON file.
     """
 
-    FRACTAL_API_V1_MODE: Literal[
-        "include", "include_read_only", "exclude"
-    ] = "include"
-    """
-    Whether to include the v1 API.
-    """
-
     FRACTAL_PIP_CACHE_DIR: Optional[str] = None
     """
     Absolute path to the cache directory for `pip`; if unset,

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/main.py

@@ -39,20 +39,11 @@ def collect_routers(app: FastAPI) -> None:
             The application to register the routers to.
     """
     from .app.routes.api import router_api
-    from .app.routes.api.v1 import router_api_v1
     from .app.routes.api.v2 import router_api_v2
-    from .app.routes.admin.v1 import router_admin_v1
     from .app.routes.admin.v2 import router_admin_v2
     from .app.routes.auth.router import router_auth
 
-    settings = Inject(get_settings)
-
     app.include_router(router_api, prefix="/api")
-    if settings.FRACTAL_API_V1_MODE.startswith("include"):
-        app.include_router(router_api_v1, prefix="/api/v1")
-        app.include_router(
-            router_admin_v1, prefix="/admin/v1", tags=["V1 Admin area"]
-        )
     app.include_router(router_api_v2, prefix="/api/v2")
     app.include_router(
         router_admin_v2, prefix="/admin/v2", tags=["V2 Admin area"]
@@ -84,7 +75,6 @@ def check_settings() -> None:
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    app.state.jobsV1 = []
     app.state.jobsV2 = []
     logger = set_logger("fractal_server.lifespan")
     logger.info("Start application startup")
@@ -123,12 +113,11 @@ async def lifespan(app: FastAPI):
 
     logger.info(
         f"Current worker with pid {os.getpid()} is shutting down. "
-        f"Current jobs: {app.state.jobsV1=}, {app.state.jobsV2=}"
+        f"Current jobs: {app.state.jobsV2=}"
     )
     if _backend_supports_shutdown(settings.FRACTAL_RUNNER_BACKEND):
         try:
             await cleanup_after_shutdown(
-                jobsV1=app.state.jobsV1,
                 jobsV2=app.state.jobsV2,
                 logger_name="fractal_server.lifespan",
             )

fractal_server-2.12.0a1/fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py

@@ -0,0 +1,67 @@
+"""Drop V1 tables
+
+Revision ID: 1eac13a26c83
+Revises: af8673379a5c
+Create Date: 2025-01-10 13:17:47.838607
+
+"""
+import logging
+
+from alembic import op
+from sqlmodel import SQLModel
+
+from fractal_server.migrations.naming_convention import NAMING_CONVENTION
+
+# revision identifiers, used by Alembic.
+revision = "1eac13a26c83"
+down_revision = "af8673379a5c"
+branch_labels = None
+depends_on = None
+
+
+TABLES_V1 = [
+    "resource",
+    "applyworkflow",
+    "task",
+    "workflow",
+    "workflowtask",
+    "linkuserproject",
+    "dataset",
+    "project",
+    "state",
+]
+
+
+def upgrade() -> None:
+
+    logger = logging.getLogger("alembic.runtime.migration")
+
+    target_metadata = SQLModel.metadata
+    target_metadata.naming_convention = NAMING_CONVENTION
+
+    connection = op.get_bind()
+    target_metadata.reflect(
+        bind=connection,
+        extend_existing=True,
+        only=TABLES_V1,
+    )
+
+    logger.info("Starting non-reversible upgrade")
+    logger.info("Dropping all V1 ForeignKey constraints")
+    fk_names = []
+    for table_name in TABLES_V1:
+        table = target_metadata.tables[table_name]
+        for fk in table.foreign_keys:
+            op.drop_constraint(fk.name, table_name, type_="foreignkey")
+            fk_names.append(fk.name)
+    logger.info(f"Dropped all V1 ForeignKey constraints: {fk_names}")
+    logger.info(f"Dropping all V1 tables: {TABLES_V1}")
+    for table_name in TABLES_V1:
+        op.drop_table(table_name)
+
+
+def downgrade() -> None:
+    raise RuntimeError(
+        "Cannot downgrade from 1eac13a26c83 to db09233ad13a, "
+        "because it's fully breaking."
+    )

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/fractal_server/string_tools.py

@@ -33,27 +33,6 @@ def sanitize_string(value: str) -> str:
     return new_value
 
 
-def slugify_task_name_for_source_v1(task_name: str) -> str:
-    """
-    NOTE: this function is used upon creation of tasks' sources, therefore
-    for the moment we cannot replace it with its more comprehensive version
-    from `fractal_server.string_tools.sanitize_string`, nor we can remove it.
-
-    As of 2.3.1, we are renaming it to `slugify_task_name_for_source`, to make
-    it clear that it should not be used for other purposes.
-
-    As of 2.7.0, we are renaming it to `slugify_task_name_for_source_v1`, to
-    make it clear that it is not used for v2.
-
-    Args:
-        task_name:
-
-    Return:
-        Slug-ified task name.
-    """
-    return task_name.replace(" ", "_").lower()
-
-
 def validate_cmd(
     command: str,
     *,

fractal_server-2.12.0a1/fractal_server/tasks/utils.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+
+COLLECTION_FILENAME = "collection.json"
+COLLECTION_LOG_FILENAME = "collection.log"
+COLLECTION_FREEZE_FILENAME = "collection_freeze.txt"
+FORBIDDEN_DEPENDENCY_STRINGS = ["github.com"]
+
+
+def get_log_path(base: Path) -> Path:
+    return base / COLLECTION_LOG_FILENAME

{fractal_server-2.11.1 → fractal_server-2.12.0a1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.11.1"
+version = "2.12.0a1"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 ]
 
 [tool.bumpver]
-current_version = "2.11.1"
+current_version = "2.12.0a1"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.11.1/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.11.1"

fractal_server-2.11.1/fractal_server/app/models/v1/__init__.py

@@ -1,13 +0,0 @@
-"""
-`models` module
-"""
-from ..linkuserproject import LinkUserProject  # noqa F401
-from .dataset import Dataset  # noqa F401
-from .dataset import Resource  # noqa F401
-from .job import ApplyWorkflow  # noqa F401
-from .job import JobStatusTypeV1  # noqa F401
-from .project import Project  # noqa F401
-from .state import State  # noqa F401
-from .task import Task  # noqa F401
-from .workflow import Workflow  # noqa F401
-from .workflow import WorkflowTask  # noqa F401

fractal_server-2.11.1/fractal_server/app/models/v1/dataset.py

@@ -1,71 +0,0 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
-from sqlalchemy import Column
-from sqlalchemy.ext.orderinglist import ordering_list
-from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
-from sqlmodel import Field
-from sqlmodel import Relationship
-from sqlmodel import SQLModel
-
-from ....utils import get_timestamp
-from ...schemas.v1.dataset import _DatasetBaseV1
-from ...schemas.v1.dataset import _ResourceBaseV1
-
-
-class Resource(_ResourceBaseV1, SQLModel, table=True):
-    id: Optional[int] = Field(default=None, primary_key=True)
-    dataset_id: int = Field(foreign_key="dataset.id")
-
-
-class Dataset(_DatasetBaseV1, SQLModel, table=True):
-    """
-    Represent a dataset
-
-    Attributes:
-        id:
-            Primary key
-        project_id:
-            ID of the project the workflow belongs to.
-        meta:
-            Metadata of the Dataset
-        history:
-            History of the Dataset
-        resource_list:
-            (Mapper attribute)
-
-    """
-
-    id: Optional[int] = Field(default=None, primary_key=True)
-    project_id: int = Field(foreign_key="project.id")
-    project: "Project" = Relationship(  # noqa: F821
-        sa_relationship_kwargs=dict(lazy="selectin"),
-    )
-
-    resource_list: list[Resource] = Relationship(
-        sa_relationship_kwargs={
-            "lazy": "selectin",
-            "order_by": "Resource.id",
-            "collection_class": ordering_list("id"),
-            "cascade": "all, delete-orphan",
-        }
-    )
-
-    meta: dict[str, Any] = Field(sa_column=Column(JSON), default={})
-    history: list[dict[str, Any]] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
-    )
-
-    timestamp_created: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(DateTime(timezone=True), nullable=False),
-    )
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    @property
-    def paths(self) -> list[str]:
-        return [r.path for r in self.resource_list]

fractal_server-2.11.1/fractal_server/app/models/v1/job.py

@@ -1,101 +0,0 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
-from sqlalchemy import Column
-from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
-from sqlmodel import Field
-from sqlmodel import SQLModel
-
-from ....utils import get_timestamp
-from ...schemas.v1 import JobStatusTypeV1
-from ...schemas.v1.applyworkflow import _ApplyWorkflowBaseV1
-
-
-class ApplyWorkflow(_ApplyWorkflowBaseV1, SQLModel, table=True):
-    """
-    Represent a workflow run
-
-    This table is responsible for storing the state of a workflow execution in
-    the database.
-
-    Attributes:
-        id:
-            Primary key.
-        project_id:
-            ID of the project the workflow belongs to, or `None` if the project
-            was deleted.
-        input_dataset_id:
-            ID of the input dataset, or `None` if the dataset was deleted.
-        output_dataset_id:
-            ID of the output dataset, or `None` if the dataset was deleted.
-        workflow_id:
-            ID of the workflow being applied, or `None` if the workflow was
-            deleted.
-        status:
-            Job status
-        workflow_dump:
-            Copy of the submitted workflow at submission.
-        input_dataset_dump:
-            Copy of the input_dataset at submission.
-        output_dataset_dump:
-            Copy of the output_dataset at submission.
-        start_timestamp:
-            Timestamp of when the run began.
-        end_timestamp:
-            Timestamp of when the run ended or failed.
-        status:
-            Status of the run.
-        log:
-            Forward of the workflow logs.
-        user_email:
-            Email address of the user who submitted the job.
-        slurm_account:
-            Account to be used when submitting the job to SLURM (see "account"
-            option in [`sbatch`
-            documentation](https://slurm.schedmd.com/sbatch.html#SECTION_OPTIONS)).
-        first_task_index:
-        last_task_index:
-    """
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    id: Optional[int] = Field(default=None, primary_key=True)
-
-    project_id: Optional[int] = Field(foreign_key="project.id")
-    workflow_id: Optional[int] = Field(foreign_key="workflow.id")
-    input_dataset_id: Optional[int] = Field(foreign_key="dataset.id")
-    output_dataset_id: Optional[int] = Field(foreign_key="dataset.id")
-
-    user_email: str = Field(nullable=False)
-    slurm_account: Optional[str]
-
-    input_dataset_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
-    )
-    output_dataset_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
-    )
-    workflow_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
-    )
-    project_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
-    )
-
-    working_dir: Optional[str]
-    working_dir_user: Optional[str]
-    first_task_index: int
-    last_task_index: int
-
-    start_timestamp: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(DateTime(timezone=True), nullable=False),
-    )
-    end_timestamp: Optional[datetime] = Field(
-        default=None, sa_column=Column(DateTime(timezone=True))
-    )
-    status: str = JobStatusTypeV1.SUBMITTED
-    log: Optional[str] = None

fractal_server-2.11.1/fractal_server/app/models/v1/project.py

@@ -1,29 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from sqlalchemy import Column
-from sqlalchemy.types import DateTime
-from sqlmodel import Field
-from sqlmodel import Relationship
-from sqlmodel import SQLModel
-
-from . import LinkUserProject
-from ....utils import get_timestamp
-from ...schemas.v1.project import _ProjectBaseV1
-from fractal_server.app.models import UserOAuth
-
-
-class Project(_ProjectBaseV1, SQLModel, table=True):
-
-    id: Optional[int] = Field(default=None, primary_key=True)
-    timestamp_created: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(DateTime(timezone=True), nullable=False),
-    )
-
-    user_list: list[UserOAuth] = Relationship(
-        link_model=LinkUserProject,
-        sa_relationship_kwargs={
-            "lazy": "selectin",
-        },
-    )

fractal_server-2.11.1/fractal_server/app/models/v1/state.py

@@ -1,34 +0,0 @@
-from datetime import datetime
-from typing import Any
-from typing import Optional
-
-from sqlalchemy import Column
-from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
-from sqlmodel import Field
-from sqlmodel import SQLModel
-
-from ....utils import get_timestamp
-
-
-class State(SQLModel, table=True):
-    """
-    Store arbitrary data in the database
-
-    This table is just a state interchange that allows the system to store
-    arbitrary data for later retrieval. This is particuarly important for long
-    background tasks, in which it is not possible to return a meaningful
-    response to the client within a single request lifespan.
-
-    Attributes:
-        id: Primary key
-        data: Content of the `State`
-        timestamp: Timestap of the `State`
-    """
-
-    id: Optional[int] = Field(default=None, primary_key=True)
-    data: dict[str, Any] = Field(sa_column=Column(JSON), default={})
-    timestamp: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(DateTime(timezone=True)),
-    )