fractal-server 2.4.2__py3-none-any.whl → 2.5.0a1__py3-none-any.whl

fractal_server/app/runner/v2/runner.py

@@ -17,7 +17,6 @@ from ..filenames import FILTERS_FILENAME
 from ..filenames import HISTORY_FILENAME
 from ..filenames import IMAGES_FILENAME
 from .runner_functions import no_op_submit_setup_call
-from .runner_functions import run_v1_task_parallel
 from .runner_functions import run_v2_task_compound
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
@@ -53,16 +52,8 @@ def execute_tasks_v2(
 
     for wftask in wf_task_list:
         task = wftask.task
-        task_legacy = wftask.task_legacy
-        if wftask.is_legacy_task:
-            task_name = task_legacy.name
-            logger.debug(
-                f"SUBMIT {wftask.order}-th task "
-                f'(legacy, name="{task_name}")'
-            )
-        else:
-            task_name = task.name
-            logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
+        task_name = task.name
+        logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
 
         # PRE TASK EXECUTION
 
@@ -78,67 +69,53 @@ def execute_tasks_v2(
             filters=Filters(**pre_filters),
         )
         # Verify that filtered images comply with task input_types
-        if not wftask.is_legacy_task:
-            for image in filtered_images:
-                if not match_filter(image, Filters(types=task.input_types)):
-                    raise JobExecutionError(
-                        "Invalid filtered image list\n"
-                        f"Task input types: {task.input_types=}\n"
-                        f'Image zarr_url: {image["zarr_url"]}\n'
-                        f'Image types: {image["types"]}\n'
-                    )
+        for image in filtered_images:
+            if not match_filter(image, Filters(types=task.input_types)):
+                raise JobExecutionError(
+                    "Invalid filtered image list\n"
+                    f"Task input types: {task.input_types=}\n"
+                    f'Image zarr_url: {image["zarr_url"]}\n'
+                    f'Image types: {image["types"]}\n'
+                )
 
         # TASK EXECUTION (V2)
-        if not wftask.is_legacy_task:
-            if task.type == "non_parallel":
-                current_task_output = run_v2_task_non_parallel(
-                    images=filtered_images,
-                    zarr_dir=zarr_dir,
-                    wftask=wftask,
-                    task=task,
-                    workflow_dir_local=workflow_dir_local,
-                    workflow_dir_remote=workflow_dir_remote,
-                    executor=executor,
-                    logger_name=logger_name,
-                    submit_setup_call=submit_setup_call,
-                )
-            elif task.type == "parallel":
-                current_task_output = run_v2_task_parallel(
-                    images=filtered_images,
-                    wftask=wftask,
-                    task=task,
-                    workflow_dir_local=workflow_dir_local,
-                    workflow_dir_remote=workflow_dir_remote,
-                    executor=executor,
-                    logger_name=logger_name,
-                    submit_setup_call=submit_setup_call,
-                )
-            elif task.type == "compound":
-                current_task_output = run_v2_task_compound(
-                    images=filtered_images,
-                    zarr_dir=zarr_dir,
-                    wftask=wftask,
-                    task=task,
-                    workflow_dir_local=workflow_dir_local,
-                    workflow_dir_remote=workflow_dir_remote,
-                    executor=executor,
-                    logger_name=logger_name,
-                    submit_setup_call=submit_setup_call,
-                )
-            else:
-                raise ValueError(f"Unexpected error: Invalid {task.type=}.")
-        # TASK EXECUTION (V1)
-        else:
-            current_task_output = run_v1_task_parallel(
+        if task.type == "non_parallel":
+            current_task_output = run_v2_task_non_parallel(
                 images=filtered_images,
+                zarr_dir=zarr_dir,
                 wftask=wftask,
-                task_legacy=task_legacy,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
                 executor=executor,
                 logger_name=logger_name,
+                submit_setup_call=submit_setup_call,
+            )
+        elif task.type == "parallel":
+            current_task_output = run_v2_task_parallel(
+                images=filtered_images,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=executor,
+                logger_name=logger_name,
+                submit_setup_call=submit_setup_call,
+            )
+        elif task.type == "compound":
+            current_task_output = run_v2_task_compound(
+                images=filtered_images,
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
+                executor=executor,
+                logger_name=logger_name,
                 submit_setup_call=submit_setup_call,
             )
+        else:
+            raise ValueError(f"Unexpected error: Invalid {task.type=}.")
 
         # POST TASK EXECUTION
 
@@ -191,8 +168,7 @@ def execute_tasks_v2(
                 # Update image attributes/types with task output and manifest
                 updated_attributes.update(image["attributes"])
                 updated_types.update(image["types"])
-                if not wftask.is_legacy_task:
-                    updated_types.update(task.output_types)
+                updated_types.update(task.output_types)
 
                 # Unset attributes with None value
                 updated_attributes = {
@@ -249,8 +225,7 @@ def execute_tasks_v2(
                     if value is not None
                 }
                 updated_types.update(image["types"])
-                if not wftask.is_legacy_task:
-                    updated_types.update(task.output_types)
+                updated_types.update(task.output_types)
                 new_image = dict(
                     zarr_url=image["zarr_url"],
                     origin=image["origin"],
@@ -282,10 +257,7 @@ def execute_tasks_v2(
             )
 
         # Find manifest ouptut types
-        if wftask.is_legacy_task:
-            types_from_manifest = {}
-        else:
-            types_from_manifest = task.output_types
+        types_from_manifest = task.output_types
 
         # Find task-output types
         if current_task_output.filters is not None:

fractal_server/app/runner/v2/runner_functions.py

@@ -16,8 +16,6 @@ from .merge_outputs import merge_outputs
 from .runner_functions_low_level import run_single_task
 from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
-from .v1_compat import convert_v2_args_into_v1
-from fractal_server.app.models.v1 import Task as TaskV1
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.components import _COMPONENT_KEY_
@@ -28,7 +26,6 @@ __all__ = [
     "run_v2_task_non_parallel",
     "run_v2_task_parallel",
     "run_v2_task_compound",
-    "run_v1_task_parallel",
 ]
 
 MAX_PARALLELIZATION_LIST_SIZE = 20_000
@@ -317,58 +314,3 @@ def run_v2_task_compound(
 
     merged_output = merge_outputs(outputs)
     return merged_output
-
-
-def run_v1_task_parallel(
-    *,
-    images: list[dict[str, Any]],
-    task_legacy: TaskV1,
-    wftask: WorkflowTaskV2,
-    executor: Executor,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Optional[Path] = None,
-    logger_name: Optional[str] = None,
-    submit_setup_call: Callable = no_op_submit_setup_call,
-) -> TaskOutput:
-
-    _check_parallelization_list_size(images)
-
-    executor_options = _get_executor_options(
-        wftask=wftask,
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
-        submit_setup_call=submit_setup_call,
-        which_type="parallel",
-    )
-
-    list_function_kwargs = []
-    for ind, image in enumerate(images):
-        list_function_kwargs.append(
-            convert_v2_args_into_v1(
-                kwargs_v2=dict(
-                    zarr_url=image["zarr_url"],
-                    **(wftask.args_parallel or {}),
-                ),
-                parallelization_level=task_legacy.parallelization_level,
-            ),
-        )
-        list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
-
-    results_iterator = executor.map(
-        functools.partial(
-            run_single_task,
-            wftask=wftask,
-            command=task_legacy.command,
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_remote,
-            is_task_v1=True,
-        ),
-        list_function_kwargs,
-        **executor_options,
-    )
-    # Explicitly iterate over the whole list, so that all futures are waited
-    list(results_iterator)
-
-    # Ignore any output metadata for V1 tasks, and return an empty object
-    out = TaskOutput()
-    return out

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -61,7 +61,6 @@ def run_single_task(
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
-    is_task_v1: bool = False,
 ) -> dict[str, Any]:
     """
     Runs within an executor.
@@ -73,10 +72,7 @@ def run_single_task(
     if not workflow_dir_remote:
         workflow_dir_remote = workflow_dir_local
 
-    if is_task_v1:
-        task_name = wftask.task_legacy.name
-    else:
-        task_name = wftask.task.name
+    task_name = wftask.task.name
 
     component = args.pop(_COMPONENT_KEY_, None)
     task_files = get_task_file_paths(
@@ -92,18 +88,11 @@ def run_single_task(
         json.dump(args, f, indent=2)
 
     # Assemble full command
-    if is_task_v1:
-        full_command = (
-            f"{command} "
-            f"--json {task_files.args.as_posix()} "
-            f"--metadata-out {task_files.metadiff.as_posix()}"
-        )
-    else:
-        full_command = (
-            f"{command} "
-            f"--args-json {task_files.args.as_posix()} "
-            f"--out-json {task_files.metadiff.as_posix()}"
-        )
+    full_command = (
+        f"{command} "
+        f"--args-json {task_files.args.as_posix()} "
+        f"--out-json {task_files.metadiff.as_posix()}"
+    )
 
     try:
         _call_command_wrapper(
@@ -113,10 +102,7 @@ def run_single_task(
     except TaskExecutionError as e:
         e.workflow_task_order = wftask.order
         e.workflow_task_id = wftask.id
-        if wftask.is_legacy_task:
-            e.task_name = wftask.task_legacy.name
-        else:
-            e.task_name = wftask.task.name
+        e.task_name = wftask.task.name
         raise e
 
     try:

fractal_server/app/schemas/user.py

@@ -20,6 +20,22 @@ __all__ = (
 )
 
 
+class OAuthAccountRead(BaseModel):
+    """
+    Schema for storing essential `OAuthAccount` information within
+    `UserRead.oauth_accounts`.
+
+    Attributes:
+        id: ID of the row in fractal-owned `oauthaccount` table.
+        account_email: Email associated to OAuth account
+        oauth_name: Name of the OAuth provider (e.g. `github`)
+    """
+
+    id: int
+    account_email: str
+    oauth_name: str
+
+
 class UserRead(schemas.BaseUser[int]):
     """
     Schema for `User` read from database.
@@ -37,6 +53,7 @@ class UserRead(schemas.BaseUser[int]):
     slurm_accounts: list[str]
     group_names: Optional[list[str]] = None
     group_ids: Optional[list[int]] = None
+    oauth_accounts: list[OAuthAccountRead]
 
 
 class UserUpdate(schemas.BaseUserUpdate):

fractal_server/app/schemas/v2/__init__.py

@@ -20,7 +20,6 @@ from .project import ProjectUpdateV2  # noqa F401
 from .task import TaskCreateV2  # noqa F401
 from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
-from .task import TaskLegacyReadV2  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
 from .task_collection import CollectionStateReadV2  # noqa F401

fractal_server/app/schemas/v2/dumps.py

@@ -12,9 +12,7 @@ from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
-from pydantic import root_validator
 
-from fractal_server.app.schemas.v1.dumps import TaskDumpV1
 from fractal_server.images import Filters
 
 
@@ -45,29 +43,10 @@ class WorkflowTaskDumpV2(BaseModel):
     workflow_id: int
     order: Optional[int]
 
-    is_legacy_task: bool
-
     input_filters: Filters
 
-    task_id: Optional[int]
-    task: Optional[TaskDumpV2]
-    task_legacy_id: Optional[int]
-    task_legacy: Optional[TaskDumpV1]
-
-    # Validators
-    @root_validator
-    def task_v1_or_v2(cls, values):
-        v1 = values.get("task_legacy_id")
-        v2 = values.get("task_id")
-        if ((v1 is not None) and (v2 is not None)) or (
-            (v1 is None) and (v2 is None)
-        ):
-            message = "both" if (v1 and v2) else "none"
-            raise ValueError(
-                "One and only one must be provided between "
-                f"'task_legacy_id' and 'task_id' (you provided {message})"
-            )
-        return values
+    task_id: int
+    task: TaskDumpV2
 
 
 class WorkflowDumpV2(BaseModel, extra=Extra.forbid):

fractal_server/app/schemas/v2/task.py

@@ -11,7 +11,6 @@ from pydantic import validator
 
 from .._validators import valdictkeys
 from .._validators import valstr
-from ..v1.task import TaskReadV1
 
 
 class TaskCreateV2(BaseModel, extra=Extra.forbid):
@@ -101,10 +100,6 @@ class TaskReadV2(BaseModel):
     output_types: dict[str, bool]
 
 
-class TaskLegacyReadV2(TaskReadV1):
-    is_v2_compatible: bool
-
-
 class TaskUpdateV2(BaseModel):
 
     name: Optional[str]

fractal_server/app/schemas/v2/workflowtask.py

@@ -5,16 +5,12 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
-from pydantic import root_validator
 from pydantic import validator
 
 from .._validators import valdictkeys
 from .._validators import valint
-from ..v1.task import TaskExportV1
-from ..v1.task import TaskImportV1
 from .task import TaskExportV2
 from .task import TaskImportV2
-from .task import TaskLegacyReadV2
 from .task import TaskReadV2
 from fractal_server.images import Filters
 
@@ -49,8 +45,6 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     order: Optional[int]
     input_filters: Filters = Field(default_factory=Filters)
 
-    is_legacy_task: bool = False
-
     # Validators
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
         valdictkeys("meta_non_parallel")
@@ -88,18 +82,6 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
             )
         return value
 
-    @root_validator
-    def validate_legacy_task(cls, values):
-        if values["is_legacy_task"] and (
-            values.get("meta_non_parallel") is not None
-            or values.get("args_non_parallel") is not None
-        ):
-            raise ValueError(
-                "If Task is legacy, 'args_non_parallel' and 'meta_non_parallel"
-                "must be None"
-            )
-        return values
-
 
 class WorkflowTaskReadV2(BaseModel):
 
@@ -115,12 +97,9 @@ class WorkflowTaskReadV2(BaseModel):
 
     input_filters: Filters
 
-    is_legacy_task: bool
     task_type: str
-    task_id: Optional[int]
-    task: Optional[TaskReadV2]
-    task_legacy_id: Optional[int]
-    task_legacy: Optional[TaskLegacyReadV2]
+    task_id: int
+    task: TaskReadV2
 
 
 class WorkflowTaskUpdateV2(BaseModel):
@@ -177,9 +156,7 @@ class WorkflowTaskImportV2(BaseModel):
 
     input_filters: Optional[Filters] = None
 
-    is_legacy_task: bool = False
-    task: Optional[TaskImportV2] = None
-    task_legacy: Optional[TaskImportV1] = None
+    task: TaskImportV2
 
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
         valdictkeys("meta_non_parallel")
@@ -203,6 +180,4 @@ class WorkflowTaskExportV2(BaseModel):
     args_parallel: Optional[dict[str, Any]] = None
     input_filters: Filters = Field(default_factory=Filters)
 
-    is_legacy_task: bool = False
-    task: Optional[TaskExportV2]
-    task_legacy: Optional[TaskExportV1]
+    task: TaskExportV2

fractal_server/migrations/env.py

@@ -7,6 +7,7 @@ from sqlmodel import SQLModel
 
 from fractal_server.app import models  # noqa
 from fractal_server.config import get_settings
+from fractal_server.migrations.naming_convention import NAMING_CONVENTION
 from fractal_server.syringe import Inject
 
 # this is the Alembic Config object, which provides
@@ -25,13 +26,7 @@ if config.config_file_name is not None:
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 target_metadata = SQLModel.metadata
-target_metadata.naming_convention = {
-    "ix": "ix_%(column_0_label)s",
-    "uq": "uq_%(table_name)s_%(column_0_name)s",
-    "ck": "ck_%(table_name)s_`%(constraint_name)s`",
-    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
-    "pk": "pk_%(table_name)s",
-}
+target_metadata.naming_convention = NAMING_CONVENTION
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -58,6 +53,7 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        render_as_batch=True,
     )
 
     with context.begin_transaction():
@@ -68,6 +64,7 @@ def do_run_migrations(connection: Connection) -> None:
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
+        render_as_batch=True,
     )
 
     with context.begin_transaction():

fractal_server/migrations/naming_convention.py

@@ -0,0 +1,7 @@
+NAMING_CONVENTION = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}

fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py

@@ -1,4 +1,4 @@
-"""“Add_usergroup_and_linkusergroup_table”
+"""Add_usergroup_and_linkusergroup_table
 
 Revision ID: 091b01f51f88
 Revises: 5bf02391cfef

fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py

@@ -0,0 +1,97 @@
+"""Remove link between v1 and v2 tasks/workflowtasks tables
+
+Revision ID: d9a140db5d42
+Revises: 5bf02391cfef
+Create Date: 2024-09-09 14:15:34.415926
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+from fractal_server.migrations.naming_convention import NAMING_CONVENTION
+
+# revision identifiers, used by Alembic.
+revision = "d9a140db5d42"
+down_revision = "091b01f51f88"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+
+    with op.batch_alter_table("workflowtaskv2") as batch_op:
+        batch_op.alter_column(
+            "task_id", existing_type=sa.INTEGER(), nullable=False
+        )
+
+    # NOTE: in sqlite, this `drop_constraint` only works if
+    # `batch_alter_table` has a `naming_convention` set. Ref
+    # https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints
+    with op.batch_alter_table(
+        "workflowtaskv2", naming_convention=NAMING_CONVENTION
+    ) as batch_op:
+        batch_op.drop_constraint(
+            "fk_workflowtaskv2_task_legacy_id_task", type_="foreignkey"
+        )
+
+    # NOTE: in sqlite, the `drop_index` command fails if the existing table
+    # has zero rows, while it succeeds if there are already some rows
+    if op.get_bind().dialect.name == "sqlite":
+        import sqlite3
+        import logging
+
+        logger = logging.getLogger("alembic.runtime.migration")
+        logger.warning(
+            f"Using sqlite, with {sqlite3.version=} and "
+            f"{sqlite3.sqlite_version=}"
+        )
+        logger.warning("Now drop index 'idx_workflowtaskv2_task_legacy_id'")
+        try:
+            with op.batch_alter_table("workflowtaskv2") as batch_op:
+                batch_op.drop_index("idx_workflowtaskv2_task_legacy_id")
+        except sa.exc.OperationalError:
+            logger.warning(
+                "Could not drop index; "
+                "this is expected, when the database is empty."
+            )
+            logger.warning("Continue.")
+
+    with op.batch_alter_table(
+        "workflowtaskv2", schema=None, naming_convention=NAMING_CONVENTION
+    ) as batch_op:
+        batch_op.drop_column("is_legacy_task")
+        batch_op.drop_column("task_legacy_id")
+
+    with op.batch_alter_table("task") as batch_op:
+        batch_op.drop_column("is_v2_compatible")
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("task", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "is_v2_compatible",
+                sa.BOOLEAN(),
+                server_default=sa.text("(false)"),
+                nullable=False,
+            )
+        )
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("task_legacy_id", sa.INTEGER(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("is_legacy_task", sa.BOOLEAN(), nullable=False)
+        )
+        batch_op.create_foreign_key(
+            "fk_workflowtaskv2_task_legacy_id_task",
+            "task",
+            ["task_legacy_id"],
+            ["id"],
+        )
+        batch_op.alter_column(
+            "task_id", existing_type=sa.INTEGER(), nullable=True
+        )
+
+    # ### end Alembic commands ###

{fractal_server-2.4.2.dist-info → fractal_server-2.5.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.4.2
+Version: 2.5.0a1
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause