fractal-server 2.14.16__py3-none-any.whl → 2.15.0__py3-none-any.whl

fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py

@@ -0,0 +1,340 @@
+"""JSON to JSONB
+
+Revision ID: b3ffb095f973
+Revises: b1e7f7a1ff71
+Create Date: 2025-06-19 10:12:06.699107
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b3ffb095f973"
+down_revision = "b1e7f7a1ff71"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "history",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+        batch_op.alter_column(
+            "images",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "dataset_dump",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+        batch_op.alter_column(
+            "workflow_dump",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+        batch_op.alter_column(
+            "project_dump",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+        batch_op.alter_column(
+            "attribute_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "type_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "pinned_package_versions",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "args_schema_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_schema_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "input_types",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "output_types",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "tags",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.alter_column(
+            "slurm_accounts",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("usergroup", schema=None) as batch_op:
+        batch_op.alter_column(
+            "viewer_paths",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "type_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            type_=postgresql.JSONB(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "type_filters",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "args_non_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+
+    with op.batch_alter_table("usergroup", schema=None) as batch_op:
+        batch_op.alter_column(
+            "viewer_paths",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.alter_column(
+            "slurm_accounts",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "tags",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+        batch_op.alter_column(
+            "output_types",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "input_types",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_schema_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "args_schema_non_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "pinned_package_versions",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=True,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "type_filters",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "attribute_filters",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'{}'::json"),
+        )
+        batch_op.alter_column(
+            "project_dump",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+        batch_op.alter_column(
+            "workflow_dump",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+        batch_op.alter_column(
+            "dataset_dump",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+        )
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "images",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+        batch_op.alter_column(
+            "history",
+            existing_type=postgresql.JSONB(astext_type=sa.Text()),
+            type_=postgresql.JSON(astext_type=sa.Text()),
+            existing_nullable=False,
+            existing_server_default=sa.text("'[]'::json"),
+        )
+
+    # ### end Alembic commands ###

fractal_server/ssh/_fabric.py

@@ -204,6 +204,32 @@ class FractalSSH:
         self.logger.info(f"END reading remote JSON file {filepath}.")
         return data
 
+    def read_remote_text_file(self, filepath: str) -> dict[str, Any]:
+        """
+        Read a remote text file into a string.
+
+        Note from paramiko docs:
+        > The Python 'b' flag is ignored, since SSH treats all files as binary.
+        """
+        self.logger.info(f"START reading remote text file {filepath}.")
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label="read_remote_text_file",
+            timeout=self.default_lock_timeout,
+        ):
+            try:
+                with self._sftp_unsafe().open(filepath, "r") as f:
+                    data = f.read().decode()
+            except Exception as e:
+                self.log_and_raise(
+                    e=e,
+                    message=(
+                        f"Error in `read_remote_text_file`, for {filepath=}."
+                    ),
+                )
+        self.logger.info(f"END reading remote text file {filepath}.")
+        return data
+
     def check_connection(self) -> None:
         """
         Open the SSH connection and handle exceptions.

fractal_server/tasks/v2/local/__init__.py

@@ -1,3 +1,6 @@
 from .collect import collect_local  # noqa
+from .collect_pixi import collect_local_pixi  # noqa
 from .deactivate import deactivate_local  # noqa
+from .deactivate_pixi import deactivate_local_pixi  # noqa
 from .reactivate import reactivate_local  # noqa
+from .reactivate_pixi import reactivate_local_pixi  # noqa

fractal_server/tasks/v2/local/_utils.py

@@ -31,7 +31,7 @@ def _customize_and_run_template(
             f"Invalid {template_filename=} (it must end with '.sh')."
         )
 
-    script_filename = f"{prefix}{template_filename}"
+    script_filename = f"{prefix}_{template_filename}"
     script_path_local = Path(script_dir) / script_filename
     # Read template
     customize_template(
@@ -53,16 +53,17 @@ def check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
     Args:
         task_list:
     """
+
     for _task in task_list:
         if _task.command_non_parallel is not None:
-            _task_path = _task.command_non_parallel.split()[1]
+            _task_path = _task.command_non_parallel.split()[-1]
             if not Path(_task_path).exists():
                 raise FileNotFoundError(
                     f"Task `{_task.name}` has `command_non_parallel` "
                     f"pointing to missing file `{_task_path}`."
                 )
         if _task.command_parallel is not None:
-            _task_path = _task.command_parallel.split()[1]
+            _task_path = _task.command_parallel.split()[-1]
             if not Path(_task_path).exists():
                 raise FileNotFoundError(
                     f"Task `{_task.name}` has `command_parallel` "

fractal_server/tasks/v2/local/collect.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import shutil
 import time
 from pathlib import Path
@@ -8,20 +7,22 @@ from tempfile import TemporaryDirectory
 from ..utils_database import create_db_tasks_and_update_task_group_sync
 from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.v2 import FractalUploadedFile
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
-from fractal_server.app.schemas.v2 import WheelFile
 from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
 from fractal_server.tasks.utils import get_log_path
 from fractal_server.tasks.v2.local._utils import check_task_files_exist
-from fractal_server.tasks.v2.utils_background import _prepare_tasks_metadata
 from fractal_server.tasks.v2.utils_background import add_commit_refresh
 from fractal_server.tasks.v2.utils_background import fail_and_cleanup
+from fractal_server.tasks.v2.utils_background import (
+    get_activity_and_task_group,
+)
 from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
 from fractal_server.tasks.v2.utils_package_names import compare_package_names
 from fractal_server.tasks.v2.utils_python_interpreter import (
     get_python_interpreter_v2,
@@ -38,7 +39,7 @@ def collect_local(
     *,
     task_group_activity_id: int,
     task_group_id: int,
-    wheel_file: WheelFile | None = None,
+    wheel_file: FractalUploadedFile | None = None,
 ) -> None:
     """
     Collect a task package.
@@ -66,24 +67,17 @@ def collect_local(
             log_file_path=log_file_path,
         )
 
+        logger.info("START")
         with next(get_sync_db()) as db:
-            # Get main objects from db
-            activity = db.get(TaskGroupActivityV2, task_group_activity_id)
-            task_group = db.get(TaskGroupV2, task_group_id)
-            if activity is None or task_group is None:
-                # Use `logging` directly
-                logging.error(
-                    "Cannot find database rows with "
-                    f"{task_group_id=} and {task_group_activity_id=}:\n"
-                    f"{task_group=}\n{activity=}. Exit."
-                )
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
                 return
 
-            # Log some info
-            logger.info("START")
-            for key, value in task_group.model_dump().items():
-                logger.debug(f"task_group.{key}: {value}")
-
             # Check that the (local) task_group path does exist
             if Path(task_group.path).exists():
                 error_msg = f"{task_group.path} already exists."
@@ -103,16 +97,18 @@ def collect_local(
                 Path(task_group.path).mkdir(parents=True)
                 logger.info(f"Created {task_group.path}")
 
-                # Write wheel file and set task_group.wheel_path
+                # Write wheel file and set task_group.archive_path
                 if wheel_file is not None:
 
-                    wheel_path = (
+                    archive_path = (
                         Path(task_group.path) / wheel_file.filename
                     ).as_posix()
-                    logger.info(f"Write wheel-file contents into {wheel_path}")
-                    with open(wheel_path, "wb") as f:
+                    logger.info(
+                        f"Write wheel-file contents into {archive_path}"
+                    )
+                    with open(archive_path, "wb") as f:
                         f.write(wheel_file.contents)
-                    task_group.wheel_path = wheel_path
+                    task_group.archive_path = archive_path
                     task_group = add_commit_refresh(obj=task_group, db=db)
 
                 # Prepare replacements for templates
@@ -131,7 +127,7 @@ def collect_local(
                     ).as_posix(),
                     prefix=(
                         f"{int(time.time())}_"
-                        f"{TaskGroupActivityActionV2.COLLECT}_"
+                        f"{TaskGroupActivityActionV2.COLLECT}"
                     ),
                     logger_name=LOGGER_NAME,
                 )
@@ -220,7 +216,7 @@ def collect_local(
                 activity = add_commit_refresh(obj=activity, db=db)
 
                 logger.info("_prepare_tasks_metadata - start")
-                task_list = _prepare_tasks_metadata(
+                task_list = prepare_tasks_metadata(
                     package_manifest=pkg_manifest,
                     package_version=task_group.version,
                     package_root=Path(package_root),
@@ -241,15 +237,15 @@ def collect_local(
 
                 # Update task_group data
                 logger.info(
-                    "Add pip_freeze, venv_size and venv_file_number "
+                    "Add env_info, venv_size and venv_file_number "
                     "to TaskGroupV2 - start"
                 )
-                task_group.pip_freeze = pip_freeze_stdout
+                task_group.env_info = pip_freeze_stdout
                 task_group.venv_size_in_kB = int(venv_size)
                 task_group.venv_file_number = int(venv_file_number)
                 task_group = add_commit_refresh(obj=task_group, db=db)
                 logger.info(
-                    "Add pip_freeze, venv_size and venv_file_number "
+                    "Add env_info, venv_size and venv_file_number "
                     "to TaskGroupV2 - end"
                 )