fractal-server 2.13.1__py3-none-any.whl → 2.14.0__py3-none-any.whl

fractal_server/config.py

@@ -28,6 +28,7 @@ from pydantic import EmailStr
 from pydantic import Field
 from pydantic import field_validator
 from pydantic import model_validator
+from pydantic import SecretStr
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 from sqlalchemy.engine import URL
@@ -54,8 +55,8 @@ class MailSettings(BaseModel):
     recipients: list[EmailStr] = Field(min_length=1)
     smtp_server: str
     port: int
-    encrypted_password: Optional[str] = None
-    encryption_key: Optional[str] = None
+    encrypted_password: Optional[SecretStr] = None
+    encryption_key: Optional[SecretStr] = None
     instance_name: str
     use_starttls: bool
     use_login: bool
@@ -97,7 +98,7 @@ class OAuthClientConfig(BaseModel):
 
     CLIENT_NAME: str
     CLIENT_ID: str
-    CLIENT_SECRET: str
+    CLIENT_SECRET: SecretStr
     OIDC_CONFIGURATION_ENDPOINT: Optional[str] = None
     REDIRECT_URL: Optional[str] = None
 
@@ -137,7 +138,7 @@ class Settings(BaseSettings):
     JWT token lifetime, in seconds.
     """
 
-    JWT_SECRET_KEY: Optional[str] = None
+    JWT_SECRET_KEY: Optional[SecretStr] = None
     """
     JWT secret
 
@@ -204,7 +205,7 @@ class Settings(BaseSettings):
     """
     User to use when connecting to the PostgreSQL database.
     """
-    POSTGRES_PASSWORD: Optional[str] = None
+    POSTGRES_PASSWORD: Optional[SecretStr] = None
     """
     Password to use when connecting to the PostgreSQL database.
     """
@@ -223,10 +224,15 @@ class Settings(BaseSettings):
 
     @property
     def DATABASE_ASYNC_URL(self) -> URL:
+        if self.POSTGRES_PASSWORD is None:
+            password = None
+        else:
+            password = self.POSTGRES_PASSWORD.get_secret_value()
+
         url = URL.create(
             drivername="postgresql+psycopg",
             username=self.POSTGRES_USER,
-            password=self.POSTGRES_PASSWORD,
+            password=password,
             host=self.POSTGRES_HOST,
             port=self.POSTGRES_PORT,
             database=self.POSTGRES_DB,
@@ -250,7 +256,7 @@ class Settings(BaseSettings):
     default admin credentials.
     """
 
-    FRACTAL_DEFAULT_ADMIN_PASSWORD: str = "1234"
+    FRACTAL_DEFAULT_ADMIN_PASSWORD: SecretStr = "1234"
     """
     Admin default password, used upon creation of the first superuser during
     server startup.
@@ -483,8 +489,12 @@ class Settings(BaseSettings):
     FRACTAL_SLURM_POLL_INTERVAL: int = 5
     """
     Interval to wait (in seconds) before checking whether unfinished job are
-    still running on SLURM (see `SlurmWaitThread` in
-    [`clusterfutures`](https://github.com/sampsyo/clusterfutures/blob/master/cfut/__init__.py)).
+    still running on SLURM.
+    """
+
+    FRACTAL_SLURM_INTERVAL_BEFORE_RETRIEVAL: int = 2
+    """
+    NOTE: see issue 2444
     """
 
     FRACTAL_SLURM_SBATCH_SLEEP: float = 0
@@ -502,14 +512,6 @@ class Settings(BaseSettings):
     `JobExecutionError`.
     """
 
-    FRACTAL_RUNNER_TASKS_INCLUDE_IMAGE: str = (
-        "Copy OME-Zarr structure;Convert Metadata Components from 2D to 3D"
-    )
-    """
-    `;`-separated list of names for task that require the `metadata["image"]`
-    attribute in their input-arguments JSON file.
-    """
-
     FRACTAL_PIP_CACHE_DIR: Optional[str] = None
     """
     Absolute path to the cache directory for `pip`; if unset,
@@ -587,21 +589,21 @@ class Settings(BaseSettings):
     """
     Address of the OAuth-signup email sender.
     """
-    FRACTAL_EMAIL_PASSWORD: Optional[str] = None
+    FRACTAL_EMAIL_PASSWORD: Optional[SecretStr] = None
     """
     Password for the OAuth-signup email sender.
     """
-    FRACTAL_EMAIL_PASSWORD_KEY: Optional[str] = None
+    FRACTAL_EMAIL_PASSWORD_KEY: Optional[SecretStr] = None
     """
     Key value for `cryptography.fernet` decrypt
     """
     FRACTAL_EMAIL_SMTP_SERVER: Optional[str] = None
     """
-    SMPT server for the OAuth-signup emails.
+    SMTP server for the OAuth-signup emails.
     """
     FRACTAL_EMAIL_SMTP_PORT: Optional[int] = None
     """
-    SMPT server port for the OAuth-signup emails.
+    SMTP server port for the OAuth-signup emails.
     """
     FRACTAL_EMAIL_INSTANCE_NAME: Optional[str] = None
     """
@@ -661,8 +663,12 @@ class Settings(BaseSettings):
                     )
                 try:
                     (
-                        Fernet(self.FRACTAL_EMAIL_PASSWORD_KEY)
-                        .decrypt(self.FRACTAL_EMAIL_PASSWORD)
+                        Fernet(
+                            self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
+                        )
+                        .decrypt(
+                            self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
+                        )
                         .decode("utf-8")
                     )
                 except Exception as e:
@@ -671,14 +677,22 @@ class Settings(BaseSettings):
                         "FRACTAL_EMAIL_PASSWORD_KEY). "
                         f"Original error: {str(e)}."
                     )
+                password = self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
+            else:
+                password = None
+
+            if self.FRACTAL_EMAIL_PASSWORD_KEY is not None:
+                key = self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
+            else:
+                key = None
 
             self.email_settings = MailSettings(
                 sender=self.FRACTAL_EMAIL_SENDER,
                 recipients=self.FRACTAL_EMAIL_RECIPIENTS.split(","),
                 smtp_server=self.FRACTAL_EMAIL_SMTP_SERVER,
                 port=self.FRACTAL_EMAIL_SMTP_PORT,
-                encrypted_password=self.FRACTAL_EMAIL_PASSWORD,
-                encryption_key=self.FRACTAL_EMAIL_PASSWORD_KEY,
+                encrypted_password=password,
+                encryption_key=key,
                 instance_name=self.FRACTAL_EMAIL_INSTANCE_NAME,
                 use_starttls=use_starttls,
                 use_login=use_login,
@@ -705,7 +719,7 @@ class Settings(BaseSettings):
 
         info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
         if self.FRACTAL_RUNNER_BACKEND == "slurm":
-            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+            from fractal_server.app.runner.executors.slurm_common._slurm_config import (  # noqa: E501
                 load_slurm_config_file,
             )
 
@@ -735,7 +749,7 @@ class Settings(BaseSettings):
                     f"Must set FRACTAL_SLURM_WORKER_PYTHON when {info}"
                 )
 
-            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+            from fractal_server.app.runner.executors.slurm_common._slurm_config import (  # noqa: E501
                 load_slurm_config_file,
             )
 
@@ -796,25 +810,6 @@ class Settings(BaseSettings):
         self.check_db()
         self.check_runner()
 
-    def get_sanitized(self) -> dict:
-        def _must_be_sanitized(string) -> bool:
-            if not string.upper().startswith("FRACTAL") or any(
-                s in string.upper()
-                for s in ["PASSWORD", "SECRET", "PWD", "TOKEN", "KEY"]
-            ):
-                return True
-            else:
-                return False
-
-        sanitized_settings = {}
-        for k, v in self.model_dump().items():
-            if _must_be_sanitized(k):
-                sanitized_settings[k] = "***"
-            else:
-                sanitized_settings[k] = v
-
-        return sanitized_settings
-
 
 def get_settings(settings=Settings()) -> Settings:
     return settings

fractal_server/images/tools.py

@@ -121,3 +121,26 @@ def merge_type_filters(
     merged_dict = task_input_types
     merged_dict.update(wftask_type_filters)
     return merged_dict
+
+
+def aggregate_attributes(images: list[dict[str, Any]]) -> dict[str, list[Any]]:
+    """
+    Given a list of images, this function returns a dictionary of all image
+    attributes, each mapped to a list of present values.
+    """
+    attributes = {}
+    for image in images:
+        for k, v in image["attributes"].items():
+            attributes.setdefault(k, []).append(v)
+        for k, v in attributes.items():
+            attributes[k] = list(set(v))
+    return attributes
+
+
+def aggregate_types(images: list[dict[str, Any]]) -> list[str]:
+    """
+    Given a list of images, this function returns a list of all image types.
+    """
+    return list(
+        set(type for image in images for type in image["types"].keys())
+    )

fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py

@@ -0,0 +1,50 @@
+"""Drop dataset filters
+
+Revision ID: 47351f8c7ebc
+Revises: fbce16ff4e47
+Create Date: 2025-03-26 11:10:17.869028
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "47351f8c7ebc"
+down_revision = "fbce16ff4e47"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_column("type_filters")
+        batch_op.drop_column("attribute_filters")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "type_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py

@@ -0,0 +1,250 @@
+"""Set ondelete
+
+Revision ID: 9db60297b8b2
+Revises: e81103413827
+Create Date: 2025-04-07 13:13:14.596394
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "9db60297b8b2"
+down_revision = "e81103413827"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_datasetv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_datasetv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_jobv2_dataset_id_datasetv2", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_jobv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_jobv2_workflow_id_workflowv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_workflow_id_workflowv2"),
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_dataset_id_datasetv2"),
+            "datasetv2",
+            ["dataset_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_linkusergroup_user_id_user_oauth", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_linkusergroup_group_id_usergroup", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_linkusergroup_group_id_usergroup"),
+            "usergroup",
+            ["group_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_linkusergroup_user_id_user_oauth"),
+            "user_oauth",
+            ["user_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("taskgroupactivityv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2",
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2"),
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_taskgroupv2_user_group_id_usergroup", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupv2_user_group_id_usergroup"),
+            "usergroup",
+            ["user_group_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_workflowtaskv2_workflow_id_workflowv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_workflowtaskv2_workflow_id_workflowv2"),
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("workflowv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_workflowv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_workflowv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_workflowv2_project_id_projectv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_workflowv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_workflowtaskv2_workflow_id_workflowv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_workflowtaskv2_workflow_id_workflowv2",
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupv2_user_group_id_usergroup"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_taskgroupv2_user_group_id_usergroup",
+            "usergroup",
+            ["user_group_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("taskgroupactivityv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2",
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_linkusergroup_user_id_user_oauth"),
+            type_="foreignkey",
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_linkusergroup_group_id_usergroup"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_linkusergroup_group_id_usergroup",
+            "usergroup",
+            ["group_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_linkusergroup_user_id_user_oauth",
+            "user_oauth",
+            ["user_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_project_id_projectv2"), type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_dataset_id_datasetv2"), type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_workflow_id_workflowv2"), type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_workflow_id_workflowv2",
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_dataset_id_datasetv2",
+            "datasetv2",
+            ["dataset_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_datasetv2_project_id_projectv2"), type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            "fk_datasetv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py

@@ -0,0 +1,41 @@
+"""job id in history run
+
+Revision ID: c90a7c76e996
+Revises: f37aceb45062
+Create Date: 2025-04-16 10:44:30.219309
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "c90a7c76e996"
+down_revision = "f37aceb45062"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyrun", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("job_id", sa.Integer(), nullable=False))
+        batch_op.create_foreign_key(
+            batch_op.f("fk_historyrun_job_id_jobv2"),
+            "jobv2",
+            ["job_id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyrun", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_historyrun_job_id_jobv2"), type_="foreignkey"
+        )
+        batch_op.drop_column("job_id")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/e81103413827_add_job_type_filters.py

@@ -0,0 +1,36 @@
+"""Add job.type_filters
+
+Revision ID: e81103413827
+Revises: 47351f8c7ebc
+Create Date: 2025-03-26 11:10:41.748248
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "e81103413827"
+down_revision = "47351f8c7ebc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_column("type_filters")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py

@@ -0,0 +1,39 @@
+"""Make HistoryUnit.logfile required
+
+Revision ID: f37aceb45062
+Revises: 9db60297b8b2
+Create Date: 2025-04-14 13:49:40.910342
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "f37aceb45062"
+down_revision = "9db60297b8b2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyunit", schema=None) as batch_op:
+        batch_op.alter_column(
+            "logfile",
+            existing_type=sa.VARCHAR(),
+            nullable=False,
+            server_default="__LOGFILE_PLACEHOLDER__",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyunit", schema=None) as batch_op:
+        batch_op.alter_column(
+            "logfile", existing_type=sa.VARCHAR(), nullable=True
+        )
+
+    # ### end Alembic commands ###