fractal-server 2.13.0__py3-none-any.whl → 2.14.0__py3-none-any.whl

fractal_server/config.py

@@ -28,6 +28,7 @@ from pydantic import EmailStr
 from pydantic import Field
 from pydantic import field_validator
 from pydantic import model_validator
+from pydantic import SecretStr
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 from sqlalchemy.engine import URL
@@ -54,8 +55,8 @@ class MailSettings(BaseModel):
     recipients: list[EmailStr] = Field(min_length=1)
     smtp_server: str
     port: int
-    encrypted_password: Optional[str] = None
-    encryption_key: Optional[str] = None
+    encrypted_password: Optional[SecretStr] = None
+    encryption_key: Optional[SecretStr] = None
     instance_name: str
     use_starttls: bool
     use_login: bool
@@ -97,7 +98,7 @@ class OAuthClientConfig(BaseModel):
 
     CLIENT_NAME: str
     CLIENT_ID: str
-    CLIENT_SECRET: str
+    CLIENT_SECRET: SecretStr
     OIDC_CONFIGURATION_ENDPOINT: Optional[str] = None
     REDIRECT_URL: Optional[str] = None
 
@@ -137,7 +138,7 @@ class Settings(BaseSettings):
     JWT token lifetime, in seconds.
     """
 
-    JWT_SECRET_KEY: Optional[str] = None
+    JWT_SECRET_KEY: Optional[SecretStr] = None
     """
     JWT secret
 
@@ -204,7 +205,7 @@ class Settings(BaseSettings):
     """
     User to use when connecting to the PostgreSQL database.
     """
-    POSTGRES_PASSWORD: Optional[str] = None
+    POSTGRES_PASSWORD: Optional[SecretStr] = None
     """
     Password to use when connecting to the PostgreSQL database.
     """
@@ -223,10 +224,15 @@ class Settings(BaseSettings):
 
     @property
     def DATABASE_ASYNC_URL(self) -> URL:
+        if self.POSTGRES_PASSWORD is None:
+            password = None
+        else:
+            password = self.POSTGRES_PASSWORD.get_secret_value()
+
         url = URL.create(
             drivername="postgresql+psycopg",
             username=self.POSTGRES_USER,
-            password=self.POSTGRES_PASSWORD,
+            password=password,
             host=self.POSTGRES_HOST,
             port=self.POSTGRES_PORT,
             database=self.POSTGRES_DB,
@@ -250,7 +256,7 @@ class Settings(BaseSettings):
     default admin credentials.
     """
 
-    FRACTAL_DEFAULT_ADMIN_PASSWORD: str = "1234"
+    FRACTAL_DEFAULT_ADMIN_PASSWORD: SecretStr = "1234"
     """
     Admin default password, used upon creation of the first superuser during
     server startup.
@@ -483,8 +489,12 @@ class Settings(BaseSettings):
     FRACTAL_SLURM_POLL_INTERVAL: int = 5
     """
     Interval to wait (in seconds) before checking whether unfinished job are
-    still running on SLURM (see `SlurmWaitThread` in
-    [`clusterfutures`](https://github.com/sampsyo/clusterfutures/blob/master/cfut/__init__.py)).
+    still running on SLURM.
+    """
+
+    FRACTAL_SLURM_INTERVAL_BEFORE_RETRIEVAL: int = 2
+    """
+    NOTE: see issue 2444
     """
 
     FRACTAL_SLURM_SBATCH_SLEEP: float = 0
@@ -502,14 +512,6 @@ class Settings(BaseSettings):
     `JobExecutionError`.
     """
 
-    FRACTAL_RUNNER_TASKS_INCLUDE_IMAGE: str = (
-        "Copy OME-Zarr structure;Convert Metadata Components from 2D to 3D"
-    )
-    """
-    `;`-separated list of names for task that require the `metadata["image"]`
-    attribute in their input-arguments JSON file.
-    """
-
     FRACTAL_PIP_CACHE_DIR: Optional[str] = None
     """
     Absolute path to the cache directory for `pip`; if unset,
@@ -587,21 +589,21 @@ class Settings(BaseSettings):
     """
     Address of the OAuth-signup email sender.
     """
-    FRACTAL_EMAIL_PASSWORD: Optional[str] = None
+    FRACTAL_EMAIL_PASSWORD: Optional[SecretStr] = None
     """
     Password for the OAuth-signup email sender.
     """
-    FRACTAL_EMAIL_PASSWORD_KEY: Optional[str] = None
+    FRACTAL_EMAIL_PASSWORD_KEY: Optional[SecretStr] = None
     """
     Key value for `cryptography.fernet` decrypt
     """
     FRACTAL_EMAIL_SMTP_SERVER: Optional[str] = None
     """
-    SMPT server for the OAuth-signup emails.
+    SMTP server for the OAuth-signup emails.
     """
     FRACTAL_EMAIL_SMTP_PORT: Optional[int] = None
     """
-    SMPT server port for the OAuth-signup emails.
+    SMTP server port for the OAuth-signup emails.
     """
     FRACTAL_EMAIL_INSTANCE_NAME: Optional[str] = None
     """
@@ -611,74 +613,92 @@ class Settings(BaseSettings):
     """
     Comma-separated list of recipients of the OAuth-signup emails.
     """
-    FRACTAL_EMAIL_USE_STARTTLS: Optional[bool] = True
+    FRACTAL_EMAIL_USE_STARTTLS: Literal["true", "false"] = "true"
     """
     Whether to use StartTLS when using the SMTP server.
+    Accepted values: 'true', 'false'.
     """
-    FRACTAL_EMAIL_USE_LOGIN: Optional[bool] = True
+    FRACTAL_EMAIL_USE_LOGIN: Literal["true", "false"] = "true"
     """
     Whether to use login when using the SMTP server.
+    If 'true', FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY must be
+    provided.
+    Accepted values: 'true', 'false'.
     """
     email_settings: Optional[MailSettings] = None
 
-    @model_validator(mode="before")
-    @classmethod
-    def validate_email_settings(cls, values):
-        email_values = {
-            k: v for k, v in values.items() if k.startswith("FRACTAL_EMAIL")
-        }
-        if email_values:
-
-            def assert_key(key: str):
-                if key not in email_values:
-                    raise ValueError(f"Missing '{key}'")
-
-            assert_key("FRACTAL_EMAIL_SENDER")
-            assert_key("FRACTAL_EMAIL_SMTP_SERVER")
-            assert_key("FRACTAL_EMAIL_SMTP_PORT")
-            assert_key("FRACTAL_EMAIL_INSTANCE_NAME")
-            assert_key("FRACTAL_EMAIL_RECIPIENTS")
+    @model_validator(mode="after")
+    def validate_email_settings(self):
+        email_values = [
+            self.FRACTAL_EMAIL_SENDER,
+            self.FRACTAL_EMAIL_SMTP_SERVER,
+            self.FRACTAL_EMAIL_SMTP_PORT,
+            self.FRACTAL_EMAIL_INSTANCE_NAME,
+            self.FRACTAL_EMAIL_RECIPIENTS,
+        ]
+        if len(set(email_values)) == 1:
+            # All required EMAIL attributes are None
+            pass
+        elif None in email_values:
+            # Not all required EMAIL attributes are set
+            error_msg = (
+                "Invalid FRACTAL_EMAIL configuration. "
+                f"Given values: {email_values}."
+            )
+            raise ValueError(error_msg)
+        else:
+            use_starttls = self.FRACTAL_EMAIL_USE_STARTTLS == "true"
+            use_login = self.FRACTAL_EMAIL_USE_LOGIN == "true"
 
-            if email_values.get("FRACTAL_EMAIL_USE_LOGIN", True):
-                if "FRACTAL_EMAIL_PASSWORD" not in email_values:
+            if use_login:
+                if self.FRACTAL_EMAIL_PASSWORD is None:
                     raise ValueError(
-                        "'FRACTAL_EMAIL_USE_LOGIN' is True but "
+                        "'FRACTAL_EMAIL_USE_LOGIN' is 'true' but "
                         "'FRACTAL_EMAIL_PASSWORD' is not provided."
                     )
-                elif "FRACTAL_EMAIL_PASSWORD_KEY" not in email_values:
+                if self.FRACTAL_EMAIL_PASSWORD_KEY is None:
                     raise ValueError(
-                        "'FRACTAL_EMAIL_USE_LOGIN' is True but "
+                        "'FRACTAL_EMAIL_USE_LOGIN' is 'true' but "
                         "'FRACTAL_EMAIL_PASSWORD_KEY' is not provided."
                     )
-                else:
-                    try:
-                        (
-                            Fernet(email_values["FRACTAL_EMAIL_PASSWORD_KEY"])
-                            .decrypt(email_values["FRACTAL_EMAIL_PASSWORD"])
-                            .decode("utf-8")
+                try:
+                    (
+                        Fernet(
+                            self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
                         )
-                    except Exception as e:
-                        raise ValueError(
-                            "Invalid pair (FRACTAL_EMAIL_PASSWORD, "
-                            "FRACTAL_EMAIL_PASSWORD_KEY). "
-                            f"Original error: {str(e)}."
+                        .decrypt(
+                            self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
                         )
+                        .decode("utf-8")
+                    )
+                except Exception as e:
+                    raise ValueError(
+                        "Invalid pair (FRACTAL_EMAIL_PASSWORD, "
+                        "FRACTAL_EMAIL_PASSWORD_KEY). "
+                        f"Original error: {str(e)}."
+                    )
+                password = self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
+            else:
+                password = None
 
-            values["email_settings"] = MailSettings(
-                sender=email_values["FRACTAL_EMAIL_SENDER"],
-                recipients=email_values["FRACTAL_EMAIL_RECIPIENTS"].split(","),
-                smtp_server=email_values["FRACTAL_EMAIL_SMTP_SERVER"],
-                port=email_values["FRACTAL_EMAIL_SMTP_PORT"],
-                encrypted_password=email_values.get("FRACTAL_EMAIL_PASSWORD"),
-                encryption_key=email_values.get("FRACTAL_EMAIL_PASSWORD_KEY"),
-                instance_name=email_values["FRACTAL_EMAIL_INSTANCE_NAME"],
-                use_starttls=email_values.get(
-                    "FRACTAL_EMAIL_USE_STARTTLS", True
-                ),
-                use_login=email_values.get("FRACTAL_EMAIL_USE_LOGIN", True),
+            if self.FRACTAL_EMAIL_PASSWORD_KEY is not None:
+                key = self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
+            else:
+                key = None
+
+            self.email_settings = MailSettings(
+                sender=self.FRACTAL_EMAIL_SENDER,
+                recipients=self.FRACTAL_EMAIL_RECIPIENTS.split(","),
+                smtp_server=self.FRACTAL_EMAIL_SMTP_SERVER,
+                port=self.FRACTAL_EMAIL_SMTP_PORT,
+                encrypted_password=password,
+                encryption_key=key,
+                instance_name=self.FRACTAL_EMAIL_INSTANCE_NAME,
+                use_starttls=use_starttls,
+                use_login=use_login,
             )
 
-        return values
+        return self
 
     ###########################################################################
     # BUSINESS LOGIC
@@ -699,7 +719,7 @@ class Settings(BaseSettings):
 
         info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
         if self.FRACTAL_RUNNER_BACKEND == "slurm":
-            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+            from fractal_server.app.runner.executors.slurm_common._slurm_config import (  # noqa: E501
                 load_slurm_config_file,
             )
 
@@ -729,7 +749,7 @@ class Settings(BaseSettings):
                     f"Must set FRACTAL_SLURM_WORKER_PYTHON when {info}"
                 )
 
-            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+            from fractal_server.app.runner.executors.slurm_common._slurm_config import (  # noqa: E501
                 load_slurm_config_file,
             )
 
@@ -790,25 +810,6 @@ class Settings(BaseSettings):
         self.check_db()
         self.check_runner()
 
-    def get_sanitized(self) -> dict:
-        def _must_be_sanitized(string) -> bool:
-            if not string.upper().startswith("FRACTAL") or any(
-                s in string.upper()
-                for s in ["PASSWORD", "SECRET", "PWD", "TOKEN", "KEY"]
-            ):
-                return True
-            else:
-                return False
-
-        sanitized_settings = {}
-        for k, v in self.model_dump().items():
-            if _must_be_sanitized(k):
-                sanitized_settings[k] = "***"
-            else:
-                sanitized_settings[k] = v
-
-        return sanitized_settings
-
 
 def get_settings(settings=Settings()) -> Settings:
     return settings

fractal_server/images/tools.py

@@ -121,3 +121,26 @@ def merge_type_filters(
     merged_dict = task_input_types
     merged_dict.update(wftask_type_filters)
     return merged_dict
+
+
+def aggregate_attributes(images: list[dict[str, Any]]) -> dict[str, list[Any]]:
+    """
+    Given a list of images, this function returns a dictionary of all image
+    attributes, each mapped to a list of present values.
+    """
+    attributes = {}
+    for image in images:
+        for k, v in image["attributes"].items():
+            attributes.setdefault(k, []).append(v)
+        for k, v in attributes.items():
+            attributes[k] = list(set(v))
+    return attributes
+
+
+def aggregate_types(images: list[dict[str, Any]]) -> list[str]:
+    """
+    Given a list of images, this function returns a list of all image types.
+    """
+    return list(
+        set(type for image in images for type in image["types"].keys())
+    )

fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py

@@ -0,0 +1,50 @@
+"""Drop dataset filters
+
+Revision ID: 47351f8c7ebc
+Revises: fbce16ff4e47
+Create Date: 2025-03-26 11:10:17.869028
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "47351f8c7ebc"
+down_revision = "fbce16ff4e47"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_column("type_filters")
+        batch_op.drop_column("attribute_filters")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "type_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'{}'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py

@@ -0,0 +1,250 @@
+"""Set ondelete
+
+Revision ID: 9db60297b8b2
+Revises: e81103413827
+Create Date: 2025-04-07 13:13:14.596394
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "9db60297b8b2"
+down_revision = "e81103413827"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_datasetv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_datasetv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_jobv2_dataset_id_datasetv2", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_jobv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_jobv2_workflow_id_workflowv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_workflow_id_workflowv2"),
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_dataset_id_datasetv2"),
+            "datasetv2",
+            ["dataset_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_jobv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_linkusergroup_user_id_user_oauth", type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            "fk_linkusergroup_group_id_usergroup", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_linkusergroup_group_id_usergroup"),
+            "usergroup",
+            ["group_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_linkusergroup_user_id_user_oauth"),
+            "user_oauth",
+            ["user_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("taskgroupactivityv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2",
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2"),
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_taskgroupv2_user_group_id_usergroup", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskgroupv2_user_group_id_usergroup"),
+            "usergroup",
+            ["user_group_id"],
+            ["id"],
+            ondelete="SET NULL",
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_workflowtaskv2_workflow_id_workflowv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_workflowtaskv2_workflow_id_workflowv2"),
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("workflowv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "fk_workflowv2_project_id_projectv2", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_workflowv2_project_id_projectv2"),
+            "projectv2",
+            ["project_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_workflowv2_project_id_projectv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_workflowv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_workflowtaskv2_workflow_id_workflowv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_workflowtaskv2_workflow_id_workflowv2",
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupv2_user_group_id_usergroup"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_taskgroupv2_user_group_id_usergroup",
+            "usergroup",
+            ["user_group_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("taskgroupactivityv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_taskgroupactivityv2_taskgroupv2_id_taskgroupv2",
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_linkusergroup_user_id_user_oauth"),
+            type_="foreignkey",
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_linkusergroup_group_id_usergroup"),
+            type_="foreignkey",
+        )
+        batch_op.create_foreign_key(
+            "fk_linkusergroup_group_id_usergroup",
+            "usergroup",
+            ["group_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_linkusergroup_user_id_user_oauth",
+            "user_oauth",
+            ["user_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_project_id_projectv2"), type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_dataset_id_datasetv2"), type_="foreignkey"
+        )
+        batch_op.drop_constraint(
+            batch_op.f("fk_jobv2_workflow_id_workflowv2"), type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_workflow_id_workflowv2",
+            "workflowv2",
+            ["workflow_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+        batch_op.create_foreign_key(
+            "fk_jobv2_dataset_id_datasetv2",
+            "datasetv2",
+            ["dataset_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_datasetv2_project_id_projectv2"), type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            "fk_datasetv2_project_id_projectv2",
+            "projectv2",
+            ["project_id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###