fractal-server 2.12.0a0__py3-none-any.whl → 2.12.1__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.12.0a0"
+__VERSION__ = "2.12.1"

fractal_server/__main__.py

@@ -63,45 +63,18 @@ update_db_data_parser = subparsers.add_parser(
     description="Apply data-migration script to an existing database.",
 )
 
-# fractalctl email-settings
-email_settings_parser = subparsers.add_parser(
-    "email-settings",
+# fractalctl encrypt-email-password
+encrypt_email_password_parser = subparsers.add_parser(
+    "encrypt-email-password",
     description=(
         "Generate valid values for environment variables "
-        "`FRACTAL_EMAIL_SETTINGS` and `FRACTAL_EMAIL_SETTINGS_KEY`."
+        "FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY."
     ),
 )
-email_settings_parser.add_argument(
-    "sender",
-    type=str,
-    help="Email of the sender",
-)
-email_settings_parser.add_argument(
-    "server",
-    type=str,
-    help="SMPT server used to send emails",
-)
-email_settings_parser.add_argument(
-    "port",
-    type=int,
-    help="Port of the SMPT server",
-)
-email_settings_parser.add_argument(
-    "instance",
-    type=str,
-    help="Name of the Fractal instance sending emails",
-)
-email_settings_parser.add_argument(
-    "--skip-starttls",
-    action="store_true",
-    default=False,
-    help="If set, skip the execution of `starttls` when sending emails",
-)
 
 
 def save_openapi(dest="openapi.json"):
     from fractal_server.main import start_application
-    import json
 
     app = start_application()
     openapi_schema = app.openapi()
@@ -129,6 +102,11 @@ def set_db(skip_init_data: bool = False):
     from pathlib import Path
     import fractal_server
 
+    # Check settings
+    settings = Inject(get_settings)
+    settings.check_db()
+
+    # Perform migrations
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
     alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
     print(f"START: Run alembic.config, with argv={alembic_args}")
@@ -138,12 +116,10 @@ def set_db(skip_init_data: bool = False):
     if skip_init_data:
         return
 
-    # Insert default group
+    # Create default group and user
     print()
     _create_first_group()
     print()
-    # NOTE: It will be fixed with #1739
-    settings = Inject(get_settings)
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
@@ -224,31 +200,15 @@ def update_db_data():
     current_update_db_data_module.fix_db()
 
 
-def print_mail_settings(
-    sender: str,
-    server: str,
-    port: int,
-    instance: str,
-    skip_starttls: bool,
-):
+def print_encrypted_password():
     from cryptography.fernet import Fernet
 
-    password = input(f"Insert email password for sender '{sender}': ")
+    password = input("Insert email password: ").encode("utf-8")
     key = Fernet.generate_key().decode("utf-8")
-    fractal_mail_settings = json.dumps(
-        dict(
-            sender=sender,
-            password=password,
-            smtp_server=server,
-            port=port,
-            instance_name=instance,
-            use_starttls=(not skip_starttls),
-        )
-    ).encode("utf-8")
-    email_settings = Fernet(key).encrypt(fractal_mail_settings).decode("utf-8")
+    encrypted_password = Fernet(key).encrypt(password).decode("utf-8")
 
-    print(f"\nFRACTAL_EMAIL_SETTINGS: {email_settings}")
-    print(f"FRACTAL_EMAIL_SETTINGS_KEY: {key}")
+    print(f"\nFRACTAL_EMAIL_PASSWORD={encrypted_password}")
+    print(f"FRACTAL_EMAIL_PASSWORD_KEY={key}")
 
 
 def run():
@@ -267,14 +227,8 @@ def run():
             port=args.port,
             reload=args.reload,
         )
-    elif args.cmd == "email-settings":
-        print_mail_settings(
-            sender=args.sender,
-            server=args.server,
-            port=args.port,
-            instance=args.instance,
-            skip_starttls=args.skip_starttls,
-        )
+    elif args.cmd == "encrypt-email-password":
+        print_encrypted_password()
     else:
         sys.exit(f"Error: invalid command '{args.cmd}'.")
 

fractal_server/app/routes/api/v2/images.py

@@ -118,7 +118,6 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    use_dataset_filters: bool = False,  # query param
     page: int = 1,  # query param
     page_size: Optional[int] = None,  # query param
     query: Optional[ImageQuery] = None,  # body
@@ -138,17 +137,6 @@ async def query_dataset_images(
     dataset = output["dataset"]
     images = dataset.images
 
-    if use_dataset_filters is True:
-        images = [
-            image
-            for image in images
-            if match_filter(
-                image=image,
-                type_filters=dataset.type_filters,
-                attribute_filters=dataset.attribute_filters,
-            )
-        ]
-
     attributes = {}
     for image in images:
         for k, v in image["attributes"].items():

fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py

@@ -88,7 +88,6 @@ class SlurmJob:
         self,
         num_tasks_tot: int,
         slurm_config: SlurmConfig,
-        workflow_task_file_prefix: Optional[str] = None,
         slurm_file_prefix: Optional[str] = None,
         wftask_file_prefixes: Optional[tuple[str, ...]] = None,
         single_task_submission: bool = False,

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -62,7 +62,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         shutdown_file:
         python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
         wait_thread_cls: Class for waiting thread
-        keep_pickle_files:
         workflow_dir_local:
             Directory for both the cfut/SLURM and fractal-server files and logs
         workflow_dir_remote:
@@ -84,7 +83,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
     python_remote: str
 
     wait_thread_cls = FractalSlurmWaitThread
-    keep_pickle_files: bool
 
     common_script_lines: list[str]
     slurm_account: Optional[str]
@@ -100,8 +98,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         # Folders and files
         workflow_dir_local: Path,
         workflow_dir_remote: Path,
-        # Runner options
-        keep_pickle_files: bool = False,
         # Monitoring options
         slurm_poll_interval: Optional[int] = None,
         # SLURM submission script options
@@ -120,7 +116,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             fractal_ssh:
             workflow_dir_local:
             workflow_dir_remote:
-            keep_pickle_files:
             slurm_poll_interval:
             common_script_lines:
             slurm_account:
@@ -194,7 +189,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             raise e
 
         # Set/initialize some more options
-        self.keep_pickle_files = keep_pickle_files
         self.map_jobid_to_slurm_files_local = {}
 
     def _validate_common_script_lines(self):
@@ -385,9 +379,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         args_batches = []
         batch_size = tasks_per_job
         for ind_chunk in range(0, tot_tasks, batch_size):
-            args_batches.append(
-                list_args[ind_chunk : ind_chunk + batch_size]  # noqa
-            )
+            args_batches.append(list_args[ind_chunk : ind_chunk + batch_size])
         if len(args_batches) != math.ceil(tot_tasks / tasks_per_job):
             raise RuntimeError("Something wrong here while batching tasks")
 
@@ -536,10 +528,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             _prefixes = []
             _subfolder_names = []
             for component in components:
-                if isinstance(component, dict):
+                # In Fractal, `component` is `dict` by construction (e.g.
+                # `component = {"zarr_url": "/something", "param": 1}``). The
+                # try/except covers the case of e.g. `executor.map([1, 2])`,
+                # which is useful for testing.
+                try:
                     actual_component = component.get(_COMPONENT_KEY_, None)
-                else:
-                    actual_component = component
+                except AttributeError:
+                    actual_component = str(component)
+
                 _task_file_paths = get_task_file_paths(
                     workflow_dir_local=task_files.workflow_dir_local,
                     workflow_dir_remote=task_files.workflow_dir_remote,
@@ -898,12 +895,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 pass
         for job_id in remaining_job_ids:
             self._cleanup(job_id)
-        if not self.keep_pickle_files:
-            for job in remaining_jobs:
-                for path in job.output_pickle_files_local:
-                    path.unlink()
-                for path in job.input_pickle_files_local:
-                    path.unlink()
+        for job in remaining_jobs:
+            for path in job.output_pickle_files_local:
+                path.unlink()
+            for path in job.input_pickle_files_local:
+                path.unlink()
 
     def _completion(self, job_ids: list[str]) -> None:
         """
@@ -998,8 +994,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 f"Future {future} (SLURM job ID: {job_id}) "
                                 "was already cancelled."
                             )
-                            if not self.keep_pickle_files:
-                                in_path.unlink()
+                            in_path.unlink()
                             self._cleanup(job_id)
                             self._handle_remaining_jobs(
                                 remaining_futures=remaining_futures,
@@ -1059,17 +1054,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                     remaining_job_ids=remaining_job_ids,
                                 )
                                 return
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
+                        out_path.unlink()
                     except InvalidStateError:
                         logger.warning(
                             f"Future {future} (SLURM job ID: {job_id}) was "
                             "already cancelled, exit from "
                             "FractalSlurmSSHExecutor._completion."
                         )
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
-                            in_path.unlink()
+                        out_path.unlink()
+                        in_path.unlink()
 
                         self._cleanup(job_id)
                         self._handle_remaining_jobs(
@@ -1079,8 +1072,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                         return
 
                     # Clean up input pickle file
-                    if not self.keep_pickle_files:
-                        in_path.unlink()
+                    in_path.unlink()
                 self._cleanup(job_id)
                 if job.single_task_submission:
                     future.set_result(outputs[0])
@@ -1207,8 +1199,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         script_lines = slurm_config.sort_script_lines(script_lines)
         logger.debug(script_lines)
 
-        # Always print output of `pwd`
-        script_lines.append('echo "Working directory (pwd): `pwd`"\n')
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
 
         # Complete script preamble
         script_lines.append("\n")

fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -10,6 +10,7 @@
 #
 # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
 # University of Zurich
+import json
 import math
 import shlex
 import subprocess  # nosec
@@ -161,7 +162,6 @@ class SlurmJob:
         self,
         num_tasks_tot: int,
         slurm_config: SlurmConfig,
-        workflow_task_file_prefix: Optional[str] = None,
         slurm_file_prefix: Optional[str] = None,
         wftask_file_prefixes: Optional[tuple[str, ...]] = None,
         single_task_submission: bool = False,
@@ -219,7 +219,6 @@ class FractalSlurmExecutor(SlurmExecutor):
     workflow_dir_local: Path
     workflow_dir_remote: Path
     map_jobid_to_slurm_files: dict[str, tuple[str, str, str]]
-    keep_pickle_files: bool
     slurm_account: Optional[str]
     jobs: dict[str, tuple[Future, SlurmJob]]
 
@@ -232,7 +231,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         user_cache_dir: Optional[str] = None,
         common_script_lines: Optional[list[str]] = None,
         slurm_poll_interval: Optional[int] = None,
-        keep_pickle_files: bool = False,
         slurm_account: Optional[str] = None,
         *args,
         **kwargs,
@@ -253,11 +251,18 @@ class FractalSlurmExecutor(SlurmExecutor):
         # raised within `__init__`).
         self.wait_thread.shutdown_callback = self.shutdown
 
-        self.keep_pickle_files = keep_pickle_files
         self.slurm_user = slurm_user
         self.slurm_account = slurm_account
 
         self.common_script_lines = common_script_lines or []
+        settings = Inject(get_settings)
+
+        if settings.FRACTAL_SLURM_WORKER_PYTHON is not None:
+            try:
+                self.check_remote_python_interpreter()
+            except Exception as e:
+                self._stop_and_join_wait_thread()
+                raise RuntimeError(f"Original error {str(e)}")
 
         # Check that SLURM account is not set here
         try:
@@ -289,7 +294,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         # Set the attribute slurm_poll_interval for self.wait_thread (see
         # cfut.SlurmWaitThread)
         if not slurm_poll_interval:
-            settings = Inject(get_settings)
             slurm_poll_interval = settings.FRACTAL_SLURM_POLL_INTERVAL
         self.wait_thread.slurm_poll_interval = slurm_poll_interval
         self.wait_thread.slurm_user = self.slurm_user
@@ -608,11 +612,14 @@ class FractalSlurmExecutor(SlurmExecutor):
             _prefixes = []
             _subfolder_names = []
             for component in components:
-                if isinstance(component, dict):
-                    # This is needed for V2
+                # In Fractal, `component` is a `dict` by construction (e.g.
+                # `component = {"zarr_url": "/something", "param": 1}``). The
+                # try/except covers the case of e.g. `executor.map([1, 2])`,
+                # which is useful for testing.
+                try:
                     actual_component = component.get(_COMPONENT_KEY_, None)
-                else:
-                    actual_component = component
+                except AttributeError:
+                    actual_component = str(component)
                 _task_file_paths = get_task_file_paths(
                     workflow_dir_local=task_files.workflow_dir_local,
                     workflow_dir_remote=task_files.workflow_dir_remote,
@@ -864,8 +871,7 @@ class FractalSlurmExecutor(SlurmExecutor):
                             " cancelled, exit from"
                             " FractalSlurmExecutor._completion."
                         )
-                        if not self.keep_pickle_files:
-                            in_path.unlink()
+                        in_path.unlink()
                         self._cleanup(jobid)
                         return
 
@@ -907,23 +913,20 @@ class FractalSlurmExecutor(SlurmExecutor):
                             exc = TaskExecutionError(proxy.tb, **kwargs)
                             fut.set_exception(exc)
                             return
-                    if not self.keep_pickle_files:
-                        out_path.unlink()
+                    out_path.unlink()
                 except InvalidStateError:
                     logger.warning(
                         f"Future {fut} (SLURM job ID: {jobid}) was already"
                         " cancelled, exit from"
                         " FractalSlurmExecutor._completion."
                     )
-                    if not self.keep_pickle_files:
-                        out_path.unlink()
-                        in_path.unlink()
+                    out_path.unlink()
+                    in_path.unlink()
                     self._cleanup(jobid)
                     return
 
                 # Clean up input pickle file
-                if not self.keep_pickle_files:
-                    in_path.unlink()
+                in_path.unlink()
             self._cleanup(jobid)
             if job.single_task_submission:
                 fut.set_result(outputs[0])
@@ -1159,8 +1162,10 @@ class FractalSlurmExecutor(SlurmExecutor):
         script_lines = slurm_config.sort_script_lines(script_lines)
         logger.debug(script_lines)
 
-        # Always print output of `pwd`
-        script_lines.append('echo "Working directory (pwd): `pwd`"\n')
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
 
         # Complete script preamble
         script_lines.append("\n")
@@ -1247,3 +1252,29 @@ class FractalSlurmExecutor(SlurmExecutor):
         )
         self._stop_and_join_wait_thread()
         logger.debug("[FractalSlurmExecutor.__exit__] End")
+
+    def check_remote_python_interpreter(self):
+        """
+        Check fractal-server version on the _remote_ Python interpreter.
+        """
+        settings = Inject(get_settings)
+        output = _subprocess_run_or_raise(
+            (
+                f"{settings.FRACTAL_SLURM_WORKER_PYTHON} "
+                "-m fractal_server.app.runner.versions"
+            )
+        )
+        runner_version = json.loads(output.stdout.strip("\n"))[
+            "fractal_server"
+        ]
+
+        if runner_version != __VERSION__:
+            error_msg = (
+                "Fractal-server version mismatch.\n"
+                "Local interpreter: "
+                f"({sys.executable}): {__VERSION__}.\n"
+                "Remote interpreter: "
+                f"({settings.FRACTAL_SLURM_WORKER_PYTHON}): {runner_version}."
+            )
+            logger.error(error_msg)
+            raise ValueError(error_msg)

fractal_server/app/schemas/_validators.py

@@ -48,25 +48,6 @@ def valdict_keys(attribute: str):
     return val
 
 
-def valint(attribute: str, min_val: int = 1):
-    """
-    Check that an integer attribute (e.g. if it is meant to be the ID of a
-    database entry) is greater or equal to min_val.
-    """
-
-    def val(integer: Optional[int]) -> Optional[int]:
-        if integer is None:
-            raise ValueError(f"Integer attribute '{attribute}' cannot be None")
-        if integer < min_val:
-            raise ValueError(
-                f"Integer attribute '{attribute}' cannot be less than "
-                f"{min_val} (given {integer})"
-            )
-        return integer
-
-    return val
-
-
 def val_absolute_path(attribute: str, accept_none: bool = False):
     """
     Check that a string attribute is an absolute path

fractal_server/app/security/__init__.py

@@ -252,15 +252,18 @@ class UserManager(IntegerIDMixin, BaseUserManager[UserOAuth, int]):
             # Send mail section
             settings = Inject(get_settings)
 
-            if this_user.oauth_accounts and settings.MAIL_SETTINGS is not None:
+            if (
+                this_user.oauth_accounts
+                and settings.email_settings is not None
+            ):
                 try:
                     logger.info(
                         "START sending email about new signup to "
-                        f"{settings.MAIL_SETTINGS.recipients}."
+                        f"{settings.email_settings.recipients}."
                     )
                     mail_new_oauth_signup(
                         msg=f"New user registered: '{this_user.email}'.",
-                        mail_settings=settings.MAIL_SETTINGS,
+                        email_settings=settings.email_settings,
                     )
                     logger.info("END sending email about new signup.")
                 except Exception as e:

fractal_server/app/security/signup_email.py

@@ -2,38 +2,47 @@ import smtplib
 from email.message import EmailMessage
 from email.utils import formataddr
 
+from cryptography.fernet import Fernet
+
 from fractal_server.config import MailSettings
 
 
-def mail_new_oauth_signup(msg: str, mail_settings: MailSettings):
+def mail_new_oauth_signup(msg: str, email_settings: MailSettings):
     """
     Send an email using the specified settings to notify a new OAuth signup.
     """
 
     mail_msg = EmailMessage()
     mail_msg.set_content(msg)
-    mail_msg["From"] = formataddr((mail_settings.sender, mail_settings.sender))
+    mail_msg["From"] = formataddr(
+        (email_settings.sender, email_settings.sender)
+    )
     mail_msg["To"] = ", ".join(
         [
             formataddr((recipient, recipient))
-            for recipient in mail_settings.recipients
+            for recipient in email_settings.recipients
         ]
     )
     mail_msg[
         "Subject"
-    ] = f"[Fractal, {mail_settings.instance_name}] New OAuth signup"
+    ] = f"[Fractal, {email_settings.instance_name}] New OAuth signup"
 
-    with smtplib.SMTP(mail_settings.smtp_server, mail_settings.port) as server:
+    with smtplib.SMTP(
+        email_settings.smtp_server, email_settings.port
+    ) as server:
         server.ehlo()
-        if mail_settings.use_starttls:
+        if email_settings.use_starttls:
             server.starttls()
             server.ehlo()
-
-        server.login(
-            user=mail_settings.sender, password=mail_settings.password
-        )
+        if email_settings.use_login:
+            password = (
+                Fernet(email_settings.encryption_key)
+                .decrypt(email_settings.encrypted_password)
+                .decode("utf-8")
+            )
+            server.login(user=email_settings.sender, password=password)
         server.sendmail(
-            from_addr=mail_settings.sender,
-            to_addrs=mail_settings.recipients,
+            from_addr=email_settings.sender,
+            to_addrs=email_settings.recipients,
             msg=mail_msg.as_string(),
         )