fractal-server 2.12.0a1__tar.gz → 2.12.1__tar.gz

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.12.0a1
+Version: 2.12.1
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause
@@ -27,7 +27,7 @@ Requires-Dist: pydantic (>=1.10.8,<2)
 Requires-Dist: python-dotenv (>=1.0.0,<1.1.0)
 Requires-Dist: sqlalchemy[asyncio] (>=2.0.23,<2.1)
 Requires-Dist: sqlmodel (==0.0.21)
-Requires-Dist: uvicorn (==0.29.0)
+Requires-Dist: uvicorn (>=0.29.0,<0.35.0)
 Requires-Dist: uvicorn-worker (==0.2.0)
 Project-URL: Documentation, https://fractal-analytics-platform.github.io/fractal-server
 Project-URL: Repository, https://github.com/fractal-analytics-platform/fractal-server

fractal_server-2.12.1/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.12.1"

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/__main__.py

@@ -63,45 +63,18 @@ update_db_data_parser = subparsers.add_parser(
     description="Apply data-migration script to an existing database.",
 )
 
-# fractalctl email-settings
-email_settings_parser = subparsers.add_parser(
-    "email-settings",
+# fractalctl encrypt-email-password
+encrypt_email_password_parser = subparsers.add_parser(
+    "encrypt-email-password",
     description=(
         "Generate valid values for environment variables "
-        "`FRACTAL_EMAIL_SETTINGS` and `FRACTAL_EMAIL_SETTINGS_KEY`."
+        "FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY."
     ),
 )
-email_settings_parser.add_argument(
-    "sender",
-    type=str,
-    help="Email of the sender",
-)
-email_settings_parser.add_argument(
-    "server",
-    type=str,
-    help="SMPT server used to send emails",
-)
-email_settings_parser.add_argument(
-    "port",
-    type=int,
-    help="Port of the SMPT server",
-)
-email_settings_parser.add_argument(
-    "instance",
-    type=str,
-    help="Name of the Fractal instance sending emails",
-)
-email_settings_parser.add_argument(
-    "--skip-starttls",
-    action="store_true",
-    default=False,
-    help="If set, skip the execution of `starttls` when sending emails",
-)
 
 
 def save_openapi(dest="openapi.json"):
     from fractal_server.main import start_application
-    import json
 
     app = start_application()
     openapi_schema = app.openapi()
@@ -129,6 +102,11 @@ def set_db(skip_init_data: bool = False):
     from pathlib import Path
     import fractal_server
 
+    # Check settings
+    settings = Inject(get_settings)
+    settings.check_db()
+
+    # Perform migrations
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
     alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
     print(f"START: Run alembic.config, with argv={alembic_args}")
@@ -138,12 +116,10 @@ def set_db(skip_init_data: bool = False):
     if skip_init_data:
         return
 
-    # Insert default group
+    # Create default group and user
     print()
     _create_first_group()
     print()
-    # NOTE: It will be fixed with #1739
-    settings = Inject(get_settings)
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
@@ -224,31 +200,15 @@ def update_db_data():
     current_update_db_data_module.fix_db()
 
 
-def print_mail_settings(
-    sender: str,
-    server: str,
-    port: int,
-    instance: str,
-    skip_starttls: bool,
-):
+def print_encrypted_password():
     from cryptography.fernet import Fernet
 
-    password = input(f"Insert email password for sender '{sender}': ")
+    password = input("Insert email password: ").encode("utf-8")
     key = Fernet.generate_key().decode("utf-8")
-    fractal_mail_settings = json.dumps(
-        dict(
-            sender=sender,
-            password=password,
-            smtp_server=server,
-            port=port,
-            instance_name=instance,
-            use_starttls=(not skip_starttls),
-        )
-    ).encode("utf-8")
-    email_settings = Fernet(key).encrypt(fractal_mail_settings).decode("utf-8")
+    encrypted_password = Fernet(key).encrypt(password).decode("utf-8")
 
-    print(f"\nFRACTAL_EMAIL_SETTINGS: {email_settings}")
-    print(f"FRACTAL_EMAIL_SETTINGS_KEY: {key}")
+    print(f"\nFRACTAL_EMAIL_PASSWORD={encrypted_password}")
+    print(f"FRACTAL_EMAIL_PASSWORD_KEY={key}")
 
 
 def run():
@@ -267,14 +227,8 @@ def run():
             port=args.port,
             reload=args.reload,
         )
-    elif args.cmd == "email-settings":
-        print_mail_settings(
-            sender=args.sender,
-            server=args.server,
-            port=args.port,
-            instance=args.instance,
-            skip_starttls=args.skip_starttls,
-        )
+    elif args.cmd == "encrypt-email-password":
+        print_encrypted_password()
     else:
         sys.exit(f"Error: invalid command '{args.cmd}'.")
 

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/app/routes/api/v2/images.py

@@ -118,7 +118,6 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    use_dataset_filters: bool = False,  # query param
     page: int = 1,  # query param
     page_size: Optional[int] = None,  # query param
     query: Optional[ImageQuery] = None,  # body
@@ -138,17 +137,6 @@ async def query_dataset_images(
     dataset = output["dataset"]
     images = dataset.images
 
-    if use_dataset_filters is True:
-        images = [
-            image
-            for image in images
-            if match_filter(
-                image=image,
-                type_filters=dataset.type_filters,
-                attribute_filters=dataset.attribute_filters,
-            )
-        ]
-
     attributes = {}
     for image in images:
         for k, v in image["attributes"].items():

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py

@@ -88,7 +88,6 @@ class SlurmJob:
         self,
         num_tasks_tot: int,
         slurm_config: SlurmConfig,
-        workflow_task_file_prefix: Optional[str] = None,
         slurm_file_prefix: Optional[str] = None,
         wftask_file_prefixes: Optional[tuple[str, ...]] = None,
         single_task_submission: bool = False,

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -62,7 +62,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         shutdown_file:
         python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
         wait_thread_cls: Class for waiting thread
-        keep_pickle_files:
         workflow_dir_local:
             Directory for both the cfut/SLURM and fractal-server files and logs
         workflow_dir_remote:
@@ -84,7 +83,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
     python_remote: str
 
     wait_thread_cls = FractalSlurmWaitThread
-    keep_pickle_files: bool
 
     common_script_lines: list[str]
     slurm_account: Optional[str]
@@ -100,8 +98,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         # Folders and files
         workflow_dir_local: Path,
         workflow_dir_remote: Path,
-        # Runner options
-        keep_pickle_files: bool = False,
         # Monitoring options
         slurm_poll_interval: Optional[int] = None,
         # SLURM submission script options
@@ -120,7 +116,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             fractal_ssh:
             workflow_dir_local:
             workflow_dir_remote:
-            keep_pickle_files:
             slurm_poll_interval:
             common_script_lines:
             slurm_account:
@@ -194,7 +189,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             raise e
 
         # Set/initialize some more options
-        self.keep_pickle_files = keep_pickle_files
         self.map_jobid_to_slurm_files_local = {}
 
     def _validate_common_script_lines(self):
@@ -901,12 +895,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 pass
         for job_id in remaining_job_ids:
             self._cleanup(job_id)
-        if not self.keep_pickle_files:
-            for job in remaining_jobs:
-                for path in job.output_pickle_files_local:
-                    path.unlink()
-                for path in job.input_pickle_files_local:
-                    path.unlink()
+        for job in remaining_jobs:
+            for path in job.output_pickle_files_local:
+                path.unlink()
+            for path in job.input_pickle_files_local:
+                path.unlink()
 
     def _completion(self, job_ids: list[str]) -> None:
         """
@@ -1001,8 +994,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 f"Future {future} (SLURM job ID: {job_id}) "
                                 "was already cancelled."
                             )
-                            if not self.keep_pickle_files:
-                                in_path.unlink()
+                            in_path.unlink()
                             self._cleanup(job_id)
                             self._handle_remaining_jobs(
                                 remaining_futures=remaining_futures,
@@ -1062,17 +1054,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                     remaining_job_ids=remaining_job_ids,
                                 )
                                 return
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
+                        out_path.unlink()
                     except InvalidStateError:
                         logger.warning(
                             f"Future {future} (SLURM job ID: {job_id}) was "
                             "already cancelled, exit from "
                             "FractalSlurmSSHExecutor._completion."
                         )
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
-                            in_path.unlink()
+                        out_path.unlink()
+                        in_path.unlink()
 
                         self._cleanup(job_id)
                         self._handle_remaining_jobs(
@@ -1082,8 +1072,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                         return
 
                     # Clean up input pickle file
-                    if not self.keep_pickle_files:
-                        in_path.unlink()
+                    in_path.unlink()
                 self._cleanup(job_id)
                 if job.single_task_submission:
                     future.set_result(outputs[0])
@@ -1210,8 +1199,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         script_lines = slurm_config.sort_script_lines(script_lines)
         logger.debug(script_lines)
 
-        # Always print output of `pwd`
-        script_lines.append('echo "Working directory (pwd): `pwd`"\n')
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
 
         # Complete script preamble
         script_lines.append("\n")

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -10,6 +10,7 @@
 #
 # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
 # University of Zurich
+import json
 import math
 import shlex
 import subprocess  # nosec
@@ -161,7 +162,6 @@ class SlurmJob:
         self,
         num_tasks_tot: int,
         slurm_config: SlurmConfig,
-        workflow_task_file_prefix: Optional[str] = None,
         slurm_file_prefix: Optional[str] = None,
         wftask_file_prefixes: Optional[tuple[str, ...]] = None,
         single_task_submission: bool = False,
@@ -219,7 +219,6 @@ class FractalSlurmExecutor(SlurmExecutor):
     workflow_dir_local: Path
     workflow_dir_remote: Path
     map_jobid_to_slurm_files: dict[str, tuple[str, str, str]]
-    keep_pickle_files: bool
     slurm_account: Optional[str]
     jobs: dict[str, tuple[Future, SlurmJob]]
 
@@ -232,7 +231,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         user_cache_dir: Optional[str] = None,
         common_script_lines: Optional[list[str]] = None,
         slurm_poll_interval: Optional[int] = None,
-        keep_pickle_files: bool = False,
         slurm_account: Optional[str] = None,
         *args,
         **kwargs,
@@ -253,11 +251,18 @@ class FractalSlurmExecutor(SlurmExecutor):
         # raised within `__init__`).
         self.wait_thread.shutdown_callback = self.shutdown
 
-        self.keep_pickle_files = keep_pickle_files
         self.slurm_user = slurm_user
         self.slurm_account = slurm_account
 
         self.common_script_lines = common_script_lines or []
+        settings = Inject(get_settings)
+
+        if settings.FRACTAL_SLURM_WORKER_PYTHON is not None:
+            try:
+                self.check_remote_python_interpreter()
+            except Exception as e:
+                self._stop_and_join_wait_thread()
+                raise RuntimeError(f"Original error {str(e)}")
 
         # Check that SLURM account is not set here
         try:
@@ -289,7 +294,6 @@ class FractalSlurmExecutor(SlurmExecutor):
         # Set the attribute slurm_poll_interval for self.wait_thread (see
         # cfut.SlurmWaitThread)
         if not slurm_poll_interval:
-            settings = Inject(get_settings)
             slurm_poll_interval = settings.FRACTAL_SLURM_POLL_INTERVAL
         self.wait_thread.slurm_poll_interval = slurm_poll_interval
         self.wait_thread.slurm_user = self.slurm_user
@@ -608,7 +612,14 @@ class FractalSlurmExecutor(SlurmExecutor):
             _prefixes = []
             _subfolder_names = []
             for component in components:
-                actual_component = component.get(_COMPONENT_KEY_, None)
+                # In Fractal, `component` is a `dict` by construction (e.g.
+                # `component = {"zarr_url": "/something", "param": 1}``). The
+                # try/except covers the case of e.g. `executor.map([1, 2])`,
+                # which is useful for testing.
+                try:
+                    actual_component = component.get(_COMPONENT_KEY_, None)
+                except AttributeError:
+                    actual_component = str(component)
                 _task_file_paths = get_task_file_paths(
                     workflow_dir_local=task_files.workflow_dir_local,
                     workflow_dir_remote=task_files.workflow_dir_remote,
@@ -860,8 +871,7 @@ class FractalSlurmExecutor(SlurmExecutor):
                             " cancelled, exit from"
                             " FractalSlurmExecutor._completion."
                         )
-                        if not self.keep_pickle_files:
-                            in_path.unlink()
+                        in_path.unlink()
                         self._cleanup(jobid)
                         return
 
@@ -903,23 +913,20 @@ class FractalSlurmExecutor(SlurmExecutor):
                             exc = TaskExecutionError(proxy.tb, **kwargs)
                             fut.set_exception(exc)
                             return
-                    if not self.keep_pickle_files:
-                        out_path.unlink()
+                    out_path.unlink()
                 except InvalidStateError:
                     logger.warning(
                         f"Future {fut} (SLURM job ID: {jobid}) was already"
                         " cancelled, exit from"
                         " FractalSlurmExecutor._completion."
                     )
-                    if not self.keep_pickle_files:
-                        out_path.unlink()
-                        in_path.unlink()
+                    out_path.unlink()
+                    in_path.unlink()
                     self._cleanup(jobid)
                     return
 
                 # Clean up input pickle file
-                if not self.keep_pickle_files:
-                    in_path.unlink()
+                in_path.unlink()
             self._cleanup(jobid)
             if job.single_task_submission:
                 fut.set_result(outputs[0])
@@ -1155,8 +1162,10 @@ class FractalSlurmExecutor(SlurmExecutor):
         script_lines = slurm_config.sort_script_lines(script_lines)
         logger.debug(script_lines)
 
-        # Always print output of `pwd`
-        script_lines.append('echo "Working directory (pwd): `pwd`"\n')
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
 
         # Complete script preamble
         script_lines.append("\n")
@@ -1243,3 +1252,29 @@ class FractalSlurmExecutor(SlurmExecutor):
         )
         self._stop_and_join_wait_thread()
         logger.debug("[FractalSlurmExecutor.__exit__] End")
+
+    def check_remote_python_interpreter(self):
+        """
+        Check fractal-server version on the _remote_ Python interpreter.
+        """
+        settings = Inject(get_settings)
+        output = _subprocess_run_or_raise(
+            (
+                f"{settings.FRACTAL_SLURM_WORKER_PYTHON} "
+                "-m fractal_server.app.runner.versions"
+            )
+        )
+        runner_version = json.loads(output.stdout.strip("\n"))[
+            "fractal_server"
+        ]
+
+        if runner_version != __VERSION__:
+            error_msg = (
+                "Fractal-server version mismatch.\n"
+                "Local interpreter: "
+                f"({sys.executable}): {__VERSION__}.\n"
+                "Remote interpreter: "
+                f"({settings.FRACTAL_SLURM_WORKER_PYTHON}): {runner_version}."
+            )
+            logger.error(error_msg)
+            raise ValueError(error_msg)

{fractal_server-2.12.0a1 → fractal_server-2.12.1}/fractal_server/app/security/__init__.py

@@ -252,15 +252,18 @@ class UserManager(IntegerIDMixin, BaseUserManager[UserOAuth, int]):
             # Send mail section
             settings = Inject(get_settings)
 
-            if this_user.oauth_accounts and settings.MAIL_SETTINGS is not None:
+            if (
+                this_user.oauth_accounts
+                and settings.email_settings is not None
+            ):
                 try:
                     logger.info(
                         "START sending email about new signup to "
-                        f"{settings.MAIL_SETTINGS.recipients}."
+                        f"{settings.email_settings.recipients}."
                     )
                     mail_new_oauth_signup(
                         msg=f"New user registered: '{this_user.email}'.",
-                        mail_settings=settings.MAIL_SETTINGS,
+                        email_settings=settings.email_settings,
                     )
                     logger.info("END sending email about new signup.")
                 except Exception as e:

fractal_server-2.12.1/fractal_server/app/security/signup_email.py

@@ -0,0 +1,48 @@
+import smtplib
+from email.message import EmailMessage
+from email.utils import formataddr
+
+from cryptography.fernet import Fernet
+
+from fractal_server.config import MailSettings
+
+
+def mail_new_oauth_signup(msg: str, email_settings: MailSettings):
+    """
+    Send an email using the specified settings to notify a new OAuth signup.
+    """
+
+    mail_msg = EmailMessage()
+    mail_msg.set_content(msg)
+    mail_msg["From"] = formataddr(
+        (email_settings.sender, email_settings.sender)
+    )
+    mail_msg["To"] = ", ".join(
+        [
+            formataddr((recipient, recipient))
+            for recipient in email_settings.recipients
+        ]
+    )
+    mail_msg[
+        "Subject"
+    ] = f"[Fractal, {email_settings.instance_name}] New OAuth signup"
+
+    with smtplib.SMTP(
+        email_settings.smtp_server, email_settings.port
+    ) as server:
+        server.ehlo()
+        if email_settings.use_starttls:
+            server.starttls()
+            server.ehlo()
+        if email_settings.use_login:
+            password = (
+                Fernet(email_settings.encryption_key)
+                .decrypt(email_settings.encrypted_password)
+                .decode("utf-8")
+            )
+            server.login(user=email_settings.sender, password=password)
+        server.sendmail(
+            from_addr=email_settings.sender,
+            to_addrs=email_settings.recipients,
+            msg=mail_msg.as_string(),
+        )