fractal-server 2.7.0a11__py3-none-any.whl → 2.8.0__py3-none-any.whl

fractal_server/tasks/utils.py

@@ -1,4 +1,3 @@
-import re
 from pathlib import Path
 
 from fractal_server.config import get_settings
@@ -42,38 +41,8 @@ def get_collection_log_v1(path: Path) -> str:
     return log
 
 
-def get_collection_log_v2(path: Path) -> str:
-    log_path = get_log_path(path)
-    log = log_path.open().read()
-    return log
-
-
 def get_collection_freeze_v1(venv_path: Path) -> str:
     package_path = get_absolute_venv_path_v1(venv_path)
     freeze_path = get_freeze_path(package_path)
     freeze = freeze_path.open().read()
     return freeze
-
-
-def get_collection_freeze_v2(path: Path) -> str:
-    freeze_path = get_freeze_path(path)
-    freeze = freeze_path.open().read()
-    return freeze
-
-
-def _normalize_package_name(name: str) -> str:
-    """
-    Implement PyPa specifications for package-name normalization
-
-    The name should be lowercased with all runs of the characters `.`, `-`,
-    or `_` replaced with a single `-` character. This can be implemented in
-    Python with the re module.
-    (https://packaging.python.org/en/latest/specifications/name-normalization)
-
-    Args:
-        name: The non-normalized package name.
-
-    Returns:
-        The normalized package name.
-    """
-    return re.sub(r"[-_.]+", "-", name).lower()

fractal_server/tasks/v1/background_operations.py

@@ -7,10 +7,10 @@ from pathlib import Path
 from shutil import rmtree as shell_rmtree
 
 from ...string_tools import slugify_task_name_for_source_v1
-from ..utils import _normalize_package_name
 from ..utils import get_collection_log_v1
 from ..utils import get_collection_path
 from ..utils import get_log_path
+from ..v2.utils_package_names import normalize_package_name
 from ._TaskCollectPip import _TaskCollectPip
 from .utils import _init_venv_v1
 from fractal_server.app.db import DBSyncSession
@@ -23,7 +23,7 @@ from fractal_server.app.schemas.v1 import TaskReadV1
 from fractal_server.logger import close_logger
 from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 async def _pip_install(
@@ -60,12 +60,12 @@ async def _pip_install(
     cmd_install = f"{pip} install {pip_install_str}"
     cmd_inspect = f"{pip} show {task_pkg.package}"
 
-    await execute_command(
+    await execute_command_async(
         cwd=venv_path,
         command=f"{pip} install --upgrade pip",
         logger_name=logger_name,
     )
-    await execute_command(
+    await execute_command_async(
         cwd=venv_path, command=cmd_install, logger_name=logger_name
     )
     if task_pkg.pinned_package_versions:
@@ -82,7 +82,7 @@ async def _pip_install(
                 "Preliminary check: verify that "
                 f"{pinned_pkg_version} is already installed"
             )
-            stdout_inspect = await execute_command(
+            stdout_inspect = await execute_command_async(
                 cwd=venv_path,
                 command=f"{pip} show {pinned_pkg_name}",
                 logger_name=logger_name,
@@ -99,7 +99,7 @@ async def _pip_install(
                     f"({pinned_pkg_version}); "
                     f"install version {pinned_pkg_version}."
                 )
-                await execute_command(
+                await execute_command_async(
                     cwd=venv_path,
                     command=(
                         f"{pip} install "
@@ -114,7 +114,7 @@ async def _pip_install(
                 )
 
     # Extract package installation path from `pip show`
-    stdout_inspect = await execute_command(
+    stdout_inspect = await execute_command_async(
         cwd=venv_path, command=cmd_inspect, logger_name=logger_name
     )
 
@@ -165,8 +165,8 @@ async def _create_venv_install_package(
     """
 
     # Normalize package name
-    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
-    task_pkg.package = _normalize_package_name(task_pkg.package)
+    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
+    task_pkg.package = normalize_package_name(task_pkg.package)
 
     python_bin = await _init_venv_v1(
         path=path,
@@ -192,8 +192,8 @@ async def create_package_environment_pip(
     logger = get_logger(logger_name)
 
     # Normalize package name
-    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
-    task_pkg.package = _normalize_package_name(task_pkg.package)
+    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
+    task_pkg.package = normalize_package_name(task_pkg.package)
 
     # Only proceed if package, version and manifest attributes are set
     task_pkg.check()

fractal_server/tasks/v1/endpoint_operations.py

@@ -4,14 +4,14 @@ from typing import Optional
 from typing import Union
 from zipfile import ZipFile
 
-from ..utils import _normalize_package_name
+from ..v2.utils_package_names import normalize_package_name
 from ._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV1
 from .utils import get_python_interpreter_v1
 from fractal_server.app.schemas.v1 import ManifestV1
 from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
@@ -32,7 +32,7 @@ async def download_package(
     )
     package_and_version = f"{task_pkg.package}{version}"
     cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
-    stdout = await execute_command(command=cmd, cwd=Path("."))
+    stdout = await execute_command_async(command=cmd, cwd=Path("."))
     pkg_file = next(
         line.split()[-1] for line in stdout.split("\n") if "Saved" in line
     )
@@ -123,7 +123,7 @@ def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
         logger.debug("Package name and version read correctly.")
 
     # Normalize package name:
-    pkg_name = _normalize_package_name(pkg_name)
+    pkg_name = normalize_package_name(pkg_name)
 
     info = dict(
         pkg_name=pkg_name,
@@ -148,7 +148,7 @@ def create_package_dir_pip(
             f"Cannot create venv folder for package `{task_pkg.package}` "
             "with `version=None`."
         )
-    normalized_package = _normalize_package_name(task_pkg.package)
+    normalized_package = normalize_package_name(task_pkg.package)
     package_dir = f"{normalized_package}{task_pkg.package_version}"
     venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
     if create:

fractal_server/tasks/v1/utils.py

@@ -2,7 +2,7 @@ from pathlib import Path
 from typing import Optional
 
 from fractal_server.logger import get_logger
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 def get_python_interpreter_v1(version: Optional[str] = None) -> str:
@@ -57,7 +57,7 @@ async def _init_venv_v1(
     logger.debug(f"[_init_venv] {path=}")
     interpreter = get_python_interpreter_v1(version=python_version)
     logger.debug(f"[_init_venv] {interpreter=}")
-    await execute_command(
+    await execute_command_async(
         cwd=path,
         command=f"{interpreter} -m venv venv",
         logger_name=logger_name,

fractal_server/tasks/v2/collection_local.py

@@ -0,0 +1,357 @@
+import json
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from sqlalchemy.orm.attributes import flag_modified
+
+from .database_operations import create_db_tasks_and_update_task_group
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.v2 import CollectionStatusV2
+from fractal_server.app.schemas.v2 import TaskReadV2
+from fractal_server.app.schemas.v2.manifest import ManifestV2
+from fractal_server.config import get_settings
+from fractal_server.logger import get_logger
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import _handle_failure
+from fractal_server.tasks.v2.utils_background import _prepare_tasks_metadata
+from fractal_server.tasks.v2.utils_background import _refresh_logs
+from fractal_server.tasks.v2.utils_background import (
+    _set_collection_state_data_status,
+)
+from fractal_server.tasks.v2.utils_background import check_task_files_exist
+from fractal_server.tasks.v2.utils_package_names import compare_package_names
+from fractal_server.tasks.v2.utils_python_interpreter import (
+    get_python_interpreter_v2,
+)
+from fractal_server.tasks.v2.utils_templates import customize_template
+from fractal_server.tasks.v2.utils_templates import parse_script_5_stdout
+from fractal_server.utils import execute_command_sync
+
+
+def _customize_and_run_template(
+    script_filename: str,
+    replacements: list[tuple[str, str]],
+    script_dir: str,
+    logger_name: str,
+) -> str:
+    """
+    Customize one of the template bash scripts.
+
+    Args:
+        script_filename:
+        replacements:
+        script_dir:
+        logger_name:
+    """
+    logger = get_logger(logger_name)
+    logger.debug(f"_customize_and_run_template {script_filename} - START")
+
+    script_path_local = Path(script_dir) / script_filename
+    # Read template
+    customize_template(
+        template_name=script_filename,
+        replacements=replacements,
+        script_path=script_path_local,
+    )
+
+    cmd = f"bash {script_path_local}"
+    logger.debug(f"Now run '{cmd}' ")
+
+    stdout = execute_command_sync(command=cmd)
+
+    logger.debug(f"Standard output of '{cmd}':\n{stdout}")
+    logger.debug(f"_customize_and_run_template {script_filename} - END")
+
+    return stdout
+
+
+def collect_package_local(
+    *,
+    state_id: int,
+    task_group: TaskGroupV2,
+) -> None:
+    """
+    Collect a task package.
+
+    This function is run as a background task, therefore exceptions must be
+    handled.
+
+    NOTE: by making this function sync, it will run within a thread - due to
+    starlette/fastapi handling of background tasks (see
+    https://github.com/encode/starlette/blob/master/starlette/background.py).
+
+
+    Arguments:
+        state_id:
+        task_group:
+    """
+
+    # Create the task_group path
+    with TemporaryDirectory() as tmpdir:
+
+        # Setup logger in tmpdir
+        LOGGER_NAME = "task_collection_local"
+        log_file_path = get_log_path(Path(tmpdir))
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+
+        # Log some info
+        logger.debug("START")
+        for key, value in task_group.model_dump().items():
+            logger.debug(f"task_group.{key}: {value}")
+
+        # Open a DB session
+        with next(get_sync_db()) as db:
+
+            # Check that the task_group path does not exist
+            if Path(task_group.path).exists():
+                error_msg = f"{task_group.path} already exists."
+                logger.error(error_msg)
+                _handle_failure(
+                    state_id=state_id,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=FileExistsError(error_msg),
+                    db=db,
+                    task_group_id=task_group.id,
+                )
+                return
+
+            try:
+                # Prepare replacements for task-collection scripts
+                python_bin = get_python_interpreter_v2(
+                    python_version=task_group.python_version
+                )
+                install_string = task_group.pip_install_string
+                settings = Inject(get_settings)
+                replacements = [
+                    ("__PACKAGE_NAME__", task_group.pkg_name),
+                    ("__TASK_GROUP_DIR__", task_group.path),
+                    ("__PACKAGE_ENV_DIR__", task_group.venv_path),
+                    ("__PYTHON__", python_bin),
+                    ("__INSTALL_STRING__", install_string),
+                    (
+                        "__FRACTAL_MAX_PIP_VERSION__",
+                        settings.FRACTAL_MAX_PIP_VERSION,
+                    ),
+                    (
+                        "__PINNED_PACKAGE_LIST__",
+                        task_group.pinned_package_versions_string,
+                    ),
+                ]
+
+                common_args = dict(
+                    replacements=replacements,
+                    script_dir=task_group.path,
+                    logger_name=LOGGER_NAME,
+                )
+
+                logger.debug("installing - START")
+                _set_collection_state_data_status(
+                    state_id=state_id,
+                    new_status=CollectionStatusV2.INSTALLING,
+                    logger_name=LOGGER_NAME,
+                    db=db,
+                )
+
+                # Create main path for task group
+                Path(task_group.path).mkdir(parents=True)
+                logger.debug(f"Created {task_group.path}")
+
+                # Create venv
+                logger.debug(
+                    (f"START - Create python venv {task_group.venv_path}")
+                )
+                cmd = (
+                    f"python{task_group.python_version} -m venv "
+                    f"{task_group.venv_path} --copies"
+                )
+                stdout = execute_command_sync(command=cmd)
+                logger.debug(
+                    (f"END - Create python venv folder {task_group.venv_path}")
+                )
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+                # Close db connections before long pip related operations
+                # Warning this expunge all ORM objects.
+                # https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close
+                db.close()
+
+                stdout = _customize_and_run_template(
+                    script_filename="_2_preliminary_pip_operations.sh",
+                    **common_args,
+                )
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+                stdout = _customize_and_run_template(
+                    script_filename="_3_pip_install.sh",
+                    **common_args,
+                )
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+                stdout_pip_freeze = _customize_and_run_template(
+                    script_filename="_4_pip_freeze.sh",
+                    **common_args,
+                )
+                logger.debug("installing - END")
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+
+                logger.debug("collecting - START")
+                _set_collection_state_data_status(
+                    state_id=state_id,
+                    new_status=CollectionStatusV2.COLLECTING,
+                    logger_name=LOGGER_NAME,
+                    db=db,
+                )
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+
+                stdout = _customize_and_run_template(
+                    script_filename="_5_pip_show.sh",
+                    **common_args,
+                )
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+
+                pkg_attrs = parse_script_5_stdout(stdout)
+                for key, value in pkg_attrs.items():
+                    logger.debug(
+                        f"collecting - parsed from pip-show: {key}={value}"
+                    )
+                # Check package_name match between pip show and task-group
+                package_name_pip_show = pkg_attrs.get("package_name")
+                package_name_task_group = task_group.pkg_name
+                compare_package_names(
+                    pkg_name_pip_show=package_name_pip_show,
+                    pkg_name_task_group=package_name_task_group,
+                    logger_name=LOGGER_NAME,
+                )
+                # Extract/drop parsed attributes
+                package_name = package_name_task_group
+                python_bin = pkg_attrs.pop("python_bin")
+                package_root_parent = pkg_attrs.pop("package_root_parent")
+
+                # TODO : Use more robust logic to determine `package_root`.
+                # Examples: use `importlib.util.find_spec`, or parse the output
+                # of `pip show --files {package_name}`.
+                package_name_underscore = package_name.replace("-", "_")
+                package_root = (
+                    Path(package_root_parent) / package_name_underscore
+                ).as_posix()
+
+                # Read and validate manifest file
+                manifest_path = pkg_attrs.pop("manifest_path")
+                logger.info(f"collecting - now loading {manifest_path=}")
+                with open(manifest_path) as json_data:
+                    pkg_manifest_dict = json.load(json_data)
+                logger.info(f"collecting - loaded {manifest_path=}")
+                logger.info("collecting - now validating manifest content")
+                pkg_manifest = ManifestV2(**pkg_manifest_dict)
+                logger.info("collecting - validated manifest content")
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+
+                logger.info("collecting - _prepare_tasks_metadata - start")
+                task_list = _prepare_tasks_metadata(
+                    package_manifest=pkg_manifest,
+                    package_version=task_group.version,
+                    package_root=Path(package_root),
+                    python_bin=Path(python_bin),
+                )
+                check_task_files_exist(task_list=task_list)
+                logger.info("collecting - _prepare_tasks_metadata - end")
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+
+                logger.info(
+                    "collecting - create_db_tasks_and_update_task_group - "
+                    "start"
+                )
+                task_group = create_db_tasks_and_update_task_group(
+                    task_list=task_list,
+                    task_group_id=task_group.id,
+                    db=db,
+                )
+                logger.info(
+                    "collecting - create_db_tasks_and_update_task_group - end"
+                )
+
+                logger.debug("collecting - END")
+
+                # Finalize (write metadata to DB)
+                logger.debug("finalising - START")
+
+                _refresh_logs(
+                    state_id=state_id,
+                    log_file_path=log_file_path,
+                    db=db,
+                )
+                collection_state = db.get(CollectionStateV2, state_id)
+                collection_state.data["freeze"] = stdout_pip_freeze
+                collection_state.data["status"] = CollectionStatusV2.OK
+                # FIXME: The `task_list` key is likely not used by any client,
+                # we should consider dropping it
+                task_read_list = [
+                    TaskReadV2(**task.model_dump()).dict()
+                    for task in task_group.task_list
+                ]
+                collection_state.data["task_list"] = task_read_list
+                flag_modified(collection_state, "data")
+                db.commit()
+                logger.debug("finalising - END")
+                logger.debug("END")
+
+            except Exception as collection_e:
+                # Delete corrupted package dir
+                try:
+                    logger.info(f"Now delete folder {task_group.path}")
+
+                    shutil.rmtree(task_group.path)
+                    logger.info(f"Deleted folder {task_group.path}")
+                except Exception as rm_e:
+                    logger.error(
+                        "Removing folder failed.\n"
+                        f"Original error:\n{str(rm_e)}"
+                    )
+
+                _handle_failure(
+                    state_id=state_id,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=collection_e,
+                    db=db,
+                    task_group_id=task_group.id,
+                )
+    return