fractal-server 2.6.4__py3-none-any.whl → 2.7.0__py3-none-any.whl

fractal_server/tasks/v2/background_operations_ssh.py

@@ -1,17 +1,16 @@
-import json
 import os
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from sqlalchemy.orm.attributes import flag_modified
 
-from ...app.models.v2 import CollectionStateV2
-from ._TaskCollectPip import _TaskCollectPip
 from .background_operations import _handle_failure
-from .background_operations import _insert_tasks
 from .background_operations import _prepare_tasks_metadata
 from .background_operations import _set_collection_state_data_status
+from .database_operations import create_db_tasks_and_update_task_group
 from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.schemas.v2 import CollectionStatusV2
 from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.config import get_settings
@@ -108,8 +107,9 @@ def _customize_and_run_template(
 
 
 def background_collect_pip_ssh(
+    *,
     state_id: int,
-    task_pkg: _TaskCollectPip,
+    task_group: TaskGroupV2,
     fractal_ssh: FractalSSH,
     tasks_base_dir: str,
 ) -> None:
@@ -126,7 +126,7 @@ def background_collect_pip_ssh(
 
     Arguments:
         state_id:
-        task_pkg:
+        task_group:
         fractal_ssh:
         tasks_base_dir:
     """
@@ -139,47 +139,30 @@ def background_collect_pip_ssh(
             logger_name=LOGGER_NAME,
             log_file_path=log_file_path,
         )
-
         logger.debug("START")
-        for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
-            logger.debug(f"task_pkg.{key}: {value}")
+        for key, value in task_group.model_dump().items():
+            logger.debug(f"task_group.{key}: {value}")
+
+        # `remove_venv_folder_upon_failure` is set to True only if
+        # script 1 goes through, which means that the remote folder
+        # `package_env_dir` did not already exist. If this remote
+        # folder already existed, then script 1 fails and the boolean
+        # flag `remove_venv_folder_upon_failure` remains false.
+        remove_venv_folder_upon_failure = False
 
         # Open a DB session soon, since it is needed for updating `state`
         with next(get_sync_db()) as db:
             try:
                 # Prepare replacements for task-collection scripts
                 python_bin = get_python_interpreter_v2(
-                    python_version=task_pkg.python_version
-                )
-                package_version = (
-                    ""
-                    if task_pkg.package_version is None
-                    else task_pkg.package_version
+                    python_version=task_group.python_version
                 )
-
-                install_string = task_pkg.package
-                if task_pkg.package_extras is not None:
-                    install_string = (
-                        f"{install_string}[{task_pkg.package_extras}]"
-                    )
-                if (
-                    task_pkg.package_version is not None
-                    and not task_pkg.is_local_package
-                ):
-                    install_string = (
-                        f"{install_string}=={task_pkg.package_version}"
-                    )
-                package_env_dir = (
-                    Path(tasks_base_dir)
-                    / ".fractal"
-                    / f"{task_pkg.package_name}{package_version}"
-                ).as_posix()
-                logger.debug(f"{package_env_dir=}")
+                install_string = task_group.pip_install_string
                 settings = Inject(get_settings)
                 replacements = [
-                    ("__PACKAGE_NAME__", task_pkg.package_name),
-                    ("__PACKAGE_ENV_DIR__", package_env_dir),
-                    ("__PACKAGE__", task_pkg.package),
+                    ("__PACKAGE_NAME__", task_group.pkg_name),
+                    ("__TASK_GROUP_DIR__", task_group.path),
+                    ("__PACKAGE_ENV_DIR__", task_group.venv_path),
                     ("__PYTHON__", python_bin),
                     ("__INSTALL_STRING__", install_string),
                     (
@@ -210,12 +193,11 @@ def background_collect_pip_ssh(
                 # long operations that do not use the db
                 db.close()
 
-                # `remove_venv_folder_upon_failure` is set to True only if
-                # script 1 goes through, which means that the remote folder
-                # `package_env_dir` did not already exist. If this remote
-                # folder already existed, then script 1 fails and the boolean
-                # flag `remove_venv_folder_upon_failure` remains false.
-                remove_venv_folder_upon_failure = False
+                # Create remote folder (note that because of `parents=True` we
+                # are in the `no error if existing, make parent directories as
+                # needed` scenario)
+                fractal_ssh.mkdir(folder=tasks_base_dir, parents=True)
+
                 stdout = _customize_and_run_template(
                     script_filename="_1_create_venv.sh",
                     **common_args,
@@ -223,7 +205,7 @@ def background_collect_pip_ssh(
                 remove_venv_folder_upon_failure = True
 
                 stdout = _customize_and_run_template(
-                    script_filename="_2_upgrade_pip.sh",
+                    script_filename="_2_preliminary_pip_operations.sh",
                     **common_args,
                 )
                 stdout = _customize_and_run_template(
@@ -258,13 +240,13 @@ def background_collect_pip_ssh(
                         f"collecting - parsed from pip-show: {key}={value}"
                     )
                 # Check package_name match
-                # FIXME SSH: Does this work for non-canonical `package_name`?
+                # FIXME SSH: Does this work well for non-canonical names?
                 package_name_pip_show = pkg_attrs.get("package_name")
-                package_name_task_pkg = task_pkg.package_name
-                if package_name_pip_show != package_name_task_pkg:
+                package_name_task_group = task_group.pkg_name
+                if package_name_pip_show != package_name_task_group:
                     error_msg = (
                         f"`package_name` mismatch: "
-                        f"{package_name_task_pkg=} but "
+                        f"{package_name_task_group=} but "
                         f"{package_name_pip_show=}"
                     )
                     logger.error(error_msg)
@@ -286,31 +268,35 @@ def background_collect_pip_ssh(
                 ).as_posix()
 
                 # Read and validate remote manifest file
-                with fractal_ssh.sftp().open(manifest_path_remote, "r") as f:
-                    manifest = json.load(f)
+                pkg_manifest_dict = fractal_ssh.read_remote_json_file(
+                    manifest_path_remote
+                )
                 logger.info(f"collecting - loaded {manifest_path_remote=}")
-                ManifestV2(**manifest)
+                pkg_manifest = ManifestV2(**pkg_manifest_dict)
                 logger.info("collecting - manifest is a valid ManifestV2")
 
-                # Create new _TaskCollectPip object
-                new_pkg = _TaskCollectPip(
-                    **task_pkg.dict(
-                        exclude={"package_version", "package_name"},
-                        exclude_unset=True,
-                        exclude_none=True,
-                    ),
-                    package_manifest=manifest,
-                    **pkg_attrs,
-                )
-
+                logger.info("collecting - _prepare_tasks_metadata - start")
                 task_list = _prepare_tasks_metadata(
-                    package_manifest=new_pkg.package_manifest,
-                    package_version=new_pkg.package_version,
-                    package_source=new_pkg.package_source,
+                    package_manifest=pkg_manifest,
+                    package_version=task_group.version,
                     package_root=Path(package_root_remote),
                     python_bin=Path(python_bin),
                 )
-                _insert_tasks(task_list=task_list, db=db)
+                logger.info("collecting - _prepare_tasks_metadata - end")
+
+                logger.info(
+                    "collecting - create_db_tasks_and_update_task_group - "
+                    "start"
+                )
+                create_db_tasks_and_update_task_group(
+                    task_list=task_list,
+                    task_group_id=task_group.id,
+                    db=db,
+                )
+                logger.info(
+                    "collecting - create_db_tasks_and_update_task_group - end"
+                )
+
                 logger.debug("collecting - END")
 
                 # Finalize (write metadata to DB)
@@ -333,18 +319,19 @@ def background_collect_pip_ssh(
                     logger_name=LOGGER_NAME,
                     exception=e,
                     db=db,
+                    task_group_id=task_group.id,
                 )
                 if remove_venv_folder_upon_failure:
                     try:
                         logger.info(
-                            f"Now delete remote folder {package_env_dir}"
+                            f"Now delete remote folder {task_group.path}"
                         )
                         fractal_ssh.remove_folder(
-                            folder=package_env_dir,
+                            folder=task_group.path,
                             safe_root=tasks_base_dir,
                         )
                         logger.info(
-                            f"Deleted remoted folder {package_env_dir}"
+                            f"Deleted remoted folder {task_group.path}"
                         )
                     except Exception as e:
                         logger.error(
@@ -354,6 +341,6 @@ def background_collect_pip_ssh(
                     else:
                         logger.info(
                             "Not trying to remove remote folder "
-                            f"{package_env_dir}."
+                            f"{task_group.path}."
                         )
                 return

fractal_server/tasks/v2/database_operations.py

@@ -0,0 +1,44 @@
+from sqlalchemy.orm import Session as DBSyncSession
+
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import TaskV2
+from fractal_server.app.schemas.v2 import TaskCreateV2
+
+
+def _get_task_type(task: TaskCreateV2) -> str:
+    if task.command_non_parallel is None:
+        return "parallel"
+    elif task.command_parallel is None:
+        return "non_parallel"
+    else:
+        return "compound"
+
+
+def create_db_tasks_and_update_task_group(
+    *,
+    task_group_id: int,
+    task_list: list[TaskCreateV2],
+    db: DBSyncSession,
+) -> TaskGroupV2:
+    """
+    Create a `TaskGroupV2` with N `TaskV2`s, and insert them into the database.
+
+    Arguments:
+        task_group: ID of an existing TaskGroupV2 object.
+        task_list: A list of TaskCreateV2 objects to be inserted into the db.
+        db: A synchronous database session
+    """
+    actual_task_list = [
+        TaskV2(
+            **task.dict(),
+            type=_get_task_type(task),
+        )
+        for task in task_list
+    ]
+    task_group = db.get(TaskGroupV2, task_group_id)
+    task_group.task_list = actual_task_list
+    db.add(task_group)
+    db.commit()
+    db.refresh(task_group)
+
+    return task_group

fractal_server/tasks/v2/endpoint_operations.py

@@ -1,136 +1,124 @@
-import json
-from pathlib import Path
-from typing import Literal
 from typing import Optional
-from typing import Union
-from zipfile import ZipFile
 
-from ._TaskCollectPip import _TaskCollectPip
-from .utils import _parse_wheel_filename
-from .utils import get_python_interpreter_v2
-from fractal_server.app.schemas.v2 import ManifestV2
-from fractal_server.config import get_settings
-from fractal_server.logger import get_logger
-from fractal_server.syringe import Inject
-from fractal_server.utils import execute_command
+from fastapi import HTTPException
+from fastapi import status
+from httpx import AsyncClient
+from httpx import TimeoutException
 
+from fractal_server.logger import set_logger
 
-FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
 
-
-async def download_package(
-    *,
-    task_pkg: _TaskCollectPip,
-    dest: Union[str, Path],
-) -> Path:
-    """
-    Download package to destination and return wheel-file path.
-    """
-    interpreter = get_python_interpreter_v2(
-        python_version=task_pkg.python_version
-    )
-    pip = f"{interpreter} -m pip"
-    if task_pkg.package_version is None:
-        package_and_version = f"{task_pkg.package_name}"
-    else:
-        package_and_version = (
-            f"{task_pkg.package_name}=={task_pkg.package_version}"
-        )
-    cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
-    stdout = await execute_command(command=cmd, cwd=Path("."))
-    pkg_file = next(
-        line.split()[-1] for line in stdout.split("\n") if "Saved" in line
-    )
-    return Path(pkg_file)
+logger = set_logger(__name__)
 
 
-def _load_manifest_from_wheel(
-    path: Path, wheel: ZipFile, logger_name: Optional[str] = None
-) -> ManifestV2:
-    logger = get_logger(logger_name)
-    namelist = wheel.namelist()
-    try:
-        manifest = next(
-            name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
-        )
-    except StopIteration:
-        msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
-        logger.error(msg)
-        raise ValueError(msg)
-    with wheel.open(manifest) as manifest_fd:
-        manifest_dict = json.load(manifest_fd)
-    manifest_version = str(manifest_dict["manifest_version"])
-    if manifest_version == "2":
-        pkg_manifest = ManifestV2(**manifest_dict)
-        return pkg_manifest
-    else:
-        msg = f"Manifest version {manifest_version=} not supported"
-        logger.error(msg)
-        raise ValueError(msg)
-
-
-def inspect_package(
-    path: Path, logger_name: Optional[str] = None
-) -> dict[Literal["pkg_version", "pkg_manifest"], str]:
+async def get_package_version_from_pypi(
+    name: str,
+    version: Optional[str] = None,
+) -> str:
     """
-    Inspect task package to extract version and manifest
+    Make a GET call to PyPI JSON API and get latest *compatible* version.
+
+    There are three cases:
 
-    Note that this only works with wheel files, which have a well-defined
-    dist-info section. If we need to generalize to to tar.gz archives, we would
-    need to go and look for `PKG-INFO`.
+    1. `version` is set and it is found on PyPI as-is.
+    2. `version` is set but it is not found on PyPI as-is.
+    3. `version` is unset, and we query `PyPI` for latest.
 
-    Args:
-        path: Path of the package wheel file.
-        logger_name:
+    Ref https://warehouse.pypa.io/api-reference/json.html.
 
-    Returns:
-        A dictionary with keys `pkg_version` and `pkg_manifest`.
+    Arguments:
+        name: Package name.
+        version:
+            Could be a correct version (`1.3.0`), an incomplete one
+            (`1.3`) or `None`.
     """
 
-    logger = get_logger(logger_name)
+    url = f"https://pypi.org/pypi/{name}/json"
+    hint = f"Hint: specify the required version for '{name}'."
 
-    if not path.as_posix().endswith(".whl"):
-        raise ValueError(
-            "Only wheel packages are supported in Fractal "
-            f"(given {path.name})."
+    # Make request to PyPI
+    try:
+        async with AsyncClient(timeout=5.0) as client:
+            res = await client.get(url)
+    except TimeoutException as e:
+        error_msg = (
+            f"A TimeoutException occurred while getting {url}.\n"
+            f"Original error: {str(e)}."
+        )
+        logger.error(error_msg)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=error_msg,
+        )
+    except BaseException as e:
+        error_msg = (
+            f"An unknown error occurred while getting {url}. "
+            f"Original error: {str(e)}."
+        )
+        logger.error(error_msg)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=error_msg,
         )
 
-    # Extract package name and version from wheel filename
-    _info = _parse_wheel_filename(wheel_filename=path.name)
-    pkg_version = _info["version"]
-
-    # Read and validate task manifest
-    logger.debug(f"Now reading manifest for {path.as_posix()}")
-    with ZipFile(path) as wheel:
-        pkg_manifest = _load_manifest_from_wheel(
-            path, wheel, logger_name=logger_name
+    # Parse response
+    if res.status_code != 200:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Could not get {url} (status_code {res.status_code})."
+                f"\n{hint}"
+            ),
+        )
+    try:
+        response_data = res.json()
+        latest_version = response_data["info"]["version"]
+        available_releases = response_data["releases"].keys()
+    except KeyError as e:
+        logger.error(
+            f"A KeyError occurred while getting {url}. "
+            f"Original error: {str(e)}."
+        )
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"A KeyError error occurred while getting {url}.\n{hint}",
         )
-    logger.debug("Manifest read correctly.")
 
-    info = dict(
-        pkg_version=pkg_version,
-        pkg_manifest=pkg_manifest,
+    logger.info(
+        f"Obtained data from {url}: "
+        f"{len(available_releases)} releases, "
+        f"latest={latest_version}."
     )
-    return info
-
 
-def create_package_dir_pip(
-    *,
-    task_pkg: _TaskCollectPip,
-    create: bool = True,
-) -> Path:
-    """
-    Create venv folder for a task package and return corresponding Path object
-    """
-    settings = Inject(get_settings)
-    user = FRACTAL_PUBLIC_TASK_SUBDIR
-    if task_pkg.package_version is None:
-        raise ValueError(
-            f"Cannot create venv folder for package `{task_pkg.package}` "
-            "with `version=None`."
-        )
-    package_dir = f"{task_pkg.package_name}{task_pkg.package_version}"
-    venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
-    if create:
-        venv_path.mkdir(exist_ok=False, parents=True)
-    return venv_path
+    if version is not None:
+        if version in available_releases:
+            logger.info(f"Requested {version=} available on PyPI.")
+            # Case 1: `version` is set and it is found on PyPI as-is
+            return version
+        else:
+            # Case 2: `version` is set but it is not found on PyPI as-is
+            # Filter using `version` as prefix, and sort
+            matching_versions = [
+                v for v in available_releases if v.startswith(version)
+            ]
+            logger.info(
+                f"Requested {version=} not available on PyPI, "
+                f"found {len(matching_versions)} versions matching "
+                f"`{version}*`."
+            )
+            if len(matching_versions) == 0:
+                logger.info(f"No version starting with {version} found.")
+                raise HTTPException(
+                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                    detail=(
+                        f"No version starting with {version} found.\n"
+                        f"{hint}"
+                    ),
+                )
+            else:
+                latest_matching_version = sorted(matching_versions)[-1]
+                return latest_matching_version
+    else:
+        # Case 3: `version` is unset and we use latest
+        logger.info(f"No version requested, returning {latest_version=}.")
+        return latest_version

fractal_server/tasks/v2/templates/_1_create_venv.sh

@@ -7,16 +7,20 @@ write_log(){
 
 
 # Variables to be filled within fractal-server
+TASK_GROUP_DIR=__TASK_GROUP_DIR__
 PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
 PYTHON=__PYTHON__
 
 TIME_START=$(date +%s)
 
-# Check that package folder does not exist
-if [ -d "$PACKAGE_ENV_DIR" ]; then
-    write_log "ERROR: Folder $PACKAGE_ENV_DIR already exists. Exit."
-    exit 1
-fi
+# Check that task-group and venv folders do not exist
+for DIR_TO_BE_CHECKED in "$TASK_GROUP_DIR" "$PACKAGE_ENV_DIR";
+do
+    if [ -d "$DIR_TO_BE_CHECKED" ]; then
+        write_log "ERROR: Folder $DIR_TO_BE_CHECKED already exists. Exit."
+        exit 1
+    fi
+done
 
 write_log "START mkdir -p $PACKAGE_ENV_DIR"
 mkdir -p $PACKAGE_ENV_DIR

fractal_server/tasks/v2/templates/{_2_upgrade_pip.sh → _2_preliminary_pip_operations.sh}

@@ -15,6 +15,7 @@ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
 # Upgrade pip
 write_log "START upgrade pip"
 "$VENVPYTHON" -m pip install "pip<=__FRACTAL_MAX_PIP_VERSION__" --upgrade
+"$VENVPYTHON" -m pip install setuptools
 write_log "END   upgrade pip"
 echo
 

fractal_server/tasks/v2/utils.py

@@ -45,6 +45,11 @@ def _parse_wheel_filename(wheel_filename: str) -> dict[str, str]:
     Note that we transform exceptions in `ValueError`s, since this function is
     also used within Pydantic validators.
     """
+    if "/" in wheel_filename:
+        raise ValueError(
+            "[_parse_wheel_filename] Input must be a filename, not a full "
+            f"path (given: {wheel_filename})."
+        )
     try:
         parts = wheel_filename.split("-")
         return dict(distribution=parts[0], version=parts[1])

fractal_server/utils.py

@@ -32,8 +32,8 @@ def get_timestamp() -> datetime:
 
 async def execute_command(
     *,
-    cwd: Path,
     command: str,
+    cwd: Optional[Path] = None,
     logger_name: Optional[str] = None,
 ) -> str:
     """
@@ -60,12 +60,13 @@ async def execute_command(
     cmd, *args = command_split
 
     logger = get_logger(logger_name)
+    cwd_kwarg = dict() if cwd is None else dict(cwd=cwd)
     proc = await asyncio.create_subprocess_exec(
         cmd,
         *args,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
-        cwd=cwd,
+        **cwd_kwarg,
     )
     stdout, stderr = await proc.communicate()
     logger.debug(f"Subprocess call to: {command}")

{fractal_server-2.6.4.dist-info → fractal_server-2.7.0.dist-info}/METADATA

@@ -1,34 +1,30 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.6.4
+Version: 2.7.0
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause
 Author: Tommaso Comparin
 Author-email: tommaso.comparin@exact-lab.it
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: gunicorn
-Provides-Extra: postgres
 Provides-Extra: postgres-psycopg-binary
 Requires-Dist: aiosqlite (>=0.19.0,<0.20.0)
 Requires-Dist: alembic (>=1.13.1,<2.0.0)
-Requires-Dist: asyncpg (>=0.29.0,<0.30.0) ; extra == "postgres"
 Requires-Dist: bcrypt (==4.0.1)
 Requires-Dist: cloudpickle (>=3.0.0,<3.1.0)
 Requires-Dist: clusterfutures (>=0.5,<0.6)
 Requires-Dist: fabric (>=3.2.2,<4.0.0)
-Requires-Dist: fastapi (>=0.112.0,<0.113.0)
+Requires-Dist: fastapi (>=0.115.0,<0.116.0)
 Requires-Dist: fastapi-users[oauth] (>=12.1.0,<13.0.0)
 Requires-Dist: gunicorn (>=21.2,<23.0) ; extra == "gunicorn"
 Requires-Dist: packaging (>=23.2,<24.0)
 Requires-Dist: psutil (>=5.9.8,<6.0.0)
-Requires-Dist: psycopg2 (>=2.9.5,<3.0.0) ; extra == "postgres"
 Requires-Dist: psycopg[binary] (>=3.1.0,<4.0.0) ; extra == "postgres-psycopg-binary"
 Requires-Dist: pydantic (>=1.10.8,<2)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)