fractal-task-tools 0.0.2__py3-none-any.whl

fractal_task_tools/__init__.py

@@ -0,0 +1,5 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+__VERSION__ = "0.0.2"

fractal_task_tools/_args_schemas.py

@@ -0,0 +1,214 @@
+import logging
+import os
+from collections import Counter
+from pathlib import Path
+from typing import Any
+from typing import Callable
+from typing import Optional
+
+from docstring_parser import parse as docparse
+from pydantic._internal import _generate_schema
+from pydantic._internal import _typing_extra
+from pydantic._internal._config import ConfigWrapper
+
+from ._descriptions import _get_class_attrs_descriptions
+from ._descriptions import _get_function_args_descriptions
+from ._descriptions import _insert_class_attrs_descriptions
+from ._descriptions import _insert_function_args_descriptions
+from ._pydantic_generatejsonschema import CustomGenerateJsonSchema
+from ._signature_constraints import _extract_function
+from ._signature_constraints import _validate_function_signature
+from ._titles import _include_titles
+
+
+_Schema = dict[str, Any]
+
+
+def _remove_attributes_from_descriptions(old_schema: _Schema) -> _Schema:
+    """
+    Keeps only the description part of the docstrings: e.g from
+    ```
+    'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'
+    '\\n'
+    'Attributes:\\n'
+    'min: Do not change. It will be set to `0` by default.\\n'
+    'max: Do not change. It will be set according to bitdepth of the images\\n'
+    '    by default (e.g. 65535 for 16 bit images).\\n'
+    'start: Lower-bound rescaling value for visualization.\\n'
+    'end: Upper-bound rescaling value for visualization.'
+    ```
+    to `'Custom class for Omero-channel window, based on OME-NGFF v0.4.\\n'`.
+
+    Args:
+        old_schema: TBD
+    """
+    new_schema = old_schema.copy()
+    if "$defs" in new_schema:
+        for name, definition in new_schema["$defs"].items():
+            if "description" in definition.keys():
+                parsed_docstring = docparse(definition["description"])
+                new_schema["$defs"][name][
+                    "description"
+                ] = parsed_docstring.short_description
+            elif "title" in definition.keys():
+                title = definition["title"]
+                new_schema["$defs"][name][
+                    "description"
+                ] = f"Missing description for {title}."
+            else:
+                new_schema["$defs"][name][
+                    "description"
+                ] = "Missing description"
+    logging.info("[_remove_attributes_from_descriptions] END")
+    return new_schema
+
+
+def _create_schema_for_function(function: Callable) -> _Schema:
+    namespace = _typing_extra.add_module_globals(function, None)
+    gen_core_schema = _generate_schema.GenerateSchema(
+        ConfigWrapper(None), namespace
+    )
+    core_schema = gen_core_schema.generate_schema(function)
+    clean_core_schema = gen_core_schema.clean_schema(core_schema)
+    gen_json_schema = CustomGenerateJsonSchema()
+    json_schema = gen_json_schema.generate(
+        clean_core_schema, mode="validation"
+    )
+    return json_schema
+
+
+def create_schema_for_single_task(
+    executable: str,
+    package: Optional[str] = None,
+    pydantic_models: Optional[list[tuple[str, str, str]]] = None,
+    task_function: Optional[Callable] = None,
+    verbose: bool = False,
+) -> _Schema:
+    """
+    Main function to create a JSON Schema of task arguments
+
+    This function can be used in two ways:
+
+    1. `task_function` argument is `None`, `package` is set, and `executable`
+        is a path relative to that package.
+    2. `task_function` argument is provided, `executable` is an absolute path
+        to the function module, and `package` is `None. This is useful for
+        testing.
+    """
+
+    DEFINITIONS_KEY = "$defs"
+
+    logging.info("[create_schema_for_single_task] START")
+    if task_function is None:
+        usage = "1"
+        # Usage 1 (standard)
+        if package is None:
+            raise ValueError(
+                "Cannot call `create_schema_for_single_task with "
+                f"{task_function=} and {package=}. Exit."
+            )
+        if os.path.isabs(executable):
+            raise ValueError(
+                "Cannot call `create_schema_for_single_task with "
+                f"{task_function=} and absolute {executable=}. Exit."
+            )
+    else:
+        usage = "2"
+        # Usage 2 (testing)
+        if package is not None:
+            raise ValueError(
+                "Cannot call `create_schema_for_single_task with "
+                f"{task_function=} and non-None {package=}. Exit."
+            )
+        if not os.path.isabs(executable):
+            raise ValueError(
+                "Cannot call `create_schema_for_single_task with "
+                f"{task_function=} and non-absolute {executable=}. Exit."
+            )
+
+    # Extract function from module
+    if usage == "1":
+        # Extract the function name (for the moment we assume the function has
+        # the same name as the module)
+        function_name = Path(executable).with_suffix("").name
+        # Extract the function object
+        task_function = _extract_function(
+            package_name=package,
+            module_relative_path=executable,
+            function_name=function_name,
+            verbose=verbose,
+        )
+    else:
+        # The function object is already available, extract its name
+        function_name = task_function.__name__
+
+    if verbose:
+        logging.info(f"[create_schema_for_single_task] {function_name=}")
+        logging.info(f"[create_schema_for_single_task] {task_function=}")
+
+    # Validate function signature against some custom constraints
+    _validate_function_signature(task_function)
+
+    # Create and clean up schema
+    schema = _create_schema_for_function(task_function)
+    schema = _remove_attributes_from_descriptions(schema)
+
+    # Include titles for custom-model-typed arguments
+    schema = _include_titles(
+        schema, definitions_key=DEFINITIONS_KEY, verbose=verbose
+    )
+
+    # Include main title
+    if schema.get("title") is None:
+
+        def to_camel_case(snake_str):
+            return "".join(
+                x.capitalize() for x in snake_str.lower().split("_")
+            )
+
+        schema["title"] = to_camel_case(task_function.__name__)
+
+    # Include descriptions of function. Note: this function works both
+    # for usages 1 or 2 (see docstring).
+    function_args_descriptions = _get_function_args_descriptions(
+        package_name=package,
+        module_path=executable,
+        function_name=function_name,
+        verbose=verbose,
+    )
+
+    schema = _insert_function_args_descriptions(
+        schema=schema, descriptions=function_args_descriptions
+    )
+
+    if pydantic_models is not None:
+        # Check that model names are unique
+        pydantic_models_names = [item[2] for item in pydantic_models]
+        duplicate_class_names = [
+            name
+            for name, count in Counter(pydantic_models_names).items()
+            if count > 1
+        ]
+        if duplicate_class_names:
+            pydantic_models_str = "  " + "\n  ".join(map(str, pydantic_models))
+            raise ValueError(
+                "Cannot parse docstrings for models with non-unique names "
+                f"{duplicate_class_names}, in\n{pydantic_models_str}"
+            )
+
+        # Extract model-attribute descriptions and insert them into schema
+        for package_name, module_relative_path, class_name in pydantic_models:
+            attrs_descriptions = _get_class_attrs_descriptions(
+                package_name=package_name,
+                module_relative_path=module_relative_path,
+                class_name=class_name,
+            )
+            schema = _insert_class_attrs_descriptions(
+                schema=schema,
+                class_name=class_name,
+                descriptions=attrs_descriptions,
+                definition_key=DEFINITIONS_KEY,
+            )
+
+    logging.info("[create_schema_for_single_task] END")
+    return schema

fractal_task_tools/_cli.py

@@ -0,0 +1,72 @@
+import argparse as ap
+import sys
+
+from fractal_task_tools._create_manifest import check_manifest_unchanged
+from fractal_task_tools._create_manifest import create_manifest
+from fractal_task_tools._create_manifest import write_manifest_to_file
+
+
+main_parser = ap.ArgumentParser(
+    description="`fractal-manifest` command-line interface",
+    allow_abbrev=False,
+)
+
+subparsers = main_parser.add_subparsers(
+    title="Available commands",
+    dest="cmd",
+)
+
+
+create_manifest_parser = subparsers.add_parser(
+    "create",
+    description="Create new manifest file",
+    allow_abbrev=False,
+)
+
+check_manifest_parser = subparsers.add_parser(
+    "check",
+    description="Check existing manifest file",
+    allow_abbrev=False,
+)
+
+
+for subparser in (create_manifest_parser, check_manifest_parser):
+    subparser.add_argument(
+        "--package",
+        type=str,
+        help="Example: 'fractal_tasks_core'",
+        required=True,
+    )
+    subparser.add_argument(
+        "--task-list-path",
+        type=str,
+        help=(
+            "Dot-separated path to the `task_list.py` module, "
+            "relative to the package root (default value: 'dev.task_list')."
+        ),
+        default="dev.task_list",
+        required=False,
+    )
+
+
+def main():
+    args = main_parser.parse_args(sys.argv[1:])
+    if args.cmd == "create":
+        manifest = create_manifest(
+            raw_package_name=args.package,
+            task_list_path=args.task_list_path,
+        )
+        write_manifest_to_file(
+            raw_package_name=args.package,
+            manifest=manifest,
+        )
+
+    elif args.cmd == "check":
+        manifest = create_manifest(
+            raw_package_name=args.package,
+            task_list_path=args.task_list_path,
+        )
+        check_manifest_unchanged(
+            raw_package_name=args.package,
+            manifest=manifest,
+        )

fractal_task_tools/_create_manifest.py

@@ -0,0 +1,190 @@
+"""
+Generate JSON schemas for task arguments and combine them into a manifest.
+"""
+import json
+import logging
+from importlib import import_module
+from pathlib import Path
+from typing import Any
+
+from ._args_schemas import create_schema_for_single_task
+from ._package_name_tools import normalize_package_name
+from ._task_docs import create_docs_info
+from ._task_docs import read_docs_info_from_file
+
+
+ARGS_SCHEMA_VERSION = "pydantic_v2"
+MANIFEST_FILENAME = "__FRACTAL_MANIFEST__.json"
+MANIFEST_VERSION = "2"
+
+
+def create_manifest(
+    *,
+    raw_package_name: str,
+    task_list_path: str,
+) -> dict[str, Any]:
+    """
+    Create the package manifest based on a `task_list.py` module
+
+    Arguments:
+        raw_package_name:
+            The name of the package. Note that this name must be importable
+            (after normalization).
+        task_list_path:
+            Relative path to the `task_list.py` module, with respect to the
+            package root (example `dev.task_list`).
+
+    Returns:
+        Task-package manifest.
+    """
+
+    # Preliminary validation
+    if "/" in task_list_path or task_list_path.endswith(".py"):
+        raise ValueError(
+            f"Invalid {task_list_path=} (valid example: `dev.task_list`)."
+        )
+
+    # Normalize package name
+    package_name = normalize_package_name(raw_package_name)
+
+    logging.info(f"Start generating a new manifest for {package_name}")
+
+    # Prepare an empty manifest
+    manifest = dict(
+        manifest_version=MANIFEST_VERSION,
+        has_args_schemas=True,
+        args_schema_version=ARGS_SCHEMA_VERSION,
+        task_list=[],
+    )
+
+    # Import the task-list module
+    task_list_module = import_module(f"{package_name}.{task_list_path}")
+
+    # Load TASK_LIST
+    TASK_LIST = getattr(task_list_module, "TASK_LIST")
+
+    # Load INPUT_MODELS
+    try:
+        INPUT_MODELS = getattr(task_list_module, "INPUT_MODELS")
+    except AttributeError:
+        INPUT_MODELS = []
+        logging.warning(
+            "No `INPUT_MODELS` found in task_list module. Setting it to `[]`."
+        )
+
+    # Load AUTHORS
+    try:
+        manifest["authors"] = getattr(task_list_module, "AUTHORS")
+    except AttributeError:
+        logging.warning("No `AUTHORS` found in task_list module.")
+
+    # Load DOCS_LINK
+    try:
+        DOCS_LINK = getattr(task_list_module, "DOCS_LINK")
+    except AttributeError:
+        DOCS_LINK = None
+        logging.warning("No `DOCS_LINK` found in task_list module.")
+
+    # Loop over TASK_LIST, and append the proper task dictionaries
+    # to manifest["task_list"]
+    for task_obj in TASK_LIST:
+        # Convert Pydantic object to dictionary
+        task_dict = task_obj.model_dump(
+            exclude={"meta_init", "executable_init", "meta", "executable"},
+            exclude_unset=True,
+        )
+
+        # Copy some properties from `task_obj` to `task_dict`
+        if task_obj.executable_non_parallel is not None:
+            task_dict[
+                "executable_non_parallel"
+            ] = task_obj.executable_non_parallel
+        if task_obj.executable_parallel is not None:
+            task_dict["executable_parallel"] = task_obj.executable_parallel
+        if task_obj.meta_non_parallel is not None:
+            task_dict["meta_non_parallel"] = task_obj.meta_non_parallel
+        if task_obj.meta_parallel is not None:
+            task_dict["meta_parallel"] = task_obj.meta_parallel
+
+        # Autogenerate JSON Schemas for non-parallel/parallel task arguments
+        for kind in ["non_parallel", "parallel"]:
+            executable = task_dict.get(f"executable_{kind}")
+            if executable is not None:
+                logging.info(f"[{executable}] START")
+                schema = create_schema_for_single_task(
+                    executable,
+                    package=package_name,
+                    pydantic_models=INPUT_MODELS,
+                )
+                logging.info(f"[{executable}] END (new schema)")
+                task_dict[f"args_schema_{kind}"] = schema
+
+        # Compute and set `docs_info`
+        docs_info = task_dict.get("docs_info")
+        if docs_info is None:
+            docs_info = create_docs_info(
+                executable_non_parallel=task_obj.executable_non_parallel,
+                executable_parallel=task_obj.executable_parallel,
+                package=package_name,
+            )
+        elif docs_info.startswith("file:"):
+            docs_info = read_docs_info_from_file(
+                docs_info=docs_info,
+                task_list_path=task_list_module.__file__,
+            )
+        if docs_info is not None:
+            task_dict["docs_info"] = docs_info
+
+        # Set `docs_link`
+        if DOCS_LINK is not None:
+            task_dict["docs_link"] = DOCS_LINK
+
+        # Append task
+        manifest["task_list"].append(task_dict)
+        print()
+    return manifest
+
+
+def write_manifest_to_file(
+    *,
+    raw_package_name: str,
+    manifest: str,
+) -> None:
+    """
+    Write manifest to file.
+
+    Arguments:
+        raw_package_name:
+        manifest: The manifest object
+    """
+    package_name = normalize_package_name(raw_package_name)
+    imported_package = import_module(package_name)
+    package_root_dir = Path(imported_package.__file__).parent
+    manifest_path = package_root_dir / MANIFEST_FILENAME
+    with manifest_path.open("w") as f:
+        json.dump(manifest, f, indent=2)
+        f.write("\n")
+    logging.info(f"Manifest stored in {manifest_path.as_posix()}")
+
+
+def check_manifest_unchanged(
+    *,
+    raw_package_name: str,
+    manifest: str,
+) -> None:
+    """
+    Write manifest to file.
+
+    Arguments:
+        raw_package_name:
+        manifest: The manifest object
+    """
+    package_name = normalize_package_name(raw_package_name)
+    imported_package = import_module(package_name)
+    package_root_dir = Path(imported_package.__file__).parent
+    manifest_path = package_root_dir / MANIFEST_FILENAME
+    with manifest_path.open("f") as f:
+        old_manifest = json.load(f)
+    logging.info(f"Manifest read from {manifest_path.as_posix()}")
+    if manifest != old_manifest:
+        raise ValueError("New/old manifests differ")