fractal-task-tools 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

fractal_task_tools/__init__.py

@@ -2,4 +2,4 @@ import logging
 
 logging.basicConfig(level=logging.INFO)
 
-__VERSION__ = "0.0.7"
+__VERSION__ = "0.0.9"

fractal_task_tools/_cli.py

@@ -59,8 +59,21 @@ check_manifest_parser.add_argument(
 )
 
 
+def _parse_arguments(sys_argv: list[str] | None = None) -> ap.Namespace:
+    """
+    Parse `sys.argv` or custom CLI arguments.
+
+    Arguments:
+        sys_argv: If set, overrides `sys.argv` (useful for testing).
+    """
+    if sys_argv is None:
+        sys_argv = sys.argv[:]
+    args = main_parser.parse_args(sys_argv[1:])
+    return args
+
+
 def main():
-    args = main_parser.parse_args(sys.argv[1:])
+    args = _parse_arguments()
     if args.cmd == "create":
         manifest = create_manifest(
             raw_package_name=args.package,

fractal_task_tools/_create_manifest.py

@@ -7,9 +7,10 @@ from typing import Any
 
 from ._args_schemas import create_schema_for_single_task
 from ._package_name_tools import normalize_package_name
+from ._task_arguments import validate_arguments
 from ._task_docs import create_docs_info
 from ._task_docs import read_docs_info_from_file
-
+from .task_models import _BaseTask
 
 ARGS_SCHEMA_VERSION = "pydantic_v2"
 MANIFEST_FILENAME = "__FRACTAL_MANIFEST__.json"
@@ -60,7 +61,7 @@ def create_manifest(
     task_list_module = import_module(f"{package_name}.{task_list_path}")
 
     # Load TASK_LIST
-    TASK_LIST = getattr(task_list_module, "TASK_LIST")
+    TASK_LIST: list[_BaseTask] = getattr(task_list_module, "TASK_LIST")
 
     # Load INPUT_MODELS
     try:
@@ -89,9 +90,15 @@ def create_manifest(
     for task_obj in TASK_LIST:
         # Convert Pydantic object to dictionary
         task_dict = task_obj.model_dump(
-            exclude={"meta_init", "executable_init", "meta", "executable"},
+            exclude={
+                "meta_init",
+                "executable_init",
+                "meta",
+                "executable",
+            },
             exclude_unset=True,
         )
+        task_dict["type"] = task_obj.type
 
         # Copy some properties from `task_obj` to `task_dict`
         if task_obj.executable_non_parallel is not None:
@@ -115,6 +122,13 @@ def create_manifest(
                     package=package_name,
                     pydantic_models=INPUT_MODELS,
                 )
+
+                validate_arguments(
+                    task_type=task_obj.type,
+                    schema=schema,
+                    executable_kind=kind,
+                )
+
                 logging.info(f"[{executable}] END (new schema)")
                 task_dict[f"args_schema_{kind}"] = schema
 

fractal_task_tools/_deepdiff.py

@@ -17,7 +17,7 @@ def deepdiff(
     if type(old_object) is not type(new_object):
         raise ValueError(
             f"[{path}] Type difference:\n"
-            f"\tOld: {type(old_object)}\n\tNew:{type(new_object)}"
+            f"\tOld: {type(old_object)}\n\tNew: {type(new_object)}"
         )
 
     if type(old_object) not in [list, dict, str, int, float, bool, type(None)]:
@@ -64,5 +64,5 @@ def deepdiff(
         if old_object != new_object:
             raise ValueError(
                 f"{path} Values are different:\n"
-                f"\tOld: '{old_object}'\n\tNew:'{new_object}'"
+                f"\tOld: '{old_object}'\n\tNew: '{new_object}'"
             )

fractal_task_tools/_descriptions.py

@@ -122,24 +122,18 @@ def _get_function_args_descriptions(
     return descriptions
 
 
-def _get_class_attrs_descriptions(
-    package_name: str, module_relative_path: str, class_name: str
+def _get_class_attrs_descriptions_from_file(
+    *,
+    module_path: Path,
+    class_name: str,
 ) -> dict[str, str]:
     """
-    Extract attribute descriptions from a class.
+    Extract class-attribute descriptions from a Python script
 
     Args:
-        package_name: Example `fractal_tasks_core`.
-        module_relative_path: Example `lib_channels.py`.
+        module_path: Example `/something/my_class.py`.
         class_name: Example `OmeroChannel`.
     """
-
-    if not module_relative_path.endswith(".py"):
-        raise ValueError(f"Module {module_relative_path} must end with '.py'")
-
-    # Get the class ast.ClassDef object
-    package_path = Path(import_module(package_name).__file__).parent
-    module_path = package_path / module_relative_path
     tree = ast.parse(module_path.read_text())
     try:
         _class = next(
@@ -148,10 +142,7 @@ def _get_class_attrs_descriptions(
             if (isinstance(c, ast.ClassDef) and c.name == class_name)
         )
     except StopIteration:
-        raise RuntimeError(
-            f"Cannot find {class_name=} for {package_name=} "
-            f"and {module_relative_path=}"
-        )
+        raise RuntimeError(f"Cannot find {class_name=} in {module_path}.")
     docstring = ast.get_docstring(_class)
     parsed_docstring = docparse(docstring)
     descriptions = {
@@ -160,6 +151,31 @@ def _get_class_attrs_descriptions(
         else "Missing description"
         for x in parsed_docstring.params
     }
+    return descriptions
+
+
+def _get_class_attrs_descriptions(
+    package_name: str, module_relative_path: str, class_name: str
+) -> dict[str, str]:
+    """
+    Extract class-attribute descriptions from an imported module
+
+    Args:
+        package_name: Example `fractal_tasks_core`.
+        module_relative_path: Example `lib_channels.py`.
+        class_name: Example `OmeroChannel`.
+    """
+
+    if not module_relative_path.endswith(".py"):
+        raise ValueError(f"Module {module_relative_path} must end with '.py'")
+
+    # Get the class ast.ClassDef object
+    package_path = Path(import_module(package_name).__file__).parent
+    module_path = package_path / module_relative_path
+    descriptions = _get_class_attrs_descriptions_from_file(
+        module_path=module_path,
+        class_name=class_name,
+    )
     logging.info(f"[_get_class_attrs_descriptions] END ({class_name=})")
     return descriptions
 

fractal_task_tools/_package_name_tools.py

@@ -12,7 +12,7 @@ def normalize_package_name(pkg_name: str) -> str:
         imported-module name.
 
     Args:
-        name: The non-normalized package name.
+        pkg_name: The non-normalized package name.
 
     Returns:
         The normalized package name.

fractal_task_tools/_task_arguments.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Literal
+
+
+REQUIRED_ARGUMENTS: dict[tuple[str, str], set[str]] = {
+    ("non_parallel", "non_parallel"): {"zarr_urls", "zarr_dir"},
+    ("compound", "non_parallel"): {"zarr_urls", "zarr_dir"},
+    ("parallel", "parallel"): {"zarr_url"},
+    ("compound", "parallel"): {"zarr_url", "init_args"},
+    ("converter_non_parallel", "non_parallel"): {"zarr_dir"},
+    ("converter_compound", "non_parallel"): {"zarr_dir"},
+    ("converter_compound", "parallel"): {"zarr_url", "init_args"},
+}
+FORBIDDEN_ARGUMENTS: dict[tuple[str, str], set[str]] = {
+    ("non_parallel", "non_parallel"): {"zarr_url"},
+    ("compound", "non_parallel"): {"zarr_url"},
+    ("parallel", "parallel"): {"zarr_urls", "zarr_dir"},
+    ("compound", "parallel"): {"zarr_urls", "zarr_dir"},
+    ("converter_non_parallel", "non_parallel"): {"zarr_url", "zarr_urls"},
+    ("converter_compound", "non_parallel"): {"zarr_url", "zarr_urls"},
+    ("converter_compound", "parallel"): {"zarr_urls", "zarr_dir"},
+}
+
+
+def validate_arguments(
+    *,
+    task_type: Literal["parallel", "non_parallel", "compound"],
+    executable_kind: Literal["parallel", "non_parallel"],
+    schema: dict[str, Any],
+) -> None:
+    """
+    Validate schema arguments against required/forbidden ones.
+
+    Arguments:
+        task_type:
+        executable_kind: The `parallel`/`non_parallel` part of the task.
+        schema:
+    """
+
+    key = (task_type, executable_kind)
+    if not (key in REQUIRED_ARGUMENTS and key in FORBIDDEN_ARGUMENTS):
+        logging.error(f"Invalid {task_type=}, {executable_kind=}.")
+        raise ValueError(f"Invalid {task_type=}, {executable_kind=}.")
+
+    required_args = REQUIRED_ARGUMENTS[key]
+    forbidden_args = FORBIDDEN_ARGUMENTS[key]
+
+    schema_properties = set(schema["properties"].keys())
+
+    logging.info(
+        f"[validate_arguments] Task has arguments: {schema_properties}"
+    )
+    logging.info(f"[validate_arguments] Required arguments: {required_args}")
+    logging.info(f"[validate_arguments] Forbidden arguments: {forbidden_args}")
+
+    missing_required_arguments = {
+        arg for arg in required_args if arg not in schema_properties
+    }
+    if missing_required_arguments:
+        error_msg = (
+            "[validate_arguments] Required arguments "
+            f"{missing_required_arguments} are missing."
+        )
+        logging.error(error_msg)
+        raise ValueError(error_msg)
+
+    present_forbidden_args = forbidden_args.intersection(schema_properties)
+    if present_forbidden_args:
+        error_msg = (
+            "[validate_arguments] Forbidden arguments "
+            f"{present_forbidden_args} are present."
+        )
+        logging.error(error_msg)
+        raise ValueError(error_msg)

fractal_task_tools/task_models.py

@@ -1,4 +1,5 @@
 from typing import Any
+from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
@@ -18,6 +19,23 @@ class _BaseTask(BaseModel):
     modality: Optional[str] = None
     tags: list[str] = Field(default_factory=list)
     docs_info: Optional[str] = None
+    type: str
+
+    @property
+    def executable_non_parallel(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def meta_non_parallel(self) -> Optional[dict[str, Any]]:
+        raise NotImplementedError()
+
+    @property
+    def executable_parallel(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def meta_parallel(self) -> Optional[dict[str, Any]]:
+        raise NotImplementedError()
 
 
 class CompoundTask(_BaseTask):
@@ -29,6 +47,34 @@ class CompoundTask(_BaseTask):
 
     executable_init: str
     meta_init: Optional[dict[str, Any]] = None
+    type: Literal["compound"] = "compound"
+
+    @property
+    def executable_non_parallel(self) -> str:
+        return self.executable_init
+
+    @property
+    def meta_non_parallel(self) -> Optional[dict[str, Any]]:
+        return self.meta_init
+
+    @property
+    def executable_parallel(self) -> str:
+        return self.executable
+
+    @property
+    def meta_parallel(self) -> Optional[dict[str, Any]]:
+        return self.meta
+
+
+class ConverterCompoundTask(_BaseTask):
+    """
+    A `ConverterCompoundTask` task is the same as a `CompoundTask`, but it
+    is executed differently in the Fractal backend.
+    """
+
+    executable_init: str
+    meta_init: Optional[dict[str, Any]] = None
+    type: Literal["converter_compound"] = "converter_compound"
 
     @property
     def executable_non_parallel(self) -> str:
@@ -53,6 +99,33 @@ class NonParallelTask(_BaseTask):
     may include the `meta` attribute.
     """
 
+    type: Literal["non_parallel"] = "non_parallel"
+
+    @property
+    def executable_non_parallel(self) -> str:
+        return self.executable
+
+    @property
+    def meta_non_parallel(self) -> Optional[dict[str, Any]]:
+        return self.meta
+
+    @property
+    def executable_parallel(self) -> None:
+        return None
+
+    @property
+    def meta_parallel(self) -> None:
+        return None
+
+
+class ConverterNonParallelTask(_BaseTask):
+    """
+    A `ConverterNonParallelTask` task is the same as a `NonParallelTask`, but
+    it is executed differently in the Fractal backend.
+    """
+
+    type: Literal["converter_non_parallel"] = "converter_non_parallel"
+
     @property
     def executable_non_parallel(self) -> str:
         return self.executable
@@ -76,6 +149,8 @@ class ParallelTask(_BaseTask):
     include the `meta` attribute.
     """
 
+    type: Literal["parallel"] = "parallel"
+
     @property
     def executable_non_parallel(self) -> None:
         return None

{fractal_task_tools-0.0.7.dist-info → fractal_task_tools-0.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: fractal-task-tools
-Version: 0.0.7
+Version: 0.0.9
 Summary: Shared tools for Fractal tasks
 Author-email: Tommaso Comparin <tommaso.comparin@exact-lab.it>
 License: BSD-3-Clause
@@ -17,6 +17,14 @@ Requires-Dist: bumpver==2024.1130; extra == "dev"
 Requires-Dist: devtools==0.12.2; extra == "dev"
 Requires-Dist: pytest<9.0.0,>=8.3.0; extra == "dev"
 Requires-Dist: coverage<7.7.0,>=7.6.0; extra == "dev"
+Requires-Dist: mkdocs==1.5.2; extra == "dev"
+Requires-Dist: mkdocs-material==9.1.21; extra == "dev"
+Requires-Dist: mkdocs-literate-nav==0.5.0; extra == "dev"
+Requires-Dist: mkdocs-gen-files==0.4.0; extra == "dev"
+Requires-Dist: mkdocs-section-index==0.3.5; extra == "dev"
+Requires-Dist: mkdocstrings[python]==0.25.2; extra == "dev"
+Requires-Dist: mkdocs-include-markdown-plugin<6.0.0,>=5.0.0; extra == "dev"
+Dynamic: license-file
 
 # Fractal task tools
 

fractal_task_tools-0.0.9.dist-info/RECORD

@@ -0,0 +1,21 @@
+fractal_task_tools/__init__.py,sha256=sLICpL6OJuGAjnPq5Y06u4kqhfz653Ng7rRdEl3mlxk,79
+fractal_task_tools/_args_schemas.py,sha256=Ka5IAAEdC7xg03ZHMl5jFbv4Sk8JR1QKbxFl7soevlQ,7829
+fractal_task_tools/_cli.py,sha256=2yxGcktEwHYXrRg_KxhO94xqovYkapBSQZNes94v_so,2458
+fractal_task_tools/_cli_tools.py,sha256=6G7zXvL6o5t7n4HvVAqP6an0o7qx5fEau5KjtoDXFuc,2622
+fractal_task_tools/_create_manifest.py,sha256=kN_WPJMYz-4vPQ8RYUcLbGW6g8kWjs24J-rtrSXU1Eo,5219
+fractal_task_tools/_deepdiff.py,sha256=Or9wO0E9Icb_ki_DTjIdgJKd4JpSJrk8UL3DqqUf-5g,2241
+fractal_task_tools/_descriptions.py,sha256=qYwmpN11UYUmW-PH85ZrHJ3gB6bVjJZ52V9I9Rad3-w,7975
+fractal_task_tools/_package_name_tools.py,sha256=9CyzFwOw4akvQc5GViBm4g-vU8pCc0ULh9hs3b_7VCY,823
+fractal_task_tools/_pydantic_generatejsonschema.py,sha256=qZuID7YUXOdAcL8OqsWjNFNumOIAgdJillc1lA2cHIY,3136
+fractal_task_tools/_signature_constraints.py,sha256=RzJwoL-NMD0HXEACR4_4aD8OOc59lxqEreT7oTc6xJo,3188
+fractal_task_tools/_task_arguments.py,sha256=gw9xN3E50pyyvyIqkWfd-WE5D6CRIRIk618Nw96WmDg,2753
+fractal_task_tools/_task_docs.py,sha256=aEXozSKf3a7weOwJMHyTVJTvHlCKgDr1qoU-AAO3bZI,3401
+fractal_task_tools/_titles.py,sha256=GLWn-06fgQD6qzOM75H59EV0MMCXc8jVpHqGanYzNbw,3000
+fractal_task_tools/task_models.py,sha256=4qBDUAoqIgdfAV-7CKkVO5--Y_2Y6CflNVMIiwJxn9g,4177
+fractal_task_tools/task_wrapper.py,sha256=dhphKgxDm4EUxZnsrAy20hJPD6bGdqx7tNg9N_QzlCo,1878
+fractal_task_tools-0.0.9.dist-info/licenses/LICENSE,sha256=1SGAsQ3Jm_nIU7c2TgtTZe_IOKjm9BDsrcf2r98xrdk,1584
+fractal_task_tools-0.0.9.dist-info/METADATA,sha256=FTnMNU5H7uqrUnTIQsjmHa_3hTFeac7z2EmoCmBnlb4,4638
+fractal_task_tools-0.0.9.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+fractal_task_tools-0.0.9.dist-info/entry_points.txt,sha256=zE4qv7QhuiqN6DaPkmJV18X1xyYoUi0HIJ-uAg2M6TU,66
+fractal_task_tools-0.0.9.dist-info/top_level.txt,sha256=2VBpiMDIBMJGOEPiHHX3njYEZGLhr4L0nu8vfkcNVzw,19
+fractal_task_tools-0.0.9.dist-info/RECORD,,

{fractal_task_tools-0.0.7.dist-info → fractal_task_tools-0.0.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fractal_task_tools-0.0.7.dist-info/RECORD

@@ -1,20 +0,0 @@
-fractal_task_tools/__init__.py,sha256=6mDzYc6ONWs6jFCx1ww0ZebQhqb6Sjw_rNkZFsdhupE,79
-fractal_task_tools/_args_schemas.py,sha256=Ka5IAAEdC7xg03ZHMl5jFbv4Sk8JR1QKbxFl7soevlQ,7829
-fractal_task_tools/_cli.py,sha256=OULWSnI4_ziNWYoZWAiPQu2PgeQwJJRIh3wLkX2H7wA,2134
-fractal_task_tools/_cli_tools.py,sha256=6G7zXvL6o5t7n4HvVAqP6an0o7qx5fEau5KjtoDXFuc,2622
-fractal_task_tools/_create_manifest.py,sha256=p5Z7ZZy8EkwWGPZHsqfNtkmLEAmHfL5QDfuvqXWwWEY,4821
-fractal_task_tools/_deepdiff.py,sha256=M3YZbAhZZx6iiAFvORZpxNENBpWmdEqkG29UJ7dFAcA,2239
-fractal_task_tools/_descriptions.py,sha256=n7WcoIBQ4FEd16BI26iL-Sh6bEQ76jlU32JfJBF4U0A,7566
-fractal_task_tools/_package_name_tools.py,sha256=_UT2cThh742V6M0XT9m0_aByhj5fXRSjATKt_MIXFVg,819
-fractal_task_tools/_pydantic_generatejsonschema.py,sha256=qZuID7YUXOdAcL8OqsWjNFNumOIAgdJillc1lA2cHIY,3136
-fractal_task_tools/_signature_constraints.py,sha256=RzJwoL-NMD0HXEACR4_4aD8OOc59lxqEreT7oTc6xJo,3188
-fractal_task_tools/_task_docs.py,sha256=aEXozSKf3a7weOwJMHyTVJTvHlCKgDr1qoU-AAO3bZI,3401
-fractal_task_tools/_titles.py,sha256=GLWn-06fgQD6qzOM75H59EV0MMCXc8jVpHqGanYzNbw,3000
-fractal_task_tools/task_models.py,sha256=kgvEv5NQD8e8I9tyaFrnvxUxbKm6vH63r-cnID1jHE8,2269
-fractal_task_tools/task_wrapper.py,sha256=dhphKgxDm4EUxZnsrAy20hJPD6bGdqx7tNg9N_QzlCo,1878
-fractal_task_tools-0.0.7.dist-info/LICENSE,sha256=1SGAsQ3Jm_nIU7c2TgtTZe_IOKjm9BDsrcf2r98xrdk,1584
-fractal_task_tools-0.0.7.dist-info/METADATA,sha256=93L2GIU-i1gVuUE4IjAh-x1vgp-iHhWuIQRol1gucmk,4208
-fractal_task_tools-0.0.7.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-fractal_task_tools-0.0.7.dist-info/entry_points.txt,sha256=zE4qv7QhuiqN6DaPkmJV18X1xyYoUi0HIJ-uAg2M6TU,66
-fractal_task_tools-0.0.7.dist-info/top_level.txt,sha256=2VBpiMDIBMJGOEPiHHX3njYEZGLhr4L0nu8vfkcNVzw,19
-fractal_task_tools-0.0.7.dist-info/RECORD,,