pluginify 1.0.0.post11.dev0__py3-none-any.whl → 1.0.0.post41.dev0__py3-none-any.whl

pluginify/create_plugin_registries.py

@@ -17,7 +17,7 @@ import logging
 from importlib import metadata, resources, util, import_module
 from inspect import signature
 import json
-from os import getpid, makedirs, remove, replace
+from os import makedirs, remove, replace
 from os.path import (
     basename,
     dirname,
@@ -29,7 +29,9 @@ from os.path import (
 )
 from pathlib import Path
 import re
+import tempfile
 import warnings
+from typing import Any
 
 import yaml
 
@@ -39,7 +41,7 @@ from pluginify.errors import PluginRegistryError
 LOG = logging.getLogger(__name__)
 
 
-def format_docstring(docstring, use_regex=True):
+def format_docstring(docstring: str, use_regex: bool = True) -> str:
     """Format the provided docstring placement in the plugin registry.
 
     Found when using the CLI and inspecting the registry, some docstrings are formatted
@@ -49,12 +51,12 @@ def format_docstring(docstring, use_regex=True):
 
     Parameters
     ----------
-    docstring: str
-        - The docstring which we are going to format.
-    use_regex: bool, optional (default=False)
-        - Whether or not we want to apply regex formatting to the docstring. Usually
-          recommended as it will replace 'newline' chars but not purposeful
-          '.newline' strings.
+    docstring : str
+        The docstring which we are going to format.
+    use_regex : bool, optional (default=False)
+        Whether or not we want to apply regex formatting to the docstring. Usually
+        recommended as it will replace 'newline' chars but not purposeful
+        '.newline' strings.
     """
     if docstring:
         if use_regex:
@@ -70,23 +72,19 @@ def format_docstring(docstring, use_regex=True):
     return docstring
 
 
-def remove_registries(namespace, plugin_packages):
+def remove_registries(namespace: str, plugin_packages: list) -> None:
     """Remove all plugin registries if a PluginRegistryError is raised.
 
     Parameters
     ----------
-    namespace: str
+    namespace : str
         Namespace that your plugin packages fall under. The argument parser defaults
         this value to 'pluginify.plugin_packages', but a user can create separate
         namespaces if developing interfaces outside of pluginify.
-    plugin_packages: list[EntryPoints]
+    plugin_packages : list[EntryPoints]
         A list of EntryPoints pointing to each installed package --> ie.
         [EntryPoint(name='pluginify', value='pluginify',
         group='pluginify.plugin_packages'), ...]
-
-    Returns
-    -------
-    None
     """
     # Remove registries is called whenever an improperly formatted plugin or
     # package is encountered.  This is not called until all errors have been
@@ -110,7 +108,9 @@ def remove_registries(namespace, plugin_packages):
             remove(json_plug_path)
 
 
-def registry_sanity_check(plugin_packages, save_type, namespace):
+def registry_sanity_check(
+    plugin_packages: list, save_type: str, namespace: str
+) -> None:
     """Check that each plugin package registry has no duplicate lowest depth entries.
 
     If it does, raise a `PluginRegistryError` for that specific package, then remove all
@@ -120,19 +120,15 @@ def registry_sanity_check(plugin_packages, save_type, namespace):
 
     Parameters
     ----------
-    plugin_packages: list EntryPoints
+    plugin_packages : list EntryPoints
         A list of EntryPoints pointing to each installed package --> ie.
         `[EntryPoint(name='pluginify', value='pluginify',
         group='pluginify.plugin_packages')]`
-    save_type: str
+    save_type : str
         The file format to save to `[json, yaml]`
 
-    Returns
-    -------
-    No returns
-
-    Exceptions
-    ----------
+    Raises
+    ------
     PluginRegistryError
         If `error_message` has contents, then raise PluginRegistryError(error_message).
         The `error_message` string will collect and report on all errors within
@@ -262,7 +258,9 @@ def registry_sanity_check(plugin_packages, save_type, namespace):
         raise PluginRegistryError(error_message)
 
 
-def check_plugin_exists(package, plugins, interface_name, plugin_name, relpath):
+def check_plugin_exists(
+    package: str, plugins: dict, interface_name: str, plugin_name: str, relpath: str
+) -> str:
     """Check if plugin already exists. If it does raise a `PluginRegistryError`.
 
     Note this only checks for duplicate plugins within a single plugin package.
@@ -271,14 +269,16 @@ def check_plugin_exists(package, plugins, interface_name, plugin_name, relpath):
 
     Parameters
     ----------
-    package: str
+    package : str
         The package being tested against
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed plugins in the current plugin package.
-    interface_name: str
+    interface_name : str
         A string representing the interface being checked against
-    plugin_name: str
+    plugin_name : str
         A string representing the name of the plugin within the interface
+    relpath : str
+        The relative path to the plugin being checked.
 
     Returns
     -------
@@ -299,21 +299,19 @@ def check_plugin_exists(package, plugins, interface_name, plugin_name, relpath):
     return ""
 
 
-def write_registered_plugins(pkg_dir, plugins, save_type):
+def write_registered_plugins(
+    pkg_dir: str | Path, plugins: dict, save_type: str
+) -> None:
     """Write dictionary of all plugins available from installed packages.
 
     Parameters
     ----------
-    pkg_dir: str
+    pkg_dir : str
         Path in which to write registered_plugins
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed package plugins
-    save_type: str
+    save_type : str
         The file format to save to [json, yaml]
-
-    Returns
-    -------
-    No returns, file written to `pkg_dir`
     """
     if save_type == "yaml":
         reg_plug_abspath = os_join(pkg_dir, "registered_plugins.yaml")
@@ -323,21 +321,32 @@ def write_registered_plugins(pkg_dir, plugins, save_type):
     # Write to a temporary file and move it into place, so that other processes
     # reading the registry (for example parallel test workers) never see a
     # partially written file.
-    tmp_abspath = f"{reg_plug_abspath}.{getpid()}.tmp"
+    tmp_abspath = None
     try:
-        with open(tmp_abspath, "w") as plugin_registry:
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            dir=dirname(reg_plug_abspath),
+            suffix=".tmp",
+            delete=False,
+        ) as plugin_registry:
+            tmp_abspath = plugin_registry.name
             if save_type == "yaml":
                 yaml.safe_dump(plugins, plugin_registry, default_flow_style=False)
             else:
                 json.dump(plugins, plugin_registry, indent=4)
+
         replace(tmp_abspath, reg_plug_abspath)
-    except BaseException:
-        if exists(tmp_abspath):
-            remove(tmp_abspath)
-        raise
+    finally:
+        if tmp_abspath is not None:
+            try:
+                remove(tmp_abspath)
+            except FileNotFoundError:
+                pass
 
 
-def create_plugin_registries(plugin_packages, save_type, namespace):
+def create_plugin_registries(
+    plugin_packages: list, save_type: str, namespace: str
+) -> None:
     """Generate all plugin paths associated with every installed packages.
 
     These paths include text plugins, class_based plugins
@@ -347,13 +356,13 @@ def create_plugin_registries(plugin_packages, save_type, namespace):
 
     Parameters
     ----------
-    plugin_packages: list EntryPoints
+    plugin_packages : list EntryPoints
         A list of EntryPoints pointing to each installed package --> ie.
         [EntryPoint(name='pluginify', value='pluginify',
         group='pluginify.plugin_packages'), ...]
-    save_type: str
+    save_type : str
         The file format to save to [json, yaml]
-    namespace: str
+    namespace : str
         Namespace that your plugin packages fall under. The argument parser defaults
         this value to 'pluginify.plugin_packages', but a user can create separate
         namespaces if developing interfaces outside of pluginify.
@@ -361,8 +370,8 @@ def create_plugin_registries(plugin_packages, save_type, namespace):
     # It appears when there is *.egg-info directory, it picks that package up
     # twice in the list.  If the same package path exists twice, only keep one
     # of them.  This appears to be a bug with Python 3.9 entry points.
-    pkg_dirs = []
-    unique_package_entry_points = []
+    pkg_dirs: list = []
+    unique_package_entry_points: list = []
     # Note we only use ep.value (resources.files finds the actual plugins),
     # so we do not need to worry about saving the "wrong" package here.
     # We are actually looping through all the files in each package, so
@@ -381,7 +390,7 @@ def create_plugin_registries(plugin_packages, save_type, namespace):
     for pkg in unique_package_entry_points:
         # This is passed by reference and populated with each call to parse
         # plugin packages.
-        plugins = {
+        plugins: dict = {
             "text_based": {},
             "yaml_based": {},
             "class_based": {},
@@ -398,7 +407,7 @@ def create_plugin_registries(plugin_packages, save_type, namespace):
         # that holds all plugins, and the package_schema entry point directly
         # to the directory that holds all scheam, so the subdirectories are
         # not hard coded here in the create_plugin_registries code.
-        pkg_plugin_path = resources.files(package) / "plugins"
+        pkg_plugin_path = Path(str(resources.files(package) / "plugins"))
         pkg_dir = str(resources.files(package))
         # Generate a path to the parent directory where we should write the plugin
         # registry file for this package to.
@@ -455,7 +464,13 @@ def create_plugin_registries(plugin_packages, save_type, namespace):
     registry_sanity_check(unique_package_entry_points, save_type, namespace)
 
 
-def parse_plugin_paths(plugin_paths, package, package_dir, plugins, namespace):
+def parse_plugin_paths(
+    plugin_paths: dict,
+    package: str,
+    package_dir: str,
+    plugins: dict,
+    namespace: str,
+) -> str:
     """Parse the plugin_paths provided from the current installed package.
 
     Then, add them to the plugins dictionary based on the path of the plugin.
@@ -464,22 +479,22 @@ def parse_plugin_paths(plugin_paths, package, package_dir, plugins, namespace):
 
     Parameters
     ----------
-    plugin_paths: dict
+    plugin_paths : dict
         A dictionary of filepaths, with keys referring to the type of plugin
-    package: str
+    package : str
         The current package being parsed
-    package_dir: str
+    package_dir : str
         The path to the current package (for determining relative paths)
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed package plugins
-    namespace: str
+    namespace : str
         Namespace that your plugin packages fall under. The argument parser defaults
         this value to 'pluginify.plugin_packages', but a user can create separate
         namespaces if developing interfaces outside of pluginify.
 
     Returns
     -------
-    error_message: str
+    error_message : str
         String containing informative error messages from any plugins that
         were improperly formatted.  An exception will be raised at the
         very end if error_message is not the empty string - this allows
@@ -531,27 +546,29 @@ def parse_plugin_paths(plugin_paths, package, package_dir, plugins, namespace):
     return error_message
 
 
-def add_yaml_plugin(filepath, relpath, package, plugins, namespace):
+def add_yaml_plugin(
+    filepath: str, relpath: str, package: str, plugins: dict, namespace: str
+) -> str:
     """Add the yaml plugin associated with the filepaths and package to plugins.
 
     Parameters
     ----------
-    filepath: str
+    filepath : str
         The path of the plugin derived from resouces.files(package) / plugin
-    relpath: str
+    relpath : str
         The relative path to the filepath provided
-    package: str
+    package : str
         The current package being parsed
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed package plugins
-    namespace: str
+    namespace : str
         Namespace that your plugin packages fall under. The argument parser defaults
         this value to 'pluginify.plugin_packages', but a user can create separate
         namespaces if developing interfaces outside of pluginify.
 
     Returns
     -------
-    error_message: str
+    error_message : str
         String containing informative error messages from any plugins that
         were improperly formatted.  An exception will be raised at the
         very end if error_message is not the empty string - this allows
@@ -686,7 +703,7 @@ def add_yaml_plugin(filepath, relpath, package, plugins, namespace):
                     #         if not family:
                     #             family = plugins["product_defaults"][pd]["family"]
                     plugins[interface_name][subplg_source][subplg_product] = {
-                        "docstring": str(format_docstring(docstring)),
+                        "docstring": str(format_docstring(docstring or "")),
                         "family": family,
                         "interface": interface_module.name,
                         "package": plugin["package"],
@@ -727,21 +744,21 @@ def add_yaml_plugin(filepath, relpath, package, plugins, namespace):
     return error_message
 
 
-def add_text_plugin(package, relpath, plugins):
+def add_text_plugin(package: str, relpath: str, plugins: dict) -> str:
     """Add all text plugins into plugin registries.
 
     Parameters
     ----------
-    package: str
+    package : str
         The current package being parsed
-    relpath: str
+    relpath : str
         The relpath path to the text plugin
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed package plugins
 
     Returns
     -------
-    error_message: str
+    error_message : str
         String containing informative error messages from any plugins that
         were improperly formatted.  An exception will be raised at the
         very end if error_message is not the empty string - this allows
@@ -770,8 +787,12 @@ def add_text_plugin(package, relpath, plugins):
 
 
 def collect_module_plugin_metadata(
-    module, module_name, package, relpath, error_message
-):
+    module: Any,
+    module_name: str,
+    package: str,
+    relpath: str,
+    error_message: str,
+) -> dict | str:
     """Collect metadata for the module-based plugin for the plugin registry.
 
     Metadata necessary to collect here includes the interface of the plugin, the
@@ -783,27 +804,27 @@ def collect_module_plugin_metadata(
 
     Parameters
     ----------
-    module: ModuleType
-        - The object of the module which has been loaded in realtime. This should have
-          top level attributes such as 'interface', 'name', 'family' and so on.
-    module_name: str
-        - The name of the .py file.
-    package: str
-        - The name of the package this plugin comes from
-    relpath: str
-        - The relative path to the plugin found in 'package'
-    error_message: str
-        - The current state of errors found during building the registry, stored as a
-          string.
+    module : ModuleType
+        The object of the module which has been loaded in realtime. This should have
+        top level attributes such as 'interface', 'name', 'family' and so on.
+    module_name : str
+        The name of the .py file.
+    package : str
+        The name of the package this plugin comes from
+    relpath : str
+        The relative path to the plugin found in 'package'
+    error_message : str
+        The current state of errors found during building the registry, stored as a
+        string.
 
     Returns
     -------
-    metadata: dict
-        - A dictionary of metadata for the provided plugin class to store in the plugin
-          registry.
-    error_message: str
-        - The current state of errors found during building the registry, stored as a
-          string.
+    metadata : dict
+        A dictionary of metadata for the provided plugin class to store in the plugin
+        registry.
+    error_message : str
+        The current state of errors found during building the registry, stored as a
+        string.
     """
     # Try to get "interface" variable from the module.  This is required
     # on ALL files within the python module based plugins directory, to
@@ -874,7 +895,7 @@ def collect_module_plugin_metadata(
     return metadata
 
 
-def collect_class_plugin_metadata(plugin_class):
+def collect_class_plugin_metadata(plugin_class, package, relpath, error_message):
     """Collect metadata linked to the class plugin in 'module' for the plugin registry.
 
     Metadata necessary to collect here includes the interface of the plugin, the
@@ -883,24 +904,60 @@ def collect_class_plugin_metadata(plugin_class):
 
     Parameters
     ----------
-    plugin_class: Object, default=None
-        - The plugin class from 'module' to collect metadata from. If None, this
-          function will attempt to locate the correct plugin class from the provided
-          module.
+    plugin_class : Object, default=None
+        The plugin class from 'module' to collect metadata from. If None, this
+        function will attempt to locate the correct plugin class from the provided
+        module.
 
     Returns
     -------
-    metadata: dict
-        - A dictionary of metadata for the provided plugin class to store in the plugin
-          registry.
+    metadata : dict
+        A dictionary of metadata for the provided plugin class to store in the plugin
+        registry.
     """
-    # NOTE: Probably should add some attribute checks here as
-    # 'collect_module_plugin_metadata' does
+    error_occurred = False
+
+    try:
+        family = plugin_class.family
+    except AttributeError:
+        if not getattr(plugin_class, "data_tree"):
+            error_occurred = True
+            error_message += (
+                f"Error: missing 'family' attribute in non-data-tree class based "
+                f"plugin found at '{relpath}' in package '{package}'. This attribute "
+                "must be specified for non-data-tree class-based plugins."
+            )
+        else:
+            family = "N/A: data tree based plugin"
+
+    try:
+        name = plugin_class.name
+    except AttributeError:
+        error_occurred = True
+        error_message += (
+            f"Error: missing 'name' attribute in class based "
+            f"plugin found at '{relpath}' in package '{package}'. This attribute must "
+            "be specified in class-based plugins."
+        )
+
+    try:
+        interface = plugin_class.interface
+    except AttributeError:
+        error_occurred = True
+        error_message += (
+            f"Error: missing 'interface' attribute in class based "
+            f"plugin found at '{relpath}' in package '{package}'. This attribute must "
+            "be specified in class-based plugins."
+        )
+
+    if error_occurred:
+        return error_message
+
     metadata = {
         "docstring": format_docstring(plugin_class.__doc__),
-        "name": plugin_class.name,
-        "family": plugin_class.family,
-        "interface": plugin_class.interface,
+        "name": name,
+        "family": family,
+        "interface": interface,
         "plugin_type": "class_based",
         "signature": str(signature(plugin_class.call)),
         "is_derived_from_module": False,
@@ -909,7 +966,7 @@ def collect_class_plugin_metadata(plugin_class):
     return metadata
 
 
-def add_class_plugin(package, relpath, plugins):
+def add_class_plugin(package: str, relpath: str, plugins: dict) -> str:
     """Add the class-based plugin associated with the filepaths and package to plugins.
 
     NOTE: This function will work for 'legacy' module-based plugins that are dynamically
@@ -920,16 +977,16 @@ def add_class_plugin(package, relpath, plugins):
 
     Parameters
     ----------
-    package: str
+    package : str
         The current package being parsed
-    relpath: str
+    relpath : str
         The relpath path to the class-based plugin
-    plugins: dict
+    plugins : dict
         A dictionary object of all installed package plugins
 
     Returns
     -------
-    error_message: str
+    error_message : str
         String containing informative error messages from any plugins that
         were improperly formatted. An exception will be raised at the
         very end if error_message is not the empty string - this allows
@@ -960,7 +1017,11 @@ def add_class_plugin(package, relpath, plugins):
     module_path = f"{package}.{module_path}"
     abspath = resources.files(package) / relpath
 
-    spec = util.spec_from_file_location(module_path, abspath)
+    spec = util.spec_from_file_location(module_path, str(abspath))
+    if spec is None or spec.loader is None:
+        raise PluginRegistryError(
+            f"Could not create a module spec for '{module_path}' at '{abspath}'."
+        )
     module = util.module_from_spec(spec)
     # Attempting importing module, catch ImportError
     # We have to fix these to be able to import the module to
@@ -1012,7 +1073,14 @@ def add_class_plugin(package, relpath, plugins):
             error_message += str(plugin_class_error_message)
             return error_message
 
-        plugin_metadata = collect_class_plugin_metadata(PLUGIN_CLASS)
+        plugin_metadata = collect_class_plugin_metadata(
+            PLUGIN_CLASS, package, relpath, error_message
+        )
+        if isinstance(plugin_metadata, str):
+            # Error message was returned instead of a metadata dictionary. Just return
+            # that message now.
+            return plugin_metadata
+
         interface_name = plugin_metadata["interface"]
         name = plugin_metadata.pop("name")
 

pluginify/errors.py

@@ -1,16 +1,24 @@
 # # # This source code is subject to the license referenced at
 # # # https://github.com/NRLMMD-GEOIPS.
 
-"""Pluginify error module."""
+"""Exceptions raised by the pluginify package.
+
+Provides two error classes:
+
+- ``PluginError`` — raised when an individual plugin is malformed, missing
+  required attributes, or otherwise invalid.
+- ``PluginRegistryError`` — raised when a plugin registry itself is missing,
+  corrupt, or contains duplicate/conflicting entries across packages.
+"""
 
 
 class PluginError(Exception):
-    """Exception to be raised when there is an error in a plugin."""
+    """Exception raised for errors related to an individual plugin."""
 
     pass
 
 
 class PluginRegistryError(Exception):
-    """Exception to be raised when there is an error in a plugin registry."""
+    """Exception raised for errors related to a plugin registry."""
 
     pass

pluginify/interfaces/__init__.py

@@ -1,7 +1,16 @@
 # # # This source code is subject to the license referenced at
 # # # https://github.com/NRLMMD-GEOIPS.
 
-"""Initialization module for pluginify interfaces."""
+"""Interface discovery and initialization module for pluginify.
+
+Imports all available class-based and YAML-based interface instances so that
+other parts of the pluginify ecosystem can discover them via
+``from pluginify.interfaces import <interface_name>``.
+
+The ``class_based_interfaces`` and ``yaml_based_interfaces`` lists are used by
+``PluginRegistryValidator.validate_registry_interfaces`` to validate that
+registry entries reference known interface names.
+"""
 
 from pluginify.interfaces.class_based.data_modifiers import data_modifiers  # noqa: F401
 from pluginify.interfaces.yaml_based.configs import configs  # noqa: F401