pluginify 0.0.0.post1.dev0__py3-none-any.whl → 0.1.0.post11.dev0__py3-none-any.whl

pluginify/_version.py

@@ -3,5 +3,5 @@
 
 # DO NOT EDIT
 # managed by poetry-dynamic-versioning
-__version__ = "0.0.0.post1.dev0"
-__version_tuple__ = (0, 0, 0, "post1", "dev0")
+__version__ = "0.1.0.post11.dev0"
+__version_tuple__ = (0, 1, 0, "post11", "dev0")

pluginify/commandline_typer.py

@@ -56,8 +56,8 @@ def configure_logging(level=logging.INFO):
     root = logging.getLogger()
 
     # Avoid duplicate handlers if CLI is called multiple times
-    if root.handlers:
-        return
+    # if root.handlers:
+    #     return
 
     root.setLevel(level)
 
@@ -131,6 +131,10 @@ class DocstringTyper(typer.Typer):
             if doc.long_description:
                 new_doc += "\n\n" + doc.long_description
 
+            # Keep examples portion in command's help output if specified
+            if doc.examples:
+                new_doc += "\n\nExamples\n--------\n" + doc.examples[0].description
+
             func.__doc__ = new_doc
 
             new_params = []
@@ -165,10 +169,32 @@ class DocstringTyper(typer.Typer):
         return wrapper
 
 
-app = DocstringTyper(context_settings={"help_option_names": ["-h", "--help"]})
+app = DocstringTyper(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    help="pluginify command line interface.",
+)
 config_app = DocstringTyper()
-
-app.add_typer(config_app, name="config", help="Configuration commands for pluginify.")
+config_set_app = DocstringTyper()
+
+config_variable_help = (
+    "NAMESPACE specifies the default namespace in which pluginify will manage "
+    "registries.\n"
+    "REGISTRY_DIRECTORY specifies the base directory where registries and their "
+    "supporting directory structure will be written.\n"
+    "REBUILD_REGISTRIES informs pluginify whether or not to rebuild registries at "
+    "runtime if a plugin cannot be located."
+)
+
+app.add_typer(
+    config_app,
+    name="config",
+    help=f"Configure pluginify.\n\n{config_variable_help}",
+)
+config_app.add_typer(
+    config_set_app,
+    name="set",
+    help=f"Set pluginify configuration variables.\n\n{config_variable_help}",
+)
 
 
 @app.command()
@@ -177,7 +203,37 @@ def create(
     packages: Optional[List[str]] = None,
     save_type: Literal["json", "yaml"] = "json",
 ):
-    """Create plugin registry files for one or more packages under a given namespace.
+    """Create plugin registries.
+
+    Plugin registries are low-level files which contain dictionaries of metadata for all
+    plugins implemented in one or more packages registered under a common namespace.
+
+    Pluginify creates and makes use of these registry files to properly load and
+    instantiate your plugins.
+
+    Examples
+    --------
+    Namespace: packageX.plugin_packages
+    Directory structure:
+
+        packageX/
+            plugins/
+            interfaces/
+            utils/
+            registered_plugins.json [will create]
+        packageY/
+            plugins/
+            registered_plugins.json [will create]
+
+    JSON registry files are faster to load and are used by default. YAML
+    extensions are also supported for easier viewing.
+
+    CLI examples
+    ------------
+    pluginify create
+    pluginify create -n packageX.plugin_packages
+    pluginify create -s yaml
+    pluginify create -n packageX.plugin_packages -p packageY
 
     Parameters
     ----------
@@ -199,7 +255,35 @@ def delete(
     namespace: str = NAMESPACE,
     packages: Optional[List[str]] = None,
 ):
-    """Delete plugin registry files for one or more packages under a given namespace.
+    """Delete plugin registries.
+
+    Plugin registries are low-level files which contain dictionaries of metadata for all
+    plugins implemented in one or more packages registered under a common namespace.
+
+    By default, this command will delete every .json and .yaml instance of registry
+    files for every package registered under a given namespace. If you only want to
+    delete registry files from a subset of packages, use the '-p' flag to specify those
+    packages.
+
+    Examples
+    --------
+    Namespace: packageX.plugin_packages
+    Directory structure:
+
+        packageX/
+            plugins/
+            interfaces/
+            utils/
+            registered_plugins.json [will delete]
+        packageY/
+            plugins/
+            registered_plugins.json [will delete]
+
+    CLI examples
+    ------------
+    pluginify delete
+    pluginify delete -n packageX.plugin_packages
+    pluginify delete -n packageX.plugin_packages -p packageY
 
     Parameters
     ----------
@@ -213,7 +297,7 @@ def delete(
     plugin_registry.delete_registries(packages=packages)
 
 
-@config_app.command("set-rebuild-registries")
+@config_set_app.command("rebuild-registries")
 def set_rebuild_registries(rebuild_registries: bool):
     """Set pluginify's REBUILD_REGISTRIES config variable.
 
@@ -226,7 +310,7 @@ def set_rebuild_registries(rebuild_registries: bool):
     update_existing_fields({"REBUILD_REGISTRIES": rebuild_registries})
 
 
-@config_app.command("set-namespace")
+@config_set_app.command("namespace")
 def set_namespace(namespace: str):
     """Set pluginify's NAMESPACE config variable.
 
@@ -239,7 +323,7 @@ def set_namespace(namespace: str):
     update_existing_fields({"NAMESPACE": namespace})
 
 
-@config_app.command("set-registry-directory")
+@config_set_app.command("registry-directory")
 def set_registry_directory(registry_directory: Path):
     """Set pluginify's REGISTRY_DIRECTORY config variable.
 

pluginify/create_plugin_registries.py

@@ -27,6 +27,7 @@ from os.path import (
     split,
     splitext,
 )
+from pathlib import Path
 import re
 import warnings
 
@@ -481,6 +482,13 @@ def parse_plugin_paths(plugin_paths, package, package_dir, plugins, namespace):
     for plugin_type in plugin_paths:
         # Loop through each file of the current plugin type.
         for filepath in plugin_paths[plugin_type]:
+            # If any 'part' of the full filepath starts with a '.' (dot) directory or
+            # file, do not use this filepath. Just continue to the next filepath
+            # provided. Resolving path to prevent false-positives on "." or ".."
+            # paths for relative paths as an edge case.
+            if any(part.startswith(".") for part in Path(filepath).resolve().parts):
+                continue
+
             filepath = str(filepath)
             # Path relative to the package directory
             relpath = os_relpath(filepath, start=package_dir)

pluginify/plugin_registry.py

@@ -36,6 +36,34 @@ from pluginify.utils import merge_nested_dicts
 LOG = logging.getLogger(__name__)
 
 
+def env_constructor(loader, node):
+    """YAML constructor for the ``!ENV`` tag that expands environment variables.
+
+    This function allows YAML files to include environment variables using
+    the ``!ENV`` tag. The scalar value associated with the tag is read and
+    any environment variables within the string (e.g. ``${VAR_NAME}``) are
+    expanded using :func:`os.path.expandvars`.
+
+    Parameters
+    ----------
+    loader : yaml.Loader
+        The YAML loader instance currently parsing the document.
+    node : yaml.Node
+        The YAML node containing the scalar value associated with the ``!ENV`` tag.
+
+    Returns
+    -------
+    str
+        The scalar value with any environment variables expanded using the
+        current process environment.
+    """
+    value = loader.construct_scalar(node)
+    return os.path.expandvars(value)
+
+
+yaml.SafeLoader.add_constructor("!ENV", env_constructor)
+
+
 class PluginRegistry:
     """Plugin Registry class definition.
 
@@ -372,7 +400,7 @@ class PluginRegistry:
 
         return metadata
 
-    def load_plugin(self, data: dict) -> BaseModel:
+    def load_plugin(self, data: dict, _expand: bool = False) -> BaseModel:
         """
         Dynamically load and validate pydantic models based on apiVersion and interface.
 
@@ -387,6 +415,10 @@ class PluginRegistry:
             Dictionary representing a plugin definition. Must include the `interface`
             field. May optionally include `apiVersion`. If not present, "pluginify/v1"
             is assumed.
+        _expand : private bool (default=False)
+            If true, fully expand the workflow plugin in place. Otherwise, load as is
+            done usually. This should only be used for the
+            'geoips expand <workflow>' command.
 
         Returns
         -------
@@ -400,7 +432,9 @@ class PluginRegistry:
         ImportError
             If the specified module for the given model version cannot be imported.
         """
-        api_version = data.get("apiVersion", "pluginify/v1")
+        # Make this GeoIPS for now, change later once apiVersion is set on all yaml
+        # plugins.
+        api_version = data.get("apiVersion", "geoips/v1")
 
         # Split "package_name/model_version"
         # Use package_name to select the appropriate package to search for the api.
@@ -418,9 +452,15 @@ class PluginRegistry:
 
         # Construct module path and import
         try:
-            module = import_module(
-                f"{package_name}.pydantic_models.{model_version}.{interface}"
-            )
+            if interface == "product_defaults":
+                # product_defaults model defined in products pydantic module
+                module = import_module(
+                    f"{package_name}.pydantic_models.{model_version}.products"
+                )
+            else:
+                module = import_module(
+                    f"{package_name}.pydantic_models.{model_version}.{interface}"
+                )
         except ImportError as e:
             raise ImportError(
                 f"Could not import models from '{api_version}': {e}"
@@ -431,15 +471,20 @@ class PluginRegistry:
 
         try:
             model_class = getattr(module, model_name)
-            print("model class \t", model_class)
         except AttributeError as e:
             raise ValueError(
                 f"Model '{model_name}' not found in '{api_version}'"
             ) from e
 
-        return model_class.model_validate(data)
+        if _expand:
+            # Only applies to workflow plugins
+            return model_class.model_validate(data, context={"expand": True})
 
-    def get_yaml_plugin(self, interface_obj, name, rebuild_registries=None):
+        return model_class(**data)
+
+    def get_yaml_plugin(
+        self, interface_obj, name, rebuild_registries=None, _expand=False
+    ):
         """Get a YAML plugin by its name.
 
         Parameters
@@ -456,7 +501,16 @@ class PluginRegistry:
               is true and get_plugin fails, rebuild the plugin registry, call then call
               get_plugin once more with rebuild_registries toggled off, so it only gets
               rebuilt once.
+        _expand: private bool (default=False)
+            - If true, fully expand the workflow plugin in place. Otherwise, load as is
+              done usually. This should only be used for the
+              'geoips expand <workflow>' command.
         """
+        if _expand and interface_obj.name != "workflows":
+            raise AssertionError(
+                "Error: you cannot set argument 'expand' to true unless you are "
+                "requesting a workflow plugin."
+            )
         try:
             registered_yaml_plugins = self.registered_plugins["yaml_based"]
         except KeyError:
@@ -562,7 +616,21 @@ class PluginRegistry:
         plugin["relpath"] = relpath
 
         if getattr(interface_obj, "use_pydantic", False):
-            return self.load_plugin(plugin).model_dump()
+
+            def remove_none(d: dict) -> dict:
+                """Recursively remove all keys with value None from a dictionary."""
+                if not isinstance(d, dict):
+                    return d
+                return {k: remove_none(v) for k, v in d.items() if v is not None}
+
+            validated = self.load_plugin(plugin, _expand).model_dump()
+            validated = remove_none(validated)
+
+            if "package" not in validated or "relpath" not in validated:
+                validated["package"] = package
+                validated["relpath"] = relpath
+
+            return interface_obj._plugin_yaml_to_obj(name, validated)
         else:
             validated = interface_obj.validator.validate(plugin)
             return interface_obj._plugin_yaml_to_obj(name, validated)

pluginify/utils/validators.py

@@ -0,0 +1,183 @@
+"""Pluginify validators module.
+
+Currently only implements a PluginRegistryValidator class.
+"""
+
+from importlib import metadata, import_module
+import json
+from pprint import pformat
+
+import pytest
+
+from pluginify.errors import PluginRegistryError
+from pluginify.plugin_registry import PluginRegistry
+
+
+class PluginRegistryValidator(PluginRegistry):
+    """Subclass of PluginRegistry which adds functionality for unit testing."""
+
+    def __init__(self, namespace, fpaths=None):
+        """Initialize TestPluginRegistry Class."""
+        self.namespace = namespace
+        super().__init__(self.namespace, _test_registry_files=fpaths)
+
+    def validate_plugin_types_exist(self, reg_dict, reg_path):
+        """Test that all top level plugin types exist in each registry file."""
+        expected_plugin_types = ["yaml_based", "class_based", "text_based"]
+        for p_type in expected_plugin_types:
+            if p_type not in reg_dict:
+                error_str = f"Expected plugin type '{p_type}' to be in the registry but"
+                error_str += f" wasn't. This was in file '{reg_path}'."
+                raise PluginRegistryError(error_str)
+
+    def validate_all_registries(self):
+        """Validate all registries in the current installation.
+
+        This should be run during testing, but not at runtime.
+        Ensure we do not fail catastrophically for a single bad plugin
+        at runtime, so test up front to test validity.
+        """
+        for reg_path in self.registry_files:
+            pkg_plugins = json.load(open(reg_path, "r"))
+            self.validate_registry(pkg_plugins, reg_path)
+
+    def validate_registry(self, current_registry, fpath):
+        """Test all plugins found in registered plugins for their validity."""
+        try:
+            self.validate_plugin_types_exist(current_registry, fpath)
+        except PluginRegistryError as e:
+            # xfail if this is a test, otherwise just raise PluginRegistryError
+            if self._is_test:
+                pytest.xfail(str(e))
+            else:
+                raise PluginRegistryError(e)
+        try:
+            self.validate_registry_interfaces(current_registry)
+        except PluginRegistryError as e:
+            # xfail if this is a test, otherwise just raise PluginRegistryError
+            if self._is_test:
+                pytest.xfail(str(e))
+            else:
+                raise PluginRegistryError(e)
+        for plugin_type in current_registry:
+            for interface in current_registry[plugin_type]:
+                for plugin in current_registry[plugin_type][interface]:
+                    try:
+                        if interface == "products":
+                            for subplg in current_registry[plugin_type][interface][
+                                plugin
+                            ]:
+                                self.validate_plugin_attrs(
+                                    plugin_type,
+                                    interface,
+                                    (plugin, subplg),
+                                    current_registry[plugin_type][interface][plugin][
+                                        subplg
+                                    ],
+                                )
+                        elif plugin_type != "text_based":
+                            self.validate_plugin_attrs(
+                                plugin_type,
+                                interface,
+                                plugin,
+                                current_registry[plugin_type][interface][plugin],
+                            )
+                    except PluginRegistryError as e:
+                        # xfail if this is a test,
+                        # otherwise just raise PluginRegistryError
+                        if self._is_test:
+                            pytest.xfail(str(e))
+                        else:
+                            raise PluginRegistryError(e)
+
+    def validate_plugin_attrs(self, plugin_type, interface, name, plugin):
+        """Test non-product plugin for all required attributes."""
+        missing = []
+        if plugin_type == "yaml_based" and interface != "products":
+            attrs = [
+                "docstring",
+                "family",
+                "interface",
+                "package",
+                "plugin_type",
+                "relpath",
+            ]
+        elif plugin_type == "yaml_based":
+            attrs = [
+                "docstring",
+                "family",
+                "interface",
+                "package",
+                "plugin_type",
+                "product_defaults",
+                "source_names",
+                "relpath",
+            ]
+        else:
+            attrs = [
+                "docstring",
+                "family",
+                "interface",
+                "package",
+                "plugin_type",
+                "signature",
+                "relpath",
+            ]
+        for attr in attrs:
+            try:
+                plugin[attr]
+            except KeyError:
+                missing.append(attr)
+        if missing:
+            raise PluginRegistryError(
+                f"Plugin '{name}' is missing the following required "
+                f"top-level properties: '{missing}'"
+            )
+
+    def validate_registry_interfaces(self, current_registry):
+        """Test Plugin Registry interfaces validity."""
+        yaml_interfaces = []
+        class_interfaces = []
+
+        # NOTE: all <pkg>/interfaces/__init__.py files MUST
+        # contain a "class_based_interfaces" list and a
+        # "yaml_based_interfaces" list - this is how we
+        # identify the valid interface names.
+        # We must avoid actually importing the interfaces within
+        # the pluginify repo - or we will end up with a circular import
+        # due to BaseInterface.
+        for pkg in metadata.entry_points(group=self.namespace):
+            try:
+                mod = import_module(f"{pkg.value}.interfaces")
+            except ModuleNotFoundError as resp:
+                if f"No module named '{pkg.value}.interfaces'" in str(resp):
+                    continue
+                else:
+                    raise ModuleNotFoundError(resp)
+            class_interfaces += mod.class_based_interfaces
+            yaml_interfaces += mod.yaml_based_interfaces
+
+        bad_interfaces = []
+        for plugin_type in ["class_based", "yaml_based"]:
+            for interface in current_registry[plugin_type]:
+                if (
+                    interface not in class_interfaces
+                    and interface not in yaml_interfaces
+                ):
+                    error_str = f"\nPlugin type '{plugin_type}' does not allow "
+                    error_str += f"interface '{interface}'.\n"
+                    error_str += "\nValid interfaces: "
+                    if plugin_type == "class_based":
+                        interface_list = class_interfaces
+                    else:
+                        interface_list = yaml_interfaces
+                    error_str += f"\n{interface_list}\n"
+                    error_str += "\nPlease update the following plugins "
+                    error_str += "to use a valid interface:\n"
+                    error_str += pformat(current_registry[plugin_type][interface])
+                    bad_interfaces.append(error_str)
+        if bad_interfaces:
+            error_str = "The following interfaces were not valid:\n"
+            for error in bad_interfaces:
+                error_str += error
+            raise PluginRegistryError(error_str)

{pluginify-0.0.0.post1.dev0.dist-info → pluginify-0.1.0.post11.dev0.dist-info}/METADATA

@@ -1,15 +1,17 @@
 Metadata-Version: 2.4
 Name: pluginify
-Version: 0.0.0.post1.dev0
+Version: 0.1.0.post11.dev0
 Summary: Pluginify package
 License: LICENSE
 License-File: LICENSE
 Author: Evan Rose
-Requires-Python: >=3.11.0,<3.13.0
+Requires-Python: >=3.11.0
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: doc
 Requires-Dist: docstring_parser
 Requires-Dist: jsonschema (>4.18.0)
@@ -49,9 +51,9 @@ git clone https://github.com/NRLMMD-GEOIPS/pluginify.git
 # cd to pluginify's top level dir
 pip install -e .
 ```
-In the future:
+OR
 ```bash
-    pip install pluginify
+pip install pluginify
 ```
 
 Use pluginify

{pluginify-0.0.0.post1.dev0.dist-info → pluginify-0.1.0.post11.dev0.dist-info}/RECORD

@@ -1,22 +1,23 @@
 pluginify/__init__.py,sha256=NvdicJtfy-dZZ8tWjR74lDBHGV6rlaH8pAJ7UHyTTog,391
-pluginify/_version.py,sha256=I_xg_zrKOMl6Y8jLXjH6TifTHdty9HEaI8c-DJniO18,237
-pluginify/commandline_typer.py,sha256=v9r7oBnD39pFLBqhoUNefGEwyRFCzDKpiPa3cvXXzCk,8676
+pluginify/_version.py,sha256=n4Mesoqrw8DK8LkjhySdwEOZeCLfBv_cuSDOZyMBxiU,239
+pluginify/commandline_typer.py,sha256=g_pT1V8xwAF9KK-ndE-ydIYk2ivZYoO24PBW80CjsoM,11257
 pluginify/config.py,sha256=co4HkXsOrA68M0PlRaUAhz5KMp5pa60ssb7DwLvUuxk,4894
-pluginify/create_plugin_registries.py,sha256=9NGFRzyZIaF7Mj1BMbNLTbyRVtnZWjmwSnCL0qnra_0,48494
+pluginify/create_plugin_registries.py,sha256=rZmZV_iIBc3FHfhhCW4axMpoBaT1dq_P3OPS7jVrQlQ,48934
 pluginify/errors.py,sha256=kkdipLfxvj8f-Pp53SD3RPnD9QS3samBLlQBjBskZP8,373
 pluginify/interfaces/__init__.py,sha256=XcPl52V8oMihP7NoRs_NgbiIvqoL3A4YeyfTOp3NqBA,329
 pluginify/interfaces/base.py,sha256=ubimAsOgAXCyhKhn40-a1HIfrjMPqHtKr4rBnwm645M,26724
 pluginify/interfaces/class_based/data_modifiers.py,sha256=p-d80oywjo9Rr6txowr1BIlg08qeOm3dAMm_4JQ0iH8,1072
 pluginify/interfaces/class_based_plugin.py,sha256=TMuJ2INaaFaUlxPX05L5r2gH3ISu1W_2qWllGA63oq4,8209
 pluginify/interfaces/yaml_based/configs.py,sha256=3U_nKCm2mIVfA_cRl-VUmu8JrQQQ3u24eshH46a9GI0,808
-pluginify/plugin_registry.py,sha256=yN8ooSHB7HfM31AQoeDUB7oJc87K7ZJHR_XllVr-bCM,40546
+pluginify/plugin_registry.py,sha256=CLcR8xXPPaB3fd2KOODUZRiOSg3hhMyKiyKyAv9KpVE,43291
 pluginify/plugins/classes/data_modifiers/cuboid.py,sha256=WyvhPrRptpSHtPqeOdF1oiHm6zFwHahsLwNvsMdmQEE,1107
 pluginify/plugins/yaml/configs/stucco.yaml,sha256=OXLWzmmzxSAWs-MzSlmEa-OLGJzVJzrVQZwgGPL2jqE,376
 pluginify/pydantic_models/v1/configs.py,sha256=As15amNtdFwAobufh0d6mDlF3mMS23ThwTKQy9q25bc,2122
 pluginify/utils/__init__.py,sha256=gdy4E4yEI9lrHbWPeBn0Bspj7iz4z6r8BAjbmHf_hpM,2088
 pluginify/utils/context_managers.py,sha256=1-bREbijNNc5PrqMIihKqxlxNaK-YZ3hj5XGpHiYmfw,926
-pluginify-0.0.0.post1.dev0.dist-info/METADATA,sha256=YoQUcPZOkt58ozii6zhO33T9-N11vPIxl-uxXl1aWsU,2086
-pluginify-0.0.0.post1.dev0.dist-info/WHEEL,sha256=Vz2fHgx6HFtSwhs8KvkHLqH5Ea4w1_rner5uNVGCeIE,88
-pluginify-0.0.0.post1.dev0.dist-info/entry_points.txt,sha256=kCm7HOEEzSnPyDpy98W-LYYmlLYyTnRqOj40UqxX6-w,111
-pluginify-0.0.0.post1.dev0.dist-info/licenses/LICENSE,sha256=sIBqTiU78gw5bC9MZMySNxzCE7DIlOcDqUWdmflkDsY,7266
-pluginify-0.0.0.post1.dev0.dist-info/RECORD,,
+pluginify/utils/validators.py,sha256=gCESprY4mZgE0KXigjsgHOipjZAg8OUF5n3-SGQ7PG4,7577
+pluginify-0.1.0.post11.dev0.dist-info/METADATA,sha256=aY4LQhrTUPYsmTcEh9QyW6SkMiqlDM-dgX-HW7DmIZs,2165
+pluginify-0.1.0.post11.dev0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
+pluginify-0.1.0.post11.dev0.dist-info/entry_points.txt,sha256=kCm7HOEEzSnPyDpy98W-LYYmlLYyTnRqOj40UqxX6-w,111
+pluginify-0.1.0.post11.dev0.dist-info/licenses/LICENSE,sha256=sIBqTiU78gw5bC9MZMySNxzCE7DIlOcDqUWdmflkDsY,7266
+pluginify-0.1.0.post11.dev0.dist-info/RECORD,,

{pluginify-0.0.0.post1.dev0.dist-info → pluginify-0.1.0.post11.dev0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.3.2
+Generator: poetry-core 2.4.0
 Root-Is-Purelib: true
 Tag: py3-none-any