amd-node-scraper 0.0.1__py3-none-any.whl

nodescraper/pluginexecutor.py

@@ -0,0 +1,274 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+from __future__ import annotations
+
+import copy
+import logging
+from collections import deque
+from typing import Optional, Type, Union
+
+from pydantic import BaseModel
+
+from nodescraper.constants import DEFAULT_LOGGER
+from nodescraper.interfaces import ConnectionManager, DataPlugin, PluginInterface
+from nodescraper.models import PluginConfig, SystemInfo
+from nodescraper.models.pluginresult import PluginResult
+from nodescraper.pluginregistry import PluginRegistry
+from nodescraper.taskresulthooks import FileSystemLogHook
+from nodescraper.typeutils import TypeUtils
+
+
+class PluginExecutor:
+    """Class to manage execution of data collectors and error detectors"""
+
+    def __init__(
+        self,
+        plugin_configs: list[PluginConfig],
+        connections: Optional[dict[str, Union[dict, BaseModel]]] = None,
+        system_info: Optional[SystemInfo] = None,
+        logger: Optional[logging.Logger] = None,
+        plugin_registry: Optional[PluginRegistry] = None,
+        log_path: Optional[str] = None,
+    ):
+
+        if logger is None:
+            logger = logging.getLogger(DEFAULT_LOGGER)
+        self.logger = logger
+
+        self.plugin_registry = plugin_registry or PluginRegistry()
+
+        if system_info is None:
+            system_info = SystemInfo()
+        self.system_info = system_info
+
+        self.plugin_config = self.merge_configs(plugin_configs)
+
+        self.connection_library: dict[type[ConnectionManager], ConnectionManager] = {}
+
+        self.log_path = log_path
+
+        self.connection_result_hooks = []
+        if log_path:
+            self.connection_result_hooks.append(FileSystemLogHook(log_base_path=log_path))
+
+        if connections:
+            for connection, connection_args in connections.items():
+                if connection not in self.plugin_registry.connection_managers:
+                    self.logger.error(
+                        "Unable to find registered connection manager class for %s", connection
+                    )
+                    continue
+
+                connection_manager = self.plugin_registry.connection_managers[connection]
+
+                self.connection_library[connection_manager] = connection_manager(
+                    system_info=self.system_info,
+                    logger=self.logger,
+                    connection_args=connection_args,
+                    task_result_hooks=self.connection_result_hooks,
+                )
+
+        self.logger.info("System Name: %s", self.system_info.name)
+        self.logger.info("System SKU: %s", self.system_info.sku)
+        self.logger.info("System Platform: %s", self.system_info.platform)
+        self.logger.info("System location: %s", self.system_info.location)
+
+    @staticmethod
+    def merge_configs(plugin_configs: list[PluginConfig]) -> PluginConfig:
+        merged_config = PluginConfig()
+        for config in plugin_configs:
+            merged_config.global_args.update(config.global_args)
+            merged_config.plugins.update(config.plugins)
+            merged_config.result_collators.update(config.result_collators)
+
+        return merged_config
+
+    def run_queue(self) -> list[PluginResult]:
+        """Run the plugin queue and return results
+
+        Returns:
+            list[PluginResult]: List of results from running the plugins in the queue
+        """
+        plugin_results = []
+        plugin_queue = deque(self.plugin_config.plugins.items())
+        try:
+            while len(plugin_queue) > 0:
+                plugin_name, plugin_args = plugin_queue.popleft()
+                if plugin_name not in self.plugin_registry.plugins:
+                    self.logger.error("Unable to find registered plugin for name %s", plugin_name)
+                    continue
+
+                plugin_class = self.plugin_registry.plugins[plugin_name]
+
+                init_payload = {
+                    "system_info": self.system_info,
+                    "logger": self.logger,
+                    "queue_callback": plugin_queue.append,
+                    "log_path": self.log_path,
+                }
+
+                if plugin_class.CONNECTION_TYPE:
+                    connection_manager_class: Type[ConnectionManager] = plugin_class.CONNECTION_TYPE
+                    if (
+                        connection_manager_class.__name__
+                        not in self.plugin_registry.connection_managers
+                    ):
+                        self.logger.error(
+                            "Unable to find registered connection manager class for %s that is required by",
+                            connection_manager_class.__name__,
+                        )
+                        continue
+
+                    if connection_manager_class not in self.connection_library:
+                        self.logger.info(
+                            "Initializing connection manager for %s with default args",
+                            connection_manager_class.__name__,
+                        )
+                        self.connection_library[connection_manager_class] = (
+                            connection_manager_class(
+                                system_info=self.system_info,
+                                logger=self.logger,
+                                task_result_hooks=self.connection_result_hooks,
+                            )
+                        )
+
+                    init_payload["connection_manager"] = self.connection_library[
+                        connection_manager_class
+                    ]
+
+                try:
+                    plugin_inst = plugin_class(**init_payload)
+
+                    run_payload = copy.deepcopy(plugin_args)
+                    run_args = TypeUtils.get_func_arg_types(plugin_class.run, plugin_class)
+
+                    for arg in run_args.keys():
+                        if arg == "preserve_connection" and issubclass(plugin_class, DataPlugin):
+                            run_payload[arg] = True
+
+                    try:
+                        global_run_args = self.apply_global_args_to_plugin(
+                            plugin_inst, plugin_class, self.plugin_config.global_args
+                        )
+                        # Merge analysis_args and collection_args
+                        for args_key in ["analysis_args", "collection_args"]:
+                            if args_key in global_run_args and args_key in run_payload:
+                                # Merge: global args override plugin-specific args keys specified in both global and plugin-specific args
+                                run_payload[args_key].update(global_run_args[args_key])
+                                del global_run_args[args_key]
+                        run_payload.update(global_run_args)
+                    except ValueError as ve:
+                        self.logger.error(
+                            "Invalid global_args for plugin %s: %s. Skipping plugin.",
+                            plugin_name,
+                            str(ve),
+                        )
+                        continue
+
+                    self.logger.info("-" * 50)
+                    plugin_results.append(plugin_inst.run(**run_payload))
+                except Exception as e:
+                    self.logger.exception(
+                        "Unexpected exception when running plugin %s: %s", plugin_name, e
+                    )
+        except Exception as e:
+            self.logger.exception("Unexpected exception running plugin queue: %s", str(e))
+        finally:
+            self.logger.info("Closing connections")
+
+            if self.plugin_config.result_collators:
+                self.logger.info("Running result collators")
+                for collator, collator_args in self.plugin_config.result_collators.items():
+                    collator_class = self.plugin_registry.result_collators.get(collator)
+                    if collator_class is None:
+                        self.logger.warning(
+                            "No result collator found in registry for name: %s", collator
+                        )
+                        continue
+
+                    self.logger.info("Running %s result collator", collator)
+                    collator_inst = collator_class(logger=self.logger, log_path=self.log_path)
+                    collator_inst.collate_results(
+                        plugin_results,
+                        [
+                            connection_manager.result
+                            for connection_manager in self.connection_library.values()
+                        ],
+                        **collator_args,
+                    )
+            for connection_manager in self.connection_library.values():
+                connection_manager.disconnect()
+
+        return plugin_results
+
+    def apply_global_args_to_plugin(
+        self,
+        plugin_inst: PluginInterface,
+        plugin_class: type,
+        global_args: dict,
+    ) -> dict:
+        """
+        Applies global arguments to the plugin instance, including standard attributes
+        and merging Pydantic model arguments (collection_args, analysis_args).
+
+        Args:
+            plugin_inst: The plugin instance to update.
+            plugin_class: The plugin class (needed for model instantiation).
+            global_args: Dict of global argument overrides.
+        """
+
+        run_args = {}
+        for key in global_args:
+            if key in ["collection_args", "analysis_args"] and isinstance(plugin_inst, DataPlugin):
+                continue
+            else:
+                run_args[key] = global_args[key]
+
+        if (
+            "collection_args" in global_args
+            and hasattr(plugin_class, "COLLECTOR_ARGS")
+            and plugin_class.COLLECTOR_ARGS is not None
+        ):
+
+            plugin_fields = set(plugin_class.COLLECTOR_ARGS.model_fields.keys())
+            filtered = {
+                k: v for k, v in global_args["collection_args"].items() if k in plugin_fields
+            }
+            if filtered:
+                run_args["collection_args"] = filtered
+
+        if (
+            "analysis_args" in global_args
+            and hasattr(plugin_class, "ANALYZER_ARGS")
+            and plugin_class.ANALYZER_ARGS is not None
+        ):
+
+            plugin_fields = set(plugin_class.ANALYZER_ARGS.model_fields.keys())
+            filtered = {k: v for k, v in global_args["analysis_args"].items() if k in plugin_fields}
+            if filtered:
+                run_args["analysis_args"] = filtered
+
+        return run_args

nodescraper/pluginregistry.py

@@ -0,0 +1,152 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import importlib
+import importlib.metadata
+import inspect
+import pkgutil
+import types
+from typing import Optional
+
+import nodescraper.connection as internal_connections
+import nodescraper.plugins as internal_plugins
+import nodescraper.resultcollators as internal_collators
+from nodescraper.interfaces import (
+    ConnectionManager,
+    PluginInterface,
+    PluginResultCollator,
+)
+
+
+class PluginRegistry:
+
+    def __init__(
+        self,
+        plugin_pkg: Optional[list[types.ModuleType]] = None,
+        load_internal_plugins: bool = True,
+        load_entry_point_plugins: bool = True,
+    ) -> None:
+        """Initialize the PluginRegistry with optional plugin packages.
+
+        Args:
+            plugin_pkg (Optional[list[types.ModuleType]], optional): The module to search for plugins in. Defaults to None.
+            load_internal_plugins (bool, optional): Whether internal plugin should be loaded. Defaults to True.
+            load_entry_point_plugins (bool, optional): Whether to load plugins from entry points. Defaults to True.
+        """
+        if load_internal_plugins:
+            self.plugin_pkg = [internal_plugins, internal_connections, internal_collators]
+        else:
+            self.plugin_pkg = []
+
+        if plugin_pkg:
+            self.plugin_pkg += plugin_pkg
+
+        self.plugins: dict[str, type[PluginInterface]] = PluginRegistry.load_plugins(
+            PluginInterface, self.plugin_pkg
+        )
+        self.connection_managers: dict[str, type[ConnectionManager]] = PluginRegistry.load_plugins(
+            ConnectionManager, self.plugin_pkg
+        )
+        self.result_collators: dict[str, type[PluginResultCollator]] = PluginRegistry.load_plugins(
+            PluginResultCollator, self.plugin_pkg
+        )
+
+        if load_entry_point_plugins:
+            entry_point_plugins = self.load_plugins_from_entry_points()
+            self.plugins.update(entry_point_plugins)
+
+    @staticmethod
+    def load_plugins(
+        base_class: type,
+        search_modules: list[types.ModuleType],
+    ) -> dict[str, type]:
+        """Load plugins from the specified modules that are subclasses of the given base class.
+
+        Args:
+            base_class (type): The base class that the plugins should inherit from.
+            search_modules (list[types.ModuleType]): List of modules to search for plugins.
+
+        Returns:
+            dict[str, type]: A dictionary mapping plugin names to their classes.
+        """
+        registry = {}
+
+        def _recurse_pkg(pkg: types.ModuleType, base_class: type) -> None:
+            for _, module_name, ispkg in pkgutil.iter_modules(pkg.__path__, pkg.__name__ + "."):
+                module = importlib.import_module(module_name)
+                for _, plugin in inspect.getmembers(
+                    module,
+                    lambda x: inspect.isclass(x)
+                    and issubclass(x, base_class)
+                    and not inspect.isabstract(x),
+                ):
+                    if hasattr(plugin, "is_valid") and not plugin.is_valid():
+                        continue
+                    registry[plugin.__name__] = plugin
+                if ispkg:
+                    _recurse_pkg(module, base_class)
+
+        for pkg in search_modules:
+            _recurse_pkg(pkg, base_class)
+        return registry
+
+    @staticmethod
+    def load_plugins_from_entry_points() -> dict[str, type]:
+        """Load plugins registered via entry points.
+
+        Returns:
+            dict[str, type]: A dictionary mapping plugin names to their classes.
+        """
+        plugins = {}
+
+        try:
+            # Python 3.10+ supports group parameter
+            try:
+                eps = importlib.metadata.entry_points(group="nodescraper.plugins")  # type: ignore[call-arg]
+            except TypeError:
+                # Python 3.9 - entry_points() returns dict-like object
+                all_eps = importlib.metadata.entry_points()  # type: ignore[assignment]
+                eps = all_eps.get("nodescraper.plugins", [])  # type: ignore[assignment, attr-defined]
+
+            for entry_point in eps:
+                try:
+                    plugin_class = entry_point.load()  # type: ignore[attr-defined]
+
+                    if (
+                        inspect.isclass(plugin_class)
+                        and issubclass(plugin_class, PluginInterface)
+                        and not inspect.isabstract(plugin_class)
+                    ):
+                        if hasattr(plugin_class, "is_valid") and not plugin_class.is_valid():
+                            continue
+
+                        plugins[plugin_class.__name__] = plugin_class
+                except Exception:
+                    pass
+
+        except Exception:
+            pass
+
+        return plugins

nodescraper/plugins/__init__.py

@@ -0,0 +1,25 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################

nodescraper/plugins/inband/__init__.py

@@ -0,0 +1,25 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################

nodescraper/plugins/inband/amdsmi/__init__.py

@@ -0,0 +1,28 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+from .amdsmi_plugin import AmdSmiPlugin
+
+__all__ = ["AmdSmiPlugin"]