amd-node-scraper 0.0.1__py3-none-any.whl

nodescraper/cli/inputargtypes.py

@@ -0,0 +1,129 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import argparse
+import json
+from typing import Generic, Optional, Type
+
+from pydantic import ValidationError
+
+from nodescraper.generictypes import TModelType
+
+
+def log_path_arg(log_path: str) -> Optional[str]:
+    """Type function for a log path arg, allows 'none' to be specified to disable logging
+
+    Args:
+        log_path (str): log path string
+
+    Returns:
+        Optional[str]: log path or None
+    """
+    if log_path.lower() == "none":
+        return None
+    return log_path
+
+
+def bool_arg(str_input: str) -> bool:
+    """Converts a string arg input into a bool
+
+    Args:
+        str_input (str): string input
+
+    Returns:
+        bool: bool value for string
+    """
+    if str_input.lower() == "true":
+        return True
+    elif str_input.lower() == "false":
+        return False
+    raise argparse.ArgumentTypeError("Invalid input, boolean value (True or False) expected")
+
+
+def dict_arg(str_input: str) -> dict:
+    """converts a json string into a dict
+
+    Args:
+        str_input (str): input string
+
+    Raises:
+        argparse.ArgumentTypeError: if error was seen when loading string into json dict
+
+    Returns:
+        dict: dict representation of the json string
+    """
+    try:
+        return json.loads(str_input)
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError("Invalid json input for arg") from e
+
+
+class ModelArgHandler(Generic[TModelType]):
+    """Class to handle loading json files into pydantic models"""
+
+    def __init__(self, model: Type[TModelType]) -> None:
+        self.model = model
+
+    def process_file_arg(self, file_path: str) -> TModelType:
+        """load a json file into a pydantic model
+
+        Args:
+            file_path (str): json file path
+
+        Raises:
+            argparse.ArgumentTypeError: If validation errors were seen when building model
+
+        Returns:
+            TModelType: model instance
+        """
+        data = json_arg(file_path)
+        try:
+            return self.model(**data)
+        except ValidationError as e:
+            raise argparse.ArgumentTypeError(
+                f"Validation errors when processing {file_path}: {e.errors(include_url=False)}"
+            ) from e
+
+
+def json_arg(json_path: str) -> dict:
+    """loads a json file into a dict
+
+    Args:
+        json_path (str): path to json file
+
+    Raises:
+        argparse.ArgumentTypeError: If file does not exist or could not be decoded
+
+    Returns:
+        dict: output dict
+    """
+    try:
+        with open(json_path, "r", encoding="utf-8") as input_file:
+            data = json.load(input_file)
+        return data
+    except json.JSONDecodeError as e:
+        raise argparse.ArgumentTypeError(f"File {json_path} contains invalid JSON") from e
+    except FileNotFoundError as e:
+        raise argparse.ArgumentTypeError(f"Unable to find file: {json_path}") from e

nodescraper/configbuilder.py

@@ -0,0 +1,123 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import enum
+import logging
+from typing import Any, Optional, Type, Union
+
+from pydantic import BaseModel
+
+from nodescraper.interfaces import PluginInterface
+from nodescraper.models import DataModel, PluginConfig
+from nodescraper.pluginregistry import PluginRegistry
+from nodescraper.typeutils import TypeData, TypeUtils
+
+
+class ConfigBuilder:
+    """Class used to dynamically generate plugin configs"""
+
+    def __init__(self, plugin_registry: PluginRegistry, logger: Optional[logging.Logger] = None):
+        self.plugin_registry = plugin_registry
+        self.logger = logger if logger else logging.getLogger()
+
+    def gen_config(self, plugin_names: list[str]) -> PluginConfig:
+        """Generate a plugin config dict for a list of plugin names
+
+        Args:
+            plugin_names (list[str]): list of plugin names to include in the config
+
+        Returns:
+            dict: plugin config dict
+        """
+        config = PluginConfig()
+        for plugin in plugin_names:
+            if plugin in self.plugin_registry.plugins:
+                config.plugins[plugin] = self._build_plugin_config(
+                    self.plugin_registry.plugins[plugin]
+                )
+            else:
+                self.logger.warning("No plugin found with name: %s", plugin)
+        return config
+
+    @classmethod
+    def _build_plugin_config(cls, plugin_class: Type[PluginInterface]) -> dict:
+        type_map = TypeUtils.get_func_arg_types(plugin_class.run, plugin_class)
+        config = {}
+
+        for arg, arg_data in type_map.items():
+            cls._update_config(arg, arg_data, config)
+
+        return config
+
+    @classmethod
+    def _update_config(cls, config_key, type_data: TypeData, config: dict):
+        if config_key in ["self", "preserve_connection", "max_event_priority_level"]:
+            return
+
+        type_class_map = {
+            type_class.type_class: type_class for type_class in type_data.type_classes
+        }
+        if type(None) in type_class_map:
+            return
+
+        model_arg = next(
+            (
+                type_class
+                for type_class in type_class_map
+                if (isinstance(type_class, type) and issubclass(type_class, BaseModel))
+                and not issubclass(type_class, DataModel)
+            ),
+            None,
+        )
+
+        if model_arg:
+            model_config = {}
+            for attr, attr_data in TypeUtils.get_model_types(model_arg).items():
+                cls._update_config(attr, attr_data, model_config)
+            config[config_key] = model_config
+        else:
+            config[config_key] = cls._process_value(type_data.default)
+
+    @classmethod
+    def _process_value(cls, value: Any) -> Optional[Union[dict, str, int, float, list]]:
+        if isinstance(value, enum.Enum):
+            return value.name
+
+        if isinstance(value, dict):
+            return_dict = {}
+            for key, val in value.items():
+                return_dict[key] = cls._process_value(val)
+
+        elif not isinstance(
+            value,
+            (
+                str,
+                int,
+                float,
+            ),
+        ):
+            return None
+
+        return value

nodescraper/configregistry.py

@@ -0,0 +1,66 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+from pydantic import ValidationError
+
+from nodescraper.models import PluginConfig
+
+
+class ConfigRegistry:
+    """Class to load json plugin configs into models"""
+
+    INTERNAL_SEARCH_PATH = os.path.join(os.path.dirname(__file__), "configs")
+
+    def __init__(self, config_path: Optional[str] = None) -> None:
+        self.configs = {}
+        self.load_configs(config_path)
+
+    def load_configs(self, config_path: Optional[str] = None):
+        """load plugin config json files into pydantic models
+
+        Args:
+            config_path (Optional[str], optional): Path in which to search for config files. Defaults to None.
+        """
+        if not config_path:
+            config_path = self.INTERNAL_SEARCH_PATH
+
+        config_path = Path(config_path)
+
+        for config_file in config_path.glob("*.json"):
+            with open(config_file, "r", encoding="utf-8") as in_file:
+                try:
+                    file_data = json.load(in_file)
+                    config_model = PluginConfig(**file_data)
+                    if config_model.name:
+                        self.configs[config_model.name] = config_model
+                    else:
+                        self.configs[config_file.name] = config_model
+                except (ValidationError, json.JSONDecodeError):
+                    pass

nodescraper/configs/node_status.json

@@ -0,0 +1,19 @@
+{
+    "name": "NodeStatus",
+    "desc": "Check configuration and status of the node",
+    "global_args": {},
+    "plugins": {
+        "BiosPlugin": {},
+        "CmdlinePlugin": {},
+        "DimmPlugin": {},
+        "DkmsPlugin": {},
+        "DmesgPlugin": {},
+        "KernelPlugin": {},
+        "MemoryPlugin": {},
+        "OsPlugin": {},
+        "RocmPlugin": {},
+        "StoragePlugin": {},
+        "UptimePlugin": {}
+    },
+    "result_collators": {}
+}

nodescraper/connection/__init__.py

@@ -0,0 +1,25 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################

nodescraper/connection/inband/__init__.py

@@ -0,0 +1,46 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+from .inband import (
+    BaseFileArtifact,
+    BinaryFileArtifact,
+    CommandArtifact,
+    InBandConnection,
+    TextFileArtifact,
+)
+from .inbandlocal import LocalShell
+from .inbandmanager import InBandConnectionManager
+from .sshparams import SSHConnectionParams
+
+__all__ = [
+    "SSHConnectionParams",
+    "LocalShell",
+    "InBandConnectionManager",
+    "InBandConnection",
+    "BaseFileArtifact",
+    "TextFileArtifact",
+    "BinaryFileArtifact",
+    "CommandArtifact",
+]

nodescraper/connection/inband/inband.py

@@ -0,0 +1,171 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import abc
+import os
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class CommandArtifact(BaseModel):
+    """Artifact for the result of shell command execution"""
+
+    command: str
+    stdout: str
+    stderr: str
+    exit_code: int
+
+
+class BaseFileArtifact(BaseModel, abc.ABC):
+    """Base class for files"""
+
+    filename: str
+
+    @abc.abstractmethod
+    def log_model(self, log_path: str) -> None:
+        """Write file to path
+
+        Args:
+            log_path (str): Path for file
+        """
+        pass
+
+    @abc.abstractmethod
+    def contents_str(self) -> str:
+        pass
+
+    @classmethod
+    def from_bytes(
+        cls,
+        filename: str,
+        raw_contents: bytes,
+        encoding: Optional[str] = "utf-8",
+        strip: bool = True,
+    ) -> "BaseFileArtifact":
+        """factory method
+
+        Args:
+            filename (str): name of file to be read
+            raw_contents (bytes): Raw file content
+            encoding (Optional[str], optional): Optional encoding. Defaults to "utf-8".
+            strip (bool, optional): Remove padding. Defaults to True.
+
+        Returns:
+            BaseFileArtifact: _Returns instance of Artifact file
+        """
+        if encoding is None:
+            return BinaryFileArtifact(filename=filename, contents=raw_contents)
+
+        try:
+            text = raw_contents.decode(encoding)
+            return TextFileArtifact(filename=filename, contents=text.strip() if strip else text)
+        except UnicodeDecodeError:
+            return BinaryFileArtifact(filename=filename, contents=raw_contents)
+
+
+class TextFileArtifact(BaseFileArtifact):
+    """Class for text file artifacts"""
+
+    contents: str
+
+    def log_model(self, log_path: str) -> None:
+        """Write file to disk
+
+        Args:
+            log_path (str): Path for file
+        """
+        path = os.path.join(log_path, self.filename)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(self.contents)
+
+    def contents_str(self) -> str:
+        """Get content as str
+
+        Returns:
+            str: Str instance of file content
+        """
+        return self.contents
+
+
+class BinaryFileArtifact(BaseFileArtifact):
+    """Class for binary file artifacts"""
+
+    contents: bytes
+
+    def log_model(self, log_path: str) -> None:
+        """Write file to disk
+
+        Args:
+            log_path (str): Path for file
+        """
+        log_name = os.path.join(log_path, self.filename)
+        with open(log_name, "wb") as f:
+            f.write(self.contents)
+
+    def contents_str(self) -> str:
+        """File content
+
+        Returns:
+            str: Str instance of file content
+        """
+        try:
+            return self.contents.decode("utf-8")
+        except UnicodeDecodeError:
+            return f"<binary data: {len(self.contents)} bytes>"
+
+
+class InBandConnection(abc.ABC):
+
+    @abc.abstractmethod
+    def run_command(
+        self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True
+    ) -> CommandArtifact:
+        """Run an in band shell command
+
+        Args:
+            command (str): command to run
+            sudo (bool, optional): run command with sudo (Linux only). Defaults to False.
+            timeout (int, optional): timeout for command in seconds. Defaults to 300.
+            strip (bool, optional): strip output of command. Defaults to True.
+
+        Returns:
+            CommandArtifact: command result object
+        """
+
+    @abc.abstractmethod
+    def read_file(
+        self, filename: str, encoding: str = "utf-8", strip: bool = True
+    ) -> BaseFileArtifact:
+        """Read a file into a BaseFileArtifact
+
+        Args:
+            filename (str): filename
+            encoding (str, optional): encoding to use when opening file. Defaults to "utf-8".
+            strip (bool): automatically strip file contents
+
+        Returns:
+            BaseFileArtifact: file artifact
+        """

nodescraper/connection/inband/inbandlocal.py

@@ -0,0 +1,93 @@
+###############################################################################
+#
+# MIT License
+#
+# Copyright (c) 2025 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+###############################################################################
+import os
+import subprocess
+
+from .inband import (
+    BaseFileArtifact,
+    CommandArtifact,
+    InBandConnection,
+)
+
+
+class LocalShell(InBandConnection):
+
+    def run_command(
+        self, command: str, sudo: bool = False, timeout: int = 300, strip: bool = True
+    ) -> CommandArtifact:
+        """Run a local in band shell command
+
+        Args:
+            command (str): command to run
+            sudo (bool, optional): run command with sudo (Linux only). Defaults to False.
+            timeout (int, optional): timeout for command in seconds. Defaults to 300.
+            strip (bool, optional): strip output of command. Defaults to True.
+
+        Returns:
+            CommandArtifact: command result object
+        """
+        if sudo:
+            command = f"sudo {command}"
+
+        res = subprocess.run(
+            command,
+            encoding="utf-8",
+            shell=True,
+            errors="replace",
+            timeout=timeout,
+            capture_output=True,
+            check=False,
+        )
+
+        return CommandArtifact(
+            command=command,
+            stdout=res.stdout.strip() if strip else res.stdout,
+            stderr=res.stderr.strip() if strip else res.stderr,
+            exit_code=res.returncode,
+        )
+
+    def read_file(
+        self, filename: str, encoding: str = "utf-8", strip: bool = True
+    ) -> BaseFileArtifact:
+        """Read a local file into a BaseFileArtifact
+
+        Args:
+            filename (str): filename
+            encoding (str, optional): encoding to use when opening file. Defaults to "utf-8".
+            strip (bool): automatically strip file contents
+
+        Returns:
+            BaseFileArtifact: file artifact
+        """
+        with open(filename, "rb") as f:
+            raw_contents = f.read()
+
+        return BaseFileArtifact.from_bytes(
+            filename=os.path.basename(filename),
+            raw_contents=raw_contents,
+            encoding=encoding,
+            strip=strip,
+        )