besapi 4.3.1__tar.gz → 4.4.1__tar.gz

{besapi-4.3.1/src/besapi.egg-info → besapi-4.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: besapi
-Version: 4.3.1
+Version: 4.4.1
 Summary: Library for working with the BigFix REST API
 Home-page: https://github.com/jgstew/besapi
 Author: Matt Hansen, James Stewart

{besapi-4.3.1 → besapi-4.4.1}/src/besapi/besapi.py

@@ -31,7 +31,7 @@ import lxml.objectify
 import requests
 import urllib3.poolmanager
 
-__version__ = "4.3.1"
+__version__ = "4.4.1"
 
 besapi_logger = logging.getLogger("besapi")
 

{besapi-4.3.1 → besapi-4.4.1}/src/besapi/plugin_utilities.py

@@ -1,6 +1,42 @@
-"""This is a set of utility functions for use in multiple plugins.
-
-see example here: https://github.com/jgstew/besapi/blob/master/examples/export_all_sites.py
+"""Utility functions shared by BigFix plugins, such as Server Plugin Services.
+
+A plugin is a script run on a schedule, often by the BigFix Server Plugin
+Service on the root server (as root on Linux, SYSTEM on Windows), with an
+unrelated working directory and a minimal PATH. These utilities handle the
+setup every plugin needs, so a plugin can be as short as:
+
+    import besapi.plugin_utilities
+
+    with besapi.plugin_utilities.init_plugin(__version__) as (args, bes_conn):
+        config = besapi.plugin_utilities.get_plugin_config()
+        ...
+
+Main areas:
+
+- Setup: init_plugin() parses the standard args, configures logging to
+  `<plugin name>.log` next to the plugin, logs session start and end banners
+  and makes the BigFix connection. setup_plugin_argparse(),
+  get_plugin_logging_config() and plugin_session() are the parts it uses.
+- Connection: get_besapi_connection() tries explicit args first, then the
+  local root server's own credentials, then env vars, then the besapi config
+  file. The root server credentials come from the platform specific module,
+  besapi.plugin_utilities_win (Windows Registry and DPAPI) or
+  besapi.plugin_utilities_linux (MasterOperatorCredentials and CryptoUtility),
+  and are only used if this happens to be a root server.
+- Files: resolve_plugin_path() finds files relative to the plugin, not the
+  working directory. get_plugin_config() loads `<plugin name>.config.yaml`,
+  and consume_trigger_file() lets an action request a plugin run.
+- Secrets: protect_plugin_config_secrets() encrypts plaintext secrets in the
+  config file in place, with the same keys as the root server's own REST API
+  password, once the plugin has used them successfully.
+  get_plugin_config(secret_keys=...) decrypts them again.
+- Commands: find_executable() and run_logged() run external tools with
+  their output logged.
+
+The platform specific module is imported here only if available, and a
+failure in it never prevents the other connection methods.
+
+See examples/bigfix_plugin_mqtt_homeassistant.py for a complete plugin.
 """
 
 import argparse
@@ -14,6 +50,7 @@ import platform
 import shutil
 import subprocess  # nosec B404
 import sys
+import tempfile
 from typing import Any, Iterator, List, Sequence, Tuple, Union
 
 import besapi
@@ -38,6 +75,10 @@ except BaseException as import_error:  # pylint: disable=broad-exception-caught
     logging.debug("platform specific plugin utilities unavailable: %s", import_error)
 
 
+# marks a secret encrypted by protect_secret(), one prefix per platform module,
+# matching PROTECTED_SECRET_PREFIX in plugin_utilities_linux and _win:
+PROTECTED_SECRET_PREFIXES = ("{cryptoutility}", "{dpapi}")
+
 # custom log level for session start / end banners in plugin logs:
 SESSION_LOG_LEVEL = 99
 
@@ -235,24 +276,19 @@ def resolve_plugin_path(path: str) -> Union[str, None]:
     return None
 
 
-def get_plugin_config(file_name: Union[str, None] = None) -> Any:
-    """Load the plugin's YAML config file.
-
-    Requires the optional `ruamel.yaml` library: `pip install besapi[plugins]`
-
-    Arguments:
-        file_name: config file, resolved with resolve_plugin_path(). Defaults
-            to `<plugin name>.config.yaml` next to the plugin.
-
-    Returns:
-        The parsed YAML, usually a dict.
-    """
+def _find_plugin_config(file_name: Union[str, None] = None) -> str:
+    """Find the plugin's YAML config file, see get_plugin_config()."""
     file_name = file_name or get_invoke_file_name() + ".config.yaml"
 
     config_path = resolve_plugin_path(file_name)
     if not config_path:
         raise FileNotFoundError(f"plugin config file not found: {file_name}")
 
+    return config_path
+
+
+def _import_ruamel_yaml():
+    """Import the optional ruamel.yaml library, with a helpful error."""
     try:
         import ruamel.yaml  # pylint: disable=import-outside-toplevel
     except ImportError as err:
@@ -261,9 +297,141 @@ def get_plugin_config(file_name: Union[str, None] = None) -> Any:
             "install it with: pip install besapi[plugins]"
         ) from err
 
+    return ruamel.yaml
+
+
+def _get_config_parent(config: Any, key_path: Sequence[str]) -> Any:
+    """Get the dict holding the last key of key_path, None if it is not set."""
+    parent = config
+    for key in key_path[:-1]:
+        parent = parent.get(key) if isinstance(parent, dict) else None
+
+    if not isinstance(parent, dict) or not parent.get(key_path[-1]):
+        return None
+
+    return parent
+
+
+def get_plugin_config(
+    file_name: Union[str, None] = None,
+    secret_keys: Union[Sequence[Sequence[str]], None] = None,
+) -> Any:
+    """Load the plugin's YAML config file.
+
+    Requires the optional `ruamel.yaml` library: `pip install besapi[plugins]`
+
+    Arguments:
+        file_name: config file, resolved with resolve_plugin_path(). Defaults
+            to `<plugin name>.config.yaml` next to the plugin.
+        secret_keys: key paths of secrets in the config, such as
+            `[("mqtt", "password")]`. Each one encrypted by
+            protect_plugin_config_secrets() is returned decrypted. The file is
+            never changed here, since a plaintext secret is not known to work.
+
+    Returns:
+        The parsed YAML, usually a dict.
+
+    Raises:
+        ValueError: if a secret is encrypted but cannot be decrypted, since
+            sending the encrypted value as a password can never work.
+    """
+    config_path = _find_plugin_config(file_name)
+    ruamel_yaml = _import_ruamel_yaml()
+
     logging.info("loading config from: `%s`", config_path)
     with open(config_path, encoding="utf-8") as stream:
-        return ruamel.yaml.YAML(typ="safe", pure=True).load(stream)
+        config = ruamel_yaml.YAML(typ="safe", pure=True).load(stream)
+
+    for key_path in secret_keys or []:
+        parent = _get_config_parent(config, key_path)
+        if parent is None or not is_protected_secret(parent[key_path[-1]]):
+            continue
+
+        plaintext = unprotect_secret(parent[key_path[-1]])
+        if not plaintext:
+            name = ".".join(str(key) for key in key_path)
+            raise ValueError(
+                f"failed to decrypt `{name}` in plugin config `{config_path}`, "
+                "replace it with the plaintext value to have it encrypted again"
+            )
+        parent[key_path[-1]] = plaintext
+
+    return config
+
+
+def protect_plugin_config_secrets(
+    secret_keys: Sequence[Sequence[str]], file_name: Union[str, None] = None
+) -> List[str]:
+    """Encrypt plaintext secrets in the plugin's YAML config file, in place.
+
+    Call this only after the plugin has used the secrets successfully, such as
+    after logging in with a password, so a wrong one is never encrypted and is
+    left as plaintext to fix. Encrypting only works on a root server with the
+    privileges of the plugin service, see protect_secret(), otherwise the file
+    is left alone. Comments and layout in the file are kept.
+
+    This is best effort, a failure is logged and never raised, since the
+    plugin has already done its work with the plaintext.
+
+    Arguments:
+        secret_keys: key paths of secrets in the config, such as
+            `[("mqtt", "password")]`.
+        file_name: config file, see get_plugin_config().
+
+    Returns:
+        The dotted names of the secrets encrypted, such as `["mqtt.password"]`.
+    """
+    protected_names: List[str] = []
+
+    try:
+        config_path = _find_plugin_config(file_name)
+        ruamel_yaml = _import_ruamel_yaml()
+
+        # round trip mode, unlike the safe loader, keeps comments:
+        yaml = ruamel_yaml.YAML()
+        yaml.preserve_quotes = True
+
+        with open(config_path, encoding="utf-8") as stream:
+            contents = stream.read()
+
+        data = yaml.load(contents)
+        # round trip mode does not keep the `---` document start by itself:
+        yaml.explicit_start = contents.lstrip().startswith("---")
+
+        for key_path in secret_keys:
+            parent = _get_config_parent(data, key_path)
+            if parent is None or is_protected_secret(parent[key_path[-1]]):
+                continue
+
+            protected = protect_secret(str(parent[key_path[-1]]))
+            if not protected:
+                continue
+
+            parent[key_path[-1]] = protected
+            protected_names.append(".".join(str(key) for key in key_path))
+
+        if not protected_names:
+            logging.debug("no plaintext secrets encrypted in plugin config.")
+            return []
+
+        # write beside the original then swap, so a failure never truncates it:
+        file_handle, temp_path = tempfile.mkstemp(
+            dir=os.path.dirname(config_path), prefix=".", suffix=".tmp"
+        )
+        try:
+            with os.fdopen(file_handle, "w", encoding="utf-8") as stream:
+                yaml.dump(data, stream)
+            shutil.copymode(config_path, temp_path)
+            os.replace(temp_path, config_path)
+        finally:
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+    except Exception as err:  # pylint: disable=broad-exception-caught
+        logging.warning("failed to encrypt secrets in plugin config: %s", err)
+        return []
+
+    logging.info("encrypted plaintext %s in plugin config.", protected_names)
+    return protected_names
 
 
 def consume_trigger_file(path: str) -> bool:
@@ -360,7 +528,9 @@ def get_besapi_connection_env_then_config():
     return bes_conn
 
 
-def _try_platform_utility(function_name: str):
+def _try_platform_utility(
+    function_name: str, *args, failure_log_level: int = logging.DEBUG
+):
     """Call a function from the platform specific utilities module, if possible.
 
     These are best effort conveniences for the case where the plugin happens to
@@ -368,7 +538,10 @@ def _try_platform_utility(function_name: str):
     not a usable root server, so the caller falls back to the other methods.
 
     Args:
-        function_name: The name of the function to call, with no arguments.
+        function_name: The name of the function to call.
+        *args: Arguments to pass to the function.
+        failure_log_level: The level to log an error raised by the function at.
+            DEBUG by default, since that usually just means not a root server.
 
     Returns:
         Whatever the function returned, or None if it could not be used.
@@ -386,10 +559,10 @@ def _try_platform_utility(function_name: str):
         return None
 
     try:
-        return platform_function()
+        return platform_function(*args)
     except BaseException as err:  # pylint: disable=broad-exception-caught
         # NOTE: intentionally broad, this must never prevent the other methods:
-        logging.debug("`%s` failed, ignoring: %s", function_name, err)
+        logging.log(failure_log_level, "`%s` failed, ignoring: %s", function_name, err)
         return None
 
 
@@ -406,6 +579,57 @@ def get_root_server_rest_pass() -> Union[str, None]:
     return _try_platform_utility("get_linux_credentials_rest_pass")
 
 
+def is_protected_secret(value: Any) -> bool:
+    """Check if a value is a secret encrypted by protect_secret()."""
+    return isinstance(value, str) and value.startswith(PROTECTED_SECRET_PREFIXES)
+
+
+def protect_secret(plaintext: str) -> Union[str, None]:
+    """Encrypt a secret, such as a password in a plugin config file.
+
+    This only works on a root server, with the privileges of the plugin service
+    (root on Linux, SYSTEM or an administrator on Windows), since the keys used
+    are the same as for the server's own REST API password. The result is
+    decrypted again before it is returned, so it is known to be usable.
+
+    NOTE: this checks what the process can do, not which user it runs as, by
+    requiring get_root_server_rest_pass() to succeed. A check for root or
+    SYSTEM would allow a machine that is not a root server, which lacks the
+    keys, and would refuse an administrator on a Windows root server, where
+    machine scope DPAPI works and SYSTEM can still decrypt the result. On
+    Linux, reading the server's credentials file means root in practice.
+
+    Returns:
+        The encrypted secret, otherwise None if it could not be encrypted.
+    """
+    # proves this is a root server, and that the keys are readable:
+    if not get_root_server_rest_pass():
+        logging.debug("not a usable root server, cannot protect secret.")
+        return None
+
+    # NOTE: this is a root server by now, so an error here is a real problem:
+    protected = _try_platform_utility(
+        "protect_secret", plaintext, failure_log_level=logging.ERROR
+    )
+
+    if not protected:
+        return None
+
+    # never trust an encrypted value that cannot be decrypted back:
+    if unprotect_secret(protected) != plaintext:
+        logging.warning("encrypted secret did not decrypt back, not using it.")
+        return None
+
+    return protected
+
+
+def unprotect_secret(protected: str) -> Union[str, None]:
+    """Decrypt a secret from protect_secret(), otherwise return None."""
+    return _try_platform_utility(
+        "unprotect_secret", protected, failure_log_level=logging.ERROR
+    )
+
+
 def get_besconn_root_server() -> Union[besapi.besapi.BESConnection, None]:
     """Get a connection using local root server credentials, if this is one.
 

{besapi-4.3.1 → besapi-4.4.1}/src/besapi/plugin_utilities_linux.py

@@ -33,6 +33,14 @@ BES_APPLICATION_DIRS = [
 CRYPTO_UTILITY_NAME = "CryptoUtility"
 CREDENTIALS_FILE_NAME = "MasterOperatorCredentials"
 
+CRYPTO_UTILITY_DECRYPT_ARGS = ["-d", "-i"]
+# NOTE: encrypting is the CryptoUtility default, there is no `-e` flag. From its
+# usage: `CryptoUtility [-d] [-u] [-i <input text> | -f <input file path>]`
+CRYPTO_UTILITY_ENCRYPT_ARGS = ["-i"]
+
+# marks a secret encrypted by protect_secret(), such as in a plugin config file:
+PROTECTED_SECRET_PREFIX = "{cryptoutility}"
+
 # env var overrides, keyed by the file name they override:
 BES_APPLICATION_FILE_ENV_VARS = {
     CRYPTO_UTILITY_NAME: "BESAPI_CRYPTO_UTILITY",
@@ -135,38 +143,34 @@ def parse_credentials_file(file_path: Union[str, None] = None) -> Dict[str, str]
     return dict(config[_INI_SECTION])
 
 
-def crypto_utility_decrypt(
-    encrypted_value: str,
+def _run_crypto_utility(
+    crypto_args: List[str],
+    value: str,
     crypto_utility_path: Union[str, None] = None,
     timeout: int = 30,
 ) -> Union[str, None]:
-    """Decrypt a value using the BigFix server CryptoUtility binary.
-
-    Equivalent to: `CryptoUtility -d -i "<encrypted_value>"`
+    """Run the BigFix server CryptoUtility binary on one value.
 
     Args:
-        encrypted_value: The encrypted value, including its `{aes,1}` prefix.
+        crypto_args: The args that go before the value, such as `["-d", "-i"]`.
+        value: The value to encrypt or decrypt.
         crypto_utility_path: Path to CryptoUtility, located automatically if not given.
         timeout: Seconds to wait for CryptoUtility before giving up.
 
     Returns:
-        The decrypted string, otherwise None.
+        The stripped stdout of CryptoUtility, otherwise None.
     """
-    if not encrypted_value or encrypted_value.strip() == "":
-        logger.warning("No encrypted value provided for decryption.")
-        return None
-
     if not crypto_utility_path:
         crypto_utility_path = find_bes_application_file(CRYPTO_UTILITY_NAME)
         if not crypto_utility_path:
-            logger.debug("CryptoUtility not found, cannot decrypt.")
+            logger.debug("CryptoUtility not found, cannot run it.")
             return None
 
     # NOTE: the command is a list and shell is False,
-    # so the encrypted value is never interpreted by a shell:
+    # so the value is never interpreted by a shell:
     try:
         result = subprocess.run(  # nosec B603
-            [crypto_utility_path, "-d", "-i", encrypted_value],
+            [crypto_utility_path, *crypto_args, value],
             capture_output=True,
             text=True,
             check=False,
@@ -180,7 +184,7 @@ def crypto_utility_decrypt(
         return None
 
     if result.returncode != 0:
-        # NOTE: never log stdout here, it would contain the plaintext:
+        # NOTE: never log stdout here, it could contain the plaintext:
         logger.error(
             "CryptoUtility failed with return code %s: %s",
             result.returncode,
@@ -188,15 +192,102 @@ def crypto_utility_decrypt(
         )
         return None
 
-    decrypted = (result.stdout or "").strip()
+    output = (result.stdout or "").strip()
 
-    if not decrypted:
+    if not output:
         logger.debug("CryptoUtility returned no data.")
         return None
 
-    # NOTE: only the length is logged, never the plaintext:
-    logger.debug("CryptoUtility decrypted value length: %s", len(decrypted))
-    return decrypted
+    # NOTE: only the length is logged, never the value:
+    logger.debug("CryptoUtility output length: %s", len(output))
+    return output
+
+
+def crypto_utility_decrypt(
+    encrypted_value: str,
+    crypto_utility_path: Union[str, None] = None,
+    timeout: int = 30,
+) -> Union[str, None]:
+    """Decrypt a value using the BigFix server CryptoUtility binary.
+
+    Equivalent to: `CryptoUtility -d -i "<encrypted_value>"`
+
+    Args:
+        encrypted_value: The encrypted value, including its `{aes,1}` prefix.
+        crypto_utility_path: Path to CryptoUtility, located automatically if not given.
+        timeout: Seconds to wait for CryptoUtility before giving up.
+
+    Returns:
+        The decrypted string, otherwise None.
+    """
+    if not encrypted_value or encrypted_value.strip() == "":
+        logger.warning("No encrypted value provided for decryption.")
+        return None
+
+    return _run_crypto_utility(
+        CRYPTO_UTILITY_DECRYPT_ARGS, encrypted_value, crypto_utility_path, timeout
+    )
+
+
+def crypto_utility_encrypt(
+    plaintext: str,
+    crypto_utility_path: Union[str, None] = None,
+    timeout: int = 30,
+) -> Union[str, None]:
+    """Encrypt a value using the BigFix server CryptoUtility binary.
+
+    Equivalent to: `CryptoUtility -i "<plaintext>"`
+
+    The output is a `{aes,1}` prefixed value.
+
+    Args:
+        plaintext: The value to encrypt.
+        crypto_utility_path: Path to CryptoUtility, located automatically if not given.
+        timeout: Seconds to wait for CryptoUtility before giving up.
+
+    Returns:
+        The encrypted string exactly as CryptoUtility printed it, otherwise None.
+    """
+    if not plaintext or plaintext.strip() == "":
+        logger.warning("No plaintext provided for encryption.")
+        return None
+
+    return _run_crypto_utility(
+        CRYPTO_UTILITY_ENCRYPT_ARGS, plaintext, crypto_utility_path, timeout
+    )
+
+
+def protect_secret(plaintext: str) -> Union[str, None]:
+    """Encrypt a secret, such as a password in a plugin config file.
+
+    Args:
+        plaintext: The secret to encrypt.
+
+    Returns:
+        The encrypted secret with the PROTECTED_SECRET_PREFIX, otherwise None.
+    """
+    encrypted = crypto_utility_encrypt(plaintext)
+
+    if not encrypted:
+        return None
+
+    return PROTECTED_SECRET_PREFIX + encrypted
+
+
+def unprotect_secret(protected: str) -> Union[str, None]:
+    """Decrypt a secret from protect_secret().
+
+    Args:
+        protected: The encrypted secret, including the PROTECTED_SECRET_PREFIX.
+
+    Returns:
+        The decrypted secret, otherwise None, including if the prefix is missing.
+    """
+    if not protected or not protected.startswith(PROTECTED_SECRET_PREFIX):
+        logger.debug("value is not a protected secret, not decrypting.")
+        return None
+
+    return crypto_utility_decrypt(protected[len(PROTECTED_SECRET_PREFIX) :])
 
 
 def get_linux_credentials_rest_pass(

{besapi-4.3.1 → besapi-4.4.1}/src/besapi/plugin_utilities_win.py

@@ -31,6 +31,9 @@ except (ImportError, ModuleNotFoundError) as e:
         "This script requires the pywin32 package. Install it via 'pip install pywin32'."
     ) from e
 
+# marks a secret encrypted by protect_secret(), such as in a plugin config file:
+PROTECTED_SECRET_PREFIX = "{dpapi}"
+
 
 def win_dpapi_encrypt_str(
     plaintext: str, scope_flags: int = 4, entropy: Union[str, bytes, None] = None
@@ -115,6 +118,46 @@ def win_dpapi_decrypt_base64(
     return None
 
 
+def protect_secret(plaintext: str) -> Union[str, None]:
+    """Encrypt a secret, such as a password in a plugin config file.
+
+    Uses machine scope DPAPI, the same as the BigFix server RESTPassword.
+
+    Args:
+        plaintext: The secret to encrypt.
+
+    Returns:
+        The encrypted secret with the PROTECTED_SECRET_PREFIX, otherwise None.
+    """
+    encrypted = win_dpapi_encrypt_str(plaintext)
+
+    if not encrypted:
+        return None
+
+    return PROTECTED_SECRET_PREFIX + encrypted
+
+
+def unprotect_secret(protected: str) -> Union[str, None]:
+    """Decrypt a secret from protect_secret().
+
+    Args:
+        protected: The encrypted secret, including the PROTECTED_SECRET_PREFIX.
+
+    Returns:
+        The decrypted secret, otherwise None, including if the prefix is missing.
+    """
+    if not protected or not protected.startswith(PROTECTED_SECRET_PREFIX):
+        logger.debug("value is not a protected secret, not decrypting.")
+        return None
+
+    try:
+        return win_dpapi_decrypt_base64(protected[len(PROTECTED_SECRET_PREFIX) :])
+    # NOTE: broad, bad base64 and DPAPI failures raise unrelated error types:
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.error("Failed to decrypt protected secret: %s", e)
+        return None
+
+
 def win_registry_value_read(hive, subkey, value_name):
     """
     Reads a value from the Windows Registry.

{besapi-4.3.1 → besapi-4.4.1/src/besapi.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: besapi
-Version: 4.3.1
+Version: 4.4.1
 Summary: Library for working with the BigFix REST API
 Home-page: https://github.com/jgstew/besapi
 Author: Matt Hansen, James Stewart

{besapi-4.3.1 → besapi-4.4.1}/src/besapi.egg-info/SOURCES.txt

@@ -32,5 +32,7 @@ tests/test_plugin_utilities_coverage.py
 tests/test_plugin_utilities_linux.py
 tests/test_plugin_utilities_logging.py
 tests/test_plugin_utilities_sdk.py
+tests/test_plugin_utilities_secrets.py
+tests/test_plugin_utilities_win_fake.py
 tests/test_sample_plugin.py
 tests/tests.py