edq-utils 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

edq/__init__.py

@@ -2,4 +2,4 @@
 General Python tools used by several EduLinq projects.
 """
 
-__version__ = '0.0.5'
+__version__ = '0.0.6'

edq/cli/config/__init__.py

@@ -0,0 +1,3 @@
+"""
+The edq.cli.config package provides tools for working with project configuration options.
+"""

edq/cli/config/list.py

@@ -0,0 +1,69 @@
+"""
+List the current configuration options.
+"""
+
+import argparse
+import sys
+
+import edq.core.argparser
+
+CONFIG_FIELD_SEPARATOR: str = "\t"
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    rows = []
+
+    for (key, value) in args._config.items():
+        row = [key, str(value)]
+        if (args.show_origin):
+            config_source_obj = args._config_sources.get(key)
+
+            origin = config_source_obj.path
+            if (origin is None):
+                origin = config_source_obj.label
+
+            row.append(origin)
+
+        rows.append(CONFIG_FIELD_SEPARATOR.join(row))
+
+    rows.sort()
+
+    if (not args.skip_header):
+        header = ["Key", "Value"]
+        if (args.show_origin):
+            header.append("Origin")
+
+        rows.insert(0, (CONFIG_FIELD_SEPARATOR.join(header)))
+
+    print("\n".join(rows))
+    return 0
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> argparse.ArgumentParser:
+    """ Get a parser and add addition flags. """
+
+    parser = edq.core.argparser.get_default_parser(__doc__.strip())
+    modify_parser(parser)
+
+    return parser
+
+def modify_parser(parser: argparse.ArgumentParser) -> None:
+    """ Add this CLI's flags to the given parser. """
+
+    parser.add_argument("--show-origin", dest = 'show_origin',
+        action = 'store_true',
+        help = "Display where each configuration's value was obtained from.",
+    )
+
+    parser.add_argument("--skip-header", dest = 'skip_header',
+        action = 'store_true',
+        help = 'Skip headers when displaying configs.',
+    )
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/cli/http/__init__.py

@@ -0,0 +1,3 @@
+"""
+The edq.cli.http package provides CLI utilities involving HTTP.
+"""

edq/cli/http/exchange-server.py

@@ -0,0 +1,71 @@
+# pylint: disable=invalid-name
+
+"""
+Start an HTTP test server that serves the specified HTTP exchanges.
+"""
+
+import argparse
+import os
+import sys
+
+import edq.core.argparser
+import edq.testing.httpserver
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    match_options = {
+        'params_to_skip': args.ignore_params,
+        'headers_to_skip': args.ignore_headers,
+    }
+
+    server = edq.testing.httpserver.HTTPTestServer(
+            port = args.port,
+            match_options = match_options,
+            verbose = True,
+            raise_on_404 = False,
+    )
+
+    for path in args.paths:
+        path = os.path.abspath(path)
+
+        if (os.path.isfile(path)):
+            server.load_exchange(path)
+        else:
+            server.load_exchanges_dir(path)
+
+    server.start_and_wait()
+
+    return 0
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> argparse.ArgumentParser:
+    """ Get the parser. """
+
+    parser = edq.core.argparser.get_default_parser(__doc__.strip(),
+            include_net = True,
+    )
+
+    parser.add_argument('paths', metavar = 'PATH',
+        type = str, nargs = '+',
+        help = 'Path to exchange files or dirs (which will be recursively searched for all exchange files).')
+
+    parser.add_argument('--port', dest = 'port',
+        action = 'store', type = int, default = None,
+        help = 'The port to run this test server on. If not set, a random open port will be chosen.')
+
+    parser.add_argument('--ignore-param', dest = 'ignore_params',
+        action = 'append', type = str, default = [],
+        help = 'Ignore this parameter during exchange matching.')
+
+    parser.add_argument('--ignore-header', dest = 'ignore_headers',
+        action = 'append', type = str, default = [],
+        help = 'Ignore this header during exchange matching.')
+
+    return parser
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/cli/http/send-exchange.py

@@ -0,0 +1,45 @@
+# pylint: disable=invalid-name
+
+"""
+Send an HTTP exchange to the target server.
+"""
+
+import argparse
+import sys
+
+import edq.core.argparser
+import edq.util.net
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    exchange = edq.util.net.HTTPExchange.from_path(args.path)
+    _, body = exchange.make_request(args.server)
+
+    print(body)
+
+    return 0
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> argparse.ArgumentParser:
+    """ Get the parser. """
+
+    parser = edq.core.argparser.get_default_parser(__doc__.strip(),
+            include_net = True,
+    )
+
+    parser.add_argument('server', metavar = 'SERVER',
+        action = 'store', type = str,
+        help = 'Server to send the exahnge to.')
+
+    parser.add_argument('path', metavar = 'PATH',
+        action = 'store', type = str,
+        help = 'Path to the exchange file.')
+
+    return parser
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/cli/http/verify-exchanges.py

@@ -0,0 +1,38 @@
+# pylint: disable=invalid-name
+
+"""
+Verify that exchanges sent to a given server have the same response.
+"""
+
+import argparse
+import sys
+
+import edq.core.argparser
+import edq.procedure.verify_exchanges
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    return edq.procedure.verify_exchanges.run(args.paths, args.server)
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> argparse.ArgumentParser:
+    """ Get the parser. """
+
+    parser = edq.core.argparser.get_default_parser(__doc__.strip())
+
+    parser.add_argument('server', metavar = 'SERVER',
+        action = 'store', type = str, default = None,
+        help = 'Address of the server to send exchanges to.')
+
+    parser.add_argument('paths', metavar = 'PATH',
+        type = str, nargs = '+',
+        help = 'Path to exchange files or dirs (which will be recursively searched for all exchange files).')
+
+    return parser
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/cli/testing/__init__.py

@@ -0,0 +1,3 @@
+"""
+The edq.cli.testing package provides testing-related utilities.
+"""

edq/cli/testing/cli-test.py

@@ -1,4 +1,4 @@
-#  # pylint: disable=invalid-name
+# pylint: disable=invalid-name
 
 """
 Run specified CLI test files.
@@ -10,15 +10,18 @@ import unittest
 
 import edq.core.argparser
 import edq.testing.cli
-import edq.testing.cli_test
+import edq.testing.unittest
+
+class CLITest(edq.testing.unittest.BaseTest):
+    """ Test CLI invocations. """
 
 def run_cli(args: argparse.Namespace) -> int:
     """ Run the CLI. """
 
-    edq.testing.cli.add_test_paths(edq.testing.cli_test.CLITest, args.data_dir, args.paths)
+    edq.testing.cli.add_test_paths(CLITest, args.data_dir, args.paths)
 
     runner = unittest.TextTestRunner(verbosity = 2)
-    tests = unittest.defaultTestLoader.loadTestsFromTestCase(edq.testing.cli_test.CLITest)
+    tests = unittest.defaultTestLoader.loadTestsFromTestCase(CLITest)
     results = runner.run(tests)
 
     return len(results.errors) + len(results.failures)
@@ -27,7 +30,7 @@ def main() -> int:
     """ Get a parser, parse the args, and call run. """
     return run_cli(_get_parser().parse_args())
 
-def _get_parser() -> edq.core.argparser.Parser:
+def _get_parser() -> argparse.ArgumentParser:
     """ Get the parser. """
 
     parser = edq.core.argparser.get_default_parser(__doc__.strip())

edq/cli/version.py

@@ -16,9 +16,10 @@ def run_cli(args: argparse.Namespace) -> int:
 
 def main() -> int:
     """ Get a parser, parse the args, and call run. """
+
     return run_cli(_get_parser().parse_args())
 
-def _get_parser() -> edq.core.argparser.Parser:
+def _get_parser() -> argparse.ArgumentParser:
     """ Get the parser. """
 
     return edq.core.argparser.get_default_parser(__doc__.strip())

edq/core/argparser.py

@@ -8,9 +8,12 @@ while post-callbacks are generally intended to act on the results of parsing.
 """
 
 import argparse
+import functools
 import typing
 
+import edq.core.config
 import edq.core.log
+import edq.util.net
 
 @typing.runtime_checkable
 class PreParseFunction(typing.Protocol):
@@ -102,11 +105,33 @@ class Parser(argparse.ArgumentParser):
 
         return parsed_args  # type: ignore[no-any-return]
 
-def get_default_parser(description: str) -> Parser:
-    """ Get a parser with the default callbacks already attached. """
+def get_default_parser(description: str,
+        version: typing.Union[str, None] = None,
+        include_log: bool = True,
+        include_config: bool = True,
+        include_net: bool = False,
+        config_options: typing.Union[typing.Dict[str, typing.Any], None] = None,
+        ) -> Parser:
+    """ Get a parser with the requested default callbacks already attached. """
+
+    if (config_options is None):
+        config_options = {}
 
     parser = Parser(description = description)
 
-    parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args)
+    if (version is not None):
+        parser.add_argument('--version',
+                action = 'version', version = version)
+
+    if (include_log):
+        parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args)
+
+    if (include_config):
+        config_pre_func = functools.partial(edq.core.config.set_cli_args, **config_options)
+        config_post_func = functools.partial(edq.core.config.load_config_into_args, **config_options)
+        parser.register_callbacks('config', config_pre_func, config_post_func)
+
+    if (include_net):
+        parser.register_callbacks('net', edq.util.net.set_cli_args, edq.util.net.init_from_args)
 
     return parser

edq/core/config.py

@@ -0,0 +1,268 @@
+import argparse
+import os
+import typing
+
+import platformdirs
+
+import edq.util.dirent
+import edq.util.json
+
+CONFIG_SOURCE_GLOBAL: str = "<global config file>"
+CONFIG_SOURCE_LOCAL: str = "<local config file>"
+CONFIG_SOURCE_CLI_FILE: str = "<cli config file>"
+CONFIG_SOURCE_CLI: str = "<cli argument>"
+
+CONFIG_PATHS_KEY: str = 'config_paths'
+CONFIGS_KEY: str = 'configs'
+GLOBAL_CONFIG_KEY: str = 'global_config_path'
+IGNORE_CONFIGS_KEY: str = 'ignore_configs'
+DEFAULT_CONFIG_FILENAME: str = "edq-config.json"
+
+class ConfigSource:
+    """ A class for storing config source information. """
+
+    def __init__(self, label: str, path: typing.Union[str, None] = None) -> None:
+        self.label = label
+        """ The label identifying the config (see CONFIG_SOURCE_* constants). """
+
+        self.path = path
+        """ The path of where the config was sourced from. """
+
+    def __eq__(self, other: object) -> bool:
+        if (not isinstance(other, ConfigSource)):
+            return False
+
+        return ((self.label == other.label) and (self.path == other.path))
+
+    def __str__(self) -> str:
+        return f"({self.label}, {self.path})"
+
+def get_global_config_path(config_filename: str) -> str:
+    """ Get the path for the global config file. """
+
+    return platformdirs.user_config_dir(config_filename)
+
+def get_tiered_config(
+        config_filename: str = DEFAULT_CONFIG_FILENAME,
+        legacy_config_filename: typing.Union[str, None] = None,
+        cli_arguments: typing.Union[dict, argparse.Namespace, None] = None,
+        local_config_root_cutoff: typing.Union[str, None] = None,
+    ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]:
+    """
+    Load all configuration options from files and command-line arguments.
+    Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin.
+    """
+
+    if (cli_arguments is None):
+        cli_arguments = {}
+
+    config: typing.Dict[str, str] = {}
+    sources: typing.Dict[str, ConfigSource] = {}
+
+    # Ensure CLI arguments are always a dict, even if provided as argparse.Namespace.
+    if (isinstance(cli_arguments, argparse.Namespace)):
+        cli_arguments = vars(cli_arguments)
+
+    global_config_path = cli_arguments.get(GLOBAL_CONFIG_KEY, get_global_config_path(config_filename))
+
+    # Check the global user config file.
+    if (os.path.isfile(global_config_path)):
+        _load_config_file(global_config_path, config, sources, CONFIG_SOURCE_GLOBAL)
+
+    # Check the local user config file.
+    local_config_path = _get_local_config_path(
+        config_filename = config_filename,
+        legacy_config_filename = legacy_config_filename,
+        local_config_root_cutoff = local_config_root_cutoff,
+    )
+
+    if (local_config_path is not None):
+        _load_config_file(local_config_path, config, sources, CONFIG_SOURCE_LOCAL)
+
+    # Check the config file specified on the command-line.
+    config_paths = cli_arguments.get(CONFIG_PATHS_KEY, [])
+    for path in config_paths:
+        _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE)
+
+    # Check the command-line config options.
+    cli_configs = cli_arguments.get(CONFIGS_KEY, [])
+    for cli_config in cli_configs:
+        if ("=" not in cli_config):
+            raise ValueError(
+                f"Invalid configuration option '{cli_config}'."
+                + " Configuration options must be provided in the format `<key>=<value>` when passed via the CLI."
+            )
+
+        (key, value) = cli_config.split("=", maxsplit = 1)
+
+        key = key.strip()
+        if (key == ""):
+            raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.")
+
+        config[key] = value
+        sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI)
+
+    # Finally, ignore any configs that is specified from CLI command.
+    cli_ignore_configs = cli_arguments.get(IGNORE_CONFIGS_KEY, [])
+    for ignore_config in cli_ignore_configs:
+        config.pop(ignore_config, None)
+        sources.pop(ignore_config, None)
+
+    return config, sources
+
+def _load_config_file(
+        config_path: str,
+        config: typing.Dict[str, str],
+        sources: typing.Dict[str, ConfigSource],
+        source_label: str,
+    ) -> None:
+    """ Loads config variables and the source from the given config JSON file. """
+
+    config_path = os.path.abspath(config_path)
+    for (key, value) in edq.util.json.load_path(config_path).items():
+        key = key.strip()
+        if (key == ""):
+            raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.")
+
+        config[key] = value
+        sources[key] = ConfigSource(label = source_label, path = config_path)
+
+def _get_local_config_path(
+        config_filename: str,
+        legacy_config_filename: typing.Union[str, None] = None,
+        local_config_root_cutoff: typing.Union[str, None] = None,
+    ) -> typing.Union[str, None]:
+    """
+    Search for a config file in hierarchical order.
+    Begins with the provided config file name,
+    optionally checks the legacy config file name if specified,
+    then continues up the directory tree looking for the provided config file name.
+    Returns the path to the first config file found.
+
+    If no config file is found, returns None.
+
+    The cutoff parameter limits the search depth, preventing detection of config file in higher-level directories during testing.
+    """
+
+    # Provided config file is in current directory.
+    if (os.path.isfile(config_filename)):
+        return os.path.abspath(config_filename)
+
+    # Provided legacy config file is in current directory.
+    if (legacy_config_filename is not None):
+        if (os.path.isfile(legacy_config_filename)):
+            return os.path.abspath(legacy_config_filename)
+
+    # Provided config file is found in an ancestor directory up to the root or cutoff limit.
+    parent_dir = os.path.dirname(os.getcwd())
+    return _get_ancestor_config_file_path(
+        parent_dir,
+        config_filename = config_filename,
+        local_config_root_cutoff = local_config_root_cutoff,
+    )
+
+def _get_ancestor_config_file_path(
+        current_directory: str,
+        config_filename: str,
+        local_config_root_cutoff: typing.Union[str, None] = None,
+    ) -> typing.Union[str, None]:
+    """
+    Search through the parent directories (until root or a given cutoff directory(inclusive)) for a config file.
+    Stops at the first occurrence of the specified config file along the path to root.
+    Returns the path if a config file is found.
+    Otherwise, returns None.
+    """
+
+    if (local_config_root_cutoff is not None):
+        local_config_root_cutoff = os.path.abspath(local_config_root_cutoff)
+
+    current_directory = os.path.abspath(current_directory)
+    for _ in range(edq.util.dirent.DEPTH_LIMIT):
+        config_file_path = os.path.join(current_directory, config_filename)
+        if (os.path.isfile(config_file_path)):
+            return config_file_path
+
+        # Check if current directory is root.
+        parent_dir = os.path.dirname(current_directory)
+        if (parent_dir == current_directory):
+            break
+
+        if (local_config_root_cutoff == current_directory):
+            break
+
+        current_directory = parent_dir
+
+    return None
+
+def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, typing.Any],
+        config_filename: str = DEFAULT_CONFIG_FILENAME,
+        **kwargs: typing.Any,
+    ) -> None:
+    """
+    Set common CLI arguments for configuration.
+    """
+
+    parser.add_argument('--config-global', dest = GLOBAL_CONFIG_KEY,
+        action = 'store', type = str, default = get_global_config_path(config_filename),
+        help = 'Set the default global config file path (default: %(default)s).',
+    )
+
+    parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY,
+        action = 'append', type = str, default = [],
+        help = ('Load config options from a JSON file.'
+            + ' This flag can be specified multiple times.'
+            + ' Files are applied in the order provided and later files override earlier ones.'
+            + ' Will override options form both global and local config files.')
+    )
+
+    parser.add_argument('--config', dest = CONFIGS_KEY,
+        action = 'append', type = str, default = [],
+        help = ('Set a configuration option from the command-line.'
+            + ' Specify options as <key>=<value> pairs.'
+            + ' This flag can be specified multiple times.'
+            + ' The options are applied in the order provided and later options override earlier ones.'
+            + ' Will override options form all config files.')
+    )
+
+    parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY,
+        action = 'append', type = str, default = [],
+        help = ('Ignore any config option with the specified key.'
+            + ' The system-provided default value will be used for that option if one exists.'
+            + ' This flag can be specified multiple times.'
+            + ' Ignored options are processed last.')
+    )
+
+def load_config_into_args(
+        parser: argparse.ArgumentParser,
+        args: argparse.Namespace,
+        extra_state: typing.Dict[str, typing.Any],
+        config_filename: str = DEFAULT_CONFIG_FILENAME,
+        cli_arg_config_map: typing.Union[typing.Dict[str, str], None] = None,
+        **kwargs: typing.Any,
+    ) -> None:
+    """
+    Take in args from a parser that was passed to set_cli_args(),
+    and get the tired configuration with the appropriate parameters, and attache it to args.
+
+    Arguments that appear on the CLI as flags (e.g. `--foo bar`) can be copied over to the config options via `cli_arg_config_map`.
+    The keys of `cli_arg_config_map` represent attributes in the CLI arguments (`args`),
+    while the values represent the desired config name this argument should be set as.
+    For example, a `cli_arg_config_map` of `{'foo': 'baz'}` will make the CLI argument `--foo bar`
+    be equivalent to `--config baz=bar`.
+    """
+
+    if (cli_arg_config_map is None):
+        cli_arg_config_map = {}
+
+    for (cli_key, config_key) in cli_arg_config_map.items():
+        value = getattr(args, cli_key, None)
+        if (value is not None):
+            getattr(args, CONFIGS_KEY).append(f"{config_key}={value}")
+
+    (config_dict, sources_dict) = get_tiered_config(
+        cli_arguments = args,
+        config_filename = config_filename,
+    )
+
+    setattr(args, "_config", config_dict)
+    setattr(args, "_config_sources", sources_dict)