edq-utils 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

edq/__init__.py

@@ -1 +1,5 @@
-__version__ = '0.0.2'
+"""
+General Python tools used by several EduLinq projects.
+"""
+
+__version__ = '0.0.4'

edq/cli/testing/cli-test.py

@@ -0,0 +1,42 @@
+#  # pylint: disable=invalid-name
+
+"""
+Run specified CLI test files.
+"""
+
+import argparse
+import sys
+import unittest
+
+import edq.core.argparser
+import edq.testing.cli
+import edq.testing.cli_test
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    edq.testing.cli.add_test_paths(edq.testing.cli_test.CLITest, args.paths)
+
+    runner = unittest.TextTestRunner(verbosity = 2)
+    tests = unittest.defaultTestLoader.loadTestsFromTestCase(edq.testing.cli_test.CLITest)
+    results = runner.run(tests)
+
+    return len(results.errors) + len(results.failures)
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> edq.core.argparser.Parser:
+    """ Get the parser. """
+
+    parser = edq.core.argparser.get_default_parser(__doc__.strip())
+
+    parser.add_argument('paths', metavar = 'PATH',
+        type = str, nargs = '+',
+        help = 'Path to CLI test case files.')
+
+    return parser
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/cli/version.py

@@ -0,0 +1,27 @@
+"""
+Get the version of the EduLinq Python utils package.
+"""
+
+import argparse
+import sys
+
+import edq.core.argparser
+import edq.core.version
+
+def run_cli(args: argparse.Namespace) -> int:
+    """ Run the CLI. """
+
+    print(f"v{edq.core.version.get_version()}")
+    return 0
+
+def main() -> int:
+    """ Get a parser, parse the args, and call run. """
+    return run_cli(_get_parser().parse_args())
+
+def _get_parser() -> edq.core.argparser.Parser:
+    """ Get the parser. """
+
+    return edq.core.argparser.get_default_parser(__doc__.strip())
+
+if (__name__ == '__main__'):
+    sys.exit(main())

edq/core/argparser.py

@@ -0,0 +1,112 @@
+"""
+A place to handle common CLI arguments.
+"parsers" in this file are always assumed to be argparse parsers.
+
+The general idea is that callers can register callbacks to be called before and after parsing CLI arguments.
+Pre-callbacks are generally intended to add arguments to the parser,
+while post-callbacks are generally intended to act on the results of parsing.
+"""
+
+import argparse
+import typing
+
+import edq.core.log
+
+@typing.runtime_checkable
+class PreParseFunction(typing.Protocol):
+    """
+    A function that can be called before parsing arguments.
+    """
+
+    def __call__(self, parser: argparse.ArgumentParser, extra_state: typing.Dict[str, typing.Any]) -> None:
+        """
+        Prepare a parser for parsing.
+        This is generally used for adding your module's arguments to the parser,
+        for example a logging module may add arguments to set a logging level.
+
+        The extra state is shared between all pre-parse functions
+        and will be placed in the final parsed output under `_pre_extra_state_`.
+        """
+
+@typing.runtime_checkable
+class PostParseFunction(typing.Protocol):
+    """
+    A function that can be called after parsing arguments.
+    """
+
+    def __call__(self,
+            parser: argparse.ArgumentParser,
+            args: argparse.Namespace,
+            extra_state: typing.Dict[str, typing.Any]) -> None:
+        """
+        Take actions after arguments are parsed.
+        This is generally used for initializing your module with options,
+        for example a logging module may set a logging level.
+
+        The extra state is shared between all post-parse functions
+        and will be placed in the final parsed output under `_post_extra_state_`.
+        """
+
+class Parser(argparse.ArgumentParser):
+    """
+    Extend an argparse parser to call the pre and post functions.
+    """
+
+    def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None:
+        super().__init__(*args, **kwargs)
+
+        self._pre_parse_callbacks: typing.Dict[str, PreParseFunction] = {}
+        self._post_parse_callbacks: typing.Dict[str, PostParseFunction] = {}
+
+    def register_callbacks(self,
+            key: str,
+            pre_parse_callback: typing.Union[PreParseFunction, None] = None,
+            post_parse_callback: typing.Union[PostParseFunction, None] = None,
+            ) -> None:
+        """
+        Register callback functions to run before/after argument parsing.
+        Any existing callbacks under the specified key will be replaced.
+        """
+
+        if (pre_parse_callback is not None):
+            self._pre_parse_callbacks[key] = pre_parse_callback
+
+        if (post_parse_callback is not None):
+            self._post_parse_callbacks[key] = post_parse_callback
+
+    def parse_args(self,  # type: ignore[override]
+            *args: typing.Any,
+            skip_keys: typing.Union[typing.List[str], None] = None,
+            **kwargs: typing.Any) -> argparse.Namespace:
+        if (skip_keys is None):
+            skip_keys = []
+
+        # Call pre-parse callbacks.
+        pre_extra_state: typing.Dict[str, typing.Any] = {}
+        for (key, pre_parse_callback) in self._pre_parse_callbacks.items():
+            if (key not in skip_keys):
+                pre_parse_callback(self, pre_extra_state)
+
+        # Parse the args.
+        parsed_args = super().parse_args(*args, **kwargs)
+
+        # Call post-parse callbacks.
+        post_extra_state: typing.Dict[str, typing.Any] = {}
+        for (key, post_parse_callback) in self._post_parse_callbacks.items():
+            if (key not in skip_keys):
+                post_parse_callback(self, parsed_args, post_extra_state)
+
+        # Attach the additional state to the args.
+        setattr(parsed_args, '_pre_extra_state_', pre_extra_state)
+        setattr(parsed_args, '_post_extra_state_', post_extra_state)
+
+        return parsed_args  # type: ignore[no-any-return]
+
+def get_default_parser(description: str) -> Parser:
+    """ Get a parser with the default callbacks already attached. """
+
+    parser = Parser(description = description)
+
+    parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args)
+
+    return parser

edq/core/argparser_test.py

@@ -0,0 +1,124 @@
+import functools
+
+import edq.core.argparser
+import edq.testing.unittest
+
+class TestArgParser(edq.testing.unittest.BaseTest):
+    """ Test argument parsing. """
+
+    def test_callbacks_base(self):
+        """ Test the argument parsing callbacks. """
+
+        # [(parse text, [(key, pre, post), ...], skip keys, expected (as dict)), ...]
+        test_cases = [
+            # Empty
+            (
+                "",
+                [],
+                [],
+                {
+                    '_pre_extra_state_': {},
+                    '_post_extra_state_': {},
+                },
+            ),
+
+            # Single Callbacks
+            (
+                "",
+                [
+                    ('test', functools.partial(_pre_callback_append, value = 1), functools.partial(_post_callback_append, value = 2)),
+                ],
+                [],
+                {
+                    '_pre_extra_state_': {
+                        'append': [1],
+                    },
+                    '_post_extra_state_': {
+                        'append': [2],
+                    },
+                },
+            ),
+
+            # Double Callbacks
+            (
+                "",
+                [
+                    ('test1', functools.partial(_pre_callback_append, value = 1), functools.partial(_post_callback_append, value = 2)),
+                    ('test2', functools.partial(_pre_callback_append, value = 3), functools.partial(_post_callback_append, value = 4)),
+                ],
+                [],
+                {
+                    '_pre_extra_state_': {
+                        'append': [1, 3],
+                    },
+                    '_post_extra_state_': {
+                        'append': [2, 4],
+                    },
+                },
+            ),
+
+            # Split Callbacks
+            (
+                "",
+                [
+                    ('test1', functools.partial(_pre_callback_append, value = 1), None),
+                    ('test2', None, functools.partial(_post_callback_append, value = 4)),
+                ],
+                [],
+                {
+                    '_pre_extra_state_': {
+                        'append': [1],
+                    },
+                    '_post_extra_state_': {
+                        'append': [4],
+                    },
+                },
+            ),
+
+            # Override Callbacks
+            (
+                "",
+                [
+                    ('test', functools.partial(_pre_callback_append, value = 1), functools.partial(_post_callback_append, value = 2)),
+                    ('test', functools.partial(_pre_callback_append, value = 3), functools.partial(_post_callback_append, value = 4)),
+                ],
+                [],
+                {
+                    '_pre_extra_state_': {
+                        'append': [3],
+                    },
+                    '_post_extra_state_': {
+                        'append': [4],
+                    },
+                },
+            ),
+        ]
+
+        for (i, test_case) in enumerate(test_cases):
+            (text, registrations, skip_keys, expected) = test_case
+
+            with self.subTest(msg = f"Case {i} ('{text}'):"):
+                parser = edq.core.argparser.Parser(f"Case {i}")
+                for (key, pre, post) in registrations:
+                    parser.register_callbacks(key, pre, post)
+
+                args = parser.parse_args(text.split(), skip_keys = skip_keys)
+
+                actual = vars(args)
+                self.assertJSONDictEqual(expected, actual)
+
+def _pre_callback_append(parser, extra_state, key = 'append', value = None) -> None:
+    """ Append the given value into the extra state. """
+
+    if (key not in extra_state):
+        extra_state[key] = []
+
+    extra_state[key].append(value)
+
+def _post_callback_append(parser, args, extra_state, key = 'append', value = None) -> None:
+    """ Append the given value into the extra state. """
+
+    if (key not in extra_state):
+        extra_state[key] = []
+
+    extra_state[key].append(value)

edq/core/log.py

@@ -0,0 +1,101 @@
+import argparse
+import logging
+import typing
+
+DEFAULT_LOGGING_LEVEL: str = logging.getLevelName(logging.INFO)
+DEFAULT_LOGGING_FORMAT: str = '%(asctime)s [%(levelname)-8s] - %(filename)s:%(lineno)s -- %(message)s'
+
+LEVELS: typing.List[str] = [
+    'TRACE',
+    logging.getLevelName(logging.DEBUG),
+    logging.getLevelName(logging.INFO),
+    logging.getLevelName(logging.WARNING),
+    logging.getLevelName(logging.ERROR),
+    logging.getLevelName(logging.CRITICAL),
+]
+
+def init(level: str = DEFAULT_LOGGING_LEVEL, log_format: str = DEFAULT_LOGGING_FORMAT,
+        warn_loggers: typing.Union[typing.List[str], None] = None,
+        **kwargs: typing.Any) -> None:
+    """
+    Initialize or re-initialize the logging infrastructure.
+    The list of warning loggers is a list of identifiers for loggers (usually third-party) to move up to warning on init.
+    """
+
+    # Add trace.
+    _add_logging_level('TRACE', logging.DEBUG - 5)
+
+    logging.basicConfig(level = level, format = log_format, force = True)
+
+    if (warn_loggers is not None):
+        for warn_logger in warn_loggers:
+            logging.getLogger(warn_logger).setLevel(logging.WARNING)
+
+    logging.trace("Logging initialized with level '%s'.", level)  # type: ignore[attr-defined]
+
+def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, typing.Any]) -> None:
+    """
+    Set common CLI arguments.
+    This is a sibling to init_from_args(), as the arguments set here can be interpreted there.
+    """
+
+    parser.add_argument('--log-level', dest = 'log_level',
+            action = 'store', type = str, default = logging.getLevelName(logging.INFO),
+            choices = LEVELS,
+            help = 'Set the logging level (default: %(default)s).')
+
+    parser.add_argument('--quiet', dest = 'quiet',
+            action = 'store_true', default = False,
+            help = 'Set the logging level to warning (overrides --log-level) (default: %(default)s).')
+
+    parser.add_argument('--debug', dest = 'debug',
+            action = 'store_true', default = False,
+            help = 'Set the logging level to debug (overrides --log-level and --quiet) (default: %(default)s).')
+
+def init_from_args(
+        parser: argparse.ArgumentParser,
+        args: argparse.Namespace,
+        extra_state: typing.Dict[str, typing.Any]) -> None:
+    """
+    Take in args from a parser that was passed to set_cli_args(),
+    and call init() with the appropriate arguments.
+    """
+
+    level = args.log_level
+
+    if (args.quiet):
+        level = logging.getLevelName(logging.WARNING)
+
+    if (args.debug):
+        level = logging.getLevelName(logging.DEBUG)
+
+    init(level)
+
+def _add_logging_level(level_name: str, level_number: int, method_name: typing.Union[str, None] = None) -> None:
+    """
+    Add a new logging level.
+
+    See https://stackoverflow.com/questions/2183233/how-to-add-a-custom-loglevel-to-pythons-logging-facility/35804945#35804945 .
+    """
+
+    if (method_name is None):
+        method_name = level_name.lower()
+
+    # Level has already been defined.
+    if hasattr(logging, level_name):
+        return
+
+    def log_for_level(self: typing.Any, message: str, *args: typing.Any, **kwargs: typing.Any) -> None:
+        if self.isEnabledFor(level_number):
+            self._log(level_number, message, args, **kwargs)
+
+    def log_to_root(message: str, *args: typing.Any, **kwargs: typing.Any) -> None:
+        logging.log(level_number, message, *args, **kwargs)
+
+    logging.addLevelName(level_number, level_name)
+    setattr(logging, level_name, level_number)
+    setattr(logging.getLoggerClass(), method_name, log_for_level)
+    setattr(logging, method_name, log_to_root)
+
+# Load the default logging when this module is loaded.
+init()

edq/core/version.py

@@ -0,0 +1,6 @@
+import edq
+
+def get_version() -> str:
+    """ Get the version for the EduLinq Python utils. """
+
+    return edq.__version__

edq/testing/__init__.py

@@ -0,0 +1,3 @@
+"""
+Testing infrastructure.
+"""

edq/testing/asserts.py

@@ -0,0 +1,66 @@
+"""
+More complex testing assertions.
+Often used as output checks in CLI tests.
+"""
+
+import re
+import typing
+
+import edq.testing.unittest
+
+TRACEBACK_LINE_REGEX: str = r'^\s*File "[^"]+", line \d+,.*$\n.*$(\n\s*[\^~]+\s*$)?'
+TRACEBACK_LINE_REPLACEMENT: str = '<TRACEBACK_LINE>'
+
+TEXT_NORMALIZATIONS: typing.List[typing.Tuple[str, str]] = [
+    (r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d+ \[\S+ *\] - .*\.py:\d+ -- ', '<LOG_PREFIX> -- '),
+    (r'\d+\.\d+ seconds', '<DURATION_SECONDS>'),
+    (r'\bv\d+\.\d+\.\d+\b', '<VERSION>'),
+    (r'^\s*File "[^"]+", line \d+,.*$\n.*$(\n\s*[\^~]+\s*$)?', '<TRACEBACK_LINE>'),
+    (rf'{TRACEBACK_LINE_REPLACEMENT}(\n{TRACEBACK_LINE_REPLACEMENT})*', '<TRACEBACK>'),
+]
+"""
+Normalization to make to the CLI output.
+Formatted as: [(regex, replacement), ...]
+"""
+
+@typing.runtime_checkable
+class StringComparisonAssertion(typing.Protocol):
+    """
+    A function that can be used as a comparison assertion for a test.
+    """
+
+    def __call__(self,
+            test: edq.testing.unittest.BaseTest,
+            expected: str, actual: str,
+            **kwargs: typing.Any) -> None:
+        """
+        Perform an assertion between expected and actual data.
+        """
+
+
+def content_equals_raw(test: edq.testing.unittest.BaseTest, expected: str, actual: str, **kwargs: typing.Any) -> None:
+    """ Check for equality using a simple string comparison. """
+
+    test.assertEqual(expected, actual)
+
+def content_equals_normalize(test: edq.testing.unittest.BaseTest, expected: str, actual: str, **kwargs: typing.Any) -> None:
+    """
+    Perform some standard text normalizations (see TEXT_NORMALIZATIONS) before using simple string comparison.
+    """
+
+    for (regex, replacement) in TEXT_NORMALIZATIONS:
+        expected = re.sub(regex, replacement, expected, flags = re.MULTILINE)
+        actual = re.sub(regex, replacement, actual, flags = re.MULTILINE)
+
+    content_equals_raw(test, expected, actual)
+
+def has_content_100(test: edq.testing.unittest.BaseTest, expected: str, actual: str, **kwargs: typing.Any) -> None:
+    """ Check the that output has at least 100 characters. """
+
+    return has_content(test, expected, actual, min_length = 100)
+
+def has_content(test: edq.testing.unittest.BaseTest, expected: str, actual: str, min_length: int = 100) -> None:
+    """ Ensure that the output has content of at least some length. """
+
+    message = f"Output does not meet minimum length of {min_length}, it is only {len(actual)}."
+    test.assertTrue((len(actual) >= min_length), msg = message)