smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/__main__.py

@@ -0,0 +1,60 @@
+import importlib
+import importlib.util
+import sys
+from glob import glob
+from os.path import basename, dirname, join
+
+from smart_tests.app import Application
+from smart_tests.args4p.command import Group
+from smart_tests.commands.compare import compare
+from smart_tests.commands.detect_flakes import detect_flakes
+from smart_tests.commands.inspect import inspect
+from smart_tests.commands.record import record
+from smart_tests.commands.stats import stats
+from smart_tests.commands.subset import subset
+from smart_tests.commands.verify import verify
+
+cli = Group(name="cli", callback=Application)
+cli.add_command(record)
+cli.add_command(subset)
+# TODO: main.add_command(split_subset)
+cli.add_command(verify)
+cli.add_command(inspect)
+cli.add_command(stats)
+cli.add_command(compare)
+cli.add_command(detect_flakes)
+
+
+def _load_test_runners():
+    # load all test runners
+    for f in glob(join(dirname(__file__), 'test_runners', "*.py")):
+        f = basename(f)[:-3]
+        if f == '__init__':
+            continue
+        importlib.import_module(f'smart_tests.test_runners.{f}')
+
+    # load all plugins. Here we do a bit of command line parsing ourselves,
+    # because the command line could look something like `smart-tests record tests myprofile --plugins ...
+    plugin_dir = None
+    if "--plugins" in sys.argv:
+        idx = sys.argv.index("--plugins")
+        if idx + 1 < len(sys.argv):
+            plugin_dir = sys.argv[idx + 1]
+
+    if plugin_dir:
+        for f in glob(join(plugin_dir, '*.py')):
+            spec = importlib.util.spec_from_file_location(
+                f"smart_tests.plugins.{basename(f)[:-3]}", f)
+            plugin = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(plugin)
+
+
+_load_test_runners()
+
+
+def main():
+    cli.main()
+
+
+if __name__ == '__main__':
+    main()

smart_tests/app.py

@@ -0,0 +1,67 @@
+# Object representing the most global state possible, which represents a single invocation of CLI
+# Currently it's used to keep global configurations.
+#
+# From command implementations, this is available via dependency injection
+
+import logging
+import os
+from typing import Annotated
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.utils import logger
+from smart_tests.utils.env_keys import SKIP_CERT_VERIFICATION
+from smart_tests.version import __version__
+
+
+class Application:
+    # Group commands that take the CLI profile as a sub-command shall set this parameter
+    test_runner: str | None = None
+
+    # this maps to the main entry point of the CLI command
+    def __init__(
+            self,
+            log_level: Annotated[str, typer.Option(
+                help="Set logger's log level (CRITICAL, ERROR, WARNING, AUDIT, INFO, DEBUG)."
+            )] = logger.LOG_LEVEL_DEFAULT_STR,
+            plugin_dir: Annotated[str | None, typer.Option(
+                "--plugins",
+                help="Directory to load plugins from"
+            )] = None,
+            dry_run: Annotated[bool, typer.Option(
+                help="Dry-run mode. No data is sent to the server. However, sometimes "
+                     "GET requests without payload data or side effects could be sent."
+                     "note: Since the dry run log is output together with the AUDIT log, "
+                     "even if the log-level is set to warning or higher, the log level will "
+                     "be forced to be set to AUDIT."
+            )] = False,
+            skip_cert_verification: Annotated[bool, typer.Option(
+                help="Skip the SSL certificate check. This lets you bypass system setup issues "
+                     "like CERTIFICATE_VERIFY_FAILED, at the expense of vulnerability against "
+                     "a possible man-in-the-middle attack. Use it as an escape hatch, but with caution."
+            )] = False,
+            version: Annotated[bool, typer.Option(
+                "--version", help="Show version and exit"
+            )] = False,
+    ):
+        if version:
+            click.echo(f"smart-tests-cli {__version__}")
+            raise typer.Exit(0)
+
+        level = logger.get_log_level(log_level)
+        # In the case of dry-run, it is forced to set the level below the AUDIT.
+        # This is because the dry-run log will be output along with the audit log.
+        if dry_run and level > logger.LOG_LEVEL_AUDIT:
+            level = logger.LOG_LEVEL_AUDIT
+        logging.basicConfig(level=level)
+
+        # plugin_dir is processed earlier. If we do it here, it's too late
+
+        # Dry run mode. This command is used by customers to inspect data we'd send to our server,
+        # but without actually doing so.
+        self.dry_run = dry_run
+
+        if not skip_cert_verification:
+            skip_cert_verification = (os.environ.get(SKIP_CERT_VERIFICATION) is not None)
+        self.skip_cert_verification = skip_cert_verification

smart_tests/args4p/README.md

@@ -0,0 +1,102 @@
+# User guide
+
+## Option/argument annotations
+
+To control how args4p passes option/argument values to your function, you can either use the `@args4p.option` or `@args4p.argument` decorators, or use the `Option` and `Argument` in parameter annotations.
+
+    @args4p.option('-v', 'verbose', ...)
+    @args4p.command
+    def foo(verbose: bool, ...):
+        ...
+
+    import smart_tests.args4p.typer as typer
+    
+    @args4p.command
+    def foo(verbose: Annotated[bool, typer.Option('-v', ...)
+
+The only difference between those two variants is that the former requires the parameter name be passed as string, while the latter takes that from the actual parameter declaration.
+
+These invocations take the following parameters:
+
+**\*param_decls: List[str]**: The first portion of the invocation is a var-arg of strings, and they designate the names of options. When used as a decorator, this list must be followed by the parameter name itself.
+
+If just the parameter name is given but no option names are given, the option name is generated from the parameter name.
+
+    # this creates the --verbose option
+    @args4p.option("verbose")
+    def f (verbose: bool):
+        ...
+
+    # same
+    def f (verbose: Annotated[bool, args4p.Option()]):
+        ...
+
+Arguments would take the parameter name if used as a decorator, or nothing if used as annotation.
+
+**help**: Human readable description of the option, used to render the help message
+
+**type**: Specify the type of the option/argument. The type is individual, meaning even if you allow the option/argument to be specified multiple times, the type of option/argument is always that of a single value (e.g., `int`, not `List[int]`)
+
+You can also specify any `Callable` that takes a `str` and produces a value of your desired type. This is handy for defining custom types.
+
+If this parameter is omitted, the type is inferred from the type annotation of the function parameter, with the following massaging:
+
+- if the type is optional, such as `str|None` or `Optional[str]`, the type is inferred as the non-None type (e.g., `str` in this case)
+
+**default**: Default value if the option/argument is not provided. bool options are treated as having `False` as the default.
+
+**required**: Whether the option/argument is required. Default is `False`.
+
+**metavar**: User-friendly name for the option value place holder, used in help messages.
+
+**multiple**: Whether the option/argument can be specified multiple times. Default is `False`. If true, the parameter type must be a list type (e.g. `List[str]`)
+
+**hidden**: If true, this option is hidden from help messages.
+
+## Group (sub-commands)
+This library encourages organizing a complex CLI through sub-commands,
+ala `git`, like this:
+
+```
+@args4p.group
+@args4p.option('-v', 'verbose', ...)
+def cli(verbose: bool):
+    return {'verbose': verbose}
+
+@cli.command   # NOT args4p.command
+@args4p.argument('name')
+def subcmd1(parent, name):
+    if parent.verbose:
+        print(f"Hello {name}")
+```
+
+Sub-commands receive the result of the parent command as its first argument. This allows you to pass options to the parent command and access them in the sub-command.
+
+A particularly useful idiom is to create a group out of a class initializer, like this:
+
+```
+class App:
+    def __init__(self, verbose: Annotated[bool,Option('-v')]):
+        self.verbose = verbose
+
+cli=Group(App)
+
+@cli.command   # NOT args4p.command
+@args4p.argument('name')
+def subcmd1(app: App, name):
+    if app.verbose:
+        print(f"Hello {name}")
+```
+
+(Note that decorators won't work with this idiom, so you have to use annotations.)
+
+
+Alternatively, use `Group.add_command` to add a top-level command as a sub-command to a group.
+
+```
+@args4p.command
+def subcmd2(...):
+    ...
+
+cli.add_command(subcmd2)
+```

smart_tests/args4p/__init__.py

@@ -0,0 +1,13 @@
+from typing import Callable, TypeVar
+
+F = TypeVar("F", bound=Callable)
+
+
+def decorator(func: F) -> F:
+    """
+    Functions marked with this decorator are meant to be used as decorators themselves.
+    """
+    return func
+
+
+from .decorators import argument, command, group, option  # noqa: F401, E402

smart_tests/args4p/argument.py

@@ -0,0 +1,45 @@
+from typing import Any, Callable, Optional
+
+from .exceptions import BadCmdLineException
+from .option import NO_DEFAULT
+from .parameter import Parameter, normalize_type
+
+
+class Argument(Parameter):
+    clazz = "argument"
+
+    def __init__(self, name: Optional[str], type: type | Callable[..., Any] | None = None, multiple: bool = False,
+                 required: bool = True, metavar: str | None = None, help: str | None = None, default: Any = NO_DEFAULT):
+        self.name = name  # type: ignore[assignment]  # once properly constructed, name is never None
+        self.type = normalize_type(type)
+        self.multiple = multiple
+        self.required = required
+        self.metavar = metavar
+        self.help = help
+        self.default = default
+
+    def append(self, existing: Any, arg: str):
+        '''
+        Given the current value 'existing' that represents the present value to invoke the user function with,
+        this method is called when another argument 'arg' is consumed from the command line to create the updated
+        value that replaces 'existing'.
+        '''
+
+        try:
+            v = self.type(arg)
+        except ValueError as e:
+            raise BadCmdLineException(f"Invalid value '{arg}' for argument '{self.name}'") from e
+
+        if self.multiple:
+            if existing is None:
+                existing = []
+            existing.append(v)
+            return existing
+        else:
+            return v
+
+    def attach_to_command(self, command):  # typing command makes reference circular
+        super().attach_to_command(command)
+
+        if self.metavar is None:
+            self.metavar = str(self.name).upper()