openfilter 0.1.0__py3-none-any.whl

openfilter/cli/__main__.py

@@ -0,0 +1,31 @@
+import sys
+
+from .common import SCRIPT
+from .cmd_logs_metrics import cmd_logs, cmd_metrics
+from .cmd_info import cmd_info
+from .cmd_run import cmd_run
+
+
+def main():
+    args = sys.argv[2:]
+    cmd  = sys.argv[1] if len(sys.argv) > 1 else ''
+
+    if cmd_func := getattr(sys.modules[__name__], f'cmd_{cmd}', None):
+        cmd_func(args)
+
+    else:
+        print(f"""
+usage: {SCRIPT} COMMAND ...
+
+Commands:
+  run       Directly run one or more filter(s)
+  logs      Show filter(s) logs
+  metrics   Show filter(s) metrics
+  info      Get help on a specific filter
+
+Run '{SCRIPT} COMMAND --help' for more information on a command.
+""".strip())
+
+
+if __name__ == '__main__':
+    main()

openfilter/cli/cmd_info.py

@@ -0,0 +1,70 @@
+import argparse
+import inspect
+import logging
+import os
+
+from openfilter.filter_runtime.filter import Filter
+
+from .common import SCRIPT, get_filter
+
+logger = logging.getLogger(__name__)
+
+logger.setLevel(int(getattr(logging, (os.getenv('LOG_LEVEL') or 'INFO').upper())))
+
+SHORTERHAND = {
+    'filter':       'Filter',
+    'mqtt':         'MQTTOut',
+    'mqttout':      'MQTTOut',
+    'mqtt_out':     'MQTTOut',
+    'recorder':     'Recorder',
+    'rest':         'REST',
+    'util':         'Util',
+    'video':        'Video',
+    'vidin':        'VideoIn',
+    'vid_in':       'VideoIn',
+    'videoin':      'VideoIn',
+    'video_in':     'VideoIn',
+    'vidout':       'VideoOut',
+    'vid_out':      'VideoOut',
+    'videoout':     'VideoOut',
+    'video_out':    'VideoOut',
+    'webvis':       'Webvis',
+}
+
+
+# --- info -------------------------------------------------------------------------------------------------------------
+
+def cmd_info(args):
+    parser = argparse.ArgumentParser(prog=f'{SCRIPT} info', description='Get info on a specific Filter.')
+
+    parser.add_argument('FILTER',
+        help='Filter to show info on',
+    )
+
+    opts = parser.parse_args(args)
+
+    logger.debug(f'opts: {opts}')
+
+    # do the thing
+
+    module_name, filter_name, module, filter_cls = get_filter(SHORTERHAND.get(opts.FILTER.lower(), opts.FILTER))
+
+    for cls in inspect.getmro(filter_cls):
+        if not issubclass(cls, Filter):
+            continue
+
+        if cls is not filter_cls:
+            if cls is not Filter:
+                print('\n\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nThe filter above is a subclass of the following:\n\n')
+
+            else:
+                print("\n\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nThe filter above is a subclass of the base Filter, see 'Filter' for information on that.")
+
+                break
+
+        print(f'{(s := f"{cls.__module__}.{cls.__qualname__}:")}\n{"-" * len(s)}\n')
+
+        if not cls.__doc__:
+            print('\nFilter class does not have a docstring.')
+        else:
+            print(inspect.cleandoc(cls.__doc__).strip())

openfilter/cli/cmd_logs_metrics.py

@@ -0,0 +1,212 @@
+import argparse
+import logging
+import os
+import time
+from datetime import datetime, timezone
+
+from openfilter.filter_runtime.filter import LOG_UTC
+from openfilter.filter_runtime.rolllog import RollLog
+from openfilter.filter_runtime.logging import LOG_PATH, Logger
+from openfilter.filter_runtime.utils import sanitize_filename, parse_date_and_or_time
+
+from .common import SCRIPT
+
+logger = logging.getLogger(__name__)
+
+logger.setLevel(int(getattr(logging, (os.getenv('LOG_LEVEL') or 'INFO').upper())))
+
+
+# --- logs OR metrics --------------------------------------------------------------------------------------------------
+
+def _cmd_logs_or_metrics(args, category, default_path, setup_func, entry_func):
+
+    # parse options from command line
+
+    parser = argparse.ArgumentParser(prog=f'{SCRIPT} {category}', formatter_class=argparse.RawTextHelpFormatter,
+        description=f'Show Filter {category}.',
+        epilog=f"""
+notes:
+  * Dates are in year/month/day order, separator is '/' or '-', accepted formats are 'yyyy/mm/dd', 'yy/mm/dd', 'mm/dd' or 'dd'.
+  * Accepted times are 24 hour clock 'hh:mm:ss.ms', 'hh:mm:ss', 'hh:mm', 'mm:ss.ms' or 'ss.ms'.
+  * Date with time is accepted separated by a space or a 'T', e.g. 'yy-mm-dd hh:mm:ss' or 'mm/ddTss.ms'.
+  * Datetime can also be ISO format.
+        """.strip(),
+    )
+
+    parser.add_argument('-f', '--from',
+        type = str,
+        help = f"date/time to show {category} from, 'start' from very beginning (default: show last file)",
+    )
+    parser.add_argument('-t', '--to',
+        type = str,
+        help = f'date/time to show {category} to (default: show last file)',
+    )
+    parser.add_argument('-p', '--path',
+        type    = str,
+        default = default_path,
+        help    = f'path to {category} (default: %(default)s)',
+    )
+    parser.add_argument('--utc',
+        action  = 'store_true',
+        default = None,
+        help    = 'all dates/times in UTC regardless of environment setting',
+    )
+    parser.add_argument('--no-utc',
+        action = 'store_false',
+        dest   = 'utc',
+        help   = 'all dates/times in local time regardless of environment setting',
+    )
+    parser.add_argument('FILTER',
+        nargs='*',
+        help='Filters to show, all if nothing specified',
+    )
+
+    opts = parser.parse_args(args)
+
+    logger.debug(f'opts: {opts}')
+
+    # do the thing
+
+    setup_func(opts.utc)
+
+    path    = opts.path
+    ts_from = None if (f := getattr(opts, 'from')) is None else \
+        (dt_from := parse_date_and_or_time(f, opts.utc)).timestamp() if f.lower() != 'start' else 0
+    ts_to   = None if (t := opts.to) is None else (dt_to := parse_date_and_or_time(t, opts.utc)).timestamp()
+    FILTERS = [sanitize_filename(f) for f in opts.FILTER]
+    filters = set()
+
+    if ts_from:  # also checks for 0
+        logger.debug(f'from: {dt_from}')
+
+    if ts_to is not None:
+        logger.debug(f'to: {dt_to}')
+
+    if not os.path.isdir(path):
+        print(f'Path does not exist: {path}')
+
+        return
+
+    if ts_from is not None and ts_to is not None and ts_to <= ts_from:
+        raise ValueError("'--to' timestamp can not be before or same as '--from' timestamp")
+
+    for name in os.listdir(path):
+        if os.path.isdir(os.path.join(path, name)):
+            filters.add(name)
+
+    if not FILTERS:
+        if not filters:
+            print(f'No {category} directories')
+
+    else:
+        for filter in FILTERS:
+            if filter not in filters:
+                print(f'No {category} directory for: {filter}')
+
+        filters &= set(FILTERS)
+
+    if not filters:
+        return
+
+    filters     = sorted(filters)
+    filter_logs = {}  # {'filter': RollLog, ...}
+
+    for filter in filters:
+        rl = RollLog(mode='json', rdonly=True, **Logger.path_prefix_and_suffix(path, filter, category))
+
+        if rl.logfiles:
+            filter_logs[filter] = rl
+        else:
+            print(f'No {category} for: {filter}')
+
+    for filter, rl in filter_logs.items():
+        if len(filter_logs) > 1:
+            print(f'\n{filter}\n')
+
+        rl.seek_block(rl.logfiles[-1].timestamp if ts_from is None else ts_from)
+
+        if ts_from is not None:
+            while entry := rl.read():
+                entry['ts'] = dt = datetime.fromisoformat(entry['ts'])
+
+                if dt.timestamp() >= ts_from:
+                    if ts_to is None or dt.timestamp() < ts_to:
+                        entry_func(entry)
+
+                    break
+
+            else:
+                continue
+
+        while entry := rl.read():
+            entry['ts'] = dt = datetime.fromisoformat(entry['ts'])
+
+            if ts_to is None or dt.timestamp() < ts_to:
+                entry_func(entry)
+            else:
+                break
+
+
+# --- logs -------------------------------------------------------------------------------------------------------------
+
+def cmd_logs(args):
+    LogRecord = logging.LogRecord
+    levels    = {
+        'CRITICAL': 50,
+        'FATAL':    50,
+        'ERROR':    40,
+        'WARN':     30,
+        'WARNING':  30,
+        'INFO':     20,
+        'DEBUG':    10,
+        'NOTSET':   0,
+    }
+
+    format = None
+
+    def setup_func(utc: bool | None):
+        nonlocal format
+
+        root_formatter      = logging.getLogger().handlers[0].formatter  # root formatter
+        formatter           = logging.Formatter(  # clone it
+            fmt     = root_formatter._fmt,
+            datefmt = root_formatter.datefmt,
+            style   = root_formatter._style._fmt[0],
+        )
+        formatter.converter = root_formatter.converter if utc is None else time.gmtime if utc else time.localtime
+        format              = formatter.format
+
+    def entry_func(entry: dict):  # format it exactly the same way as the logger
+        log_record         = LogRecord(None, levels.get(entry['lvl'], 0), None, None, entry['msg'], None, None)
+        log_record.process = entry['pid']
+        log_record.thread  = log_record.threadName = entry['thid']
+        log_record.created = ct = entry['ts'].timestamp()
+        log_record.msecs   = int((ct - int(ct)) * 1000) + 0.0  # see gh-89047
+        log_record.lineno  = 0
+
+        # log_record.threadName = ''
+        # log_record.filename   = ''
+        # log_record.funcName   = ''
+
+        print(format(log_record))
+
+    return _cmd_logs_or_metrics(args, 'logs', LOG_PATH, setup_func, entry_func)
+
+
+# --- metrics ----------------------------------------------------------------------------------------------------------
+
+def cmd_metrics(args):
+    tz = datefmt = None
+
+    def setup_func(utc: bool | None):
+        nonlocal tz, datefmt
+
+        tz      = timezone.utc if (LOG_UTC if utc is None else utc) else datetime.now().astimezone().tzinfo
+        datefmt = logging.getLogger().handlers[0].formatter.datefmt
+
+    def entry_func(entry: dict):
+        entry['ts'] = entry['ts'].astimezone(tz).strftime(datefmt)
+
+        print(', '.join(f'{k}: {v}' for k, v in entry.items()))
+
+    return _cmd_logs_or_metrics(args, 'metrics', LOG_PATH, setup_func, entry_func)

openfilter/cli/cmd_run.py

@@ -0,0 +1,111 @@
+import argparse
+import logging
+import multiprocessing as mp
+import os
+from pprint import pp
+
+from openfilter.filter_runtime.filter import Filter, PROP_EXIT_FLAGS, PROP_EXIT, OBEY_EXIT
+from openfilter.filter_runtime.utils import dict_without
+
+from .common import SCRIPT, parse_filters
+
+logger = logging.getLogger(__name__)
+
+logger.setLevel(int(getattr(logging, (os.getenv('LOG_LEVEL') or 'INFO').upper())))
+
+
+# --- run --------------------------------------------------------------------------------------------------------------
+
+def cmd_run(args):
+    try:
+        idx = args.index('-')
+    except ValueError:
+        idx = len(args)
+
+    opts   = args[:idx]
+    parser = argparse.ArgumentParser(prog=f'{SCRIPT} run', formatter_class=argparse.RawTextHelpFormatter,
+        usage=f"""
+usage: {SCRIPT} run [-h] [--ipc] [-s] [-f] [-p {{all,clean,error,none}}] [-o {{all,clean,error,none}}] [--dry] FILTER [FILTER ...]
+        """.strip(),
+        description="""
+Run one or more Filters.
+
+positional arguments:
+  FILTER                specified as "- package.filter.Filter [--config_param value ...]"
+        """.strip(),
+        epilog=f"""
+examples:
+  Plug into a running pipeline Filter at localhost:5554 and log its output without disturbint it too much:
+    {SCRIPT} run - Util --sources tcp://localhost:5554?? --log
+
+  Read a video file and visualize it:
+    {SCRIPT} run - VideoIn --sources file://video.mp4 --outputs tcp://0 - Webvis --sources tcp://localhost
+      autochain filters:
+    {SCRIPT} run - VideoIn --sources file://video.mp4 - Webvis
+
+  Connect via ids:
+    {SCRIPT} run - VideoIn --id myvideo --sources file://video.mp4 - webvis --sources myvideo
+      autogenerated id:
+    {SCRIPT} run - VideoIn --sources file://video.mp4 - Webvis --sources VideoIn
+
+notes:
+  * For --outputs, 'tcp://*', 'tcp://0.0.0.0' and 'tcp://0' are all equivalent.
+        """.strip(),
+    )
+
+    parser.add_argument('--ipc',
+        action = 'store_true',
+        help   = 'when creating new connections make them ipc:// instead of tcp://',
+    )
+    parser.add_argument('-s', '--solo',
+        action = 'store_true',
+        help   = 'run a single Filter in same process',
+    )
+    parser.add_argument('-f', '--fork',
+        action = 'store_true',
+        help   = "run Filters using 'fork' method instead of 'spawn', doesn't work with CUDA",
+    )
+    parser.add_argument('-p', '--prop-exit',
+        type    = str,
+        default = PROP_EXIT,
+        choices = list(PROP_EXIT_FLAGS),
+        help    = 'exit conditions Filters will propagate (default: %(default)s)',
+    )
+    parser.add_argument('-o', '--obey-exit',
+        type    = str,
+        default = OBEY_EXIT,
+        choices = list(PROP_EXIT_FLAGS),
+        help    = 'exit conditions Filters will obey (default: %(default)s)',
+    )
+    parser.add_argument('--dry',
+        action = 'store_true',
+        help   = 'dry run, just prepare and show the configs that would be used',
+    )
+
+    opts = parser.parse_args(opts)
+
+    logger.debug(f'opts: {opts}')
+
+    # parse filters
+
+    filters = parse_filters(args[:idx:-1], opts.ipc)
+
+    # run
+
+    if not opts.fork:
+        mp.set_start_method('spawn')
+
+    if opts.solo and len(filters) > 1:
+        raise ValueError("can only run a single Filter '--solo'")
+
+    if opts.dry:
+        for _, config, name in filters:
+            print(f'\n{name}:\n{"-" * (len(name) + 1)}')
+            pp(config)
+
+    elif opts.solo:
+        filters[0][0].run(dict_without(filters[0][1], '__env_compose'),
+            prop_exit=opts.prop_exit, obey_exit=opts.obey_exit)
+    else:
+        Filter.run_multi([(cls, dict_without(config, '__env_compose')) for cls, config, _ in filters],
+            prop_exit=opts.prop_exit, obey_exit=opts.obey_exit)