python-jack-knife 0.5.0__py3-none-any.whl

pjk/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+from .version import __version__
+
+__all__ = ["__version__"]

pjk/base.py

@@ -0,0 +1,377 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional, List, Set
+
+class TokenError(ValueError):
+    @classmethod
+    def from_list(cls, lines: List[str]):
+        text = '\n'.join(lines)
+        return TokenError(text)
+
+    def __init__(self, text: str):
+        super().__init__(text)
+        self.text = text
+
+    def get_text(self):
+        return self.text
+    
+class UsageError(ValueError):
+    def __init__(self, message: str,
+                 tokens: List[str] = None,
+                 token_no: int = 0,
+                 token_error: TokenError = None):
+        super().__init__(message)
+        self.message = message
+        self.tokens = tokens
+        self.token_no = token_no
+        self.token_error = token_error
+
+    def __str__(self):
+        lines = []
+        token_copies = [self._quote(t) for t in self.tokens]
+        lines.append('pjk ' + ' '.join(token_copies))
+        lines.append(self._get_underline(token_copies))
+        lines.append(self.message)
+        lines.append('')
+        lines.append(self.token_error.get_text())
+        return '\n'.join(lines)
+    
+    # quote json inline 
+    def _quote(self, token):
+        if token.startswith('[') or token.startswith('{'):
+            return '"' + token + '"'
+        else:
+            return token
+
+    def _get_underline(self, tokens: List, marker='^') -> str:
+        offset = 4 + sum(len(t) + 1 for t in tokens[:self.token_no])  # +1 for space, 4 for pjk
+        underline = ' ' * offset + marker * len(tokens[self.token_no])
+        return underline
+
+class ParsedToken:
+    def __init__(self, token: str):
+        self.token = token
+        self._params = {}
+        self._args = []
+        at_parts = token.split('@', 1)  # Separate params off
+        if len(at_parts) > 1:
+            param_list = at_parts[1].split('@')
+            for param in param_list:
+                parts = param.split('=')
+                value = parts[1] if len(parts) == 2 else None
+                self._params[parts[0]] = value
+
+        self._all_but_params = at_parts[0]
+
+        # args
+        colon_parts = at_parts[0].split(':')
+        self._pre_colon = colon_parts[0]
+
+        for arg in colon_parts[1:]: # treat a '' arg as missing and ignore all args after that
+            if arg != '':
+                self._args.append(arg)
+            else:
+                break
+
+    @property
+    def pre_colon(self):
+        return self._pre_colon
+    
+    @property
+    def whole_token(self):
+        return self.token
+    
+    @property # avoid colon parsing
+    def all_but_params(self):
+        return self._all_but_params
+    
+    def num_args(self):
+        return len(self._args)
+    
+    # args are mandatory
+    def get_arg(self, arg_no: int):
+        return self._args[arg_no] if arg_no < len(self._args) else None
+
+    # params are optional
+    def get_params(self) -> dict:
+        return self._params
+    
+class Usage:
+    def __init__(self, name: str, desc: str, component_class: type):
+        self.name = name
+        self.desc = desc
+        self.comp_class = component_class
+        self.args = {}
+        self.params = {}
+        self.syntax = None
+
+        self.arg_defs = []
+        self.param_usages = {}
+        self.examples = []
+
+    def get_component_class(self):
+        return self.comp_class
+    
+    def get_base_class(self, as_string: bool = False):
+        if issubclass(self.comp_class, Sink):
+            return 'sink' if as_string else Sink
+        elif issubclass(self.comp_class, Pipe):
+            return 'pipe' if as_string else Pipe
+        elif issubclass(self.comp_class, Source):
+            return 'source' if as_string else Source
+        raise 'improper class'
+
+    # args and param values default as str
+    def def_arg(self, name: str, usage: str, is_num: bool = False, valid_values: Optional[Set[str]] = None):
+        self.arg_defs.append((name, usage, is_num, valid_values))
+
+    def def_param(self, name:str, usage: str, is_num: bool = False, valid_values: Optional[Set[str]] = None, default:str = None):
+        self.param_usages[name] = (usage, is_num, valid_values, default)
+        if default:
+            self.params[name] = self._get_val(default, is_num, valid_values)
+
+    def def_example(self, expr_tokens:list[str], expect:str):
+        self.examples.append((expr_tokens, expect))
+
+    def def_syntax(self, syntax: str):
+        self.syntax = syntax
+
+    def get_examples(self):
+        return self.examples
+
+    def get_arg(self, name: str):
+        return self.args.get(name, None)
+    
+    def get_param(self, name: str):
+        return self.params.get(name)
+    
+    def get_usage_text(self):
+        lines = []
+        lines.append(self.desc)
+
+        syntax_str = self.get_token_syntax() # might be ''
+        if len(syntax_str) > 0:
+            lines.append('')
+            lines.append(f'syntax:')
+            lines.append(f'  {self.get_token_syntax()}')
+
+        lines.extend(f"{line}" for line in self.get_arg_param_desc())
+        return '\n'.join(lines)
+
+    def get_token_syntax(self):
+        if self.syntax != None:
+            return self.syntax # else piece it together
+
+        token = f'{self.name}'
+        for name, usage, is_num, valid_values in self.arg_defs:
+            token += f':<{name}>'
+
+        for name, (usage, is_num, valid_values, default) in self.param_usages.items():
+            value_display = name
+            if valid_values:
+                value_display  = '|'.join(list(valid_values))
+            token += f'@{name}=<{value_display}>'
+        return token
+    
+    def get_arg_param_desc(self):
+        notes = []
+        if self.arg_defs:
+            notes.append('mandatory args:')
+            for name, usage, is_num, valid_values in self.arg_defs:
+                notes.append(f'  {name} = {usage}')
+
+        if self.param_usages:
+            notes.append('optional params:')
+            for name, usage in self.param_usages.items():
+                text, is_num, valid_values, default = usage
+                notes.append(f'  {name} = {text} (default={default})')
+        return notes
+
+    def bind(self, ptok: ParsedToken):
+        if ptok.num_args() > len(self.arg_defs):
+            extra = []
+            for i in range(len(self.arg_defs), ptok.num_args()):
+                name = ptok.get_arg(i)
+                extra.append(name)
+
+            raise TokenError.from_list([f"extra arg{'s' if len(extra) > 1 else ''}: {','.join(extra)}.", 
+                                        '', self.get_usage_text()])
+        
+        if ptok.num_args() < len(self.arg_defs):
+            missing = []
+            for i in range(ptok.num_args(), len(self.arg_defs)):
+                name, usage, is_num, valid_values = self.arg_defs[i]
+                missing.append(name)
+
+            raise TokenError.from_list([f"missing arg{'s' if len(missing) > 1 else ''}: {','.join(missing)}.", 
+                                        '', self.get_usage_text()])
+
+        for i, adef in enumerate(self.arg_defs):
+            name, usage, is_num, valid_values = adef
+
+            try:
+                val_str = ptok.get_arg(i)
+                self.args[name] = self._get_val(val_str, is_num, valid_values)
+            except (ValueError, TypeError) as e:
+                raise TokenError.from_list([f"wrong value for '{name}' arg.", '', self.get_usage_text()])
+            
+        for name, str_val in ptok.get_params().items():
+            usage = self.param_usages.get(name, None)
+            if not usage:
+                raise TokenError.from_list([f"unknown param: '{name}'.", '', self.get_usage_text()])
+            if not str_val:
+                raise TokenError.from_list([f"missing value for '{name}' param.", '', self.get_usage_text()])
+
+            text, is_num, valid_values, default = usage
+            try:
+                self.params[name] = self._get_val(str_val, is_num, valid_values)
+            except (ValueError, TypeError) as e:
+                raise TokenError.from_list([f"wrong value type for '{name}' param.", '', self.get_usage_text()])
+
+    def _get_val(self, val_str: str, is_num: bool, valid_values: Optional[Set[str]] = None):
+        if not val_str:
+            raise ValueError('missing value')
+        if not is_num: # is string
+            if valid_values is None: # no constraints
+                return val_str
+            if not val_str in valid_values:
+                raise ValueError(f'illegal value: {val_str}')
+            return val_str
+            
+        else: # is_num
+            try:
+                return int(val_str)
+            except ValueError as e: # coud be a float that errors, but is ok
+                return float(val_str)
+
+# until all usages are implemented a default that doesn't bind
+# they continue to use ParsedToken ptok
+class NoBindUsage(Usage):
+    def __init__(self, name: str, desc: str, component_class: type):
+        super().__init__(name=name, desc=desc, component_class=component_class)
+    def bind(self, ptok: ParsedToken):
+        return
+
+# mixin
+class KeyedSource(ABC):
+    @classmethod
+    def usage(cls):
+        return Usage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component"
+        )
+    
+    @abstractmethod
+    def lookup(self, left_rec) -> Optional[dict]:
+        """Return the record associated with the given key, or None."""
+        pass
+
+    def get_unlookedup_records(self) -> List[Any]:
+        # for outer join
+        pass
+
+    def deep_copy(self):
+        return None
+
+class Source(ABC):
+    is_format = False
+
+    @classmethod
+    def usage(cls):
+        return NoBindUsage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component",
+            component_class=cls
+        )
+
+    @abstractmethod
+    def __iter__(self):
+        raise NotImplementedError("__iter__ must be implemented by subclasses")
+
+    def __next__(self):
+        # lazily create an internal iterator the first time next() is called
+        if not hasattr(self, "_iter"):
+            self._iter = iter(self)
+        return next(self._iter)
+
+
+    def deep_copy(self):
+        return None  # Default: not copyable unless overridden
+
+    
+class Pipe(Source):
+    deep_copyable: bool = False # default to false
+    arity: int = 1
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+        self.ptok = ptok
+        self.left = None  # left source for convience
+        self.right = None # right source for convience
+        self.inputs: List[Source] = []
+
+    def add_source(self, source: Source) -> None:
+        self.inputs.append(source)
+        # first two are assigned left, right
+        if self.left is None:
+            self.left = source
+        elif self.right is None:
+            self.right = self.left
+            self.left = source
+
+    def reset(self):
+        pass  # optional hook
+
+    def deep_copy(self) -> Optional["Pipe"]:
+        if not self.deep_copyable:
+            return None
+        if not self.inputs:
+            raise RuntimeError(f"{self.__class__.__name__} has no inputs set")
+
+        clone = self.__class__(self.ptok, self.__class__.usage())
+
+        for input in self.inputs:
+            strand = input.deep_copy()
+            if strand is None:
+                return None
+            clone.add_source(strand)
+
+        return clone
+
+class Sink(ABC):
+    is_format = False
+
+    @classmethod
+    def usage(cls):
+        return NoBindUsage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component",
+            component_class=cls
+        )
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+        self.ptok = ptok
+        self.usage = usage
+
+    def drain(self):
+        self.process()
+
+    def print_info(self):
+        pass
+
+    def add_source(self, source: Source) -> None:
+        self.input = source
+        
+    @abstractmethod
+    def process(self) -> None:
+        pass
+
+    def deep_copy(self):
+        return None
+
+# identity source for sub-pipeline seeding
+class IdentitySource(Source):
+    def next(self):
+        raise RuntimeError("IdentitySource should never be executed")
+

pjk/common.py

@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import sys, shutil, subprocess, contextlib, signal
+import os
+import yaml
+
+class SafeNamespace:
+    def __init__(self, obj):
+        for k, v in obj.items():
+            if isinstance(v, dict):
+                v = SafeNamespace(v)
+            elif isinstance(v, list):
+                v = [SafeNamespace(x) if isinstance(x, dict) else x for x in v]
+            setattr(self, k, v)
+
+    def __getattr__(self, key):
+        return None  # gracefully handle missing keys
+
+class ReducingNamespace:
+    def __init__(self, record):
+        self._record = record
+
+    def __getattr__(self, name):
+        value = self._record[name]
+        if isinstance(value, (list, tuple, set)):
+            return value
+        return [value]  # promote scalars to singleton lists
+
+@contextlib.contextmanager
+def pager_stdout(use_pager=True):
+    if use_pager and shutil.which("less"):
+        # Avoid BrokenPipeError noise if user quits less early
+        try:
+            signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+        except Exception:
+            pass  # not available on Windows
+
+        pager = subprocess.Popen(["less", "-FRSX"], stdin=subprocess.PIPE, text=True)
+        old_stdout = sys.stdout
+        try:
+            sys.stdout = pager.stdin
+            yield
+        finally:
+            try:
+                sys.stdout.flush()
+            except Exception:
+                pass
+            sys.stdout = old_stdout
+            if pager.stdin:
+                pager.stdin.close()
+            pager.wait()
+    else:
+        yield
+
+COLOR_CODES = {
+        'bold': '\033[1m',
+        'underline': '\033[4m',
+        'red': '\033[31m',
+        'green': '\033[32m',
+        'yellow': '\033[33m',
+        'blue': '\033[34m',
+        'magenta': '\033[35m',
+        'cyan': '\033[36m',
+        'gray': '\033[90m',
+    }
+
+RESET = '\033[0m'
+
+def highlight(text: str, color: str = 'bold', value: str = None) -> str:
+    value = text if not value else value
+    style = COLOR_CODES.get(color.lower(), COLOR_CODES['bold'])
+    return text.replace(value, f"{style}{value}{RESET}")
+
+class Lookups:
+    def __init__(self):
+        self.lookups_yaml = os.path.expanduser('~/.pjk/lookups.yaml')
+        self._data = {}
+        self._load()
+
+    def _load(self):
+        """Load lookups from YAML file if it exists."""
+        if os.path.exists(self.lookups_yaml):
+            with open(self.lookups_yaml, 'r') as f:
+                self._data = yaml.safe_load(f) or {}
+        else:
+            self._data = {}
+
+    def save(self):
+        """Save current lookups back to YAML file."""
+        os.makedirs(os.path.dirname(self.lookups_yaml), exist_ok=True)
+        with open(self.lookups_yaml, 'w') as f:
+            yaml.safe_dump(self._data, f)
+
+    def get(self, key, default=None):
+        """Retrieve a lookup value by key."""
+        return self._data.get(key, default)
+
+    def set(self, key, value):
+        """Set a lookup value and persist it."""
+        self._data[key] = value
+        self.save()
+
+    def delete(self, key):
+        """Remove a key if it exists and save."""
+        if key in self._data:
+            del self._data[key]
+            self.save()
+
+    def all(self):
+        """Return the full lookup dictionary."""
+        return dict(self._data)
+
+class ComponentFactory:
+    def __init__(self, components: dict, comp_type_name: str):
+        self.num_orig = 0
+        self.components = components # name -> component_class
+        self.comp_type_name = comp_type_name
+        self.num_orig_comps = len(components)
+
+    def register(self, name, comp_class):
+        self.components[name] = comp_class
+
+    def get_comp_type_name(self):
+        return self.comp_type_name
+
+    def print_descriptions(self):
+        header = highlight(f'{self.comp_type_name}s')
+        print(header)
+
+        i = 0
+        plugin = ''
+        for name, comp_class in self.components.items():
+            usage = comp_class.usage()
+            lines = usage.desc.split('\n')
+            if i >= self.num_orig_comps:
+                plugin = '(~/.pjk/plugin)'
+            line = f'  {name:<12} {lines[0]} {plugin}'
+            line = highlight(line, 'bold', plugin) if plugin else line
+            print(line)
+            i += 1
+
+    def get_usage(self, name: str):
+        comp_class = self.components.get(name)
+        if not comp_class:
+            return None
+        return comp_class.usage()
+
+    def create(self, token: str):
+        pass

pjk/log.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import logging, os, atexit
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("djk")
+
+def _truthy(env_val: Optional[str]) -> bool:
+    return str(env_val).lower() in ("1", "true", "yes", "on")
+
+def init(force: bool = False, level: Optional[int] = None, console: Optional[bool] = None):
+    """
+    Initialize 'djk' logging.
+
+    - Rotates at DJK_LOG_MAX_MB (default 2 MB), keeps DJK_LOG_BACKUPS (default 3).
+    - Files under ~/.pjk/logs by default; override with DJK_LOG_DIR / DJK_LOG_FILE.
+    - Set DJK_DEBUG=1|true|yes for DEBUG, else INFO (or pass explicit level).
+    - To enable console output explicitly, set console=True or DJK_LOG_CONSOLE=1.
+    - Set force=True to replace existing handlers.
+    """
+    if logger.handlers and not force:
+        return
+
+    logger.handlers.clear()
+
+    if level is None:
+        level = logging.DEBUG if _truthy(os.getenv("DJK_DEBUG")) else logging.INFO
+
+    fmt = "[%(levelname)s] [%(threadName)s] %(message)s"
+    formatter = logging.Formatter(fmt)
+
+    # Rotating file handler in ~/.pjk/logs
+    log_dir = Path(os.getenv("DJK_LOG_DIR", os.path.expanduser("~/.pjk/logs")))
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / os.getenv("DJK_LOG_FILE", "pjk.log")
+    max_bytes = int(float(os.getenv("DJK_LOG_MAX_MB", "2")) * 1024 * 1024)  # 2 MB
+    backups = int(os.getenv("DJK_LOG_BACKUPS", "3"))
+
+    fh = RotatingFileHandler(
+        log_file,
+        maxBytes=max_bytes,
+        backupCount=backups,
+        encoding="utf-8",
+        delay=False,  # open immediately so first emit writes bytes
+    )
+    fh.setLevel(level)
+    fh.setFormatter(formatter)
+
+    logger.setLevel(level)
+    logger.addHandler(fh)
+
+    # DO NOT propagate into root (prevents accidental console logs elsewhere)
+    logger.propagate = False
+
+    # Optional console (off by default)
+    enable_console = console if console is not None else _truthy(os.getenv("DJK_LOG_CONSOLE"))
+    if enable_console:
+        sh = logging.StreamHandler()
+        sh.setLevel(level)
+        sh.setFormatter(formatter)
+        logger.addHandler(sh)
+
+    # Flush/close on exit for short-lived runs
+    atexit.register(logging.shutdown)

pjk/main.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+#!/usr/bin/env python
+import sys
+import os
+import signal
+import shlex
+from typing import List
+from pjk.parser import ExpressionParser
+from pjk.base import UsageError
+from pjk.log import init as init_logging
+from datetime import datetime, timezone
+import concurrent.futures
+from pjk.registry import ComponentRegistry
+from pjk.pipes.factory import PipeFactory
+from pjk.sources.factory import SourceFactory
+from pjk.sinks.factory import SinkFactory
+from pjk.man_page import do_man, do_examples
+from pjk.sinks.expect import ExpectSink
+from pjk.version import __version__
+
+def write_history(tokens):
+    log_path = ".pjk-history.txt"
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    command = " ".join(tokens)
+    with open(log_path, "a") as f:
+        f.write(f"{timestamp}\tpjk {command}\n")
+
+def execute_threaded(sinks):
+    # Choose a max thread limit (explicitly)
+    max_workers = min(32, len(sinks))  # or set a fixed cap like 8
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(s.drain): s for s in sinks
+        }
+
+        for future in concurrent.futures.as_completed(futures):
+            sink_obj = futures[future]
+            try:
+                future.result()  # This will re-raise any exception from s.drain()
+            except Exception as e:
+                print(f"Sink {sink_obj} raised an exception:")
+                print(e)
+
+def execute(command: str):
+    tokens = shlex.split(command, comments=True, posix=True)
+    execute_tokens(tokens)
+
+def execute_tokens(tokens:List[str]):
+    init_logging()
+    signal.signal(signal.SIGINT, lambda s, f: sys.exit(0))
+
+    if '--version' in tokens:
+        print(f"pjk version {__version__}")
+        sys.exit(0)
+    
+    registry = ComponentRegistry()
+       
+    if len(tokens) < 1:
+        registry.print_usage()
+        return
+    
+    # pjk man --all | --all+ | <component>
+    if len(tokens) == 2 and tokens[0] == 'man':
+        do_man(tokens[1], registry)
+        return
+    
+    # pjk examples | examples+
+    if len(tokens) == 1 and tokens[0] in ['examples', 'examples+']:
+        do_examples(tokens[0], registry)
+        return
+
+    parser = ExpressionParser(registry)
+
+    try:
+        # Build initial sink
+        sink = parser.parse(tokens)
+
+        sinks = [sink]
+        max_threads = os.cpu_count()
+        while len(sinks) < max_threads:
+            clone = sink.deep_copy()
+            if not clone:
+                break
+            sinks.append(clone)
+
+        if len(sinks) > 1:
+            execute_threaded(sinks)
+        else:
+            sink.drain() # run single in main thread
+            sink.print_info() # rarely used, e.g. expect and devnull
+
+        write_history(sys.argv[1:])
+
+    except UsageError as e:
+        print(e, file=sys.stderr)
+        sys.exit(2)  # Exit with a non-zero code, but no traceback
+
+def main():
+    tokens = sys.argv[1:]
+    execute_tokens(tokens)
+
+if __name__ == "__main__":
+    main()