python-jack-knife 0.5.0__tar.gz → 0.5.5__tar.gz

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.5.0
+Version: 0.5.5
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/base.py

@@ -152,16 +152,17 @@ class Usage:
         lines.append(self.desc)
 
         syntax_str = self.get_token_syntax() # might be ''
-        if len(syntax_str) > 0:
-            lines.append('')
-            lines.append(f'syntax:')
-            lines.append(f'  {self.get_token_syntax()}')
-
+        if not syntax_str:
+            return '\n'.join(lines)
+        
+        lines.append('')
+        lines.append(f'syntax:')
+        lines.append(f'  {self.get_token_syntax()}')
         lines.extend(f"{line}" for line in self.get_arg_param_desc())
         return '\n'.join(lines)
 
     def get_token_syntax(self):
-        if self.syntax != None:
+        if self.syntax:
             return self.syntax # else piece it together
 
         token = f'{self.name}'
@@ -216,7 +217,10 @@ class Usage:
                 self.args[name] = self._get_val(val_str, is_num, valid_values)
             except (ValueError, TypeError) as e:
                 raise TokenError.from_list([f"wrong value for '{name}' arg.", '', self.get_usage_text()])
-            
+
+        self.bind_params(ptok)
+        
+    def bind_params(self, ptok: ParsedToken):
         for name, str_val in ptok.get_params().items():
             usage = self.param_usages.get(name, None)
             if not usage:
@@ -276,8 +280,6 @@ class KeyedSource(ABC):
         return None
 
 class Source(ABC):
-    is_format = False
-
     @classmethod
     def usage(cls):
         return NoBindUsage(
@@ -296,17 +298,21 @@ class Source(ABC):
             self._iter = iter(self)
         return next(self._iter)
 
-
     def deep_copy(self):
         return None  # Default: not copyable unless overridden
-
+    
+    def close(self):
+        pass
+    
+    def _get_sources(self, source_list: list):
+        pass
     
 class Pipe(Source):
-    deep_copyable: bool = False # default to false
     arity: int = 1
     
     def __init__(self, ptok: ParsedToken, usage: Usage = None):
         self.ptok = ptok
+        self.usage = usage
         self.left = None  # left source for convience
         self.right = None # right source for convience
         self.inputs: List[Source] = []
@@ -324,24 +330,29 @@ class Pipe(Source):
         pass  # optional hook
 
     def deep_copy(self) -> Optional["Pipe"]:
-        if not self.deep_copyable:
-            return None
-        if not self.inputs:
-            raise RuntimeError(f"{self.__class__.__name__} has no inputs set")
-
-        clone = self.__class__(self.ptok, self.__class__.usage())
+        return None
+    
+    def _get_sources(self, source_list: list):
+        for ix in self.inputs:
+            source_list.append(ix)
+            ix._get_sources(source_list)
 
-        for input in self.inputs:
-            strand = input.deep_copy()
-            if strand is None:
-                return None
-            clone.add_source(strand)
+class DeepCopyPipe(Pipe):
+    def deep_copy(self):
+        """
+        Generic deep_copy: clone left source, re-instantiate
+        this pipe class with the same ptok/usage, and attach.
+        """
+        source_clone = self.left.deep_copy()
+        if not source_clone:
+            return None
 
-        return clone
+        # re-instantiate using the actual subclass
+        pipe = type(self)(self.ptok, self.usage)
+        pipe.add_source(source_clone)
+        return pipe
 
 class Sink(ABC):
-    is_format = False
-
     @classmethod
     def usage(cls):
         return NoBindUsage(
@@ -356,8 +367,16 @@ class Sink(ABC):
 
     def drain(self):
         self.process()
+        self.close()
+
+        # get all inputs in the execution chain for closing
+        inputs = [self.input]
+        self.input._get_sources(inputs)
+        for input in inputs:
+            input.close()
 
-    def print_info(self):
+    # optional
+    def close(self):
         pass
 
     def add_source(self, source: Source) -> None:

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/common.py

@@ -4,6 +4,7 @@
 import sys, shutil, subprocess, contextlib, signal
 import os
 import yaml
+from pjk.base import TokenError
 
 class SafeNamespace:
     def __init__(self, obj):
@@ -73,11 +74,12 @@ def highlight(text: str, color: str = 'bold', value: str = None) -> str:
     return text.replace(value, f"{style}{value}{RESET}")
 
 class Lookups:
-    def __init__(self):
+    def __init__(self, component_class):
         self.lookups_yaml = os.path.expanduser('~/.pjk/lookups.yaml')
+        self.class_name = type(component_class).__name__
         self._data = {}
         self._load()
-
+        
     def _load(self):
         """Load lookups from YAML file if it exists."""
         if os.path.exists(self.lookups_yaml):
@@ -93,8 +95,13 @@ class Lookups:
             yaml.safe_dump(self._data, f)
 
     def get(self, key, default=None):
-        """Retrieve a lookup value by key."""
-        return self._data.get(key, default)
+        lookup_key = f'{self.class_name}-{key}'
+        entry = self._data.get(lookup_key, default)
+        if not entry:
+            raise TokenError(
+                f"~/.pjk/lookups.yaml must contain entry for '{lookup_key}' with host, user, password."
+            )
+        return entry
 
     def set(self, key, value):
         """Set a lookup value and persist it."""
@@ -129,14 +136,14 @@ class ComponentFactory:
         print(header)
 
         i = 0
-        plugin = ''
+        # user and outside package components are also here, but printed from registry class
         for name, comp_class in self.components.items():
             usage = comp_class.usage()
             lines = usage.desc.split('\n')
             if i >= self.num_orig_comps:
-                plugin = '(~/.pjk/plugin)'
-            line = f'  {name:<12} {lines[0]} {plugin}'
-            line = highlight(line, 'bold', plugin) if plugin else line
+                break
+
+            line = f'  {name:<12} {lines[0]}'
             print(line)
             i += 1
 

python_jack_knife-0.5.5/src/pjk/log.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import logging, os, tempfile
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("pjk")
+
+def _truthy(v: Optional[str]) -> bool:
+    return str(v).lower() in ("1", "true", "yes", "on")
+
+def init(force: bool = False, level: Optional[int] = None):
+    """
+    Initialize 'pjk' logging.
+
+    - Rotates at DJK_LOG_MAX_MB (default 2 MB), keeps DJK_LOG_BACKUPS (default 3).
+    - Files under ~/.pjk/logs by default; override with DJK_LOG_DIR / DJK_LOG_FILE.
+    - Set DJK_DEBUG=1|true|yes for DEBUG, else INFO (or pass explicit level).
+    - If the log directory is not writable, fall back to console logging
+      (stderr → CloudWatch in AWS).
+    - Set force=True to replace existing handlers.
+    """
+    if logger.handlers and not force:
+        return
+    logger.handlers.clear()
+
+    level = level or (logging.DEBUG if _truthy(os.getenv("DJK_DEBUG")) else logging.INFO)
+    fmt = "[%(levelname)s] [%(threadName)s] %(message)s"
+    formatter = logging.Formatter(fmt)
+
+    try:
+        # Preferred: rotating file handler under ~/.pjk/logs
+        log_dir = Path(os.getenv("DJK_LOG_DIR", Path.home() / ".pjk" / "logs"))
+        log_dir.mkdir(parents=True, exist_ok=True)
+
+        log_file = log_dir / os.getenv("DJK_LOG_FILE", "pjk.log")
+        max_bytes = int(float(os.getenv("DJK_LOG_MAX_MB", "2")) * 1024 * 1024)  # 2 MB
+        backups = int(os.getenv("DJK_LOG_BACKUPS", "3"))
+
+        fh = RotatingFileHandler(
+            log_file,
+            maxBytes=max_bytes,
+            backupCount=backups,
+            encoding="utf-8",
+            delay=False,
+        )
+        fh.setLevel(level)
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+    except Exception:
+        # Fallback: console handler
+        ch = logging.StreamHandler()
+        ch.setLevel(level)
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+        logger.warning("Falling back to console logging (log file not writable)")
+
+    logger.setLevel(level)
+    # Do not propagate to root
+    logger.propagate = False

python_jack_knife-0.5.5/src/pjk/main.py

@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+#!/usr/bin/env python
+import sys
+import os
+import signal
+import shlex
+from typing import List
+from pjk.parser import ExpressionParser
+from pjk.base import UsageError
+from pjk.log import init as init_logging
+from datetime import datetime
+import traceback
+import concurrent.futures
+from pjk.registry import ComponentRegistry
+from pjk.sinks.stdout import StdoutSink
+from pjk.man_page import do_man, do_examples
+from pjk.sinks.expect import ExpectSink
+from pjk.progress import ProgressDisplay
+from pjk.version import __version__
+
+def write_history(tokens):
+    if os.environ.get("PJK_NO_HISTORY") == "1":
+        return
+    
+    log_path = ".pjk-history.txt"
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    command = " ".join(tokens)
+
+    try:
+        with open(log_path, "a") as f:
+            f.write(f"{timestamp}\tpjk {command}\n")
+    except (PermissionError, OSError):
+        pass
+
+def execute_threaded(sinks, stop_progress=None):
+    max_workers = min(32, len(sinks))
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)  # no 'with'
+    futures = {executor.submit(s.drain): s for s in sinks}
+    try:
+        for future in concurrent.futures.as_completed(futures):
+            sink_obj = futures[future]
+            future.result()  # re-raises worker exception with traceback
+    except KeyboardInterrupt:
+        # stop UI first, then cancel and non-blocking shutdown
+        if stop_progress:
+            try: stop_progress()
+            except Exception: pass
+        for f in futures:
+            f.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        os._exit(130)
+
+    except Exception as e:
+        if stop_progress:
+            try: stop_progress()
+            except Exception: pass
+        sys.stderr.write(f"Sink {futures[future]} raised an exception:\n")
+        traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr)
+        for f in futures:
+            f.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    else:
+        executor.shutdown(wait=True)
+
+def execute(command: str):
+    tokens = shlex.split(command, comments=True, posix=True)
+    execute_tokens(tokens)
+
+def execute_tokens(tokens: List[str]):
+    init_logging()
+    # (remove the sys.exit SIGINT handler here)
+
+    if '--version' in tokens:
+        print(f"pjk version {__version__}")
+        sys.exit(0)
+
+    registry = ComponentRegistry()
+
+    if len(tokens) < 1:
+        registry.print_usage()
+        return
+
+    if len(tokens) == 2 and tokens[0] == 'man':
+        do_man(tokens[1], registry)
+        return
+
+    if len(tokens) == 1 and tokens[0] in ['examples', 'examples+']:
+        do_examples(tokens[0], registry)
+        return
+
+    parser = ExpressionParser(registry)
+
+    display = None
+    try:
+        sink = parser.parse(tokens)
+        if not isinstance(sink, (StdoutSink | ExpectSink)):
+            display = ProgressDisplay(interval=3.0)
+            display.start()
+
+        sinks = [sink]
+        max_threads = os.cpu_count()
+        while len(sinks) < max_threads:
+            clone = sink.deep_copy()
+            if not clone:
+                break
+            sinks.append(clone)
+
+        if len(sinks) > 1:
+            # pass a stopper so we halt the UI before tracebacks / shutdown
+            execute_threaded(sinks, stop_progress=(display.stop if display else None))
+        else:
+            sink.drain()
+
+        write_history(sys.argv[1:])
+
+    except UsageError as e:
+        print(e, file=sys.stderr)
+        sys.exit(2)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if display:
+            # short join so Ctrl-C is immediate
+            try: display.stop(timeout=0.1)
+            except Exception: pass
+
+def main():
+    tokens = sys.argv[1:]
+    execute_tokens(tokens)
+
+if __name__ == "__main__":
+    main()

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/parser.py

@@ -8,6 +8,9 @@ from typing import Optional, Any, List
 from pjk.base import Source, Pipe, Sink, TokenError, UsageError, ParsedToken, Usage
 from pjk.pipes.user_pipe_factory import UserPipeFactory
 from pjk.pipes.let_reduce import ReducePipe
+from pjk.sinks.stdout import StdoutSink
+from pjk.sinks.expect import ExpectSink
+from pjk.pipes.progress_pipe import ProgressPipe
 from pjk.registry import ComponentRegistry
 
 def expand_macros(tokens: List[str]) -> List[str]:
@@ -58,8 +61,12 @@ class ExpressionParser:
         if not sink:
             raise TokenError.from_list(['expression must end in a sink.',
                             'pjk <source> [<pipe> ...] <sink>'])
+        
+        # so each sink doesn't have to, maybe make a base class or mixin for sinks
+        progress_pipe = ProgressPipe(component_instance=sink)
+        progress_pipe.add_source(source)
 
-        sink.add_source(source)
+        sink.add_source(progress_pipe)
         return sink
 
     def parse(self, tokens: List[str]) -> Sink:
@@ -78,10 +85,12 @@ class ExpressionParser:
                     return self.get_sink(stack_helper, token)
                     
                 source = self.registry.create_source(token)
-                if source:
+                if source:                    
                     stack_helper.add_operator(source, self.stack)
+                    progress_pipe = ProgressPipe(component_instance=source, simple=True)
+                    stack_helper.add_operator(progress_pipe, self.stack)
                     continue
-
+                
                 subexp = SubExpression.create(token)
                 if subexp:
                     stack_helper.add_operator(subexp, self.stack)
@@ -94,7 +103,7 @@ class ExpressionParser:
 
                 else: # unrecognized token
                     # could be sink in WRONG position, let's see for better error message
-                    sink = self.registry.create_sink(token, None) 
+                    sink = self.registry.create_sink(token) 
                     if sink:
                         raise TokenError.from_list(['sink may only occur in final position.',
                                             'pjk <source> [<pipe> ...] <sink>'])

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/pipes/factory.py

@@ -12,12 +12,14 @@ from pjk.pipes.head import HeadPipe
 from pjk.pipes.tail import TailPipe
 from pjk.pipes.sort import SortPipe
 from pjk.pipes.where import WherePipe
-from pjk.pipes.map import MapPipe
+from pjk.pipes.map import MapByPipe
+from pjk.pipes.map import GroupByPipe
 from pjk.pipes.join import JoinPipe
 from pjk.pipes.filter import FilterPipe
 from pjk.pipes.select import SelectFields
 from pjk.pipes.denorm import DenormPipe
 from pjk.pipes.postgres_pipe import PostgresPipe
+from pjk.pipes.sample import SamplePipe
 from pjk.pipes.user_pipe_factory import UserPipeFactory
 
 COMPONENTS = {
@@ -25,7 +27,8 @@ COMPONENTS = {
         'tail': TailPipe,
         'join': JoinPipe,
         'filter': FilterPipe,
-        'map': MapPipe,            
+        'mapby': MapByPipe,            
+        'groupby': GroupByPipe,
         'as': MoveField,
         'drop': RemoveField,        
         'let': LetPipe,
@@ -33,6 +36,7 @@ COMPONENTS = {
         'sort': SortPipe,
         'where': WherePipe,
         'sel': SelectFields,
+        'sample': SamplePipe,
         'explode': DenormPipe,
         'pgres': PostgresPipe,
     }

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/pipes/filter.py

@@ -15,13 +15,13 @@ class FilterPipe(Pipe):
         )
         usage.def_arg("mode", "'+' to include matches, '-' to exclude matches",
                       valid_values={'+', '-'})
-        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> filter:<mode> <sink>")
+        usage.def_syntax("pjk <left_source> <map_source> [mapby:groupby]:<how>:<key> filter:<mode> <sink>")
 
         usage.def_example(expr_tokens=
         [
             "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
             "[{id:1}, {id:3}, {id:5}]",
-            'map:o:id',
+            'mapby:id',
             "filter:+"
         ],
         expect="[{id:1}, {id:3}, {id:5}]")
@@ -30,7 +30,7 @@ class FilterPipe(Pipe):
         [
             "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
             "[{id:1}, {id:3}, {id:5}]",
-            'map:o:id',
+            'mapby:id',
             "filter:-"
         ],
         expect="[{id:2}, {id:4}]")

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/pipes/head.py

@@ -2,8 +2,6 @@
 # Copyright 2024 Mike Schultz
 
 # djk/pipes/head.py
-
-from typing import Optional
 from pjk.base import Pipe, ParsedToken, Usage
 
 class HeadPipe(Pipe):
@@ -11,15 +9,15 @@ class HeadPipe(Pipe):
     def usage(cls):
         usage = Usage(
             name='head',
-            desc='take first records of input (when single-threaded)',
+            desc='take first records of input (single-threaded)',
             component_class=cls
         )
-        usage.def_arg(name='limit', usage='number of records', is_num=True)
         usage.def_example(expr_tokens=['[{id:1}, {id:2}]', 'head:1'], expect="{id:1}")
+        usage.def_arg(name='limit', usage='number of records', is_num=True)
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.limit = usage.get_arg('limit')
         self.count = 0
 

{python_jack_knife-0.5.0 → python_jack_knife-0.5.5}/src/pjk/pipes/join.py

@@ -20,13 +20,13 @@ class JoinPipe(Pipe):
             usage="'left', 'inner', or 'outer' join behavior",
             valid_values={'left', 'inner', 'outer'}
         )
-        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> join:<mode> <sink>")
+        usage.def_syntax("pjk <left_source> <map_source> [mapby|groupby]:<key> join:<mode> <sink>")
 
         usage.def_example(expr_tokens=
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:left"
         ],
         expect="[{color:'blue', price:50}, {color:'green'}]")
@@ -34,7 +34,7 @@ class JoinPipe(Pipe):
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:inner"
         ],
         expect="[{color:'blue', price:50}]")
@@ -43,7 +43,7 @@ class JoinPipe(Pipe):
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:outer"
         ],
         expect="[{color:'blue', price:50}, {color:'green'}, {color:'red', price: 20}]")