python-jack-knife 0.5.1__tar.gz → 0.5.5__tar.gz

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.5.1
+Version: 0.5.5
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/base.py

@@ -162,7 +162,7 @@ class Usage:
         return '\n'.join(lines)
 
     def get_token_syntax(self):
-        if not self.syntax:
+        if self.syntax:
             return self.syntax # else piece it together
 
         token = f'{self.name}'
@@ -300,10 +300,14 @@ class Source(ABC):
 
     def deep_copy(self):
         return None  # Default: not copyable unless overridden
-
+    
+    def close(self):
+        pass
+    
+    def _get_sources(self, source_list: list):
+        pass
     
 class Pipe(Source):
-    deep_copyable: bool = False # default to false
     arity: int = 1
     
     def __init__(self, ptok: ParsedToken, usage: Usage = None):
@@ -326,20 +330,12 @@ class Pipe(Source):
         pass  # optional hook
 
     def deep_copy(self) -> Optional["Pipe"]:
-        if not self.deep_copyable:
-            return None
-        if not self.inputs:
-            raise RuntimeError(f"{self.__class__.__name__} has no inputs set")
-
-        clone = self.__class__(self.ptok, self.__class__.usage())
-
-        for input in self.inputs:
-            strand = input.deep_copy()
-            if strand is None:
-                return None
-            clone.add_source(strand)
-
-        return clone
+        return None
+    
+    def _get_sources(self, source_list: list):
+        for ix in self.inputs:
+            source_list.append(ix)
+            ix._get_sources(source_list)
 
 class DeepCopyPipe(Pipe):
     def deep_copy(self):
@@ -373,13 +369,16 @@ class Sink(ABC):
         self.process()
         self.close()
 
+        # get all inputs in the execution chain for closing
+        inputs = [self.input]
+        self.input._get_sources(inputs)
+        for input in inputs:
+            input.close()
+
     # optional
     def close(self):
         pass
 
-    def print_info(self):
-        pass
-
     def add_source(self, source: Source) -> None:
         self.input = source
         

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/common.py

@@ -4,6 +4,7 @@
 import sys, shutil, subprocess, contextlib, signal
 import os
 import yaml
+from pjk.base import TokenError
 
 class SafeNamespace:
     def __init__(self, obj):
@@ -73,11 +74,12 @@ def highlight(text: str, color: str = 'bold', value: str = None) -> str:
     return text.replace(value, f"{style}{value}{RESET}")
 
 class Lookups:
-    def __init__(self):
+    def __init__(self, component_class):
         self.lookups_yaml = os.path.expanduser('~/.pjk/lookups.yaml')
+        self.class_name = type(component_class).__name__
         self._data = {}
         self._load()
-
+        
     def _load(self):
         """Load lookups from YAML file if it exists."""
         if os.path.exists(self.lookups_yaml):
@@ -93,8 +95,13 @@ class Lookups:
             yaml.safe_dump(self._data, f)
 
     def get(self, key, default=None):
-        """Retrieve a lookup value by key."""
-        return self._data.get(key, default)
+        lookup_key = f'{self.class_name}-{key}'
+        entry = self._data.get(lookup_key, default)
+        if not entry:
+            raise TokenError(
+                f"~/.pjk/lookups.yaml must contain entry for '{lookup_key}' with host, user, password."
+            )
+        return entry
 
     def set(self, key, value):
         """Set a lookup value and persist it."""
@@ -129,14 +136,14 @@ class ComponentFactory:
         print(header)
 
         i = 0
-        plugin = ''
+        # user and outside package components are also here, but printed from registry class
         for name, comp_class in self.components.items():
             usage = comp_class.usage()
             lines = usage.desc.split('\n')
             if i >= self.num_orig_comps:
-                plugin = '(~/.pjk/plugin)'
-            line = f'  {name:<12} {lines[0]} {plugin}'
-            line = highlight(line, 'bold', plugin) if plugin else line
+                break
+
+            line = f'  {name:<12} {lines[0]}'
             print(line)
             i += 1
 

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/main.py

@@ -10,17 +10,20 @@ from typing import List
 from pjk.parser import ExpressionParser
 from pjk.base import UsageError
 from pjk.log import init as init_logging
-from datetime import datetime, timezone
+from datetime import datetime
+import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
-from pjk.pipes.factory import PipeFactory
-from pjk.sources.factory import SourceFactory
-from pjk.sinks.factory import SinkFactory
+from pjk.sinks.stdout import StdoutSink
 from pjk.man_page import do_man, do_examples
 from pjk.sinks.expect import ExpectSink
+from pjk.progress import ProgressDisplay
 from pjk.version import __version__
 
 def write_history(tokens):
+    if os.environ.get("PJK_NO_HISTORY") == "1":
+        return
+    
     log_path = ".pjk-history.txt"
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
     command = " ".join(tokens)
@@ -31,56 +34,71 @@ def write_history(tokens):
     except (PermissionError, OSError):
         pass
 
-def execute_threaded(sinks):
-    # Choose a max thread limit (explicitly)
-    max_workers = min(32, len(sinks))  # or set a fixed cap like 8
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = {
-            executor.submit(s.drain): s for s in sinks
-        }
-
+def execute_threaded(sinks, stop_progress=None):
+    max_workers = min(32, len(sinks))
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)  # no 'with'
+    futures = {executor.submit(s.drain): s for s in sinks}
+    try:
         for future in concurrent.futures.as_completed(futures):
             sink_obj = futures[future]
-            try:
-                future.result()  # This will re-raise any exception from s.drain()
-            except Exception as e:
-                print(f"Sink {sink_obj} raised an exception:")
-                print(e)
+            future.result()  # re-raises worker exception with traceback
+    except KeyboardInterrupt:
+        # stop UI first, then cancel and non-blocking shutdown
+        if stop_progress:
+            try: stop_progress()
+            except Exception: pass
+        for f in futures:
+            f.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        os._exit(130)
+
+    except Exception as e:
+        if stop_progress:
+            try: stop_progress()
+            except Exception: pass
+        sys.stderr.write(f"Sink {futures[future]} raised an exception:\n")
+        traceback.print_exception(type(e), e, e.__traceback__, file=sys.stderr)
+        for f in futures:
+            f.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    else:
+        executor.shutdown(wait=True)
 
 def execute(command: str):
     tokens = shlex.split(command, comments=True, posix=True)
     execute_tokens(tokens)
 
-def execute_tokens(tokens:List[str]):
+def execute_tokens(tokens: List[str]):
     init_logging()
-    signal.signal(signal.SIGINT, lambda s, f: sys.exit(0))
+    # (remove the sys.exit SIGINT handler here)
 
     if '--version' in tokens:
         print(f"pjk version {__version__}")
         sys.exit(0)
-    
+
     registry = ComponentRegistry()
-       
+
     if len(tokens) < 1:
         registry.print_usage()
         return
-    
-    # pjk man --all | --all+ | <component>
+
     if len(tokens) == 2 and tokens[0] == 'man':
         do_man(tokens[1], registry)
         return
-    
-    # pjk examples | examples+
+
     if len(tokens) == 1 and tokens[0] in ['examples', 'examples+']:
         do_examples(tokens[0], registry)
         return
 
     parser = ExpressionParser(registry)
 
+    display = None
     try:
-        # Build initial sink
         sink = parser.parse(tokens)
+        if not isinstance(sink, (StdoutSink | ExpectSink)):
+            display = ProgressDisplay(interval=3.0)
+            display.start()
 
         sinks = [sink]
         max_threads = os.cpu_count()
@@ -91,16 +109,23 @@ def execute_tokens(tokens:List[str]):
             sinks.append(clone)
 
         if len(sinks) > 1:
-            execute_threaded(sinks)
+            # pass a stopper so we halt the UI before tracebacks / shutdown
+            execute_threaded(sinks, stop_progress=(display.stop if display else None))
         else:
-            sink.drain() # run single in main thread
-            sink.print_info() # rarely used, e.g. expect and devnull
+            sink.drain()
 
         write_history(sys.argv[1:])
 
     except UsageError as e:
         print(e, file=sys.stderr)
-        sys.exit(2)  # Exit with a non-zero code, but no traceback
+        sys.exit(2)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if display:
+            # short join so Ctrl-C is immediate
+            try: display.stop(timeout=0.1)
+            except Exception: pass
 
 def main():
     tokens = sys.argv[1:]

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/parser.py

@@ -8,6 +8,9 @@ from typing import Optional, Any, List
 from pjk.base import Source, Pipe, Sink, TokenError, UsageError, ParsedToken, Usage
 from pjk.pipes.user_pipe_factory import UserPipeFactory
 from pjk.pipes.let_reduce import ReducePipe
+from pjk.sinks.stdout import StdoutSink
+from pjk.sinks.expect import ExpectSink
+from pjk.pipes.progress_pipe import ProgressPipe
 from pjk.registry import ComponentRegistry
 
 def expand_macros(tokens: List[str]) -> List[str]:
@@ -58,8 +61,12 @@ class ExpressionParser:
         if not sink:
             raise TokenError.from_list(['expression must end in a sink.',
                             'pjk <source> [<pipe> ...] <sink>'])
+        
+        # so each sink doesn't have to, maybe make a base class or mixin for sinks
+        progress_pipe = ProgressPipe(component_instance=sink)
+        progress_pipe.add_source(source)
 
-        sink.add_source(source)
+        sink.add_source(progress_pipe)
         return sink
 
     def parse(self, tokens: List[str]) -> Sink:
@@ -78,10 +85,12 @@ class ExpressionParser:
                     return self.get_sink(stack_helper, token)
                     
                 source = self.registry.create_source(token)
-                if source:
+                if source:                    
                     stack_helper.add_operator(source, self.stack)
+                    progress_pipe = ProgressPipe(component_instance=source, simple=True)
+                    stack_helper.add_operator(progress_pipe, self.stack)
                     continue
-
+                
                 subexp = SubExpression.create(token)
                 if subexp:
                     stack_helper.add_operator(subexp, self.stack)

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/factory.py

@@ -12,12 +12,14 @@ from pjk.pipes.head import HeadPipe
 from pjk.pipes.tail import TailPipe
 from pjk.pipes.sort import SortPipe
 from pjk.pipes.where import WherePipe
-from pjk.pipes.map import MapPipe
+from pjk.pipes.map import MapByPipe
+from pjk.pipes.map import GroupByPipe
 from pjk.pipes.join import JoinPipe
 from pjk.pipes.filter import FilterPipe
 from pjk.pipes.select import SelectFields
 from pjk.pipes.denorm import DenormPipe
 from pjk.pipes.postgres_pipe import PostgresPipe
+from pjk.pipes.sample import SamplePipe
 from pjk.pipes.user_pipe_factory import UserPipeFactory
 
 COMPONENTS = {
@@ -25,7 +27,8 @@ COMPONENTS = {
         'tail': TailPipe,
         'join': JoinPipe,
         'filter': FilterPipe,
-        'map': MapPipe,            
+        'mapby': MapByPipe,            
+        'groupby': GroupByPipe,
         'as': MoveField,
         'drop': RemoveField,        
         'let': LetPipe,
@@ -33,6 +36,7 @@ COMPONENTS = {
         'sort': SortPipe,
         'where': WherePipe,
         'sel': SelectFields,
+        'sample': SamplePipe,
         'explode': DenormPipe,
         'pgres': PostgresPipe,
     }

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/filter.py

@@ -15,13 +15,13 @@ class FilterPipe(Pipe):
         )
         usage.def_arg("mode", "'+' to include matches, '-' to exclude matches",
                       valid_values={'+', '-'})
-        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> filter:<mode> <sink>")
+        usage.def_syntax("pjk <left_source> <map_source> [mapby:groupby]:<how>:<key> filter:<mode> <sink>")
 
         usage.def_example(expr_tokens=
         [
             "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
             "[{id:1}, {id:3}, {id:5}]",
-            'map:o:id',
+            'mapby:id',
             "filter:+"
         ],
         expect="[{id:1}, {id:3}, {id:5}]")
@@ -30,7 +30,7 @@ class FilterPipe(Pipe):
         [
             "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
             "[{id:1}, {id:3}, {id:5}]",
-            'map:o:id',
+            'mapby:id',
             "filter:-"
         ],
         expect="[{id:2}, {id:4}]")

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/head.py

@@ -2,20 +2,18 @@
 # Copyright 2024 Mike Schultz
 
 # djk/pipes/head.py
+from pjk.base import Pipe, ParsedToken, Usage
 
-from typing import Optional
-from pjk.base import Pipe, ParsedToken, Usage, DeepCopyPipe
-
-class HeadPipe(DeepCopyPipe):
+class HeadPipe(Pipe):
     @classmethod
     def usage(cls):
         usage = Usage(
             name='head',
-            desc='take first records of input (when single-threaded)',
+            desc='take first records of input (single-threaded)',
             component_class=cls
         )
-        usage.def_arg(name='limit', usage='number of records', is_num=True)
         usage.def_example(expr_tokens=['[{id:1}, {id:2}]', 'head:1'], expect="{id:1}")
+        usage.def_arg(name='limit', usage='number of records', is_num=True)
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/join.py

@@ -20,13 +20,13 @@ class JoinPipe(Pipe):
             usage="'left', 'inner', or 'outer' join behavior",
             valid_values={'left', 'inner', 'outer'}
         )
-        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> join:<mode> <sink>")
+        usage.def_syntax("pjk <left_source> <map_source> [mapby|groupby]:<key> join:<mode> <sink>")
 
         usage.def_example(expr_tokens=
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:left"
         ],
         expect="[{color:'blue', price:50}, {color:'green'}]")
@@ -34,7 +34,7 @@ class JoinPipe(Pipe):
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:inner"
         ],
         expect="[{color:'blue', price:50}]")
@@ -43,7 +43,7 @@ class JoinPipe(Pipe):
         [
             "[{color:'blue'},{color:'green'}]",
             "[{color:'blue', price:50}, {color:'red', price:20}]",
-            'map:o:color',
+            'mapby:color',
             "join:outer"
         ],
         expect="[{color:'blue', price:50}, {color:'green'}, {color:'red', price: 20}]")

python_jack_knife-0.5.5/src/pjk/pipes/map.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/group.py
+
+from typing import Optional
+from pjk.base import ParsedToken, Usage, Pipe, KeyedSource
+
+class MapByPipe(Pipe, KeyedSource):
+    @classmethod
+    def usage(cls):
+        u = Usage(
+            name='mapby',
+            desc="Maps records to key, taking last instance of duplicates.\nFilters out records without all key fields.\nCreates Keyed Source for join or filter.",
+            component_class=cls
+        )
+        u.def_arg(name='key', usage='comma separated fields to map by')
+        u.def_param(name='count', usage='add count of the records with key', valid_values={'true', 'false'}, default='false')
+        u.def_example(expr_tokens=["[{id: 1, color:'blue'}, {id:1, color:'green'}, {id:2, color:'red'}]", 'mapby:id'],
+                          expect="[{id:2, color:'red'}, {id:1, color:'green'}]")
+        u.def_example(expr_tokens=["[{id: 1, color:'blue', size:5}, {id:1, color:'green', size:10}]", 'mapby:id,color'], 
+                          expect="[{id:1, color:'green', size: 10}, {id:1, color:'blue', size:5}]")
+        u.def_example(expr_tokens=["[{id:'a'}, {id:'a'}, {id:'b'}, {j:3}]", "mapby:id@count=true"],
+                        expect="[{id:'a', count:2}, {id:'b', 'count': 1}]")
+
+        return u
+
+    def __init__(self, ptok: ParsedToken, usage: Usage, is_group: bool = False):
+        super().__init__(ptok)
+        self.is_group = is_group
+        self.fields = usage.get_arg('key').split(',')
+        self.rec_map = {}
+        self.matched_map = {}
+        self.is_loaded = False
+        self.do_count = usage.get_param(name='count').lower() == 'true'
+        self.counts = {}
+
+    def reset(self):
+        self.rec_map.clear()
+        self.matched_map.clear()
+        self._rec_list = None
+        self.is_loaded = False
+
+    def get_key_rec(self, record):
+        key_rec = {}
+        for field in self.fields:
+            key_val = record.pop(field, None) if self.is_group else record.get(field)
+            if not key_val:
+                return None
+            
+            key_rec[field] = key_val
+        return key_rec
+    
+    def count(self, key):
+        if not self.do_count:
+            return
+        i = self.counts.get(key, 0)
+        self.counts[key] = i+1
+
+    def load(self):
+        if self.is_loaded:
+            return
+        self.is_loaded = True
+
+        for record in self.left:
+            key_rec = self.get_key_rec(record)
+            if not key_rec: # some fields missing, filter out rec
+                continue
+
+            key = tuple(key_rec.values())
+            self.count(key)
+
+            existing = self.rec_map.get(key)
+            if not existing:
+                if self.is_group:
+                    key_rec['child'] = [record]
+                    self.rec_map[key] = key_rec
+                else:
+                    self.rec_map[key] = record
+            else:
+                if self.is_group:
+                    existing['child'].append(record)
+                else:
+                    self.rec_map[key] = record
+
+        if self.do_count:
+            for k, v in self.rec_map.items():
+                if self.do_count:
+                    c = self.counts.get(k, 0)
+                    v['count'] = c
+
+    def __iter__(self):
+        if not self.is_loaded:
+            self.load()
+        for v in self.rec_map.values():
+            yield v
+
+    def lookup(self, left_rec) -> Optional[dict]:
+        if not self.is_loaded:
+            self.load()
+
+        key = tuple(left_rec.get(f) for f in self.fields)
+        rec = self.rec_map.pop(key, None)
+        if rec is not None:
+            self.matched_map[key] = rec
+            return rec
+        return self.matched_map.get(key)
+
+    def get_unlookedup_records(self):
+        if not self.is_loaded:
+            self.load()
+        return list(self.rec_map.values())
+
+class GroupByPipe(MapByPipe):
+    @classmethod
+    def usage(cls):
+        u = Usage(
+            name='groupby',
+            desc="groups records by key. Creates Keyed Source for join or filter.",
+            component_class=cls
+        )
+        u.def_arg(name='key', usage='comma separated fields to map by')
+        u.def_param(name='count', usage='add count of the records with key', valid_values={'true', 'false'}, default='false')
+        u.def_example(expr_tokens=["[{id: 1, color:'blue'}, {id:1, color:'green'}, {id:2, color:'red'}]", 'groupby:id'], 
+                          expect="[{id:2, child:[{color:'red'}]}, {id:1, child:[{color:'blue'},{color: 'green'}]}]")
+
+        return u
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage, True)

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/move_field.py

@@ -10,7 +10,7 @@ class MoveField(Pipe):
     def usage(cls):
         usage = Usage(
             name='as',
-            desc='Move one field to another key in the record',
+            desc='rename a field in the record',
             component_class=cls
         )
         usage.def_arg(name='src', usage='Source field name')
@@ -20,7 +20,7 @@ class MoveField(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.src = usage.get_arg('src')
         self.dst = usage.get_arg('dst')
         self.count = 0

{python_jack_knife-0.5.1 → python_jack_knife-0.5.5}/src/pjk/pipes/postgres_pipe.py

@@ -99,30 +99,26 @@ class PostgresPipe(Pipe):
         )
         usage.def_arg(
             "dbname",
-            "name of db. Entry in ~/.pjk/lookups.yaml containing host, user, password"
+            f"~/.pjk/lookups.yaml must containing entry '{cls.__name__}-<dbname>' with host, user, password"
         )
         usage.def_param(
             "header",
             usage="emit header record before query results",
-            valid_values={"true", "false"}, default='true',
+            valid_values={"true", "false"}, default='false',
         )
 
         usage.def_example(expr_tokens=['myquery.sql', 'pgres:mydb'], expect=None)
         usage.def_example(expr_tokens=["{'query': 'SELECT * from MY_TABLE;'}", 'pgres:mydb'], expect=None)
+        usage.def_example(expr_tokens=["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'pgres:mydb'], expect=None)
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        lookups = Lookups()
+        lookups = Lookups(self)
         self.dbname = usage.get_arg("dbname")
-        db_params = lookups.get(self.dbname)
-        if not db_params:
-            # f-string so dbname prints correctly
-            raise TokenError(
-                f"~/.pjk/lookups.yaml must contain entry for '{self.dbname}' with host, user, password."
-            )
 
+        db_params = lookups.get(self.dbname)
         self.db_host = db_params.get("host")
         self.db_user = db_params.get("user")
         self.db_pass = db_params.get("password")

python_jack_knife-0.5.5/src/pjk/pipes/progress_pipe.py

@@ -0,0 +1,41 @@
+from typing import Iterator
+from pjk.base import Pipe
+from pjk.progress import papi
+
+# monitors flow of records wherever inserted
+
+class ProgressPipe(Pipe):
+    def __init__(self, component_instance = None, simple: bool = False):
+        super().__init__(None, None)
+        self.component_instance = component_instance
+        self.simple = simple
+
+        label = self.get_component_label(component_instance)
+        self.counter = papi.get_counter(label, var_label='recs')
+        #papi.add_rate(sink_name, self.counter, var_label='krecs/sec')
+        if not simple:
+            papi.get_counter(label, var_label='threads').increment()
+            papi.add_elapsed_time(label, var_label='elapsed')
+
+    def get_component_label(self, component_instance):
+        if hasattr(type(component_instance), 'extension'):
+            return type(component_instance).extension
+        elif hasattr(component_instance, 'usage'):
+            return type(component_instance).usage().name
+        return type(component_instance).__name__
+
+    def __iter__(self) -> Iterator:
+        # only counting here
+        for record in self.left:
+            self.counter.increment()
+            yield record
+
+    def deep_copy(self):
+        source_clone = self.left.deep_copy()
+        if not source_clone:
+            return None
+
+        pipe = ProgressPipe(self.component_instance, self.simple)
+        pipe.add_source(source_clone)
+        return pipe
+