python-jack-knife 0.6.16__tar.gz → 0.7.0__tar.gz

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.16
+Version: 0.7.0
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/components.py

@@ -35,9 +35,12 @@ class Source(ABC):
             component_class=cls
         )
 
+    def __init__(self, root = None):
+        self.root = root
+
     @abstractmethod
     def __iter__(self):
-        raise NotImplementedError("__iter__ must be implemented by subclasses")
+        pass
 
     def __next__(self):
         # lazily create an internal iterator the first time next() is called
@@ -57,7 +60,8 @@ class Source(ABC):
 class Pipe(Source):
     arity: int = 1
     
-    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+    def __init__(self, ptok: ParsedToken, usage: Usage, root = None):
+        self.root = root
         self.ptok = ptok
         self.usage = usage
         self.left = None  # left source for convience
@@ -95,7 +99,7 @@ class DeepCopyPipe(Pipe):
             return None
 
         # re-instantiate using the actual subclass
-        pipe = type(self)(self.ptok, self.usage)
+        pipe = type(self)(self.ptok, self.usage, self) # this self is the root
         pipe.add_source(source_clone)
         return pipe
 
@@ -108,7 +112,8 @@ class Sink(ABC):
             component_class=cls
         )
     
-    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+    def __init__(self, ptok: ParsedToken, usage: Usage, root = None):
+        self.root = root
         self.ptok = ptok
         self.usage = usage
 
@@ -135,4 +140,3 @@ class Sink(ABC):
 
     def deep_copy(self):
         return None
-

python_jack_knife-0.7.0/src/pjk/history.py

@@ -0,0 +1,103 @@
+import re
+import os
+import shlex
+from typing import List, Set
+from pjk.common import pager_stdout, highlight
+
+LOG_FILE = '.pjk-history.txt'
+
+def printable_command(tokens: list) -> str:
+    pattern = re.compile(r"[({]")
+
+    return ' '.join(
+        f'"{s}"' if pattern.search(s) else s
+        for s in tokens
+    )
+
+def read_history(log_path: str) -> List[int]:
+    """
+    Reads the history file into an ordered dictionary (command -> ordinal)
+    """
+    # dict preserves insertion order in modern Python
+    clist: List[str] = []
+    cset:Set = set()
+
+    try:
+        with open(log_path, "r") as f:
+            ordinal = 1
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                # Expected format: <command_string>
+                line = line.strip()
+                        
+                # 2. Add to the map. Since dict keys must be unique,
+                # this ensures the map only contains one entry per command, preserving the first seen's order.
+                if line not in cset:
+                    clist.append(line)
+                    cset.add(line)
+                            
+    except FileNotFoundError:
+        pass 
+    except (PermissionError, OSError) as e:
+        print(f"Warning: Could not read history file {log_path}: {e}")
+        
+    return clist, cset
+
+def write_history(tokens: list):
+    if os.environ.get("PJK_NO_HISTORY") == "1":
+        return
+    
+    if len(tokens) < 2: 
+        return 
+    
+    if tokens[0] == 'man':
+        return
+    
+    new_command_string = printable_command(tokens)
+
+    # 1. Read the existing history and find the highest number
+    clist, cset = read_history(LOG_FILE)
+    
+    # 2. Check for duplicates (Fast O(1) lookup using the dict key)
+    if new_command_string in cset:
+        # Command is a duplicate, nothing to do.
+        return
+    
+    # 3. Append the new command line to the file
+    try:
+        # Use 'a' to append the new line only
+        with open(LOG_FILE, "a") as f:
+            f.write(f"{new_command_string}\n")
+            
+    except (PermissionError, OSError) as e:
+        print(f"Warning: Could not write to history file {LOG_FILE}: {e}")
+
+def display_history():
+    clist, cset = read_history(LOG_FILE)
+
+    with pager_stdout():
+        print(f"Local history in '{LOG_FILE}'")
+        print("Use 'pjk +<#>' to execute command.")
+        print()
+        o = highlight('#', 'bold', '#')
+        c = highlight('command', 'bold', 'command')
+        print(f'{o}\t{c}')
+
+        ordn = 1
+        for command in reversed(clist):
+            print(f'{ordn}\t{command}')
+            ordn += 1
+
+def get_history_tokens(ord_str: str):
+    ord_in = int(ord_str)
+    clist, cset = read_history(LOG_FILE)
+    ordn = 0
+    for command in reversed(clist):
+        ordn += 1
+        if ord_in == ordn:
+            parts = shlex.split(command, comments=True, posix=True)
+            return parts
+    return None

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/integrations/opensearch_query_pipe.py

@@ -23,7 +23,7 @@ def build_body_from_string(query_string: str) -> dict:
 
 class OpenSearchQueryPipe(QueryPipe, Integration):
     name = "os_query"
-    desc = ("Opensearch query pipe. Uses record['query'] or record['os_query_object'] for os query\n"
+    desc = ("Opensearch query pipe. Uses record['query'] or record['os_query_object']\n"
     "An instance may define 'default_index' otherwise the query object must include an 'index' field.\n")
     arg0 = ("instance", "instance to query over.")
     examples = [

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/main.py

@@ -3,40 +3,22 @@
 
 #!/usr/bin/env python
 import sys
-import os
+import os, re
 import shlex
-from typing import List
+from typing import List, Dict
 from pjk.parser import ExpressionParser
 from pjk.usage import UsageError
 from pjk.log import init as init_logging
-from datetime import datetime
 import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
 from pjk.sinks.stdout import StdoutSink
-from pjk.man_page import do_man, do_examples
+from pjk.man_page import do_man, do_examples, display_configs, display_macros
+from pjk.history import write_history, display_history, get_history_tokens
 from pjk.sinks.expect import ExpectSink
 from pjk.progress import ProgressDisplay
 from pjk.version import __version__
 
-def write_history(tokens):
-    if os.environ.get("PJK_NO_HISTORY") == "1":
-        return
-    
-    log_path = ".pjk-history.txt"
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
-
-    if len(tokens) < 2: 
-        return 
-    
-    command = " ".join(tokens)
-
-    try:
-        with open(log_path, "a") as f:
-            f.write(f"{timestamp}\tpjk {command}\n")
-    except (PermissionError, OSError):
-        pass
-
 def execute_threaded(sinks, stop_progress=None):
     max_workers = min(32, len(sinks))
     executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)  # no 'with'
@@ -77,6 +59,32 @@ def initialize():
     #dst_dir.mkdir(parents=True, exist_ok=True)
     #hutil.copy(src, dst_dir / src.name)
 
+def execute_non_expression(tokens, registry):
+    command = tokens[0]
+
+    if len(tokens) == 2 and command == 'man':
+        do_man(tokens[1], registry)
+        return True
+    
+    if len(tokens) != 1:
+        return False
+
+    match command:
+        case 'examples':
+            do_examples(command, registry)
+        case 'examples+':
+            do_examples(command, registry)
+        case 'configs':
+            display_configs()
+        case 'macros':
+            display_macros()
+        case '+':
+            display_history()
+        case _:
+            return False
+
+    return True
+
 def execute(command: str):
     tokens = shlex.split(command, comments=True, posix=True)
     execute_tokens(tokens)
@@ -93,14 +101,16 @@ def execute_tokens(tokens: List[str]):
     if len(tokens) < 1:
         registry.print_usage()
         return
-
-    if len(tokens) == 2 and tokens[0] == 'man':
-        do_man(tokens[1], registry)
+    
+    if execute_non_expression(tokens, registry):
         return
 
-    if len(tokens) == 1 and tokens[0] in ['examples', 'examples+']:
-        do_examples(tokens[0], registry)
-        return
+    # execute command from .pjk-history.txt
+    if len(tokens) == 1 and re.fullmatch(r'^\+\d+$', tokens[0]):
+        tokens = get_history_tokens(tokens[0])
+        if not tokens:
+            print('No such history')
+            return
 
     parser = ExpressionParser(registry)
 

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/man_page.py

@@ -1,15 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-from pjk.pipes.factory import PipeFactory
-from pjk.sources.factory import SourceFactory
-from pjk.sinks.factory import SinkFactory
-from pjk.parser import ExpressionParser
+from pjk.parser import ExpressionParser, MACRO_PREFIX, MACROS_FILE, read_macros
 from pjk.components import Source, Pipe, Sink
-from pjk.usage import Usage, ParsedToken
+from pjk.usage import Usage, CONFIG_FILE
 from pjk.registry import ComponentRegistry
 from pjk.common import pager_stdout, highlight, ComponentOrigin
 from contextlib import nullcontext
+import yaml
+import sys
+from pathlib import Path
 
 def get_base_class(usage: Usage, as_string: bool = False):
         if issubclass(usage.comp_class, Sink):
@@ -87,6 +87,50 @@ def print_man(registry: ComponentRegistry, name: str, usage: Usage):
     for expr_tokens, expect in usage.get_examples(): # expect in InlineSource format
         print_example(registry, expr_tokens, expect, name)
 
+def display_configs():
+    path = Path(CONFIG_FILE).expanduser()
+
+    with pager_stdout():
+        with path.open("r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        if not isinstance(data, dict):
+            raise ValueError("Top-level YAML must be a mapping of records")
+        
+        print(f'Component configs defined in {CONFIG_FILE}')
+        print()
+        for name, body_dict in data.items():
+            print('=========================================')
+            print('   ', highlight(name, 'bold', name))
+            print('=========================================')
+
+            if 'password' in body_dict:
+                body_dict['password'] = '*************'
+
+            try:
+                yaml.dump(
+                    body_dict,
+                    sys.stdout,
+                    sort_keys=False,
+                    explicit_start=False,
+                    allow_unicode=True,
+                    width=10**9,
+                )
+            except BrokenPipeError:
+                break
+            print()
+
+def display_macros():
+    macros = read_macros()
+
+    with pager_stdout():
+        print(f"Macros defined in '{MACROS_FILE}'")
+
+        print(f"Usage: pjk [...] {MACRO_PREFIX}:<macro_name> [...]")
+        print()
+        for name, value in macros.items():
+            print(f'{name}: {value}')
+            print()
+
 def do_examples(token:str, registry: ComponentRegistry):
     no_pager = token.endswith('+')
     cm = nullcontext() if no_pager else pager_stdout()

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/parser.py

@@ -7,12 +7,27 @@ from typing import Any, List
 from pjk.components import Source, Pipe, Sink
 from pjk.usage import TokenError, UsageError, ParsedToken, Usage
 from pjk.pipes.let_reduce import ReducePipe
-from pjk.sources.macro_source import MACROS_FILE, MACRO_PREFIX, read_macros
 from pjk.pipes.progress_pipe import ProgressPipe
 from pjk.registry import ComponentRegistry
 from pjk.progress import papi
+from typing import Dict
+from pathlib import Path
 from pjk.progress import ProgressIgnore
 
+MACROS_FILE = '~/.pjk/macros.txt'
+MACRO_PREFIX = 'm'
+def read_macros(file_name: str = MACROS_FILE) -> Dict[str, str]:
+    out: Dict[str, str] = {}
+    path = Path(file_name).expanduser()
+    with path.open(encoding="utf-8") as f:
+        for raw in f:
+            line = raw.split("#", 1)[0].strip()
+            if not line or ":" not in line:
+                continue
+            key, val = line.split(":", 1)
+            out[key.strip()] = val.strip()
+    return out
+
 # macros are of the form MACRO_PREFIX:<instance>
 def handle_macros(token: str, expanded: List[str]):
     if not token.startswith(f'{MACRO_PREFIX}:'):
@@ -187,7 +202,7 @@ class ExpressionParser:
     
 class ReducerAggregatorPipe(Pipe):
     def __init__(self, top_level_reducers: List[Any]):
-        super().__init__(None)
+        super().__init__(None, None)
         self.top_level_reducers = top_level_reducers
         self.reduction = {}
         self.done = False
@@ -275,6 +290,7 @@ class UpstreamSource(Source):
         return u
     
     def __init__(self):
+        super().__init__(root=None)
         self.data = []
         self.inner_source = None
         self.sub_recs_in = papi.get_counter(self, var_label='sub_recs_in')
@@ -338,7 +354,7 @@ class SubExpression(Pipe, ProgressIgnore):
         return None
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.subexp_ops = []
         self.stack_helper = StackLoader()
         self.subexp_stack = OperandStack() 

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/denorm.py

@@ -52,10 +52,10 @@ class DenormPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
 
         self.field = usage.get_arg('field')
-        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_in = papi.get_counter(self, 'recs_in', display=False)
         self.recs_out = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 
         self._pending_iter = None

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/filter.py

@@ -39,11 +39,11 @@ class FilterPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.mode = usage.get_arg('mode')
         self.left = None
         self.right = None
-        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_in = papi.get_counter(self, 'recs_in', display=False)
         self.recs_out = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 
     def reset(self):

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/join.py

@@ -60,7 +60,7 @@ class JoinPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
 
         self.mode = usage.get_arg('mode')
         self.left = None
@@ -68,7 +68,7 @@ class JoinPipe(Pipe):
         self._pending_right = None
         self._check_right = False
 
-        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_in = papi.get_counter(self, 'recs_in', display=False) 
         self.matches = papi.get_percentage_counter(self, 'matches', self.recs_in)
         self.recs_out = papi.get_counter(self, 'recs_out')
 

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/let_reduce.py

@@ -3,7 +3,7 @@
 
 # djk/pipes/let_reduce.py
 
-from pjk.components import Pipe
+from pjk.components import DeepCopyPipe
 from pjk.usage import ParsedToken, Usage, UsageError, TokenError, NoBindUsage
 from pjk.common import SafeNamespace, ReducingNamespace
 import re
@@ -78,7 +78,7 @@ def eval_accumulating(expr: str, record: dict, op: str, acc=None):
     return do_eval(expr, env)
 
 # --- LetPipe (simple field assignment) ---
-class LetPipe(Pipe):
+class LetPipe(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = NoBindUsage( # can't use bound usage because of complicated parsing
@@ -93,7 +93,7 @@ class LetPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         args = parse_args(ptok.whole_token.split(':', 1)[-1])
         self.field = args['field']
         self.op = args['op']
@@ -121,7 +121,7 @@ def is_comprehension(expr: str) -> bool:
     except SyntaxError:
         return False
 
-class ReducePipe(Pipe):
+class ReducePipe(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = NoBindUsage( # can't use bound usage because of complicated parsing
@@ -161,7 +161,7 @@ class ReducePipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         args = parse_args(ptok.whole_token.split(':', 1)[-1])
         self.field = args['field']
         self.op = args['op']

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/map.py

@@ -28,7 +28,7 @@ class MapByPipe(Pipe, KeyedSource):
         return u
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.is_group = False
         self.fields = usage.get_arg('key').split(',')
         self.rec_map = {}
@@ -37,7 +37,7 @@ class MapByPipe(Pipe, KeyedSource):
         self.do_count = usage.get_param(name='count').lower() == 'true'
         self.counts = {}
         self.missing_keys = papi.get_counter(self, 'missing_keys')
-        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_in = papi.get_counter(self, 'recs_in', display=False)
         # recs_out = distinct_keys
         self.distinct_keys = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/query_pipe.py

@@ -40,7 +40,7 @@ class QueryPipe(Pipe):
         self.output_shape = usage.get_param('shape')
         self.count = usage.get_param('count')
         self.query_field = 'query' # for all subclasses
-        self.inrecs = papi.get_counter(self, var_label=None) # don't display progress
+        self.inrecs = papi.get_counter(self, var_label='recs_in') 
         self.outrecs = papi.get_percentage_counter(self, var_label='recs_out', denom_counter=self.inrecs)
 
     @abstractmethod

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/remove_field.py

@@ -19,7 +19,7 @@ class RemoveField(DeepCopyPipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         arg_string = usage.get_arg('fields')
         self.fields = [f.strip() for f in arg_string.split(',') if f.strip()]
         if not self.fields:

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/select.py

@@ -19,7 +19,7 @@ class SelectFields(DeepCopyPipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
 
         arg_string = usage.get_arg('fields')
         if not arg_string:

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/sort.py

@@ -21,7 +21,7 @@ class SortPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
 
         arg_string = usage.get_arg('field')
         if arg_string.startswith("-"):

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/tail.py

@@ -19,7 +19,7 @@ class TailPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.limit = usage.get_arg('limit')
 
         self.buffer = []

{python_jack_knife-0.6.16 → python_jack_knife-0.7.0}/src/pjk/pipes/where.py

@@ -22,10 +22,11 @@ class WherePipe(DeepCopyPipe):
         u.def_example(expr_tokens=["[{color:'blue'}, {color:'red'}, {color:'black'}]", "where:f.color.startswith('bl')"], expect="[{color:'blue'}, {color:'black'}]")
         return u
 
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
+    def __init__(self, ptok: ParsedToken, usage: Usage, root = None):
+        super().__init__(ptok, usage, root)
         self.expr = ptok.whole_token.split(':', 1)[1]
-        self.inrecs = papi.get_counter(self, var_label=None) # don't display progress
+
+        self.inrecs = papi.get_counter(self, var_label='recs_in', display=False)
         self.outrecs = papi.get_percentage_counter(self, var_label='recs_out', denom_counter=self.inrecs)
         try:
             self.code = compile(self.expr, '<where>', 'eval')
@@ -46,3 +47,4 @@ class WherePipe(DeepCopyPipe):
             except Exception:
                 continue  # ignore eval errors
 
+