python-jack-knife 0.6.17__tar.gz → 0.7.0__tar.gz

{python_jack_knife-0.6.17/src/python_jack_knife.egg-info → python_jack_knife-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.17
+Version: 0.7.0
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

python_jack_knife-0.7.0/src/pjk/history.py

@@ -0,0 +1,103 @@
+import re
+import os
+import shlex
+from typing import List, Set
+from pjk.common import pager_stdout, highlight
+
+LOG_FILE = '.pjk-history.txt'
+
+def printable_command(tokens: list) -> str:
+    pattern = re.compile(r"[({]")
+
+    return ' '.join(
+        f'"{s}"' if pattern.search(s) else s
+        for s in tokens
+    )
+
+def read_history(log_path: str) -> List[int]:
+    """
+    Reads the history file into an ordered dictionary (command -> ordinal)
+    """
+    # dict preserves insertion order in modern Python
+    clist: List[str] = []
+    cset:Set = set()
+
+    try:
+        with open(log_path, "r") as f:
+            ordinal = 1
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+
+                # Expected format: <command_string>
+                line = line.strip()
+                        
+                # 2. Add to the map. Since dict keys must be unique,
+                # this ensures the map only contains one entry per command, preserving the first seen's order.
+                if line not in cset:
+                    clist.append(line)
+                    cset.add(line)
+                            
+    except FileNotFoundError:
+        pass 
+    except (PermissionError, OSError) as e:
+        print(f"Warning: Could not read history file {log_path}: {e}")
+        
+    return clist, cset
+
+def write_history(tokens: list):
+    if os.environ.get("PJK_NO_HISTORY") == "1":
+        return
+    
+    if len(tokens) < 2: 
+        return 
+    
+    if tokens[0] == 'man':
+        return
+    
+    new_command_string = printable_command(tokens)
+
+    # 1. Read the existing history and find the highest number
+    clist, cset = read_history(LOG_FILE)
+    
+    # 2. Check for duplicates (Fast O(1) lookup using the dict key)
+    if new_command_string in cset:
+        # Command is a duplicate, nothing to do.
+        return
+    
+    # 3. Append the new command line to the file
+    try:
+        # Use 'a' to append the new line only
+        with open(LOG_FILE, "a") as f:
+            f.write(f"{new_command_string}\n")
+            
+    except (PermissionError, OSError) as e:
+        print(f"Warning: Could not write to history file {LOG_FILE}: {e}")
+
+def display_history():
+    clist, cset = read_history(LOG_FILE)
+
+    with pager_stdout():
+        print(f"Local history in '{LOG_FILE}'")
+        print("Use 'pjk +<#>' to execute command.")
+        print()
+        o = highlight('#', 'bold', '#')
+        c = highlight('command', 'bold', 'command')
+        print(f'{o}\t{c}')
+
+        ordn = 1
+        for command in reversed(clist):
+            print(f'{ordn}\t{command}')
+            ordn += 1
+
+def get_history_tokens(ord_str: str):
+    ord_in = int(ord_str)
+    clist, cset = read_history(LOG_FILE)
+    ordn = 0
+    for command in reversed(clist):
+        ordn += 1
+        if ord_in == ordn:
+            parts = shlex.split(command, comments=True, posix=True)
+            return parts
+    return None

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/integrations/opensearch_query_pipe.py

@@ -23,7 +23,7 @@ def build_body_from_string(query_string: str) -> dict:
 
 class OpenSearchQueryPipe(QueryPipe, Integration):
     name = "os_query"
-    desc = ("Opensearch query pipe. Uses record['query'] or record['os_query_object'] for os query\n"
+    desc = ("Opensearch query pipe. Uses record['query'] or record['os_query_object']\n"
     "An instance may define 'default_index' otherwise the query object must include an 'index' field.\n")
     arg0 = ("instance", "instance to query over.")
     examples = [

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/main.py

@@ -3,40 +3,22 @@
 
 #!/usr/bin/env python
 import sys
-import os
+import os, re
 import shlex
-from typing import List
+from typing import List, Dict
 from pjk.parser import ExpressionParser
 from pjk.usage import UsageError
 from pjk.log import init as init_logging
-from datetime import datetime
 import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
 from pjk.sinks.stdout import StdoutSink
-from pjk.man_page import do_man, do_examples
+from pjk.man_page import do_man, do_examples, display_configs, display_macros
+from pjk.history import write_history, display_history, get_history_tokens
 from pjk.sinks.expect import ExpectSink
 from pjk.progress import ProgressDisplay
 from pjk.version import __version__
 
-def write_history(tokens):
-    if os.environ.get("PJK_NO_HISTORY") == "1":
-        return
-    
-    log_path = ".pjk-history.txt"
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
-
-    if len(tokens) < 2: 
-        return 
-    
-    command = " ".join(tokens)
-
-    try:
-        with open(log_path, "a") as f:
-            f.write(f"{timestamp}\tpjk {command}\n")
-    except (PermissionError, OSError):
-        pass
-
 def execute_threaded(sinks, stop_progress=None):
     max_workers = min(32, len(sinks))
     executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)  # no 'with'
@@ -77,6 +59,32 @@ def initialize():
     #dst_dir.mkdir(parents=True, exist_ok=True)
     #hutil.copy(src, dst_dir / src.name)
 
+def execute_non_expression(tokens, registry):
+    command = tokens[0]
+
+    if len(tokens) == 2 and command == 'man':
+        do_man(tokens[1], registry)
+        return True
+    
+    if len(tokens) != 1:
+        return False
+
+    match command:
+        case 'examples':
+            do_examples(command, registry)
+        case 'examples+':
+            do_examples(command, registry)
+        case 'configs':
+            display_configs()
+        case 'macros':
+            display_macros()
+        case '+':
+            display_history()
+        case _:
+            return False
+
+    return True
+
 def execute(command: str):
     tokens = shlex.split(command, comments=True, posix=True)
     execute_tokens(tokens)
@@ -93,14 +101,16 @@ def execute_tokens(tokens: List[str]):
     if len(tokens) < 1:
         registry.print_usage()
         return
-
-    if len(tokens) == 2 and tokens[0] == 'man':
-        do_man(tokens[1], registry)
+    
+    if execute_non_expression(tokens, registry):
         return
 
-    if len(tokens) == 1 and tokens[0] in ['examples', 'examples+']:
-        do_examples(tokens[0], registry)
-        return
+    # execute command from .pjk-history.txt
+    if len(tokens) == 1 and re.fullmatch(r'^\+\d+$', tokens[0]):
+        tokens = get_history_tokens(tokens[0])
+        if not tokens:
+            print('No such history')
+            return
 
     parser = ExpressionParser(registry)
 

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/man_page.py

@@ -1,15 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-from pjk.pipes.factory import PipeFactory
-from pjk.sources.factory import SourceFactory
-from pjk.sinks.factory import SinkFactory
-from pjk.parser import ExpressionParser
+from pjk.parser import ExpressionParser, MACRO_PREFIX, MACROS_FILE, read_macros
 from pjk.components import Source, Pipe, Sink
-from pjk.usage import Usage, ParsedToken
+from pjk.usage import Usage, CONFIG_FILE
 from pjk.registry import ComponentRegistry
 from pjk.common import pager_stdout, highlight, ComponentOrigin
 from contextlib import nullcontext
+import yaml
+import sys
+from pathlib import Path
 
 def get_base_class(usage: Usage, as_string: bool = False):
         if issubclass(usage.comp_class, Sink):
@@ -87,6 +87,50 @@ def print_man(registry: ComponentRegistry, name: str, usage: Usage):
     for expr_tokens, expect in usage.get_examples(): # expect in InlineSource format
         print_example(registry, expr_tokens, expect, name)
 
+def display_configs():
+    path = Path(CONFIG_FILE).expanduser()
+
+    with pager_stdout():
+        with path.open("r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        if not isinstance(data, dict):
+            raise ValueError("Top-level YAML must be a mapping of records")
+        
+        print(f'Component configs defined in {CONFIG_FILE}')
+        print()
+        for name, body_dict in data.items():
+            print('=========================================')
+            print('   ', highlight(name, 'bold', name))
+            print('=========================================')
+
+            if 'password' in body_dict:
+                body_dict['password'] = '*************'
+
+            try:
+                yaml.dump(
+                    body_dict,
+                    sys.stdout,
+                    sort_keys=False,
+                    explicit_start=False,
+                    allow_unicode=True,
+                    width=10**9,
+                )
+            except BrokenPipeError:
+                break
+            print()
+
+def display_macros():
+    macros = read_macros()
+
+    with pager_stdout():
+        print(f"Macros defined in '{MACROS_FILE}'")
+
+        print(f"Usage: pjk [...] {MACRO_PREFIX}:<macro_name> [...]")
+        print()
+        for name, value in macros.items():
+            print(f'{name}: {value}')
+            print()
+
 def do_examples(token:str, registry: ComponentRegistry):
     no_pager = token.endswith('+')
     cm = nullcontext() if no_pager else pager_stdout()

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/parser.py

@@ -7,12 +7,27 @@ from typing import Any, List
 from pjk.components import Source, Pipe, Sink
 from pjk.usage import TokenError, UsageError, ParsedToken, Usage
 from pjk.pipes.let_reduce import ReducePipe
-from pjk.sources.macro_source import MACROS_FILE, MACRO_PREFIX, read_macros
 from pjk.pipes.progress_pipe import ProgressPipe
 from pjk.registry import ComponentRegistry
 from pjk.progress import papi
+from typing import Dict
+from pathlib import Path
 from pjk.progress import ProgressIgnore
 
+MACROS_FILE = '~/.pjk/macros.txt'
+MACRO_PREFIX = 'm'
+def read_macros(file_name: str = MACROS_FILE) -> Dict[str, str]:
+    out: Dict[str, str] = {}
+    path = Path(file_name).expanduser()
+    with path.open(encoding="utf-8") as f:
+        for raw in f:
+            line = raw.split("#", 1)[0].strip()
+            if not line or ":" not in line:
+                continue
+            key, val = line.split(":", 1)
+            out[key.strip()] = val.strip()
+    return out
+
 # macros are of the form MACRO_PREFIX:<instance>
 def handle_macros(token: str, expanded: List[str]):
     if not token.startswith(f'{MACRO_PREFIX}:'):

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/registry.py

@@ -11,7 +11,7 @@ from pjk.sources.format_source import FormatSource
 import importlib.util
 import importlib
 from pjk.components import Pipe, Source, Sink
-from pjk.common import ComponentFactory, highlight, ComponentOrigin
+from pjk.common import ComponentFactory, highlight, ComponentOrigin, pager_stdout
 from typing import List, Type
 
 class ExternalRegistrar:
@@ -50,22 +50,23 @@ class ComponentRegistry:
         return [self.source_factory, self.pipe_factory, self.sink_factory]
 
     def print_usage(self):
-        print('Usage: pjk <source> [<pipe> ...] <sink>')
-        print('       pjk man <component> | --all')
-        print('       pjk examples')
-        print()
-
-        print_core_formats([self.source_factory, self.sink_factory])
-        print()
-        print_factory_core(self.source_factory, header='sources')
-        print()
-        print_factory_core(self.pipe_factory, header='pipes')
-        print()
-        print_factory_core(self.sink_factory, header='sinks')
-
-        self.print_non_core([ComponentOrigin.CORE,ComponentOrigin.EXTERNAL], is_integration=True, header='integrations')
-        self.print_non_core([ComponentOrigin.EXTERNAL], is_integration=False, header='apps')
-        self.print_non_core([ComponentOrigin.USER], is_integration=None, header='user components (~/.pjk/plugins)')        
+        with pager_stdout():
+            print('Usage: pjk <source> [<pipe> ...] <sink>')
+            print('       pjk man <component> | --all')
+            print('       pjk examples | configs | macros | + (for history)')
+            print()
+
+            print_core_formats([self.source_factory, self.sink_factory])
+            print()
+            print_factory_core(self.source_factory, header='sources')
+            print()
+            print_factory_core(self.pipe_factory, header='pipes')
+            print()
+            print_factory_core(self.sink_factory, header='sinks')
+
+            self.print_non_core([ComponentOrigin.CORE,ComponentOrigin.EXTERNAL], is_integration=True, header='integrations')
+            self.print_non_core([ComponentOrigin.EXTERNAL], is_integration=False, header='apps')
+            self.print_non_core([ComponentOrigin.USER], is_integration=None, header='user components (~/.pjk/plugins)')        
 
     # is_integration = True|False|None  None=don't care
     def print_non_core(self, origin_list: List[ComponentOrigin], is_integration: bool, header:str):

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/sources/factory.py

@@ -13,8 +13,6 @@ from pjk.sources.inline_source import InlineSource
 from pjk.sources.user_source_factory import UserSourceFactory
 from pjk.sources.parquet_source import ParquetSource
 from pjk.sources.format_source import FormatSource
-from pjk.sources.configs_source import ConfigsSource
-from pjk.sources.macro_source import MacroSource
 
 COMPONENTS = {
         'inline': InlineSource,
@@ -25,8 +23,6 @@ COMPONENTS = {
         'sql': SQLSource,
         'npy': NpySource,
         'parquet': ParquetSource,
-        'configs': ConfigsSource,
-        'macros': MacroSource
     }
 
 class SourceFactory(ComponentFactory):

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.17"
+__version__ = "0.7.0"

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0/src/python_jack_knife.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.17
+Version: 0.7.0
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.17 → python_jack_knife-0.7.0}/src/python_jack_knife.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ pyproject.toml
 src/pjk/__init__.py
 src/pjk/common.py
 src/pjk/components.py
+src/pjk/history.py
 src/pjk/log.py
 src/pjk/main.py
 src/pjk/man_page.py
@@ -56,7 +57,6 @@ src/pjk/sinks/stdout.py
 src/pjk/sinks/tsv_sink.py
 src/pjk/sinks/user_sink_factory.py
 src/pjk/sources/__init__.py
-src/pjk/sources/configs_source.py
 src/pjk/sources/csv_source.py
 src/pjk/sources/dir_source.py
 src/pjk/sources/factory.py
@@ -67,7 +67,6 @@ src/pjk/sources/json_source.py
 src/pjk/sources/lazy_file.py
 src/pjk/sources/lazy_file_local.py
 src/pjk/sources/lazy_file_s3.py
-src/pjk/sources/macro_source.py
 src/pjk/sources/npy_source.py
 src/pjk/sources/parquet_source.py
 src/pjk/sources/s3_source.py

python_jack_knife-0.6.17/src/pjk/sources/configs_source.py

@@ -1,52 +0,0 @@
-# template for Source used by create sink
-
-from pjk.components import Source
-from pjk.usage import Usage, ParsedToken, CONFIG_FILE
-import yaml
-from pathlib import Path
-import yaml
-from pathlib import Path
-
-class YamlRecords:
-    def __init__(self, path):
-        self.path = Path(path).expanduser()
-
-    def __iter__(self):
-        with self.path.open("r", encoding="utf-8") as f:
-            data = yaml.safe_load(f) or {}
-        if not isinstance(data, dict):
-            raise ValueError("Top-level YAML must be a mapping of records")
-        
-        for name, body_dict in data.items():
-            component, instance = name.split('-', 1)
-            out = {
-                'entry': name,
-                'component': component,
-                'instance': instance,
-                'settings': body_dict
-            }
-            #out.update(body_dict)
-            yield out
-
-class ConfigsSource(Source):
-    @classmethod
-    def usage(cls):
-        usage = Usage(
-            name='configs',
-            desc=f'A source of pjk configuration in {CONFIG_FILE}',
-            component_class=cls
-    	)
-        usage.def_example(expr_tokens=['configs', '-'], expect=None)
-        return usage
-
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        self.config_recs = YamlRecords(CONFIG_FILE)
-
-    def __iter__(self):
-        yield from self.config_recs
-
-    def deep_copy(self):
-        return None
-
-    def close(self):
-        pass

python_jack_knife-0.6.17/src/pjk/sources/macro_source.py

@@ -1,46 +0,0 @@
-from pjk.usage import Usage, ParsedToken
-from pjk.components import Source
-from pathlib import Path
-from typing import Dict
-
-MACROS_FILE = '~/.pjk/macros.txt'
-MACRO_PREFIX = 'm'
-
-def read_macros(file_name: str = MACROS_FILE) -> Dict[str, str]:
-    out: Dict[str, str] = {}
-    path = Path(file_name).expanduser()
-    with path.open(encoding="utf-8") as f:
-        for raw in f:
-            line = raw.split("#", 1)[0].strip()
-            if not line or ":" not in line:
-                continue
-            key, val = line.split(":", 1)
-            out[key.strip()] = val.strip()
-    return out
-
-class MacroSource(Source):
-    @classmethod
-    def usage(cls):
-        u = Usage(
-            name='macros',
-            desc=f"A source of macros stored in {MACROS_FILE}\n"
-            f"Specific macros are referenced as '{MACRO_PREFIX}:<instance>', e.g. 'pjk {MACRO_PREFIX}:hw -'",
-            component_class=cls
-    	)
-        u.def_example(["macros", "-"], None)
-        return u
-
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        pass
-
-    # only the instance=+ case comes here.  See parser
-    def __iter__(self):
-        macros = read_macros()
-        for k, v in macros.items():
-            yield {k: v}
-
-    def deep_copy(self):
-        return None
-
-    def close(self):
-        pass