python-jack-knife 0.6.12__tar.gz → 0.6.14__tar.gz

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.12
+Version: 0.6.14
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/common.py

@@ -49,7 +49,7 @@ def pager_stdout(use_pager: bool = True):
 
     env = os.environ.copy()
     # -R: pass ANSI; -S: chop long lines; you can add -F/-X to taste
-    env.setdefault("LESS", "-R")
+    env.setdefault("LESS", "-RFX")
     # Ensure UTF-8
     env.setdefault("LESSCHARSET", "utf-8")
 
@@ -161,12 +161,6 @@ class ComponentFactory:
             return None
         return wrapper.comp_class
 
-    #def get_usage(self, name: str):
-    #    comp_class = self.get_component_class(name)
-    #   if not comp_class:
-    #       return None
-    #   return comp_class.usage()
-
     def create(self, token: str):
         pass
 

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/integrations/postgres_pipe.py

@@ -97,7 +97,8 @@ class PostgresPipe(QueryPipe,Integration):
     examples = [
         ['myquery.sql', 'postgres:mydb', '-'],
         ["{'query': 'SELECT * from MY_TABLE;'}", 'postgres:mydb', '-'],
-        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'postgres:mydb']
+        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'postgres:mydb'],
+        ["{'query': 'SELECT stored_procedure(%s, ...), batch_params:{...}"]
     ]
 
     # name, type, default
@@ -165,14 +166,32 @@ class PostgresPipe(QueryPipe,Integration):
         try:
             query = record.get(self.query_field)
             if not query:
-                    record['_error'] = 'missing query'
-                    yield record
-            else:        
-                params = record.get(self.params_field)
-
-                cur = client.conn.cursor()
-                try:
-                    # execute
+                record['_error'] = 'missing query'
+                yield record
+                return
+
+            params = record.get(self.params_field)          # single-exec params
+            batch  = record.get("batch_params", None)       # list[tuple|dict] for batching
+
+            cur = client.conn.cursor()
+            try:
+                did_executemany = False
+
+                # ---------- execute ----------
+                if batch is not None:
+                    # Handle batch sizes explicitly to preserve single-SELECT streaming semantics
+                    if len(batch) == 0:
+                        # No-op batch; execute a lightweight statement so we can still emit a header
+                        cur.execute("SELECT 1")
+                        header_params = {"batch_size": 0}
+                    elif len(batch) == 1:
+                        cur.execute(query, batch[0])
+                        header_params = batch[0]
+                    else:
+                        cur.executemany(query, batch)
+                        did_executemany = True
+                        header_params = {"batch_size": len(batch)}
+                else:
                     if params is None:
                         cur.execute(query)
                     else:
@@ -180,17 +199,20 @@ class PostgresPipe(QueryPipe,Integration):
                             cur.execute(query, params)
                         else:
                             cur.execute(query, (params,))
+                    header_params = params
 
-                    # yield header first
-                    yield self._make_header(cur, query, params)
-
-                    # then stream rows if it was a real SELECT with results
-                    if cur.description:
-                        cols = [d[0] for d in cur.description]
-                        if not (len(cols) == 1 and cols[0] == "ingest_event"):
-                            for row in cur:
-                                yield _row_to_dict(cur, row)
-                finally:
-                    cur.close()
+                # ---------- header ----------
+                yield self._make_header(cur, query, header_params)
+
+                # ---------- stream rows (only meaningful for single execute that returns rows) ----------
+                # Note: executemany() typically doesn't expose per-execution result sets.
+                if not did_executemany and cur.description:
+                    cols = [d[0] for d in cur.description]
+                    if not (len(cols) == 1 and cols[0] == "ingest_event"):
+                        for row in cur:
+                            yield _row_to_dict(cur, row)
+
+            finally:
+                cur.close()
         finally:
             client.close()

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/main.py

@@ -10,7 +10,6 @@ from pjk.parser import ExpressionParser
 from pjk.usage import UsageError
 from pjk.log import init as init_logging
 from datetime import datetime
-from pathlib import Path
 import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
@@ -26,6 +25,10 @@ def write_history(tokens):
     
     log_path = ".pjk-history.txt"
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+
+    if len(tokens) < 2: 
+        return 
+    
     command = " ".join(tokens)
 
     try:
@@ -67,6 +70,7 @@ def execute_threaded(sinks, stop_progress=None):
 
 def initialize():
     init_logging()
+    write_history(sys.argv[1:])
 
     #src = Path("src/pjk/resources/configs.tmpl")
     #dst_dir = Path.home() / ".pjk"
@@ -121,8 +125,6 @@ def execute_tokens(tokens: List[str]):
         else:
             sink.drain()
 
-        write_history(sys.argv[1:])
-
     except UsageError as e:
         print(e, file=sys.stderr)
         sys.exit(2)

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/pipes/let_reduce.py

@@ -24,7 +24,7 @@ def do_eval(expr, env):
         safe_env['json'] = json
         return eval(expr, {}, safe_env)
     except Exception:
-        raise UsageError(f"UsageError in expression: {expr}")
+        raise Exception(f"Error in expression: {expr}")
 
 def eval_regular(expr: str, record: dict):
     env = {'f': SafeNamespace(record)}

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/pipes/map.py

@@ -51,7 +51,7 @@ class MapByPipe(Pipe, KeyedSource):
         key_rec = {}
         for field in self.fields:
             key_val = record.pop(field, None) if self.is_group else record.get(field)
-            if not key_val:
+            if key_val is None: # not only false-ish but NONE
                 return None
             
             key_rec[field] = key_val

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/pipes/query_pipe.py

@@ -2,7 +2,7 @@ from pjk.components import Pipe
 from pjk.usage import ParsedToken, Usage, CONFIG_FILE
 from typing import Any, Dict, Iterable, Optional
 from abc import abstractmethod
-
+from pjk.progress import papi
 
 class QueryPipe(Pipe):
     name: str = None
@@ -40,6 +40,8 @@ class QueryPipe(Pipe):
         self.output_shape = usage.get_param('shape')
         self.count = usage.get_param('count')
         self.query_field = 'query' # for all subclasses
+        self.inrecs = papi.get_counter(self, var_label=None) # don't display progress
+        self.outrecs = papi.get_percentage_counter(self, var_label='recs_out', denom_counter=self.inrecs)
 
     @abstractmethod
     def execute_query_returning_S_xO_iterable(self, record) -> Iterable[Dict[str, Any]]:
@@ -53,6 +55,7 @@ class QueryPipe(Pipe):
 
     def __iter__(self):
         for in_rec in self.left:
+            self.inrecs.increment()
             iter = self.execute_query_returning_S_xO_iterable(in_rec)
 
             if self.output_shape == 'S_xO':
@@ -60,8 +63,11 @@ class QueryPipe(Pipe):
                 for out_rec in iter:
                     if not q_done:
                         q_done = True
+                        self.outrecs.increment()
                         yield self._make_q_object(in_rec, out_rec)
                         continue
+
+                    self.outrecs.increment()
                     yield out_rec
 
             elif self.output_shape == 'xO':
@@ -70,6 +76,7 @@ class QueryPipe(Pipe):
                     if not q_done:
                         q_done = True
                         continue
+                    self.outrecs.increment()
                     yield out_rec
 
             elif self.output_shape == 'Sxo':
@@ -84,6 +91,7 @@ class QueryPipe(Pipe):
                         continue
                     r_list.append(out_rec)
                 q_out['child'] = r_list
+                self.outrecs.increment()
                 yield q_out
 
 

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/pipes/remove_field.py

@@ -3,10 +3,10 @@
 
 # djk/pipes/remove_field.py
 
-from pjk.components import Pipe
+from pjk.components import DeepCopyPipe
 from pjk.usage import ParsedToken, Usage, UsageError
 
-class RemoveField(Pipe):
+class RemoveField(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = Usage(

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/registry.py

@@ -3,7 +3,6 @@
 
 import os
 import sys
-import time
 from pjk.sinks.factory import SinkFactory
 from pjk.pipes.factory import PipeFactory
 from pjk.sources.factory import SourceFactory
@@ -11,8 +10,6 @@ from pjk.sinks.format_sink import FormatSink
 from pjk.sources.format_source import FormatSource
 import importlib.util
 import importlib
-import importlib.metadata
-import sysconfig, pathlib, importlib
 from pjk.components import Pipe, Source, Sink
 from pjk.common import ComponentFactory, highlight, ComponentOrigin
 from typing import List, Type
@@ -38,7 +35,7 @@ class ComponentRegistry:
         self.pipe_factory = PipeFactory()
         self.sink_factory = SinkFactory()
         self.load_user_components()
-        self.load_package_extras()
+        self.load_namespace_extras()
 
     def create_source(self, token: str):
         return self.source_factory.create(token)
@@ -125,42 +122,35 @@ class ComponentRegistry:
                     elif is_source(obj, module):
                         self.source_factory.register(name, obj, ComponentOrigin.USER)
 
-    def iter_entry_points(self, group: str):
-        """
-        Return entry points in the given group across Python 3.9–3.12.
-        """
-        eps = importlib.metadata.entry_points()
-        return eps.select(group=group) if hasattr(eps, "select") else eps.get(group, [])
+    def load_namespace_extras(self, package: str = "pjk_extras") -> None:
+        registrar = ExternalRegistrar(self.source_factory, self.pipe_factory, self.sink_factory)
+        import importlib, importlib.metadata as im
 
-    def load_package_extras(self, group: str = "pjk.package_extras") -> None:
-        """
-        Load pip-installed extras and register their components into THIS registry.
+        for dist in im.distributions():
+            name = (dist.metadata.get("Name") or "")
+            if not name.startswith("pjk-"):
+                continue
 
-        Preferred contract:
-        entry point -> callable register(registrar)
+            modname = f"{package}.{name[4:].replace('-', '_')}"  # pjk-foo-bar -> pjk_extras.foo_bar
 
-        Fallback (legacy):
-        entry point -> module path; we import the MODULE PART ONLY for side-effects.
-        """
-        registrar = ExternalRegistrar(self.source_factory, self.pipe_factory, self.sink_factory)
+            # Import the extra; if it fails, continue to the next
+            try:
+                mod = importlib.import_module(modname)
+            except Exception as e:
+                print(f"[pjk] import failed for {modname}: {e}")
+                continue
+
+            reg = getattr(mod, "register", None)
+            if not callable(reg):
+                print(f"[pjk] extra '{modname}' has no register(registrar)")
+                continue
 
-        for ep in self.iter_entry_points(group):
+            # Run its register; if it fails, continue to the next
             try:
-                # Try the modern, explicit path first.
-                loader = getattr(ep, "load", None)
-                if callable(loader):
-                    target = ep.load()  # resolves "module:object" to the actual object
-                    if callable(target):
-                        target(registrar)  # plugin registers into your live factories
-                        #print(f"[pjk] loaded extra (callable): {ep.name} -> {ep.value}")
-                        continue
-                    # Not callable -> fall through to legacy import
-                # Legacy path: import ONLY the module portion before ':' for side-effects
-                mod = ep.value.split(":", 1)[0]
-                importlib.import_module(mod)
-                #print(f"[pjk] loaded extra (import): {ep.name} -> {ep.value}")
+                reg(registrar)   # registers class TYPES, same contract as before
             except Exception as e:
-                print(f"[pjk] failed to load extra {ep.name}: {e}")
+                print(f"[pjk] register() failed in {modname}: {e}")
+                continue
 
 def print_core_formats(factories: List[ComponentFactory]):
     print(highlight('formats'))

python_jack_knife-0.6.14/src/pjk/sinks/stdout.py

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import sys
+import yaml
+from yaml.representer import SafeRepresenter  # kept for compatibility
+from pjk.components import Sink, Source
+from pjk.usage import ParsedToken, Usage
+from pjk.common import pager_stdout
+
+
+class StdoutSink(Sink):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='-',
+            desc='display records in yaml or txt format to stdout through less',
+            component_class=cls
+        )
+        usage.def_param('less', usage='use less to display', valid_values=['true', 'false'], default='true')
+        usage.def_param('format', usage='output format', valid_values=['yaml', 'txt'], default='yaml')
+        usage.def_example(["{hello:'world!'}"], "{hello:'world!'}")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.use_pager = True if usage.get_param('less') is None else usage.get_param('less') == 'true'
+        self.output_format = usage.get_param('format') or 'yaml'
+
+    def _sanitize_scalar(self, v) -> str:
+        if v is None:
+            s = ''
+        elif isinstance(v, (list, dict)):
+            # single-line YAML to preserve structure compactly
+            s = yaml.safe_dump(v, sort_keys=False, allow_unicode=True).strip()
+        else:
+            s = str(v)
+        return s.replace('\r', ' ').replace('\n', ' ')
+
+    def _process_yaml(self):
+        for record in self.input:
+            try:
+                yaml.dump(
+                    record,
+                    sys.stdout,
+                    sort_keys=False,
+                    explicit_start=True,  # '---' before each record
+                    allow_unicode=True,
+                    width=10**9,
+                )
+            except BrokenPipeError:
+                break
+
+    def _process_txt(self):
+        for record in self.input:
+            try:
+                # record delimiter
+                sys.stdout.write('---\n')
+                if isinstance(record, dict):
+                    for k, v in record.items():
+                        key = str(k)
+                        val = self._sanitize_scalar(v)
+                        sys.stdout.write(f'{key}: {val}\n')
+                else:
+                    val = self._sanitize_scalar(record)
+                    sys.stdout.write(f'value: {val}\n')
+            except BrokenPipeError:
+                break
+
+    def process(self) -> None:
+        try:
+            with pager_stdout(self.use_pager):
+                if self.output_format == 'txt':
+                    self._process_txt()
+                else:
+                    self._process_yaml()
+        except BrokenPipeError:
+            pass

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.12"
+__version__ = "0.6.14"

{python_jack_knife-0.6.12 → python_jack_knife-0.6.14}/src/python_jack_knife.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.12
+Version: 0.6.14
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

python_jack_knife-0.6.12/src/pjk/sinks/stdout.py

@@ -1,46 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Copyright 2024 Mike Schultz
-
-import sys
-import yaml
-from yaml.representer import SafeRepresenter
-from pjk.components import Sink, Source
-from pjk.usage import ParsedToken, Usage
-from pjk.common import pager_stdout
-
-class StdoutSink(Sink):
-    @classmethod
-    def usage(cls):
-        usage = Usage(
-            name='-',
-            desc='display records in yaml format to stdout through less',
-            component_class=cls
-        )
-        usage.def_param('less', usage='use less to display', valid_values=['true', 'false'], default='true')
-        usage.def_example(["{hello:'world!'}"], "{hello:'world!'}")
-        return usage
-    
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
-        self.use_pager = True if usage.get_param('less') == None else usage.get_param('less') == 'true'
-
-    def process(self) -> None:
-        try:
-            with pager_stdout(self.use_pager):
-                for record in self.input:
-                    try:
-                        # everything else -> normal YAML
-                        yaml.dump(
-                            record,
-                            sys.stdout,
-                            sort_keys=False,
-                            explicit_start=True,
-                            allow_unicode=True,
-                            width=10**9,
-                        )
-                    except BrokenPipeError:
-                        break
-        except BrokenPipeError:
-            pass
-
-