python-jack-knife 0.7.4__tar.gz → 0.7.6__tar.gz

{python_jack_knife-0.7.4/src/python_jack_knife.egg-info → python_jack_knife-0.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.7.4
+Version: 0.7.6
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 
@@ -212,6 +212,7 @@ License-File: LICENSE
 Requires-Dist: hjson>=3.1.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.32.0
+Requires-Dist: deepdiff<9,>=8.0.0
 Provides-Extra: aws
 Requires-Dist: boto3>=1.34; extra == "aws"
 Provides-Extra: postgres
@@ -225,6 +226,9 @@ Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: opensearch-py; extra == "dev"
 Provides-Extra: all
 Requires-Dist: boto3>=1.34; extra == "all"
 Requires-Dist: pg8000>=1.30.0; extra == "all"

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "hjson>=3.1.0",
   "pyyaml>=6.0",
   "requests>=2.32.0",
+  "deepdiff>=8.0.0,<9",
 ]
 
 [project.optional-dependencies]
@@ -24,7 +25,7 @@ aws = ["boto3>=1.34"]
 postgres = ["pg8000>=1.30.0"]
 parquet = ["pyarrow>=15.0.0"]
 plot = ["matplotlib>=3.9.0", "pandas>=2.2.0"]
-dev = ["pytest", "black", "ruff"]
+dev = ["pytest", "black", "ruff", "build", "twine", "opensearch-py"]
 all = [
   "boto3>=1.34",
   "pg8000>=1.30.0",

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/__init__.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 from .version import __version__
+from .engine import PjkEngine
 
-__all__ = ["__version__"]
+__all__ = ["__version__", "PjkEngine"]

python_jack_knife-0.7.6/src/pjk/engine.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from typing import Iterator, List, Optional
+
+from pjk.parser import ExpressionParser, expand_macros
+from pjk.registry import ComponentRegistry
+from pjk.sources.dict_list_source import DictListSource
+
+
+class PjkEngine:
+    """
+    Run a pjk pipeline from a .pjk file, optionally with supplied input records.
+
+    - inrecs supplied: the source in the .pjk file is replaced with inrecs.
+      Expression may be full (source + pipes + sink) or pipes-only.
+    - inrecs=None: expression.pjk is fully self-contained (source, pipes, sink)
+    """
+
+    def __init__(self, inrecs: Optional[List[dict]] = None, pjk_file: str = ""):
+        self.inrecs = inrecs
+        self.pjk_file = pjk_file
+
+    def __iter__(self) -> Iterator[dict]:
+        registry = ComponentRegistry()
+        parser = ExpressionParser(registry)
+        expanded = expand_macros([self.pjk_file])
+
+        if self.inrecs is not None:
+            source_override = DictListSource(self.inrecs)
+            try:
+                first_is_source = registry.create_source(expanded[0]) is not None
+            except Exception:
+                first_is_source = False
+            if first_is_source:
+                expanded = ["{to_override: 'true'}"] + expanded[1:]
+            else:
+                expanded = ["{to_override: 'true'}"] + expanded
+        else:
+            source_override = None
+
+        sink = parser.parse(expanded, source_override=source_override)
+
+        inputs = [sink.input]
+        sink.input._get_sources(inputs)
+        try:
+            for record in sink.input:
+                yield record
+        finally:
+            for inp in inputs:
+                inp.close()

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/history.py

@@ -91,7 +91,7 @@ def display_history():
 
         ordn = 1
         for command in reversed(clist):
-            print(f'{ordn}\t{command}')
+            print(f'{ordn}\tpjk {command}')
             ordn += 1
 
 def get_history_tokens(ord_str: str):

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/integrations/opensearch_client.py

@@ -2,20 +2,20 @@ from pjk.usage import Usage
 
  # name, type, default
 OS_CONFIG_TUPLES = [
-    ("default_index", str, None),
+    ("default_index", str, None, True),
     ("os_auth_use_aws", bool, "true"),
     ("os_scheme", str, "https"),
     ("os_verify_certs", bool, "true"),
-    ("os_ca_certs", str, None),
-    ("os_region", str, None),
+    ("os_ca_certs", str, None, True),
+    ("os_region", str, None, True),
     ("os_service", str, "es"),
-    ("os_username", str, None),
-    ("os_password", str, None),
+    ("os_username", str, None, True),
+    ("os_password", str, None, True),
     ("os_timeout", float, 30),
     ("os_ssl_assert_hostname", bool, "true"),
     ("os_ssl_show_warn", bool, "false"),
     ("os_host", str, None),
-    ("os_port", int, None)
+    ("os_port", int, None, True),
 ]
 
 class OpenSearchClient:

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/integrations/opensearch_query_pipe.py

@@ -108,8 +108,7 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
             yield {
                 "took_ms": took,
                 "total_hits": total_hits,
-                "index": self.index,
-                "os_query_object": req_body
+                "index": self.index
             }
 
             # Emit each hit

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/integrations/postgres_pipe.py

@@ -5,6 +5,8 @@
 
 import base64
 import datetime as _dt
+import socket
+import sys
 import uuid
 import time
 from decimal import Decimal
@@ -17,6 +19,21 @@ from pjk.pipes.query_pipe import QueryPipe
 MAX_RETRIES = 3
 BASE_DELAY = 0.1  # seconds
 
+
+def _print_db_connect_failure(host: str, port: int, exc: BaseException) -> None:
+    print("Failed to connect to DB", file=sys.stderr)
+    cur: Optional[BaseException] = exc
+    while cur is not None:
+        if isinstance(cur, socket.gaierror):
+            print(
+                f"  Could not resolve hostname {host!r} (port {port}). "
+                "Private or corporate DB hosts usually require VPN or split-DNS.",
+                file=sys.stderr,
+            )
+            return
+        cur = cur.__cause__
+
+
 class DBClient:
     """Per-instance pg8000 connection wrapper. No shared state."""
 
@@ -47,8 +64,8 @@ class DBClient:
             self.conn = pg8000.connect(**kwargs)
             self.conn.autocommit = True
         except Exception as e:
-            print("Failed to connect to DB")
-            raise e
+            _print_db_connect_failure(host, port, e)
+            raise
 
     def close(self):
         if getattr(self, "conn", None) is None:

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/main.py

@@ -14,7 +14,7 @@ import concurrent.futures
 from pjk.registry import ComponentRegistry
 from pjk.sinks.stdout import StdoutSink
 from pjk.man_page import do_man, do_examples, display_configs, display_macros
-from pjk.history import write_history, display_history, get_history_tokens
+from pjk.history import write_history, display_history, get_history_tokens, printable_command
 from pjk.sinks.expect import ExpectSink
 from pjk.progress import ProgressDisplay
 from pjk.version import __version__
@@ -52,7 +52,6 @@ def execute_threaded(sinks, stop_progress=None):
 
 def initialize():
     init_logging()
-    write_history(sys.argv[1:])
 
     #src = Path("src/pjk/resources/configs.tmpl")
     #dst_dir = Path.home() / ".pjk"
@@ -111,12 +110,15 @@ def execute_tokens(tokens: List[str]):
         if not tokens:
             print('No such history')
             return
+        cmd = printable_command(tokens)
+        print(f"pjk {cmd}")
 
     parser = ExpressionParser(registry)
 
     display = None
     try:
         sink = parser.parse(tokens)
+        write_history(sys.argv[1:]) # now that it's parsed sucessfully
         if not isinstance(sink, (StdoutSink | ExpectSink)):
             display = ProgressDisplay(interval=3.0)
             display.start()

python_jack_knife-0.7.6/src/pjk/parse_pjk_file.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+import re
+import shlex
+from typing import Dict, List
+from pjk.usage import TokenError, UsageError
+
+PJK_END_TOKEN = 'END'
+PJK_SET_TOKEN = 'SET'
+
+# ${VAR} or $VAR - match anywhere in token (${VAR} first to avoid partial match)
+VAR_REF_PATTERN = re.compile(r'\$\{([a-zA-Z_][a-zA-Z0-9_]*)\}|\$([a-zA-Z_][a-zA-Z0-9_]*)')
+
+
+def _expand_token(t: str, env: Dict[str, str]) -> str:
+    """Expand $VAR or ${VAR} anywhere in token; raise if undefined."""
+
+    def repl(m):
+        name = m.group(1) or m.group(2)
+        if name not in env:
+            raise TokenError(f"Undefined variable: ${name}")
+        return env[name]
+
+    return VAR_REF_PATTERN.sub(repl, t)
+
+
+def handle_pjk_file(token: str, expanded: List[str]):
+    if not token.endswith(".pjk"):
+        return False
+
+    if not os.path.isfile(token):
+        raise TokenError(f"pjk file not found: {token}")
+
+    with open(token, "r") as f:
+        lines = f.readlines()
+
+    env: Dict[str, str] = {}
+
+    for line in lines:
+        try:
+            parts = shlex.split(line, comments=True, posix=True)
+        except ValueError as e:
+            raise UsageError(f"Error parsing {token}: {e}")
+
+        if not parts:
+            continue
+        if parts[0] == PJK_END_TOKEN:
+            break
+
+        if parts[0] == PJK_SET_TOKEN:
+            for p in parts[1:]:
+                if '=' in p:
+                    k, v = p.split('=', 1)
+                    env[k.strip()] = v.strip()
+            continue
+
+        for p in parts:
+            if p == PJK_END_TOKEN:
+                break
+            expanded.append(_expand_token(p, env))
+        else:
+            continue
+        break
+    return True

{python_jack_knife-0.7.4 → python_jack_knife-0.7.6}/src/pjk/parser.py

@@ -13,6 +13,8 @@ from pjk.progress import papi
 from typing import Dict
 from pathlib import Path
 from pjk.progress import ProgressIgnore
+from pjk.parse_pjk_file import handle_pjk_file
+from pjk.common import SafeNamespace
 
 MACROS_FILE = '~/.pjk/macros.txt'
 MACRO_PREFIX = 'm'
@@ -47,27 +49,6 @@ def handle_macros(token: str, expanded: List[str]):
     
     return True
 
-def handle_pjk_file(token: str, expanded: List[str]):
-    if not token.endswith(".pjk"):
-        return False
-    
-    if not os.path.isfile(token):
-        raise TokenError(f"pjk file not found: {token}")
-    
-    with open(token, "r") as f:
-        lines = f.readlines()
-
-    # Remove comments outside quotes, then split
-    stripped = []
-    for line in lines:
-        try:
-            parts = shlex.split(line, comments=True, posix=True)
-            stripped.extend(parts)
-        except ValueError as e:
-            raise UsageError(f"Error parsing {token}: {e}")
-    expanded.extend(stripped)
-    return True
-
 def expand_macros(tokens: List[str]) -> List[str]:
     expanded = []
     for token in tokens:
@@ -130,7 +111,7 @@ class ExpressionParser:
 
         source = self.stack.pop()
         if isinstance(source, SubExpression):
-            raise TokenError("Poorly formed sub-expression.  Begin token '[' without matching 'over' keyword." )
+            raise TokenError("Poorly formed sub-expression.  Begin token '[' without matching 'over' or 'if' keyword." )
 
         if not self.stack.empty():
             raise TokenError.from_list(['A sink can only consume one source.',
@@ -145,7 +126,7 @@ class ExpressionParser:
         sink = self.registry.create_sink(token)
         
         if not sink:
-            raise TokenError.from_list(['expression must end in a sink.',
+            raise TokenError.from_list(['non-sink in final position.',
                             'pjk <source> [<pipe> ...] <sink>'])
         
         # so each sink doesn't have to, maybe make a base class or mixin for sinks
@@ -155,7 +136,7 @@ class ExpressionParser:
         sink.add_source(progress_pipe)
         return sink
 
-    def parse(self, tokens: List[str]) -> Sink:
+    def parse(self, tokens: List[str], source_override=None) -> Sink:
         usage_error_message = "You've got a problem here."
         stack_helper = StackLoader()
         self.tokens = tokens
@@ -173,7 +154,9 @@ class ExpressionParser:
                     return self.get_sink(stack_helper, token)
                     
                 source = self.registry.create_source(token)
-                if source:                    
+                if source:
+                    if pos == 0 and source_override is not None:
+                        source = source_override
                     stack_helper.add_operator(source, self.stack)
                     progress_pipe = ProgressPipe(component=source, simple=True)
                     stack_helper.add_operator(progress_pipe, self.stack)
@@ -184,6 +167,16 @@ class ExpressionParser:
                     stack_helper.add_operator(subexp, self.stack)
                     continue
 
+                if not self.stack.empty() and isinstance(self.stack.peek(), SubExpression):
+                    if token == "else":
+                        self.stack.peek().enter_else_branch()
+                        continue
+                    if token.startswith("if:"):
+                        op = self.stack.peek().finish_conditional(token)
+                        if op:
+                            stack_helper.add_operator(op, self.stack)
+                            continue
+
                 pipe = self.registry.create_pipe(token)
                 if pipe:
                     stack_helper.add_operator(pipe, self.stack)
@@ -191,11 +184,10 @@ class ExpressionParser:
 
                 else: # unrecognized token
                     # could be sink in WRONG position, let's see for better error message
-                    sink = self.registry.create_sink(token) 
-                    if sink:
-                        raise TokenError.from_list(['sink may only occur in final position.',
+                    if self.registry.is_sink(token):
+                        raise TokenError.from_list(['sink in non-final position.',
                                             'pjk <source> [<pipe> ...] <sink>'])
-                    raise TokenError.from_list([token, 'unrecognized token'])
+                    raise TokenError.from_list([f"'{token}' unrecognized."])
         
         except TokenError as e:
             raise UsageError(usage_error_message, self.tokens, pos, e)
@@ -237,7 +229,7 @@ class StackLoader:
         if not stack.empty() and isinstance(stack.peek(), SubExpression):
             subexp = stack.peek()
 
-            if isinstance(op, SubExpressionOver) and subexp.recursion_depth() == 0:
+            if isinstance(op, (SubExpressionOver, SubExpressionIf)) and subexp.recursion_depth() == 0:
                 subexp = stack.pop()
                 op.add_source(subexp)
                 stack.push(op)
@@ -343,6 +335,64 @@ class SubExpressionOver(Pipe):
             self.left.subexp_process(record, self.over_arg)
             yield record
 
+
+class IdentityPipe(Pipe):
+    """Pass-through pipe for empty else branch."""
+
+    def __iter__(self):
+        yield from self.left
+
+
+class SubExpressionIf(Pipe):
+    """Closes a conditional block: [ then_ops else else_ops if:expr"""
+
+    @classmethod
+    def usage(cls) -> Usage:
+        u = Usage(name="if", desc="conditional sub-expression.", component_class=cls)
+        return u
+
+    def __init__(self, expr: str, then_chain, else_chain, upstream_source):
+        super().__init__(None, None)
+        self.expr = expr
+        self.then_chain = then_chain
+        self.else_chain = else_chain
+        self.upstream_source = upstream_source
+        self.inrecs = papi.get_counter(self, var_label='recs_in', display=False)
+        self.recs_true = papi.get_percentage_counter(self, var_label='recs_true', denom_counter=self.inrecs)
+        try:
+            self.code = compile(expr, '<if>', 'eval')
+        except Exception as e:
+            raise UsageError(f"Invalid if expression: {expr}") from e
+
+    def reset(self):
+        if self.then_chain:
+            self.then_chain.reset()
+        if self.else_chain:
+            self.else_chain.reset()
+
+    def __iter__(self):
+        for record in self.left:
+            self.inrecs.increment()
+            self.upstream_source.set_list([record])
+            f = SafeNamespace(record)
+            try:
+                cond_true = eval(self.code, {}, {'f': f})
+            except Exception:
+                cond_true = False
+            if cond_true:
+                self.recs_true.increment()
+                chain = self.then_chain
+            else:
+                chain = self.else_chain
+
+            if chain:
+                chain.reset()
+                for r in chain:
+                    yield r
+            else:
+                yield record
+
+
 class SubExpression(Pipe, ProgressIgnore):
     @classmethod
     def create(cls, token: str) -> Pipe:
@@ -357,17 +407,42 @@ class SubExpression(Pipe, ProgressIgnore):
         super().__init__(ptok, usage)
         self.subexp_ops = []
         self.stack_helper = StackLoader()
-        self.subexp_stack = OperandStack() 
+        self.subexp_stack = OperandStack()
         self.upstream_source = UpstreamSource()
         self.subexp_stack.push(self.upstream_source)
-        self.recursions = 0 # number of subexpression within
+        self.recursions = 0
         self.subexp_left = None
+        self.in_else_branch = False
+        self.conditional_then_chain = None
+
+    def enter_else_branch(self):
+        """Switch to collecting else branch; save then chain."""
+        self.conditional_then_chain = self.subexp_stack.pop()
+        self.subexp_stack.push(self.upstream_source)
+        self.in_else_branch = True
+
+    def finish_conditional(self, token: str):
+        """Build SubExpressionIf from collected then/else chains."""
+        expr = token.split(':', 1)[1]
+        if self.in_else_branch:
+            else_chain = self.subexp_stack.pop()
+            then_chain = self.conditional_then_chain
+        else:
+            then_chain = self.subexp_stack.pop()
+            else_chain = None
+        if else_chain is self.upstream_source:
+            else_chain = IdentityPipe(None, None)
+            else_chain.add_source(self.upstream_source)
+        self.in_else_branch = False
+        self.conditional_then_chain = None
+        self.subexp_stack.push(self.upstream_source)
+        return SubExpressionIf(expr, then_chain, else_chain, self.upstream_source)
 
     def add_subop(self, op):
         self.subexp_ops.append(op)
         if isinstance(op, SubExpression):
             self.recursions += 1
-        elif isinstance(op, SubExpressionOver):
+        elif isinstance(op, (SubExpressionOver, SubExpressionIf)):
             self.recursions -= 1
         self.stack_helper.add_operator(op, self.subexp_stack)
 

python_jack_knife-0.7.6/src/pjk/pipes/ddiff.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# pjk/pipes/ddiff.py
+
+import json
+from itertools import zip_longest
+
+from deepdiff import DeepDiff
+
+from pjk.components import Pipe
+from pjk.usage import Usage, ParsedToken
+from pjk.progress import papi
+
+_PAD = object()
+
+_DDIFF_OLD_NEW_KEYS = (
+    ("old_value", "left_value"),
+    ("new_value", "right_value"),
+    ("old_type", "left_type"),
+    ("new_type", "right_type"),
+)
+
+
+def _diff_left_right_labels(obj):
+    """Map DeepDiff old/new keys to left/right (matches stream argument order)."""
+    if isinstance(obj, dict):
+        out = {}
+        for k, v in obj.items():
+            for old_k, new_k in _DDIFF_OLD_NEW_KEYS:
+                if k == old_k:
+                    k = new_k
+                    break
+            out[k] = _diff_left_right_labels(v)
+        return out
+    if isinstance(obj, list):
+        return [_diff_left_right_labels(x) for x in obj]
+    return obj
+
+
+class DdiffPipe(Pipe):
+    arity = 2
+
+    @classmethod
+    def usage(cls):
+        u = Usage(
+            name="ddiff",
+            desc=(
+                "Lockstep deep diff of two record streams."
+            ),
+            component_class=cls,
+        )
+        u.def_syntax(
+            "pjk <left_source> <right_source> ddiff ..."
+        )
+        u.def_param(
+            name="ignore_order",
+            usage="DeepDiff ignore_order (lists and sets)",
+            valid_values={"true", "false"},
+            default="false",
+        )
+        u.def_param(
+            name="omit_equal",
+            usage="Suppress output when the two records are deeply equal",
+            valid_values={"true", "false"},
+            default="false",
+        )
+        u.def_param(
+            name="significant_digits",
+            usage="DeepDiff significant_digits for numeric comparisons",
+            is_num=True,
+            default=None,
+        )
+        ferry_ford = "[{ferry:'orca', cars:[{make: 'ford', size:9}]}]"
+        ferry_bmw = "[{ferry:'orca', cars:[{make: 'bmw', size:4}]}]"
+        ferry_ford_rec = {"ferry": "orca", "cars": [{"make": "ford", "size": 9}]}
+        ferry_bmw_rec = {"ferry": "orca", "cars": [{"make": "bmw", "size": 4}]}
+        u.def_example(
+            expr_tokens=[ferry_ford, ferry_ford, "ddiff"],
+            expect=json.dumps(
+                [{"left": ferry_ford_rec, "right": ferry_ford_rec, "diff": {}}],
+                separators=(",", ":"),
+            ),
+        )
+        u.def_example(
+            expr_tokens=[ferry_ford, ferry_bmw, "ddiff"],
+            expect=json.dumps(
+                [
+                    {
+                        "left": ferry_ford_rec,
+                        "right": ferry_bmw_rec,
+                        "diff": _diff_left_right_labels(
+                            json.loads(
+                                DeepDiff(
+                                    ferry_ford_rec, ferry_bmw_rec
+                                ).to_json()
+                            )
+                        ),
+                    }
+                ],
+                separators=(",", ":"),
+            ),
+        )
+        return u
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.recs_in = papi.get_counter(self, "recs_in", display=False)
+        self.recs_out = papi.get_counter(self, "recs_out")
+
+    def reset(self):
+        pass
+
+    @staticmethod
+    def _truthy(param) -> bool:
+        if param is None:
+            return False
+        return str(param).lower() == "true"
+
+    def __iter__(self):
+        ignore_order = self._truthy(self.usage.get_param("ignore_order"))
+        omit_equal = self._truthy(self.usage.get_param("omit_equal"))
+        sig = self.usage.get_param("significant_digits")
+
+        dd_kwargs = {}
+        if ignore_order:
+            dd_kwargs["ignore_order"] = True
+        if sig is not None:
+            dd_kwargs["significant_digits"] = sig
+
+        for left_rec, right_rec in zip_longest(self.left, self.right, fillvalue=_PAD):
+            self.recs_in.increment()
+            if left_rec is _PAD:
+                left_rec = {}
+            if right_rec is _PAD:
+                right_rec = {}
+
+            d = DeepDiff(left_rec, right_rec, **dd_kwargs)
+            # Normalize to JSON-native dict/list (to_dict() may use e.g. SetOrdered).
+            diff_map = _diff_left_right_labels(json.loads(d.to_json()))
+            if omit_equal and not diff_map:
+                continue
+            self.recs_out.increment()
+            yield {"left": left_rec, "right": right_rec, "diff": diff_map}