python-jack-knife 0.7.1__tar.gz → 0.7.5__tar.gz

{python_jack_knife-0.7.1/src/python_jack_knife.egg-info → python_jack_knife-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.7.1
+Version: 0.7.5
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 
@@ -212,6 +212,7 @@ License-File: LICENSE
 Requires-Dist: hjson>=3.1.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: requests>=2.32.0
+Requires-Dist: deepdiff<9,>=8.0.0
 Provides-Extra: aws
 Requires-Dist: boto3>=1.34; extra == "aws"
 Provides-Extra: postgres

{python_jack_knife-0.7.1 → python_jack_knife-0.7.5}/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "hjson>=3.1.0",
   "pyyaml>=6.0",
   "requests>=2.32.0",
+  "deepdiff>=8.0.0,<9",
 ]
 
 [project.optional-dependencies]

{python_jack_knife-0.7.1 → python_jack_knife-0.7.5}/src/pjk/__init__.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 from .version import __version__
+from .engine import PjkEngine
 
-__all__ = ["__version__"]
+__all__ = ["__version__", "PjkEngine"]

python_jack_knife-0.7.5/src/pjk/engine.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from typing import Iterator, List, Optional
+
+from pjk.parser import ExpressionParser, expand_macros
+from pjk.registry import ComponentRegistry
+from pjk.sources.dict_list_source import DictListSource
+
+
+class PjkEngine:
+    """
+    Run a pjk pipeline from a .pjk file, optionally with supplied input records.
+
+    - inrecs supplied: the source in the .pjk file is replaced with inrecs.
+      Expression may be full (source + pipes + sink) or pipes-only.
+    - inrecs=None: expression.pjk is fully self-contained (source, pipes, sink)
+    """
+
+    def __init__(self, inrecs: Optional[List[dict]] = None, pjk_file: str = ""):
+        self.inrecs = inrecs
+        self.pjk_file = pjk_file
+
+    def __iter__(self) -> Iterator[dict]:
+        registry = ComponentRegistry()
+        parser = ExpressionParser(registry)
+        expanded = expand_macros([self.pjk_file])
+
+        if self.inrecs is not None:
+            source_override = DictListSource(self.inrecs)
+            try:
+                first_is_source = registry.create_source(expanded[0]) is not None
+            except Exception:
+                first_is_source = False
+            if first_is_source:
+                expanded = ["{to_override: 'true'}"] + expanded[1:]
+            else:
+                expanded = ["{to_override: 'true'}"] + expanded
+        else:
+            source_override = None
+
+        sink = parser.parse(expanded, source_override=source_override)
+
+        inputs = [sink.input]
+        sink.input._get_sources(inputs)
+        try:
+            for record in sink.input:
+                yield record
+        finally:
+            for inp in inputs:
+                inp.close()

{python_jack_knife-0.7.1 → python_jack_knife-0.7.5}/src/pjk/history.py

@@ -91,7 +91,7 @@ def display_history():
 
         ordn = 1
         for command in reversed(clist):
-            print(f'{ordn}\t{command}')
+            print(f'{ordn}\tpjk {command}')
             ordn += 1
 
 def get_history_tokens(ord_str: str):

{python_jack_knife-0.7.1 → python_jack_knife-0.7.5}/src/pjk/integrations/opensearch_query_pipe.py

@@ -108,8 +108,7 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
             yield {
                 "took_ms": took,
                 "total_hits": total_hits,
-                "index": self.index,
-                "os_query_object": req_body
+                "index": self.index
             }
 
             # Emit each hit

python_jack_knife-0.7.5/src/pjk/integrations/postgres_pipe.py

@@ -0,0 +1,285 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+#
+# djk/pipes/postgres_pipe.py
+
+import base64
+import datetime as _dt
+import socket
+import sys
+import uuid
+import time
+from decimal import Decimal
+from typing import Any, Dict, Optional
+
+from pjk.usage import ParsedToken, Usage
+from pjk.common import Integration
+from pjk.pipes.query_pipe import QueryPipe
+
+MAX_RETRIES = 3
+BASE_DELAY = 0.1  # seconds
+
+
+def _print_db_connect_failure(host: str, port: int, exc: BaseException) -> None:
+    print("Failed to connect to DB", file=sys.stderr)
+    cur: Optional[BaseException] = exc
+    while cur is not None:
+        if isinstance(cur, socket.gaierror):
+            print(
+                f"  Could not resolve hostname {host!r} (port {port}). "
+                "Private or corporate DB hosts usually require VPN or split-DNS.",
+                file=sys.stderr,
+            )
+            return
+        cur = cur.__cause__
+
+
+class DBClient:
+    """Per-instance pg8000 connection wrapper. No shared state."""
+
+    def __init__(
+        self,
+        host: str,
+        username: str,
+        password: Optional[str],
+        db_name: str,
+        port: int = 5432,
+        ssl: bool = False,
+    ):
+        import pg8000  # lazy import
+
+        kwargs = dict(
+            user=username,
+            password=password,
+            host=host,
+            database=db_name,
+            port=port,
+        )
+        if ssl:
+            import ssl as _ssl
+
+            kwargs["ssl_context"] = _ssl.create_default_context()
+
+        try:
+            self.conn = pg8000.connect(**kwargs)
+            self.conn.autocommit = True
+        except Exception as e:
+            _print_db_connect_failure(host, port, e)
+            raise
+
+    def close(self):
+        if getattr(self, "conn", None) is None:
+            return
+
+        import pg8000  # lazy
+
+        try:
+            self.conn.close()
+        except pg8000.exceptions.InterfaceError:
+            # Already closed / broken; ignore.
+            pass
+        finally:
+            self.conn = None
+
+
+def _iso_dt(x: _dt.datetime) -> str:
+    """ISO 8601; normalize UTC offset to 'Z'."""
+    s = x.isoformat()
+    return s.replace("+00:00", "Z")
+
+
+def normalize(obj: Any) -> Any:
+    """
+    Make values JSON/YAML-safe and portable (schema-agnostic):
+      - Decimal -> exact string (no sci-notation)
+      - date/datetime/time -> ISO-8601 string (datetime keeps offset; UTC -> 'Z')
+      - UUID -> string
+      - bytes -> base64 string
+      - lists/tuples/sets, dicts -> normalized recursively
+      - leaves int/float/str/bool/None as-is
+    """
+    if obj is None:
+        return None
+
+    if isinstance(obj, Decimal):
+        return format(obj, "f")  # exact value as string
+
+    if isinstance(obj, _dt.datetime):
+        return _iso_dt(obj)
+
+    if isinstance(obj, (_dt.date, _dt.time)):
+        return obj.isoformat()
+
+    if isinstance(obj, uuid.UUID):
+        return str(obj)
+
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return base64.b64encode(bytes(obj)).decode("ascii")
+
+    if isinstance(obj, dict):
+        return {k: normalize(v) for k, v in obj.items()}
+
+    if isinstance(obj, (list, tuple, set)):
+        return [normalize(v) for v in obj]
+
+    return obj
+
+
+def _row_to_dict(cursor, row) -> Dict[str, Any]:
+    cols = [d[0] for d in cursor.description]
+    return {col: normalize(val) for col, val in zip(cols, row)}
+
+
+class PostgresPipe(QueryPipe, Integration):
+    name = "postgres"
+    desc = "Postgres query pipe; executes SQL over input record['query']."
+    arg0 = ("instance", "instance of database.")
+    examples = [
+        ["myquery.sql", "postgres:mydb", "-"],
+        ["{'query': 'SELECT * from MY_TABLE;'}", "postgres:mydb", "-"],
+        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", "postgres:mydb"],
+        ["{'query': 'SELECT procedure_batch(%s, ...), batch_params:{...}"],
+        ["{'query': 'SELECT procedure_jsonb(%s, ...), json_params:json_string"],
+    ]
+
+    # name, type, default
+    config_tuples = [
+        ("db_name", str, None),
+        ("host", str, None),
+        ("user", str, None),
+        ("password", str, None),
+        ("port", int, 5432),
+        ("ssl", bool, False),
+    ]
+
+    def __init__(self, ptok: ParsedToken, u: Usage, root=None):
+        super().__init__(ptok, u, root=root)
+
+        self.db_name = u.get_config("db_name")
+        self.db_host = u.get_config("host")
+        self.db_user = u.get_config("user")
+        self.db_pass = u.get_config("password")
+        self.db_port = u.get_config("port")
+        self.db_ssl = u.get_config("ssl")
+
+        # Standard params field: single-exec params (list/tuple/dict/single value)
+        self.params_field = "params"
+
+        # Legacy batch path: list[tuple|list|dict] → executemany
+        self.batch_field = "batch_params"
+
+        # Explicit JSON payload field (no query sniffing).
+        # If present, this value is passed to cur.execute(query, json_params).
+        self.json_params_field = "json_params"
+
+        # One DB client (and thus one connection) per PostgresPipe instance.
+        # Under your invariant (one thread per pipe), this is thread-safe.
+        self.client = DBClient(
+            host=self.db_host,
+            username=self.db_user,
+            password=self.db_pass,
+            db_name=self.db_name,
+            port=self.db_port,
+            ssl=self.db_ssl,
+        )
+
+    def reset(self):
+        # stateless across reset
+        pass
+
+    def close(self):
+        if self.client is not None:
+            self.client.close()
+
+    def _make_header(self, cur, query: str, params=None) -> Dict[str, Any]:
+        """
+        Inspect the cursor and build a full header record.
+        Figures out result, rowcount, function automatically.
+        """
+        h = {
+            "db": self.db_name,
+            "dbhost": self.db_host,
+        }
+        if params is not None:
+            h["params"] = params
+
+        if cur.description:
+            cols = [d[0] for d in cur.description]
+            if len(cols) == 1 and cols[0] == "ingest_event":
+                _ = cur.fetchone()  # consume void row
+                h["result"] = "ok"
+                h["function"] = "ingest_event"
+            else:
+                h["result"] = "ok"
+                h["rowcount"] = cur.rowcount if cur.rowcount != -1 else None
+        else:
+            h["result"] = "ok"
+            h["rowcount"] = cur.rowcount
+
+        return h
+
+    def execute_query_returning_S_xO_iterable(self, record):
+        query = record.get(self.query_field)
+        if not query:
+            record["_error"] = "missing query"
+            yield record
+            return
+
+        # Priority: json_params > batch_params > params
+        json_params = record.get(self.json_params_field, None)
+        batch = record.get(self.batch_field, None)
+        params = record.get(self.params_field, None)
+
+        cur = self.client.conn.cursor()
+        try:
+            did_executemany = False
+            header_params = None
+
+            # ---------- execute ----------
+            if json_params is not None:
+                # Explicit JSON payload; caller controls shape.
+                # We don't inspect query or payload.
+                if isinstance(json_params, (list, tuple, dict)):
+                    cur.execute(query, json_params)
+                else:
+                    cur.execute(query, (json_params,))
+                header_params = {self.json_params_field: json_params}
+
+            elif batch is not None:
+                # Legacy executemany path; no magic.
+                if len(batch) == 0:
+                    cur.execute("SELECT 1")
+                    header_params = {"batch_size": 0}
+                elif len(batch) == 1:
+                    cur.execute(query, batch[0])
+                    header_params = {"batch_size": 1, "params": batch[0]}
+                else:
+                    cur.executemany(query, batch)
+                    did_executemany = True
+                    header_params = {"batch_size": len(batch)}
+
+            else:
+                # Single-statement path.
+                if params is None:
+                    cur.execute(query)
+                    header_params = None
+                else:
+                    if isinstance(params, (list, tuple, dict)):
+                        cur.execute(query, params)
+                    else:
+                        cur.execute(query, (params,))
+                    header_params = params
+
+            # ---------- header ----------
+            yield self._make_header(cur, query, header_params)
+
+            # ---------- stream rows (only meaningful for single execute that returns rows) ----------
+            if not did_executemany and cur.description:
+                cols = [d[0] for d in cur.description]
+                if not (len(cols) == 1 and cols[0] == "ingest_event"):
+                    for row in cur:
+                        yield _row_to_dict(cur, row)
+
+        finally:
+            cur.close()
+            # connection stays open for this pipe; closed in .close()

{python_jack_knife-0.7.1 → python_jack_knife-0.7.5}/src/pjk/main.py

@@ -14,7 +14,7 @@ import concurrent.futures
 from pjk.registry import ComponentRegistry
 from pjk.sinks.stdout import StdoutSink
 from pjk.man_page import do_man, do_examples, display_configs, display_macros
-from pjk.history import write_history, display_history, get_history_tokens
+from pjk.history import write_history, display_history, get_history_tokens, printable_command
 from pjk.sinks.expect import ExpectSink
 from pjk.progress import ProgressDisplay
 from pjk.version import __version__
@@ -52,7 +52,6 @@ def execute_threaded(sinks, stop_progress=None):
 
 def initialize():
     init_logging()
-    write_history(sys.argv[1:])
 
     #src = Path("src/pjk/resources/configs.tmpl")
     #dst_dir = Path.home() / ".pjk"
@@ -111,12 +110,15 @@ def execute_tokens(tokens: List[str]):
         if not tokens:
             print('No such history')
             return
+        cmd = printable_command(tokens)
+        print(f"pjk {cmd}")
 
     parser = ExpressionParser(registry)
 
     display = None
     try:
         sink = parser.parse(tokens)
+        write_history(sys.argv[1:]) # now that it's parsed sucessfully
         if not isinstance(sink, (StdoutSink | ExpectSink)):
             display = ProgressDisplay(interval=3.0)
             display.start()

python_jack_knife-0.7.5/src/pjk/parse_pjk_file.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+import re
+import shlex
+from typing import Dict, List
+from pjk.usage import TokenError, UsageError
+
+PJK_END_TOKEN = 'END'
+PJK_SET_TOKEN = 'SET'
+
+# ${VAR} or $VAR - match anywhere in token (${VAR} first to avoid partial match)
+VAR_REF_PATTERN = re.compile(r'\$\{([a-zA-Z_][a-zA-Z0-9_]*)\}|\$([a-zA-Z_][a-zA-Z0-9_]*)')
+
+
+def _expand_token(t: str, env: Dict[str, str]) -> str:
+    """Expand $VAR or ${VAR} anywhere in token; raise if undefined."""
+
+    def repl(m):
+        name = m.group(1) or m.group(2)
+        if name not in env:
+            raise TokenError(f"Undefined variable: ${name}")
+        return env[name]
+
+    return VAR_REF_PATTERN.sub(repl, t)
+
+
+def handle_pjk_file(token: str, expanded: List[str]):
+    if not token.endswith(".pjk"):
+        return False
+
+    if not os.path.isfile(token):
+        raise TokenError(f"pjk file not found: {token}")
+
+    with open(token, "r") as f:
+        lines = f.readlines()
+
+    env: Dict[str, str] = {}
+
+    for line in lines:
+        try:
+            parts = shlex.split(line, comments=True, posix=True)
+        except ValueError as e:
+            raise UsageError(f"Error parsing {token}: {e}")
+
+        if not parts:
+            continue
+        if parts[0] == PJK_END_TOKEN:
+            break
+
+        if parts[0] == PJK_SET_TOKEN:
+            for p in parts[1:]:
+                if '=' in p:
+                    k, v = p.split('=', 1)
+                    env[k.strip()] = v.strip()
+            continue
+
+        for p in parts:
+            if p == PJK_END_TOKEN:
+                break
+            expanded.append(_expand_token(p, env))
+        else:
+            continue
+        break
+    return True