python-jack-knife 0.7.0__tar.gz → 0.7.4__tar.gz

{python_jack_knife-0.7.0/src/python_jack_knife.egg-info → python_jack_knife-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.7.0
+Version: 0.7.4
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/history.py

@@ -30,6 +30,9 @@ def read_history(log_path: str) -> List[int]:
                 if not line:
                     continue
 
+                if 'pjk ' in line: # legacy
+                    line = line.split('pjk ', 1)[1]
+
                 # Expected format: <command_string>
                 line = line.strip()
                         

python_jack_knife-0.7.4/src/pjk/integrations/postgres_pipe.py

@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+#
+# djk/pipes/postgres_pipe.py
+
+import base64
+import datetime as _dt
+import uuid
+import time
+from decimal import Decimal
+from typing import Any, Dict, Optional
+
+from pjk.usage import ParsedToken, Usage
+from pjk.common import Integration
+from pjk.pipes.query_pipe import QueryPipe
+
+MAX_RETRIES = 3
+BASE_DELAY = 0.1  # seconds
+
+class DBClient:
+    """Per-instance pg8000 connection wrapper. No shared state."""
+
+    def __init__(
+        self,
+        host: str,
+        username: str,
+        password: Optional[str],
+        db_name: str,
+        port: int = 5432,
+        ssl: bool = False,
+    ):
+        import pg8000  # lazy import
+
+        kwargs = dict(
+            user=username,
+            password=password,
+            host=host,
+            database=db_name,
+            port=port,
+        )
+        if ssl:
+            import ssl as _ssl
+
+            kwargs["ssl_context"] = _ssl.create_default_context()
+
+        try:
+            self.conn = pg8000.connect(**kwargs)
+            self.conn.autocommit = True
+        except Exception as e:
+            print("Failed to connect to DB")
+            raise e
+
+    def close(self):
+        if getattr(self, "conn", None) is None:
+            return
+
+        import pg8000  # lazy
+
+        try:
+            self.conn.close()
+        except pg8000.exceptions.InterfaceError:
+            # Already closed / broken; ignore.
+            pass
+        finally:
+            self.conn = None
+
+
+def _iso_dt(x: _dt.datetime) -> str:
+    """ISO 8601; normalize UTC offset to 'Z'."""
+    s = x.isoformat()
+    return s.replace("+00:00", "Z")
+
+
+def normalize(obj: Any) -> Any:
+    """
+    Make values JSON/YAML-safe and portable (schema-agnostic):
+      - Decimal -> exact string (no sci-notation)
+      - date/datetime/time -> ISO-8601 string (datetime keeps offset; UTC -> 'Z')
+      - UUID -> string
+      - bytes -> base64 string
+      - lists/tuples/sets, dicts -> normalized recursively
+      - leaves int/float/str/bool/None as-is
+    """
+    if obj is None:
+        return None
+
+    if isinstance(obj, Decimal):
+        return format(obj, "f")  # exact value as string
+
+    if isinstance(obj, _dt.datetime):
+        return _iso_dt(obj)
+
+    if isinstance(obj, (_dt.date, _dt.time)):
+        return obj.isoformat()
+
+    if isinstance(obj, uuid.UUID):
+        return str(obj)
+
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return base64.b64encode(bytes(obj)).decode("ascii")
+
+    if isinstance(obj, dict):
+        return {k: normalize(v) for k, v in obj.items()}
+
+    if isinstance(obj, (list, tuple, set)):
+        return [normalize(v) for v in obj]
+
+    return obj
+
+
+def _row_to_dict(cursor, row) -> Dict[str, Any]:
+    cols = [d[0] for d in cursor.description]
+    return {col: normalize(val) for col, val in zip(cols, row)}
+
+
+class PostgresPipe(QueryPipe, Integration):
+    name = "postgres"
+    desc = "Postgres query pipe; executes SQL over input record['query']."
+    arg0 = ("instance", "instance of database.")
+    examples = [
+        ["myquery.sql", "postgres:mydb", "-"],
+        ["{'query': 'SELECT * from MY_TABLE;'}", "postgres:mydb", "-"],
+        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", "postgres:mydb"],
+        ["{'query': 'SELECT procedure_batch(%s, ...), batch_params:{...}"],
+        ["{'query': 'SELECT procedure_jsonb(%s, ...), json_params:json_string"],
+    ]
+
+    # name, type, default
+    config_tuples = [
+        ("db_name", str, None),
+        ("host", str, None),
+        ("user", str, None),
+        ("password", str, None),
+        ("port", int, 5432),
+        ("ssl", bool, False),
+    ]
+
+    def __init__(self, ptok: ParsedToken, u: Usage, root=None):
+        super().__init__(ptok, u, root=root)
+
+        self.db_name = u.get_config("db_name")
+        self.db_host = u.get_config("host")
+        self.db_user = u.get_config("user")
+        self.db_pass = u.get_config("password")
+        self.db_port = u.get_config("port")
+        self.db_ssl = u.get_config("ssl")
+
+        # Standard params field: single-exec params (list/tuple/dict/single value)
+        self.params_field = "params"
+
+        # Legacy batch path: list[tuple|list|dict] → executemany
+        self.batch_field = "batch_params"
+
+        # Explicit JSON payload field (no query sniffing).
+        # If present, this value is passed to cur.execute(query, json_params).
+        self.json_params_field = "json_params"
+
+        # One DB client (and thus one connection) per PostgresPipe instance.
+        # Under your invariant (one thread per pipe), this is thread-safe.
+        self.client = DBClient(
+            host=self.db_host,
+            username=self.db_user,
+            password=self.db_pass,
+            db_name=self.db_name,
+            port=self.db_port,
+            ssl=self.db_ssl,
+        )
+
+    def reset(self):
+        # stateless across reset
+        pass
+
+    def close(self):
+        if self.client is not None:
+            self.client.close()
+
+    def _make_header(self, cur, query: str, params=None) -> Dict[str, Any]:
+        """
+        Inspect the cursor and build a full header record.
+        Figures out result, rowcount, function automatically.
+        """
+        h = {
+            "db": self.db_name,
+            "dbhost": self.db_host,
+        }
+        if params is not None:
+            h["params"] = params
+
+        if cur.description:
+            cols = [d[0] for d in cur.description]
+            if len(cols) == 1 and cols[0] == "ingest_event":
+                _ = cur.fetchone()  # consume void row
+                h["result"] = "ok"
+                h["function"] = "ingest_event"
+            else:
+                h["result"] = "ok"
+                h["rowcount"] = cur.rowcount if cur.rowcount != -1 else None
+        else:
+            h["result"] = "ok"
+            h["rowcount"] = cur.rowcount
+
+        return h
+
+    def execute_query_returning_S_xO_iterable(self, record):
+        query = record.get(self.query_field)
+        if not query:
+            record["_error"] = "missing query"
+            yield record
+            return
+
+        # Priority: json_params > batch_params > params
+        json_params = record.get(self.json_params_field, None)
+        batch = record.get(self.batch_field, None)
+        params = record.get(self.params_field, None)
+
+        cur = self.client.conn.cursor()
+        try:
+            did_executemany = False
+            header_params = None
+
+            # ---------- execute ----------
+            if json_params is not None:
+                # Explicit JSON payload; caller controls shape.
+                # We don't inspect query or payload.
+                if isinstance(json_params, (list, tuple, dict)):
+                    cur.execute(query, json_params)
+                else:
+                    cur.execute(query, (json_params,))
+                header_params = {self.json_params_field: json_params}
+
+            elif batch is not None:
+                # Legacy executemany path; no magic.
+                if len(batch) == 0:
+                    cur.execute("SELECT 1")
+                    header_params = {"batch_size": 0}
+                elif len(batch) == 1:
+                    cur.execute(query, batch[0])
+                    header_params = {"batch_size": 1, "params": batch[0]}
+                else:
+                    cur.executemany(query, batch)
+                    did_executemany = True
+                    header_params = {"batch_size": len(batch)}
+
+            else:
+                # Single-statement path.
+                if params is None:
+                    cur.execute(query)
+                    header_params = None
+                else:
+                    if isinstance(params, (list, tuple, dict)):
+                        cur.execute(query, params)
+                    else:
+                        cur.execute(query, (params,))
+                    header_params = params
+
+            # ---------- header ----------
+            yield self._make_header(cur, query, header_params)
+
+            # ---------- stream rows (only meaningful for single execute that returns rows) ----------
+            if not did_executemany and cur.description:
+                cols = [d[0] for d in cur.description]
+                if not (len(cols) == 1 and cols[0] == "ingest_event"):
+                    for row in cur:
+                        yield _row_to_dict(cur, row)
+
+        finally:
+            cur.close()
+            # connection stays open for this pipe; closed in .close()

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/pipes/factory.py

@@ -38,7 +38,7 @@ COMPONENTS = {
         'reduce': ReducePipe,        
         'sort': SortPipe,
         'where': WherePipe,
-        'sel': SelectFields,
+        'select': SelectFields,
         'sample': SamplePipe,
         'explode': DenormPipe,
         'postgres': PostgresPipe,

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/pipes/query_pipe.py

@@ -35,8 +35,8 @@ class QueryPipe(Pipe):
         return u
 
 
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
+    def __init__(self, ptok: ParsedToken, usage: Usage, root = None):
+        super().__init__(ptok, usage, root=root)
         self.output_shape = usage.get_param('shape')
         self.count = usage.get_param('count')
         self.query_field = 'query' # for all subclasses

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/pipes/select.py

@@ -10,12 +10,12 @@ class SelectFields(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = Usage(
-            name='sel',
+            name='select',
             desc='Select specific fields from each record.',
             component_class=cls
         )
         usage.def_arg(name='fields', usage='Comma-separated list of fields to retain')
-        usage.def_example(expr_tokens=["{id:1, dir:'up', color:'blue'}", 'sel:id,color'], expect="id: 1, color:'blue'")
+        usage.def_example(expr_tokens=["{id:1, dir:'up', color:'blue'}", 'select:id,color'], expect="id: 1, color:'blue'")
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/sinks/graph_bar_line.py

@@ -20,10 +20,6 @@ from typing import Any, Dict, Iterable, List, Optional, Sequence
 from datetime import date, datetime
 from collections import defaultdict
 
-import numpy as np
-import pandas as pd
-
-
 # ----------------------------- Public Params -----------------------------
 @dataclass
 class GraphParams:
@@ -48,6 +44,8 @@ class TimeDetector:
 
     @staticmethod
     def is_time(xs: pd.Series) -> bool:
+        import numpy as np # lazy
+        import pandas as pd # lazy
         # Already datetime dtype?
         if pd.api.types.is_datetime64_any_dtype(xs):
             return True
@@ -74,6 +72,7 @@ class TimeDetector:
 
     @staticmethod
     def parse_times(series: pd.Series) -> pd.Series:
+        import pandas as pd # lazy
         numeric = pd.to_numeric(series, errors="coerce")
         parsed = None
         if numeric.notna().mean() >= 0.9:
@@ -92,6 +91,8 @@ class MultiYAdapter:
     """Builds wide dataframe: columns = ['x'] + y_fields; sums duplicates of x."""
     @staticmethod
     def to_df(records: Iterable[Dict[str, Any]], x_field: str, y_fields: Sequence[str]) -> pd.DataFrame:
+        import pandas as pd # lazy
+        import numpy as np # lazy
         rows: List[Dict[str, Any]] = []
         for r in records:
             if x_field not in r:
@@ -120,7 +121,8 @@ class MultiYAdapter:
 class SingleYWithSetsAdapter:
     """Legacy: single y_field + optional per-row set_name to create series."""
     @staticmethod
-    def to_df(records: Iterable[Dict[str, Any]], x_field: str, y_field: str) -> pd.DataFrame:
+    def to_df(records: Iterable[Dict[str, Any]], x_field: str, y_field: str):
+        import pandas as pd # lazy
         triplets = []  # (x, y, set_name)
         for r in records:
             if x_field in r and y_field in r:
@@ -141,8 +143,10 @@ class GraphPlotter:
         self.y_fields = list(dict.fromkeys(self.pms.y_fields))  # dedupe, preserve order
 
     def plot(self, chart_type: str = "line"):
-        import matplotlib.pyplot as plt
-        import matplotlib.dates as mdates
+        import matplotlib.pyplot as plt # lazy
+        import matplotlib.dates as mdates # lazy
+        import pandas as pd # lazy
+        import numpy as np # lazy
 
         fig = plt.figure()
         ax = plt.gca()
@@ -258,7 +262,7 @@ class GraphPlotter:
     # ---------- Formatting helpers ----------
     @staticmethod
     def _format_time_axis(ax, df: pd.DataFrame) -> None:
-        import matplotlib.dates as mdates
+        import matplotlib.dates as mdates # lazy
         fig = ax.get_figure()
         ts = df["ts"]
         if ts.empty:
@@ -289,6 +293,7 @@ class GraphPlotter:
 
     def _bars_time(self, ax, df: pd.DataFrame, y_cols: Sequence[str]) -> None:
         # Grouped bars at each timestamp using index positions
+        import numpy as np # lazy
         x_vals = df["ts"].to_numpy(); idx = np.arange(len(x_vals))
         n = len(y_cols); width = 0.8 / max(n, 1)
         for i, y in enumerate(y_cols):
@@ -298,6 +303,7 @@ class GraphPlotter:
         ax.set_xticks(idx, [pd.to_datetime(t).strftime("%Y-%m-%d %H:%M") for t in x_vals], rotation=45)
 
     def _bars_categorical(self, ax, df: pd.DataFrame, y_cols: Sequence[str]) -> None:
+        import numpy as np # lazy
         seen = set(); ordered_x: List[Any] = []
         for x in df["x"].tolist():
             if x not in seen:
@@ -310,6 +316,7 @@ class GraphPlotter:
         ax.set_xticks(idx, ordered_x, rotation=45)
 
     def _lines_categorical(self, ax, df: pd.DataFrame, y_cols: Sequence[str]) -> None:
+        import numpy as np # lazy
         seen = set(); ordered_x: List[Any] = []
         for x in df["x"].tolist():
             if x not in seen:
@@ -322,7 +329,7 @@ class GraphPlotter:
 
     # ---------- Misc ----------
     def _apply_args_dict(self) -> None:
-        import matplotlib.pyplot as plt
+        import matplotlib.pyplot as plt # lazy
         for name, val in getattr(self.pms, "args_dict", {}).items():
             fn = getattr(plt, name, None)
             if callable(fn):
@@ -345,7 +352,7 @@ def graph_bar_line(obj, type):
     Returns (fig, ax) for optional downstream tweaks (safe to ignore).
     """
     # Lazy import (ensures MPL backend)
-    import matplotlib.pyplot as plt  # noqa: F401
+    import matplotlib.pyplot as plt  # noqa: F401 # lazy
 
     # Normalize y_fields from string or list
     raw_y = obj.y_field if isinstance(obj.y_field, str) else str(obj.y_field)

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/sources/factory.py

@@ -13,9 +13,11 @@ from pjk.sources.inline_source import InlineSource
 from pjk.sources.user_source_factory import UserSourceFactory
 from pjk.sources.parquet_source import ParquetSource
 from pjk.sources.format_source import FormatSource
+from pjk.sources.s3_select_source import S3SelectSource
 
 COMPONENTS = {
         'inline': InlineSource,
+        's3s': S3SelectSource,
         'json': JsonSource,
         'jsonl': JsonSource,
         'csv': CSVSource,
@@ -35,6 +37,11 @@ class SourceFactory(ComponentFactory):
     def create(self, token: str) -> Source:
         token = token.strip()
 
+        # s3s is a pseudo source only in the above list to provide easy man page
+        # it's instantiated by the parser when <file>.s3s, so disallow standard search for it.
+        if token == 's3s':
+            return None
+
         if InlineSource.is_inline(token):
             return InlineSource(token)
         
@@ -44,6 +51,12 @@ class SourceFactory(ComponentFactory):
             source = UserSourceFactory.create(ptok)
             if source:
                 return source
+            
+        # s3 select file
+        if ptok.pre_colon.endswith('.s3s'):
+            source = S3SelectSource(ptok, None)
+            if source:
+                return source
 
         source_cls = self.get_component_class(ptok.pre_colon)
         if source_cls and not issubclass(source_cls, FormatSource):

{python_jack_knife-0.7.0 → python_jack_knife-0.7.4}/src/pjk/sources/npy_source.py

@@ -4,18 +4,15 @@
 import json
 from typing import Iterator, Dict, Any
 
-import numpy as np
-from pjk.usage import NoBindUsage
-from pjk.components import Source
 from pjk.sources.lazy_file import LazyFile
 from pjk.sources.format_source import FormatSource
 from pjk.log import logger
 
-
 class NpySource(FormatSource):
     extension = 'npy'
 
     def __init__(self, lazy_file: LazyFile):
+        super().__init__(root=None)
         self.lazy_file = lazy_file
         self.num_vecs = 0
 
@@ -32,9 +29,11 @@ class NpySource(FormatSource):
 
         try:
             # Use mmap to avoid loading entire array in RAM at once.
+            import numpy as np #lazy import
             arr = np.load(path, mmap_mode="r", allow_pickle=False)
         except Exception as e:
             logger.error(f"Failed to load .npy file at {path}: {e}")
+            raise Exception(f"Failed to load .npy file at {path}: {e}")
             return
 
         if arr.size == 0: