python-jack-knife 0.5.0__py3-none-any.whl

pjk/pipes/join.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/join.py
+
+from pjk.base import Pipe, Usage, UsageError, ParsedToken, KeyedSource
+
+class JoinPipe(Pipe):
+    arity = 2  # left = record stream, right = KeyedSource
+
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='join',
+            desc="Join records against a keyed source on shared fields",
+            component_class=cls
+        )
+        usage.def_arg(
+            name='mode',
+            usage="'left', 'inner', or 'outer' join behavior",
+            valid_values={'left', 'inner', 'outer'}
+        )
+        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> join:<mode> <sink>")
+
+        usage.def_example(expr_tokens=
+        [
+            "[{color:'blue'},{color:'green'}]",
+            "[{color:'blue', price:50}, {color:'red', price:20}]",
+            'map:o:color',
+            "join:left"
+        ],
+        expect="[{color:'blue', price:50}, {color:'green'}]")
+        usage.def_example(expr_tokens=
+        [
+            "[{color:'blue'},{color:'green'}]",
+            "[{color:'blue', price:50}, {color:'red', price:20}]",
+            'map:o:color',
+            "join:inner"
+        ],
+        expect="[{color:'blue', price:50}]")
+
+        usage.def_example(expr_tokens=
+        [
+            "[{color:'blue'},{color:'green'}]",
+            "[{color:'blue', price:50}, {color:'red', price:20}]",
+            'map:o:color',
+            "join:outer"
+        ],
+        expect="[{color:'blue', price:50}, {color:'green'}, {color:'red', price: 20}]")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+
+        self.mode = usage.get_arg('mode')
+        self.left = None
+        self.right = None
+        self._pending_right = None
+        self._check_right = False
+
+    def reset(self):
+        self._pending_right = None
+        self._check_right = False
+
+    def __iter__(self):
+        if not isinstance(self.right, KeyedSource):
+            raise UsageError("right source must be a KeyedSource")
+
+        for left_rec in self.left:
+            match = self.right.lookup(left_rec)
+
+            if match is not None:
+                merged = dict(left_rec)
+                merged.update(match)
+                yield merged
+            elif self.mode == "left":
+                yield left_rec
+            elif self.mode == "outer":
+                yield left_rec 
+            elif self.mode == "inner":
+                continue
+
+        if self.mode == "outer":
+            for right_rec in self.right.get_unlookedup_records():
+                yield right_rec

pjk/pipes/let_reduce.py

@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/let_reduce.py
+
+from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, UsageError, TokenError
+from pjk.common import SafeNamespace, ReducingNamespace
+import re
+import ast
+import json
+
+# --- Shared Utilities ---
+def parse_args(token: str):
+    pattern = re.compile(r'(?P<field>\w+)(?P<op>[:=\+\-\*/]+)(?P<rest>.+)$')
+    match = pattern.fullmatch(token)
+    if not match:
+        raise ValueError(f"Invalid token syntax: {token!r}")
+    return match.groupdict()
+
+def do_eval(expr, env):
+    try:
+        safe_env = dict(env)
+        safe_env['json'] = json
+        return eval(expr, {}, safe_env)
+    except Exception:
+        raise UsageError(f"UsageError in expression: {expr}")
+
+def eval_regular(expr: str, record: dict):
+    env = {'f': SafeNamespace(record)}
+    if re.match(r'[a-zA-Z0-9_]+$', expr):
+        return expr
+    return do_eval(expr, env)
+
+def eval_accumulating(expr: str, record: dict, op: str, acc=None):
+    if op in ('-=', '*=', '/=') and 'acc' not in expr:
+        expr = f'acc {op[0]} ({expr})'
+
+    env = {
+        'f': SafeNamespace(record),
+        'acc': acc
+    }
+
+    try:
+        node = ast.parse(expr, mode='eval').body
+    except SyntaxError:
+        raise UsageError(f"Invalid expression: {expr}")
+
+    if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp)):
+        env['f'] = ReducingNamespace(record)
+
+    if isinstance(node, ast.ListComp):
+        values = eval(compile(ast.Expression(node), '<reduce:listcomp>', 'eval'), {}, env)
+        return (acc or []) + list(values)
+
+    if isinstance(node, ast.SetComp):
+        values = eval(compile(ast.Expression(node), '<reduce:setcomp>', 'eval'), {}, env)
+        return (acc or set()).union(values)
+
+    if isinstance(node, ast.DictComp):
+        values = eval(compile(ast.Expression(node), '<reduce:dictcomp>', 'eval'), {}, env)
+        return {**(acc or {}), **values}
+
+    if op == '+=':
+        value = eval(expr, {}, env)
+        if isinstance(value, (int, float)):
+            return (acc or 0) + value
+        elif isinstance(value, str):
+            return str(acc or '') + value
+        elif isinstance(value, list):
+            return (acc or []) + value
+        else:
+            return (acc or []) + [value]
+
+    if op in ('-=', '*=', '/='):
+        return do_eval(expr, env)
+
+    return do_eval(expr, env)
+
+# --- LetPipe (simple field assignment) ---
+class LetPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = NoBindUsage( # can't use bound usage because of complicated parsing
+            name='let',
+            desc="set a new field equal to a rhs python expression",
+            component_class=cls
+        )
+        usage.def_arg(name='rhs', usage="python rhs expression (use f.<field> syntax)")
+        usage.def_example(expr_tokens=['{hello:0}', 'let:there=f.hello + 1'], expect="{hello:0, there: 1}")
+        usage.def_example(expr_tokens=['{hello:0}', 'let:foo:bar'], expect="{hello:0, foo: 'bar'}")
+        usage.def_example(expr_tokens=['{hello:0}', 'let:foo=int(1)'], expect="{hello:0, foo: 1}")        
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        args = parse_args(ptok.whole_token.split(':', 1)[-1])
+        self.field = args['field']
+        self.op = args['op']
+        self.rest = args['rest']
+
+        if self.op in ('+=', '-=', '*=', '/='):
+            raise TokenError("Aggregation operator not allowed in let, use reduce:")
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        for record in self.left:
+            if self.op == ':':
+                record[self.field] = self.rest
+            else:
+                record[self.field] = eval_regular(self.rest, record)
+            yield record
+
+# --- ReducePipe (stateful accumulator) ---
+def is_comprehension(expr: str) -> bool:
+    try:
+        node = ast.parse(expr, mode='eval').body
+        return isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp))
+    except SyntaxError:
+        return False
+
+class ReducePipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = NoBindUsage( # can't use bound usage because of complicated parsing
+            name='reduce',
+            desc="set a new field equal to a reduction over records of a sub or main expression\n" +
+            "rhs operators must be accumulating, e.g. +=, -=, *=, /=\n" +
+            "or use list or dictionary comprehension",
+            component_class=cls
+        )
+        usage.def_arg(name='rhs', usage="accumulating python rhs expression (use f.<field> syntax)")
+
+        usage.def_example(expr_tokens=["{ferry:'orca', cars:[{make: 'ford', size:9}, {make:'bmw', size:4}]}",
+                                       '[', 'reduce:total_size+=f.size', 'over:cars'
+                                       ],
+                        expect="{ferry:'orca', cars:[{make: 'ford', size:9}, {make:'bmw', size:4}], total_size: 13}")
+        
+        usage.def_example(expr_tokens=["[{make: 'honda'}, {make: 'ford'}, {make:'bmw'}]",
+                                       'reduce:cars=[x for x in f.make]'
+                                       ],
+                        expect="{cars:['honda', 'ford', 'bmw']}")
+        
+        usage.def_example(expr_tokens=["[{i:[1,2]},{i:[3]}]",
+                                       'reduce:flattened=[x for x in f.i]'
+                                       ],
+                        expect="{flattened:[1, 2, 3]}")
+        
+        usage.def_example(expr_tokens=["[{i:1},{i:3}, {i:7}]",
+                                       'reduce:diff-=f.i'
+                                       ],
+                        expect="{diff:-11}")
+        
+        usage.def_example(expr_tokens=["[{i:1},{i:3}, {i:7}]",
+                                       'reduce:product*=f.i'
+                                       ],
+                        expect="{product:21}")
+
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        args = parse_args(ptok.whole_token.split(':', 1)[-1])
+        self.field = args['field']
+        self.op = args['op']
+        self.rest = args['rest']
+
+        if self.op not in ('+=', '-=', '*=', '/='):
+            if is_comprehension(self.rest):
+                self.op = '+='
+            else:
+                raise TokenError("Reduce pipe requires an accumulating operator (+=, -=, etc.), unless RHS is a comprehension")
+
+        self.accum_value = self.initial_acc_value()
+
+    def initial_acc_value(self):
+        if self.op == '+=':
+            return 0
+        elif self.op == '*=':
+            return 1
+        elif self.op == '-=':
+            return 0
+        elif self.op == '/=':
+            return 1.0
+        else:
+            return None
+
+    def reset(self):
+        self.accum_value = self.initial_acc_value()
+
+    def __iter__(self):
+        for record in self.left:
+            self.accum_value = eval_accumulating(self.rest, record, self.op, self.accum_value)
+            yield record
+
+    def get_subexp_result(self):
+        return (self.field, self.accum_value)

pjk/pipes/map.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/group.py
+
+from typing import Optional
+from pjk.base import ParsedToken, Usage, Pipe, KeyedSource
+
+class MapPipe(Pipe, KeyedSource):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='map',
+            desc="maps records to key, either overriding or grouping duplicates. Creates Keyed Source for join or filter.",
+            component_class=cls
+        )
+        usage.def_arg(name='how', usage="'o' for override, 'g' for group", valid_values={'o', 'g'})
+        usage.def_arg(name='key', usage='comma separated fields to map by')
+        usage.def_example(expr_tokens=["[{id: 1, color:'blue'}, {id:1, color:'green'}, {id:2, color:'red'}]", 'map:o:id'],
+                          expect="[{id:2, color:'red'}, {id:1, color:'green'}]")
+        usage.def_example(expr_tokens=["[{id: 1, color:'blue'}, {id:1, color:'green'}, {id:2, color:'red'}]", 'map:g:id'], 
+                          expect="[{id:2, child:[{color:'red'}]}, {id:1, child:[{color:'blue'},{color: 'green'}]}]")
+        usage.def_example(expr_tokens=["[{id: 1, color:'blue', size:5}, {id:1, color:'green', size:10}]", 'map:o:id,color'], 
+                          expect="[{id:1, color:'green', size: 10}, {id:1, color:'blue', size:5}]")
+
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.is_group = usage.get_arg('how') == 'g'
+        self.fields = usage.get_arg('key').split(',')
+        self.rec_map = {}
+        self.matched_map = {}
+        self._rec_list = None
+        self.is_loaded = False
+
+    def reset(self):
+        self.rec_map.clear()
+        self.matched_map.clear()
+        self._rec_list = None
+        self.is_loaded = False
+
+    def load(self):
+        if self.is_loaded:
+            return
+        self.is_loaded = True
+
+        for record in self.left:
+            key_rec = {}
+            for field in self.fields:
+                key_rec[field] = record.pop(field, None) if self.is_group else record.get(field)
+
+            key = tuple(key_rec.values())
+            existing = self.rec_map.get(key)
+
+            if not existing:
+                if self.is_group:
+                    key_rec['child'] = [record]
+                    self.rec_map[key] = key_rec
+                else:
+                    self.rec_map[key] = record
+            else:
+                if self.is_group:
+                    existing['child'].append(record)
+                else:
+                    self.rec_map[key] = record
+
+    def __iter__(self):
+        if not self.is_loaded:
+            self.load()
+        if self._rec_list is None:
+            self._rec_list = list(self.rec_map.values())
+
+        while self._rec_list:
+            yield self._rec_list.pop()
+
+    def lookup(self, left_rec) -> Optional[dict]:
+        if not self.is_loaded:
+            self.load()
+
+        key = tuple(left_rec.get(f) for f in self.fields)
+        rec = self.rec_map.pop(key, None)
+        if rec is not None:
+            self.matched_map[key] = rec
+            return rec
+        return self.matched_map.get(key)
+
+    def get_unlookedup_records(self):
+        if not self.is_loaded:
+            self.load()
+        return list(self.rec_map.values())

pjk/pipes/move_field.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/move_field.py
+
+from pjk.base import Pipe, ParsedToken, Usage
+
+class MoveField(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='as',
+            desc='Move one field to another key in the record',
+            component_class=cls
+        )
+        usage.def_arg(name='src', usage='Source field name')
+        usage.def_arg(name='dst', usage='Destination field name')
+        usage.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
+
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.src = usage.get_arg('src')
+        self.dst = usage.get_arg('dst')
+        self.count = 0
+
+    def reset(self):
+        self.count = 0
+
+    def __iter__(self):
+        for record in self.left:
+            self.count += 1
+            if self.src in record:
+                record[self.dst] = record.pop(self.src)
+            yield record

pjk/pipes/postgres_pipe.py

@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+#
+# djk/pipes/postgres_pipe.py
+
+import base64
+import datetime as _dt
+import uuid
+from decimal import Decimal
+from typing import Any, Dict, Optional
+
+from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, TokenError
+from pjk.common import Lookups
+
+
+class DBClient:
+    """Simple shared-connection wrapper for pg8000."""
+    _connection = None
+    
+    def __init__(self, host: str, username: str, password: Optional[str],
+                 dbname: str, port: int = 5432, ssl: bool = False):
+        import pg8000 # lazy import
+        if DBClient._connection is None:
+            try:
+                kwargs = dict(user=username, password=password, host=host, database=dbname, port=port)
+                if ssl:
+                    import ssl as _ssl
+                    kwargs["ssl_context"] = _ssl.create_default_context()
+                DBClient._connection = pg8000.connect(**kwargs)
+                DBClient._connection.autocommit = True
+            except Exception as e:
+                print("Failed to connect to DB")
+                raise e
+        self.conn = DBClient._connection
+
+    def close(self):
+        if self.conn is not None:
+            try:
+                self.conn.close()
+            finally:
+                DBClient._connection = None
+
+
+def _iso_dt(x: _dt.datetime) -> str:
+    """ISO 8601; normalize UTC offset to 'Z'."""
+    s = x.isoformat()
+    return s.replace("+00:00", "Z")
+
+
+def normalize(obj: Any) -> Any:
+    """
+    Make values JSON/YAML-safe and portable (schema-agnostic):
+      - Decimal -> exact string (no sci-notation)
+      - date/datetime/time -> ISO-8601 string (datetime keeps offset; UTC -> 'Z')
+      - UUID -> string
+      - bytes -> base64 string
+      - lists/tuples/sets, dicts -> normalized recursively
+      - leaves int/float/str/bool/None as-is
+    """
+    if obj is None:
+        return None
+
+    if isinstance(obj, Decimal):
+        return format(obj, "f")  # exact value as string
+
+    if isinstance(obj, _dt.datetime):
+        return _iso_dt(obj)
+
+    if isinstance(obj, (_dt.date, _dt.time)):
+        return obj.isoformat()
+
+    if isinstance(obj, uuid.UUID):
+        return str(obj)
+
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return base64.b64encode(bytes(obj)).decode("ascii")
+
+    if isinstance(obj, dict):
+        return {k: normalize(v) for k, v in obj.items()}
+
+    if isinstance(obj, (list, tuple, set)):
+        return [normalize(v) for v in obj]
+
+    return obj
+
+
+def _row_to_dict(cursor, row) -> Dict[str, Any]:
+    cols = [d[0] for d in cursor.description]
+    return {col: normalize(val) for col, val in zip(cols, row)}
+
+
+class PostgresPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name="pgres",
+            desc="Postgres query pipe; executes SQL from input record['query'].",
+            component_class=cls,
+        )
+        usage.def_arg(
+            "dbname",
+            "name of db. Entry in ~/.pjk/lookups.yaml containing host, user, password"
+        )
+        usage.def_param(
+            "header",
+            usage="emit header record before query results",
+            valid_values={"true", "false"}, default='true',
+        )
+
+        usage.def_example(expr_tokens=['myquery.sql', 'pgres:mydb'], expect=None)
+        usage.def_example(expr_tokens=["{'query': 'SELECT * from MY_TABLE;'}", 'pgres:mydb'], expect=None)
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+
+        lookups = Lookups()
+        self.dbname = usage.get_arg("dbname")
+        db_params = lookups.get(self.dbname)
+        if not db_params:
+            # f-string so dbname prints correctly
+            raise TokenError(
+                f"~/.pjk/lookups.yaml must contain entry for '{self.dbname}' with host, user, password."
+            )
+
+        self.db_host = db_params.get("host")
+        self.db_user = db_params.get("user")
+        self.db_pass = db_params.get("password")
+        self.db_port = int(db_params.get("port", 5432))
+        self.db_ssl  = bool(db_params.get("ssl", False))
+
+        self.query_field  = "query"   # SQL string
+        self.params_field = "params"  # optional: list/tuple (positional) or dict (named)
+        self.do_header = usage.get_param("header") == "true"
+
+    def reset(self):
+        # stateless across reset
+        pass
+
+    def _make_header(self, cur, query: str, params=None) -> Dict[str, Any]:
+        """
+        Inspect the cursor and build a full header record.
+        Figures out result, rowcount, function automatically.
+        """
+        h = {
+            "query": query,
+            "db": self.dbname,
+            "dbhost": self.db_host,
+        }
+        if params:
+            h["params"] = params
+
+        if cur.description:
+            cols = [d[0] for d in cur.description]
+            if len(cols) == 1 and cols[0] == "ingest_event":
+                _ = cur.fetchone()  # consume void row
+                h["result"] = "ok"
+                h["function"] = "ingest_event"
+            else:
+                h["result"] = "ok"
+                h["rowcount"] = cur.rowcount if cur.rowcount != -1 else None
+        else:
+            h["result"] = "ok"
+            h["rowcount"] = cur.rowcount
+
+        return {"header": h}
+
+    def __iter__(self):
+        client = DBClient(
+            host=self.db_host,
+            username=self.db_user,
+            password=self.db_pass,
+            dbname=self.dbname,
+            port=self.db_port,
+            ssl=self.db_ssl,
+        )
+        try:
+            for input_record in self.left:
+                query = input_record.get(self.query_field)
+                if not query:
+                    yield {"_error": "missing query"}
+                    continue
+                params = input_record.get(self.params_field)
+
+                cur = client.conn.cursor()
+                try:
+                    # execute
+                    if params is None:
+                        cur.execute(query)
+                    else:
+                        if isinstance(params, (list, tuple, dict)):
+                            cur.execute(query, params)
+                        else:
+                            cur.execute(query, (params,))
+
+                    # yield header first
+                    if self.do_header:
+                        yield self._make_header(cur, query, params)
+
+                    # then stream rows if it was a real SELECT with results
+                    if cur.description:
+                        cols = [d[0] for d in cur.description]
+                        if not (len(cols) == 1 and cols[0] == "ingest_event"):
+                            for row in cur:
+                                yield _row_to_dict(cur, row)
+                finally:
+                    cur.close()
+        finally:
+            client.close()

pjk/pipes/remove_field.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/remove_field.py
+
+from pjk.base import Pipe, ParsedToken, Usage, UsageError
+
+class RemoveField(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='drop',
+            desc='Remove one or more fields from each record',
+            component_class=cls
+        )
+        usage.def_arg(name='fields', usage='Comma-separated list of field names to drop')
+        usage.def_example(expr_tokens=["{id:1, dir:'up', color:'blue'}", 'drop:id,color'], expect="dir: 'up'")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        arg_string = usage.get_arg('fields')
+        self.fields = [f.strip() for f in arg_string.split(',') if f.strip()]
+        if not self.fields:
+            raise UsageError("rm must include at least one valid field name")
+        self.count = 0
+
+    def reset(self):
+        self.count = 0
+
+    def __iter__(self):
+        for record in self.left:
+            self.count += 1
+            for field in self.fields:
+                record.pop(field, None)
+            yield record