python-jack-knife 0.6.0__tar.gz → 0.6.3__tar.gz

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.0
+Version: 0.6.3
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/common.py

@@ -3,8 +3,9 @@
 
 import sys, shutil, subprocess, contextlib, signal
 import os
+import re
 import yaml
-from pjk.base import TokenError, Integration
+from pjk.base import TokenError, Integration, Source, Pipe
 
 class SafeNamespace:
     def __init__(self, obj):
@@ -73,50 +74,32 @@ def highlight(text: str, color: str = 'bold', value: str = None) -> str:
     style = COLOR_CODES.get(color.lower(), COLOR_CODES['bold'])
     return text.replace(value, f"{style}{value}{RESET}")
 
-class Lookups:
-    def __init__(self, component_class):
-        self.lookups_yaml = os.path.expanduser('~/.pjk/lookups.yaml')
+class Config:
+    def __init__(self, instance_type: str, component_class: Source|Pipe, instance: str):
+        # instance = name of the instance, e.g. 'myindexcollection', instance_type = 'index'
+        # instance_type only used by automatic config template maker MUST BE STRING LITERAL!
+        self.configs_yaml = os.path.expanduser('~/.pjk/component_configs.yaml')
         self.class_name = type(component_class).__name__
+        self.instance = instance
         self._data = {}
         self._load()
         
     def _load(self):
-        """Load lookups from YAML file if it exists."""
-        if os.path.exists(self.lookups_yaml):
-            with open(self.lookups_yaml, 'r') as f:
+        if os.path.exists(self.configs_yaml):
+            with open(self.configs_yaml, 'r') as f:
                 self._data = yaml.safe_load(f) or {}
         else:
             self._data = {}
 
-    def save(self):
-        """Save current lookups back to YAML file."""
-        os.makedirs(os.path.dirname(self.lookups_yaml), exist_ok=True)
-        with open(self.lookups_yaml, 'w') as f:
-            yaml.safe_dump(self._data, f)
-
-    def get(self, key, default=None):
-        lookup_key = f'{self.class_name}-{key}'
-        entry = self._data.get(lookup_key, default)
+    def lookup(self, param: str, default=None):
+        instance_key = f'{self.class_name}-{self.instance}'
+        entry = self._data.get(instance_key, None)
         if not entry:
             raise TokenError(
-                f"~/.pjk/lookups.yaml does not contain entry for '{lookup_key}' with required params."
+                f"~/.pjk/component_configs.yaml does not contain entry for '{instance_key}' with required params."
             )
-        return entry
-
-    def set(self, key, value):
-        """Set a lookup value and persist it."""
-        self._data[key] = value
-        self.save()
-
-    def delete(self, key):
-        """Remove a key if it exists and save."""
-        if key in self._data:
-            del self._data[key]
-            self.save()
-
-    def all(self):
-        """Return the full lookup dictionary."""
-        return dict(self._data)
+        
+        return entry.get(param, default)
 
 class ComponentFactory:
     def __init__(self, core_components: dict):
@@ -156,3 +139,6 @@ class ComponentFactory:
 
     def create(self, token: str):
         pass
+
+def is_valid_field_name(name: str):
+    return re.fullmatch(r'^[A-Za-z_][A-Za-z0-9_]*$', name)

python_jack_knife-0.6.3/src/pjk/integrations/opensearch_client.py

@@ -0,0 +1,77 @@
+from pjk.common import Config
+
+class OpenSearchClient:
+
+    @classmethod
+    def get_client(cls, config: Config):
+        aws_auth = config.lookup("os_auth_use_aws", "true") != "false"
+        scheme = config.lookup("os_scheme", "https")
+        verify_certs = config.lookup("os_verify_certs", "true") != "false"
+        ca_certs = config.lookup("os_ca_certs", None)
+        region = config.lookup("os_region", None)
+        service = config.lookup("os_service", "es")
+        username = config.lookup("os_username", None)
+        password = config.lookup("os_password", None)
+        timeout = float(config.lookup("os_timeout", 30))
+        ssl_assert_hostname = config.lookup("os_ssl_assert_hostname", "true") != "false"
+        ssl_show_warn = config.lookup("os_ssl_show_warn", "false") == "true"
+        host = config.lookup("os_host", None)
+        port = config.lookup("os_port", None)
+
+        # Reasonable port defaults
+        if port is None:
+            port = 443 if scheme == "https" else 9200
+        else:
+            port = int(port)
+
+        if host is None:
+            raise ValueError("Config os_host is required (set os_host + os_port/os_scheme, or a connection profile).")
+
+        # Lazy import so this module can still be imported if deps aren't installed.
+        try:
+            from opensearchpy import OpenSearch, RequestsHttpConnection, Urllib3HttpConnection
+        except Exception as e:
+            raise RuntimeError("opensearch-py must be installed to use OpenSearchQueryPipe") from e
+
+        http_auth = None
+        connection_class = Urllib3HttpConnection  # default
+        use_ssl = (scheme == "https")
+
+        if aws_auth:
+            # AWS SigV4 (works for OpenSearch Service / legacy ES domains)
+            try:
+                import boto3
+                from requests_aws4auth import AWS4Auth
+            except Exception as e:
+                raise RuntimeError("boto3 and requests-aws4auth are required for os_auth_method='aws'") from e
+
+            if not region:
+                raise ValueError("Config os_region is required for os_auth_method='aws'.")
+
+            session = boto3.Session()
+            credentials = session.get_credentials()
+            if credentials is None:
+                raise RuntimeError("No AWS credentials found (boto3 session.get_credentials() returned None).")
+
+            creds = credentials.get_frozen_credentials()
+            http_auth = AWS4Auth(creds.access_key, creds.secret_key, region, service, session_token=creds.token)
+            connection_class = RequestsHttpConnection  # SigV4 signing via requests path
+
+        else:
+            if not (username and password):
+                raise ValueError("os_username and os_password are required for os_auth_method='basic'.")
+            http_auth = (username, password)
+
+        # Build client
+        client = OpenSearch(
+            hosts=[{"host": host, "port": port}],
+            http_auth=http_auth,
+            use_ssl=use_ssl,
+            verify_certs=verify_certs,
+            ssl_assert_hostname=ssl_assert_hostname,
+            ssl_show_warn=ssl_show_warn,
+            ca_certs=ca_certs,
+            timeout=timeout,
+            connection_class=connection_class,
+        )
+        return client

python_jack_knife-0.6.3/src/pjk/integrations/opensearch_query_pipe.py

@@ -0,0 +1,104 @@
+import os
+import sys
+import traceback
+from copy import deepcopy
+from typing import Optional, Iterator, Dict, Any, Iterable
+
+from pjk.base import Pipe, ParsedToken, Usage, Integration
+from pjk.pipes.query_pipe import QueryPipe
+from pjk.common import Config
+from pjk.integrations.opensearch_client import OpenSearchClient
+
+def build_body_from_string(query_string: str) -> dict:
+    if query_string == "*":
+        return {"query": {"match_all": {}}}
+    else:
+        return {
+            "query": {
+                "simple_query_string": {
+                    "query": query_string
+                }
+            }
+        }
+
+class OpenSearchQueryPipe(QueryPipe, Integration):
+    name = "os_query"
+    desc = "Opensearch query pipe. Uses record['query_string'] or record['query_object'] for os query"
+    arg0 = ("index", "index to query over")
+    examples = [
+        ["{'query_string': '*'}", 'os_query:myindex', '-'],
+        ["{'query_string': 'dog'}", 'os_query:myindex', '-'],
+        ["{'query_string': 'dog AND cat'}", 'os_query:myindex', '-'],
+        ["{'query_object': {query: {...}}", 'os_query:myindex', '-'],
+    ]
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+
+        # index from arg0 or config
+        self.index = ptok.get_arg(0)
+
+        # Build the OpenSearch client (handles AWS/basic/none)
+        config = Config('index', self, self.index)
+        self.client = OpenSearchClient.get_client(config)
+
+        # Iteration state
+        self.cur_record: Optional[Dict[str, Any]] = None
+        self.hits_iter: Optional[Iterator[Dict[str, Any]]] = None
+
+    def reset(self):
+        # keep the index open between drains
+        pass
+
+    def close(self):
+        pass
+
+    def execute_query_returning_Q_xR_iterable(self, query_record: dict) -> Iterator[Dict[str, Any]]:
+        query_string = query_record.get('query_string', None)
+        query_body = None
+
+        if query_string:
+            query_body = build_body_from_string(query_string)
+        else:
+            query_body = query_record.get('query_object')
+
+        try:
+            # Build final request body
+            req_body = deepcopy(query_body)
+            req_body["size"] = self.count
+
+            res = self.client.search(index=self.index, body=req_body)
+
+            total_hits = 0
+            took = res.get("took")
+            hits = res.get("hits", {}).get("hits", [])
+            total_obj = res.get("hits", {}).get("total", {})
+            if isinstance(total_obj, dict):
+                total_hits = total_obj.get("value", 0)
+            elif isinstance(total_obj, int):
+                total_hits = total_obj
+
+            # Emit a metadata record first
+            yield {
+                "took_ms": took,
+                "total_hits": total_hits,
+                "index": self.index,
+                "os_query_body": req_body
+            }
+
+            # Emit each hit
+            for hit in hits:
+                if "_source" in hit and isinstance(hit["_source"], dict):
+                    yield hit["_source"]
+                else:
+                    # Some queries (e.g., stored fields only) might not include _source
+                    yield {"_type": "os_query_hit", "_hit": hit}
+
+        except Exception as e:
+            print("OpenSearch query error:", e, file=sys.stderr)
+            traceback.print_exc()
+            yield {
+                "_type": "os_query_error",
+                "error": str(e),
+                "query_record": query_record,
+            }

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/integrations/postgres_pipe.py

@@ -10,7 +10,7 @@ from decimal import Decimal
 from typing import Any, Dict, Optional
 
 from pjk.base import Integration, ParsedToken, Usage
-from pjk.common import Lookups
+from pjk.common import Config
 from pjk.pipes.query_pipe import QueryPipe
 
 
@@ -91,26 +91,25 @@ def _row_to_dict(cursor, row) -> Dict[str, Any]:
 
 
 class PostgresPipe(QueryPipe,Integration):
-    name = 'pgres'
+    name = 'postgres'
     desc = "Postgres query pipe; executes SQL from input."
     arg0 = ("dbname", 'database name.')
     examples = [
-        ['myquery.sql', 'pgres:mydb', '-'],
-        ["{'query': 'SELECT * from MY_TABLE;'}", 'pgres:mydb', '-'],
-        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'pgres:mydb']
+        ['myquery.sql', 'postgres:mydb', '-'],
+        ["{'query': 'SELECT * from MY_TABLE;'}", 'postgres:mydb', '-'],
+        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'postgres:mydb']
     ]
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        lookups = Lookups(self)
         self.dbname = usage.get_arg("dbname")
-
-        self.db_host = self.lookup_params.get("host")
-        self.db_user = self.lookup_params.get("user")
-        self.db_pass = self.lookup_params.get("password")
-        self.db_port = int(self.lookup_params.get("port", 5432))
-        self.db_ssl  = bool(self.lookup_params.get("ssl", False))
+        config = Config('dbname', self, self.dbname)
+        self.db_host = config.lookup("host")
+        self.db_user = config.lookup("user")
+        self.db_pass = config.lookup("password")
+        self.db_port = int(config.lookup("port", 5432))
+        self.db_ssl  = bool(config.lookup("ssl", False))
 
         self.query_field  = usage.get_param('query_field')
         self.params_field = "params"  # optional: list/tuple (positional) or dict (named)

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/integrations/snowflake_pipe.py

@@ -9,8 +9,9 @@ import uuid
 from decimal import Decimal
 from typing import Any, Dict, Optional
 
-from pjk.base import ParsedToken, Usage, TokenError, Integration
+from pjk.base import ParsedToken, TokenError, Usage, Integration
 from pjk.pipes.query_pipe import QueryPipe
+from pjk.common import Config
 
 
 # ---------- utilities ----------
@@ -55,23 +56,6 @@ def _row_to_dict(cursor, row) -> Dict[str, Any]:
     return {col: normalize(val) for col, val in zip(cols, row)}
 
 
-def _get_any(params: Dict[str, Any], *keys: str) -> Optional[Any]:
-    """
-    Fetch a value from params using any of the provided keys,
-    trying case variants and optional SNOWFLAKE_ prefix.
-    """
-    variants = []
-    for k in keys:
-        variants.extend([
-            k, k.lower(), k.upper(),
-            f"snowflake_{k}".lower(), f"SNOWFLAKE_{k}".upper()
-        ])
-    for v in variants:
-        if v in params:
-            return params[v]
-    return None
-
-
 # ---------- client ----------
 
 class SnowflakeClient:
@@ -141,29 +125,28 @@ class SnowflakeClient:
 class SnowflakePipe(QueryPipe, Integration):
     """
     Snowflake query pipe; executes SQL found in input record['query'] and streams rows.
-    Connection/session settings are pulled from ~/.pjk/lookups.yaml under the arg name.
+    Connection/session settings are pulled from ~/.pjk/component_configs.yaml under the arg name.
     """
     name = 'snowflake'
     desc = "Snowflake query pipe; executes an SQL query for each input record."
     arg0 = ('dbname', 'database name.')
     examples = [
-        ["{'query': 'SELECT CURRENT_ROLE();'}", "snow:EDLDB", "-"],
-        ["myquery.sql", "snow:EDLDB", "-"]
+        ["{'query': 'SELECT CURRENT_ROLE();'}", "snowflake:EDLDB", "-"],
+        ["myquery.sql", "snowflake:EDLDB", "-"]
     ]
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        self.dbname = usage.get_arg(type(self).arg0[0])
-
-        # Accept both bare keys and SNOWFLAKE_* variants in lookups.yaml
-        self.sf_account  = _get_any(self.lookup_params, "account")
-        self.sf_user     = _get_any(self.lookup_params, "user")
-        self.sf_auth     = _get_any(self.lookup_params, "authenticator")
-        self.sf_role     = _get_any(self.lookup_params, "role")
-        self.sf_wh       = _get_any(self.lookup_params, "warehouse")
+        self.dbname = usage.get_arg('dbname')
+        config = Config('dbname', self, self.dbname)
+        self.sf_account  = config.lookup("account")
+        self.sf_user     = config.lookup("user")
+        self.sf_auth     = config.lookup("authenticator")
+        self.sf_role     = config.lookup("role")
+        self.sf_wh       = config.lookup("warehouse")
+        self.sf_schema   = config.lookup("schema")
         self.sf_db       = self.dbname
-        self.sf_schema   = _get_any(self.lookup_params, "schema")
 
         # Basic validation
         missing = [k for k, v in [
@@ -176,7 +159,7 @@ class SnowflakePipe(QueryPipe, Integration):
         ] if not v]
         if missing:
             raise TokenError(
-                f"lookups entry '{self.dbname}' missing: {', '.join(missing)}"
+                f"config entry '{self.dbname}' missing: {', '.join(missing)}"
             )
 
         self.query_field = usage.get_param('query_field')

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/main.py

@@ -11,6 +11,8 @@ from pjk.parser import ExpressionParser
 from pjk.base import UsageError
 from pjk.log import init as init_logging
 from datetime import datetime
+from pathlib import Path
+import shutil
 import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
@@ -65,13 +67,22 @@ def execute_threaded(sinks, stop_progress=None):
     else:
         executor.shutdown(wait=True)
 
+def initialize():
+    init_logging()
+
+    '''
+    src = Path("src/pjk/resources/component_configs.tmpl")
+    dst_dir = Path.home() / ".pjk"
+    dst_dir.mkdir(parents=True, exist_ok=True)
+    shutil.copy(src, dst_dir / src.name)
+    '''
+
 def execute(command: str):
     tokens = shlex.split(command, comments=True, posix=True)
     execute_tokens(tokens)
 
 def execute_tokens(tokens: List[str]):
-    init_logging()
-    # (remove the sys.exit SIGINT handler here)
+    initialize()
 
     if '--version' in tokens:
         print(f"pjk version {__version__}")

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/parser.py

@@ -55,8 +55,20 @@ class OperandStack:
             return None
         return self.stack[-1]
     
+    def clear(self):
+        self.stack.clear()
+    
     def empty(self):
         return len(self.stack) == 0
+    
+    def print(self, toadd):
+        print('---------')
+        if toadd:
+            print(f'{type(toadd).__name__}={id(toadd)}')
+        if len(self.stack) == 0:
+            print(f'Stack={id(self)} StackEmpty')
+        for op in self.stack:
+            print(f'Stack={id(self)} {type(op).__name__}={id(op)}')
 
 class ExpressionParser:
     def __init__(self, registry: ComponentRegistry):
@@ -69,6 +81,9 @@ class ExpressionParser:
                                             'pjk <source> [<pipe> ...] <sink>'])
 
         source = self.stack.pop()
+        if isinstance(source, SubExpression):
+            raise TokenError("Poorly formed sub-expression.  Begin token '[' without matching 'over' keyword." )
+
         if not self.stack.empty():
             raise TokenError.from_list(['A sink can only consume one source.',
                                         'pjk <source> [<pipe> ...] <sink>'])
@@ -167,29 +182,38 @@ class StackLoader:
         return ReducerAggregatorPipe(top_level_reducers=self.top_level_reducers)
 
     def add_operator(self, op, stack: OperandStack):
+        #stack.print(op)
+
         if not stack.empty() and isinstance(stack.peek(), SubExpression):
-            top = stack.peek()
+            subexp = stack.peek()
 
-            if isinstance(op, SubExpressionOver):
-                subexp_begin = stack.pop()
-                subexp_begin.set_over_arg(op.get_over_arg())
-                op.add_source(subexp_begin)
+            if isinstance(op, SubExpressionOver) and subexp.recursion_depth() == 0:
+                subexp = stack.pop()
+                op.add_source(subexp)
                 stack.push(op)
 
                 global stack_level
-                # SEEMS LIKE A HACK! FIXME.  The stack should handle this but its off by one
-                stack_level-=1 
+                stack_level -=1 # not sure why this can't be handled exclusively by the stack
                 return
+            
             else: # an operator within the subexpression
-                top.add_subop(op)
+                subexp = stack.peek()
+                subexp.add_subop(op)
                 return
 
+        if isinstance(op, SubExpressionOver):
+            if stack.empty or not isinstance(stack.peek(), SubExpression):
+                raise TokenError("Poorly formed sub-expression.  'over' keyword without matching begin token '['.")
+            op.add_source(stack.pop())
+            stack.push(op)
+
         # order matters, because sources are pipes
         if isinstance(op, Pipe):
             arity = op.arity # class level attribute
             for _ in range(arity):
                 if stack.empty():
-                    raise UsageError(f"'{op}' requires {arity} input(s)")
+                    name = type(op).usage().name
+                    raise TokenError(f"'{name}' requires {arity} input(s)")
                 op.add_source(stack.pop())
             stack.push(op)
 
@@ -243,6 +267,31 @@ class UpstreamSource(Source):
                 self.sub_recs_in.increment()
                 yield item
     
+class SubExpressionOver(Pipe):
+    @classmethod
+    def usage(cls) -> Usage:
+        u = Usage(
+            name="over",
+            desc="sub-expression over.",
+            component_class=cls,
+        )
+        return u
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.over_arg = ptok.get_arg(0)
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        if not isinstance(self.left, SubExpression):
+            raise Exception('this actually cannot happen, but did')
+
+        for record in self.left:
+            self.left.subexp_process(record, self.over_arg)
+            yield record
+
 class SubExpression(Pipe, ProgressIgnore):
     @classmethod
     def create(cls, token: str) -> Pipe:
@@ -255,28 +304,34 @@ class SubExpression(Pipe, ProgressIgnore):
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok)
-        self.over_arg = None
-        self.over_field = None
         self.subexp_ops = []
-        self.over_pipe = None
         self.stack_helper = StackLoader()
         self.subexp_stack = OperandStack() 
         self.upstream_source = UpstreamSource()
         self.subexp_stack.push(self.upstream_source)
+        self.recursions = 0 # number of subexpression within
+        self.subexp_left = None
 
     def add_subop(self, op):
         self.subexp_ops.append(op)
+        if isinstance(op, SubExpression):
+            self.recursions += 1
+        elif isinstance(op, SubExpressionOver):
+            self.recursions -= 1
         self.stack_helper.add_operator(op, self.subexp_stack)
 
-    def set_over_arg(self, over_arg):  #FIXME, this should take QueryPipe
-        self.over_arg = over_arg
-        if over_arg.endswith('.py'):
-            self.over_field = 'child'
-            self.over_pipe = UserPipeFactory.create(over_arg)
-            self.upstream_source.set_source(self.over_pipe)
-            self.subexp_ops.append(self.over_pipe)
-        else:
-            self.over_field = over_arg
+    def recursion_depth(self):
+        return self.recursions
+    
+    #def bind(self, subex_over: SubExpressionOver):
+    #    self.over_arg = subex_over.get_over_arg()
+    #    if self.over_arg.endswith('.py'):
+    #        self.over_field = 'child'
+    #        self.over_pipe = UserPipeFactory.create(self.over_arg)
+    #        self.upstream_source.set_source(self.over_pipe)
+    #        self.subexp_ops.append(self.over_pipe)
+    #    else:
+    #        self.over_field = self.over_arg
 
     def reset(self):
         for op in self.subexp_ops:
@@ -284,60 +339,40 @@ class SubExpression(Pipe, ProgressIgnore):
                 op.reset()
 
     def __iter__(self):
-        for record in self.left:
-            if self.over_pipe:
-                one = UpstreamSource()
-                one.add_item(record)
-                self.over_pipe.set_sources([one])
-            else:
-                field_data = record.pop(self.over_field, None)
-                if not field_data:
-                    yield record
-                    continue
-                if isinstance(field_data, list):
-                    self.upstream_source.set_list(field_data)
-                else:
-                    self.upstream_source.set_list([field_data])
+        yield from self.left # pass thru to subexp_over which then calls process
 
-            # Reset sub-pipe stack
-            for op in self.subexp_ops:
-                op.reset()
+    def subexp_process(self, record: dict, over_field: str):
+        #for record in self.left:
+        #    if self.over_pipe:
+        #        one = UpstreamSource()
+        #        one.add_item(record)
+        #        self.over_pipe.set_sources([one])
 
-            out_recs = []
-            top = self.subexp_stack.peek()
-            for rec in top:
-                out_recs.append(rec)
+        if not self.subexp_left:
+            self.subexp_left = self.subexp_stack.pop()
 
-            record[self.over_field] = out_recs
+        field_data = record.pop(over_field, None)
+        if not field_data:
+            return
 
-            for op in self.subexp_ops:
-                get_subexp = getattr(op, "get_subexp_result", None)
-                if get_subexp:
-                    name, value = get_subexp()
-                    if name:
-                        record[name] = value
+        if isinstance(field_data, list):
+            self.upstream_source.set_list(field_data)
+        else:
+            self.upstream_source.set_list([field_data])
 
-            yield record
+        # Reset sub-pipe stack
+        for op in self.subexp_ops:
+            op.reset()
 
-class SubExpressionOver(Pipe):
-    @classmethod
-    def usage(cls) -> Usage:
-        u = Usage(
-            name="over",
-            desc="sub-expression over.",
-            component_class=cls,
-        )
-        return u
-    
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
-        self.over_arg = ptok.get_arg(0)
+        out_recs = []
 
-    def get_over_arg(self):
-        return self.over_arg
+        for rec in self.subexp_left:
+            out_recs.append(rec)
 
-    def reset(self):
-        pass  # stateless
+        record[over_field] = out_recs
 
-    def __iter__(self):
-        yield from self.left
+        for op in self.subexp_ops:
+            if isinstance(op, ReducePipe):
+                name, value = op.get_subexp_result()
+                if name:
+                    record[name] = value

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/pipes/denorm.py

@@ -5,6 +5,7 @@
 
 from pjk.base import Pipe, ParsedToken, Usage, UsageError
 from typing import Iterator
+from pjk.progress import papi
 
 class Denormer:
     def __init__(self, record, field):
@@ -23,7 +24,7 @@ class Denormer:
         elif isinstance(data, dict):
             self.subrec_list = [data]
         else:
-            raise UsageError("can only denorm sub-records")
+            raise UsageError("can only explode sub-records")
 
     def __iter__(self) -> Iterator[dict]:
         for subrec in self.subrec_list:
@@ -53,8 +54,8 @@ class DenormPipe(Pipe):
         super().__init__(ptok)
 
         self.field = usage.get_arg('field')
-        if not self.field:
-            raise UsageError("denorm must include a field name")
+        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_out = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 
         self._pending_iter = None
 
@@ -63,6 +64,8 @@ class DenormPipe(Pipe):
 
     def __iter__(self):
         for record in self.left:
+            self.recs_in.increment()
             denormer = Denormer(record, self.field)
             for out in denormer:
+                self.recs_out.increment()
                 yield out

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/pipes/factory.py

@@ -20,6 +20,7 @@ from pjk.pipes.select import SelectFields
 from pjk.pipes.denorm import DenormPipe
 from pjk.integrations.postgres_pipe import PostgresPipe
 from pjk.integrations.snowflake_pipe import SnowflakePipe
+from pjk.integrations.opensearch_query_pipe import OpenSearchQueryPipe
 from pjk.pipes.sample import SamplePipe
 from pjk.pipes.user_pipe_factory import UserPipeFactory
 
@@ -40,7 +41,8 @@ COMPONENTS = {
         'sample': SamplePipe,
         'explode': DenormPipe,
         'postgres': PostgresPipe,
-        'snowflake': SnowflakePipe
+        'snowflake': SnowflakePipe,
+        'os_query': OpenSearchQueryPipe
     }
 
 class PipeFactory(ComponentFactory):

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/pipes/move_field.py

@@ -3,34 +3,33 @@
 
 # djk/pipes/move_field.py
 
-from pjk.base import Pipe, ParsedToken, Usage
+from pjk.base import Pipe, ParsedToken, Usage, TokenError
+from pjk.common import is_valid_field_name
 
 class MoveField(Pipe):
     @classmethod
     def usage(cls):
-        usage = Usage(
+        u = Usage(
             name='as',
             desc='rename a field in the record',
             component_class=cls
         )
-        usage.def_arg(name='src', usage='Source field name')
-        usage.def_arg(name='dst', usage='Destination field name')
-        usage.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
+        u.def_arg(name='src', usage='Source field name')
+        u.def_arg(name='dst', usage='Destination field name')
+        u.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
 
-        return usage
+        return u
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
         self.src = usage.get_arg('src')
         self.dst = usage.get_arg('dst')
-        self.count = 0
 
-    def reset(self):
-        self.count = 0
+        if not is_valid_field_name(self.dst) or not is_valid_field_name(self.src):
+            raise TokenError('field names only allow letters, numbers (non-initially) and underbar')
 
     def __iter__(self):
         for record in self.left:
-            self.count += 1
             if self.src in record:
                 record[self.dst] = record.pop(self.src)
             yield record

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/pipes/query_pipe.py

@@ -1,11 +1,9 @@
-from pjk.base import Pipe, ParsedToken, Usage, TokenError
-from pjk.common import Lookups
+from pjk.base import Pipe, ParsedToken, Usage
 from typing import Any, Dict, Iterable, Optional
 from abc import abstractmethod
 
 
 class QueryPipe(Pipe):
-    requires_lookups = True
     name: str = None
     desc: str = None
     arg0: tuple[Optional[str], Optional[str]] = (None, None)
@@ -18,8 +16,7 @@ class QueryPipe(Pipe):
             desc=cls.desc,
             component_class=cls
         )
-        u.def_arg(name=cls.arg0[0], usage=f'{cls.arg0[1]} ~/.pjk/lookups.yaml must containing entry {cls.__name__}-<{cls.arg0[0]}>\n   containing necessary parameters.')
-        u.def_param("query_field", usage="field of query.", default="query")
+        u.def_arg(name=cls.arg0[0], usage=f"{cls.arg0[1]} ~/.pjk/component_configs.yaml must contain entry '{cls.__name__}-<{cls.arg0[0]}'>\n  with necessary parameters.")
         u.def_param("count", usage="Number of search results, (databases may ignore)", is_num=True, default="10")
         u.def_param("shape", usage='the shape of ouput records', is_num=False,
                        valid_values={'xR', 'Q_xR', 'Qxr'}, default='xR')
@@ -32,14 +29,8 @@ class QueryPipe(Pipe):
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
-
-        self.lookup_params = None
-        if type(self).requires_lookups:
-            lookups = Lookups(self)
-            lookup_name = usage.get_arg(type(self).arg0[0])
-            self.lookup_params = lookups.get(lookup_name)
-
         self.output_shape = usage.get_param('shape')
+        self.count = usage.get_param('count')
 
     @abstractmethod
     def execute_query_returning_Q_xR_iterable(self, record) -> Iterable[Dict[str, Any]]:
@@ -83,7 +74,7 @@ class QueryPipe(Pipe):
                         q_out = self._make_q_object(in_rec, out_rec)
                         continue
                     r_list.append(out_rec)
-                q_out['result'] = r_list
+                q_out['child'] = r_list
                 yield q_out
 
 

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.0"
+__version__ = "0.6.3"

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/python_jack_knife.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.0
+Version: 0.6.3
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.0 → python_jack_knife-0.6.3}/src/python_jack_knife.egg-info/SOURCES.txt

@@ -12,6 +12,8 @@ src/pjk/progress.py
 src/pjk/registry.py
 src/pjk/version.py
 src/pjk/integrations/ddb_sink.py
+src/pjk/integrations/opensearch_client.py
+src/pjk/integrations/opensearch_query_pipe.py
 src/pjk/integrations/postgres_pipe.py
 src/pjk/integrations/snowflake_pipe.py
 src/pjk/pipes/__init__.py