python-jack-knife 0.6.4__tar.gz → 0.6.5__tar.gz

{python_jack_knife-0.6.4/src/python_jack_knife.egg-info → python_jack_knife-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.4
+Version: 0.6.5
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/common.py

@@ -5,7 +5,13 @@ import sys, shutil, subprocess, contextlib, signal
 import os
 import re
 import yaml
-from pjk.base import TokenError, Integration, Source, Pipe
+from pjk.usage import Usage, TokenError
+from abc import ABC
+
+# mixin 
+# just for distinguishing components for display
+class Integration(ABC):
+    pass
 
 class SafeNamespace:
     def __init__(self, obj):
@@ -74,33 +80,6 @@ def highlight(text: str, color: str = 'bold', value: str = None) -> str:
     style = COLOR_CODES.get(color.lower(), COLOR_CODES['bold'])
     return text.replace(value, f"{style}{value}{RESET}")
 
-class Config:
-    def __init__(self, instance_type: str, component_class: Source|Pipe, instance: str):
-        # instance = name of the instance, e.g. 'myindexcollection', instance_type = 'index'
-        # instance_type only used by automatic config template maker MUST BE STRING LITERAL!
-        self.configs_yaml = os.path.expanduser('~/.pjk/component_configs.yaml')
-        self.class_name = type(component_class).__name__
-        self.instance = instance
-        self._data = {}
-        self._load()
-        
-    def _load(self):
-        if os.path.exists(self.configs_yaml):
-            with open(self.configs_yaml, 'r') as f:
-                self._data = yaml.safe_load(f) or {}
-        else:
-            self._data = {}
-
-    def lookup(self, param: str, default=None):
-        instance_key = f'{self.class_name}-{self.instance}'
-        entry = self._data.get(instance_key, None)
-        if not entry:
-            raise TokenError(
-                f"~/.pjk/component_configs.yaml does not contain entry for '{instance_key}' with required params."
-            )
-        
-        return entry.get(param, default)
-
 class ComponentFactory:
     def __init__(self, core_components: dict):
         self.num_orig = 0

python_jack_knife-0.6.5/src/pjk/components.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional, List
+from pjk.usage import Usage, NoBindUsage, ParsedToken
+    
+# mixin
+class KeyedSource(ABC):
+    @classmethod
+    def usage(cls):
+        return Usage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component"
+        )
+    
+    @abstractmethod
+    def lookup(self, left_rec) -> Optional[dict]:
+        """Return the record associated with the given key, or None."""
+        pass
+
+    def get_unlookedup_records(self) -> List[Any]:
+        # for outer join
+        pass
+
+    def deep_copy(self):
+        return None
+
+class Source(ABC):
+    @classmethod
+    def usage(cls):
+        return NoBindUsage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component",
+            component_class=cls
+        )
+
+    @abstractmethod
+    def __iter__(self):
+        raise NotImplementedError("__iter__ must be implemented by subclasses")
+
+    def __next__(self):
+        # lazily create an internal iterator the first time next() is called
+        if not hasattr(self, "_iter"):
+            self._iter = iter(self)
+        return next(self._iter)
+
+    def deep_copy(self):
+        return None  # Default: not copyable unless overridden
+    
+    def close(self):
+        pass
+    
+    def _get_sources(self, source_list: list):
+        pass
+    
+class Pipe(Source):
+    arity: int = 1
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+        self.ptok = ptok
+        self.usage = usage
+        self.left = None  # left source for convience
+        self.right = None # right source for convience
+        self.inputs: List[Source] = []
+
+    def add_source(self, source: Source) -> None:
+        self.inputs.append(source)
+        # first two are assigned left, right
+        if self.left is None:
+            self.left = source
+        elif self.right is None:
+            self.right = self.left
+            self.left = source
+
+    def reset(self):
+        pass  # optional hook
+
+    def deep_copy(self) -> Optional["Pipe"]:
+        return None
+    
+    def _get_sources(self, source_list: list):
+        for ix in self.inputs:
+            source_list.append(ix)
+            ix._get_sources(source_list)
+
+class DeepCopyPipe(Pipe):
+    def deep_copy(self):
+        """
+        Generic deep_copy: clone left source, re-instantiate
+        this pipe class with the same ptok/usage, and attach.
+        """
+        source_clone = self.left.deep_copy()
+        if not source_clone:
+            return None
+
+        # re-instantiate using the actual subclass
+        pipe = type(self)(self.ptok, self.usage)
+        pipe.add_source(source_clone)
+        return pipe
+
+class Sink(ABC):
+    @classmethod
+    def usage(cls):
+        return NoBindUsage(
+            name=cls.__name__,
+            desc=f"{cls.__name__} component",
+            component_class=cls
+        )
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage = None):
+        self.ptok = ptok
+        self.usage = usage
+
+    def drain(self):
+        self.process()
+        self.close()
+
+        # get all inputs in the execution chain for closing
+        inputs = [self.input]
+        self.input._get_sources(inputs)
+        for input in inputs:
+            input.close()
+
+    # optional
+    def close(self):
+        pass
+
+    def add_source(self, source: Source) -> None:
+        self.input = source
+        
+    @abstractmethod
+    def process(self) -> None:
+        pass
+
+    def deep_copy(self):
+        return None
+

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/integrations/opensearch_client.py

@@ -1,28 +1,26 @@
-from pjk.common import Config
+from pjk.usage import Usage
 
 class OpenSearchClient:
 
     @classmethod
-    def get_client(cls, config: Config):
-        aws_auth = config.lookup("os_auth_use_aws", "true") == 'true'
-        scheme = config.lookup("os_scheme", "https")
-        verify_certs = config.lookup("os_verify_certs", "true") == 'true'
-        ca_certs = config.lookup("os_ca_certs", None)
-        region = config.lookup("os_region", None)
-        service = config.lookup("os_service", "es")
-        username = config.lookup("os_username", None)
-        password = config.lookup("os_password", None)
-        timeout = float(config.lookup("os_timeout", 30))
-        ssl_assert_hostname = config.lookup("os_ssl_assert_hostname", "true") == 'true'
-        ssl_show_warn = config.lookup("os_ssl_show_warn", "false") == 'true'
-        host = config.lookup("os_host", None)
-        port = config.lookup("os_port", None)
+    def get_client(cls, u: Usage):
+        aws_auth = u.get_config("os_auth_use_aws")
+        scheme = u.get_config("os_scheme")
+        verify_certs = u.get_config("os_verify_certs")
+        ca_certs = u.get_config("os_ca_certs")
+        region = u.get_config("os_region")
+        service = u.get_config("os_service")
+        username = u.get_config("os_username")
+        password = u.get_config("os_password")
+        timeout = u.get_config("os_timeout")
+        ssl_assert_hostname = u.get_config("os_ssl_assert_hostname")
+        ssl_show_warn = u.get_config("os_ssl_show_warn")
+        host = u.get_config("os_host")
+        port = u.get_config("os_port")
 
         # Reasonable port defaults
         if port is None:
             port = 443 if scheme == "https" else 9200
-        else:
-            port = int(port)
 
         if host is None:
             raise ValueError("Config os_host is required (set os_host + os_port/os_scheme, or a connection profile).")

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/integrations/opensearch_query_pipe.py

@@ -4,9 +4,9 @@ import traceback
 from copy import deepcopy
 from typing import Optional, Iterator, Dict, Any, Iterable
 
-from pjk.base import Pipe, ParsedToken, Usage, Integration
+from pjk.usage import ParsedToken, Usage
 from pjk.pipes.query_pipe import QueryPipe
-from pjk.common import Config
+from pjk.common import Integration
 from pjk.integrations.opensearch_client import OpenSearchClient
 
 def build_body_from_string(query_string: str) -> dict:
@@ -23,24 +23,39 @@ def build_body_from_string(query_string: str) -> dict:
 
 class OpenSearchQueryPipe(QueryPipe, Integration):
     name = "os_query"
-    desc = "Opensearch query pipe. Uses record['query_string'] or record['query_object'] for os query"
-    arg0 = ("index", "index to query over")
+    desc = "Opensearch query pipe. Uses record['query'] or record['os_query_object'] for os query"
+    arg0 = ("instance", "instance to query over.")
     examples = [
-        ["{'query_string': '*'}", 'os_query:myindex', '-'],
-        ["{'query_string': 'dog'}", 'os_query:myindex', '-'],
-        ["{'query_string': 'dog AND cat'}", 'os_query:myindex', '-'],
-        ["{'query_object': {query: {...}}", 'os_query:myindex', '-'],
+        ["{'query': '_ping'}", 'os_query:myindex', '-'],
+        ["{'query': '*'}", 'os_query:myindex', '-'],
+        ["{'query': 'dog'}", 'os_query:myindex', '-'],
+        ["{'query': 'dog AND cat'}", 'os_query:myindex', '-'],
+        ["{'os_query_object': {query: {...}}", 'os_query:myindex', '-'],
+    ]
+
+     # name, type, default
+    config_tuples = [
+        ("index_name", str, None),
+        ("os_auth_use_aws", bool, "true"),
+        ("os_scheme", str, "https"),
+        ("os_verify_certs", bool, "true"),
+        ("os_ca_certs", str, None),
+        ("os_region", str, None),
+        ("os_service", str, "es"),
+        ("os_username", str, None),
+        ("os_password", str, None),
+        ("os_timeout", float, 30),
+        ("os_ssl_assert_hostname", bool, "true"),
+        ("os_ssl_show_warn", bool, "false"),
+        ("os_host", str, None),
+        ("os_port", int, None)
     ]
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        # index from arg0 or config
-        self.index = ptok.get_arg(0)
-
-        # Build the OpenSearch client (handles AWS/basic/none)
-        config = Config('index', self, self.index)
-        self.client = OpenSearchClient.get_client(config)
+        self.index = usage.get_config("index_name")
+        self.client = OpenSearchClient.get_client(usage)
 
         # Iteration state
         self.cur_record: Optional[Dict[str, Any]] = None
@@ -53,14 +68,35 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
     def close(self):
         pass
 
-    def execute_query_returning_Q_xR_iterable(self, query_record: dict) -> Iterator[Dict[str, Any]]:
-        query_string = query_record.get('query_string', None)
+    def ping(self):
+        indexes = self.client.indices.get_alias(index="*")  
+        index_list = []
+
+        yield {'num_indexes': len(indexes.keys())}
+        for index_name in sorted(indexes.keys()):
+            try:
+                count = self.client.count(index=index_name)["count"]
+                yield {'index': index_name, 'count': count}
+
+            except Exception as e:
+                print(f"{index_name}: failed to count ({e})")
+
+    def execute_query_returning_S_xO_iterable(self, query_record: dict) -> Iterator[Dict[str, Any]]:
+        query_string = query_record.get('query', None)
         query_body = None
 
         if query_string:
+            if query_string == '_ping':
+                yield from self.ping()
+                return
+
             query_body = build_body_from_string(query_string)
         else:
-            query_body = query_record.get('query_object')
+            query_body = query_record.get('os_query_object')
+
+        if not query_body:
+            yield {'_error': "query_record missing 'query' or 'os_query_object' field"}
+            return
 
         try:
             # Build final request body
@@ -83,7 +119,7 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
                 "took_ms": took,
                 "total_hits": total_hits,
                 "index": self.index,
-                "os_query_body": req_body
+                "os_query_object": req_body
             }
 
             # Emit each hit

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/integrations/postgres_pipe.py

@@ -9,8 +9,8 @@ import uuid
 from decimal import Decimal
 from typing import Any, Dict, Optional
 
-from pjk.base import Integration, ParsedToken, Usage
-from pjk.common import Config
+from pjk.usage import ParsedToken, Usage
+from pjk.common import Integration
 from pjk.pipes.query_pipe import QueryPipe
 
 
@@ -19,11 +19,11 @@ class DBClient:
     _connection = None
     
     def __init__(self, host: str, username: str, password: Optional[str],
-                 dbname: str, port: int = 5432, ssl: bool = False):
+                 db_name: str, port: int = 5432, ssl: bool = False):
         import pg8000 # lazy import
         if DBClient._connection is None:
             try:
-                kwargs = dict(user=username, password=password, host=host, database=dbname, port=port)
+                kwargs = dict(user=username, password=password, host=host, database=db_name, port=port)
                 if ssl:
                     import ssl as _ssl
                     kwargs["ssl_context"] = _ssl.create_default_context()
@@ -92,26 +92,34 @@ def _row_to_dict(cursor, row) -> Dict[str, Any]:
 
 class PostgresPipe(QueryPipe,Integration):
     name = 'postgres'
-    desc = "Postgres query pipe; executes SQL from input."
-    arg0 = ("dbname", 'database name.')
+    desc = "Postgres query pipe; executes SQL over input record['query']."
+    arg0 = ("instance", 'instance of database.')
     examples = [
         ['myquery.sql', 'postgres:mydb', '-'],
         ["{'query': 'SELECT * from MY_TABLE;'}", 'postgres:mydb', '-'],
         ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'postgres:mydb']
     ]
 
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
+    # name, type, default
+    config_tuples = [
+            ('db_name', str, None),
+            ('host', str, None),
+            ('user', str, None),
+            ('password', str, None),
+            ('port', int, 5432),
+            ('ssl', bool, False)
+    ]
+    
+    def __init__(self, ptok: ParsedToken, u: Usage):
+        super().__init__(ptok, u)
 
-        self.dbname = usage.get_arg("dbname")
-        config = Config('dbname', self, self.dbname)
-        self.db_host = config.lookup("host")
-        self.db_user = config.lookup("user")
-        self.db_pass = config.lookup("password")
-        self.db_port = int(config.lookup("port", 5432))
-        self.db_ssl  = bool(config.lookup("ssl", False))
+        self.db_name = u.get_config('db_name')
+        self.db_host = u.get_config("host")
+        self.db_user = u.get_config("user")
+        self.db_pass = u.get_config("password")
+        self.db_port = u.get_config("port")
+        self.db_ssl  = u.get_config("ssl")
 
-        self.query_field  = usage.get_param('query_field')
         self.params_field = "params"  # optional: list/tuple (positional) or dict (named)
 
     def reset(self):
@@ -124,7 +132,7 @@ class PostgresPipe(QueryPipe,Integration):
         Figures out result, rowcount, function automatically.
         """
         h = {
-            "db": self.dbname,
+            "db": self.db_name,
             "dbhost": self.db_host,
         }
         if params:
@@ -145,12 +153,12 @@ class PostgresPipe(QueryPipe,Integration):
 
         return h
 
-    def execute_query_returning_Q_xR_iterable(self, record):
+    def execute_query_returning_S_xO_iterable(self, record):
         client = DBClient(
             host=self.db_host,
             username=self.db_user,
             password=self.db_pass,
-            dbname=self.dbname,
+            db_name=self.db_name,
             port=self.db_port,
             ssl=self.db_ssl,
         )

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/integrations/snowflake_pipe.py

@@ -9,10 +9,9 @@ import uuid
 from decimal import Decimal
 from typing import Any, Dict, Optional
 
-from pjk.base import ParsedToken, TokenError, Usage, Integration
+from pjk.usage import ParsedToken, TokenError, Usage
 from pjk.pipes.query_pipe import QueryPipe
-from pjk.common import Config
-
+from pjk.common import Integration
 
 # ---------- utilities ----------
 
@@ -128,41 +127,34 @@ class SnowflakePipe(QueryPipe, Integration):
     Connection/session settings are pulled from ~/.pjk/component_configs.yaml under the arg name.
     """
     name = 'snowflake'
-    desc = "Snowflake query pipe; executes an SQL query for each input record."
-    arg0 = ('dbname', 'database name.')
+    desc = "Snowflake query pipe; executes SQL over input record['query']."
+    arg0 = ('instance', 'instance of the database.')
     examples = [
         ["{'query': 'SELECT CURRENT_ROLE();'}", "snowflake:EDLDB", "-"],
         ["myquery.sql", "snowflake:EDLDB", "-"]
     ]
 
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
-
-        self.dbname = usage.get_arg('dbname')
-        config = Config('dbname', self, self.dbname)
-        self.sf_account  = config.lookup("account")
-        self.sf_user     = config.lookup("user")
-        self.sf_auth     = config.lookup("authenticator")
-        self.sf_role     = config.lookup("role")
-        self.sf_wh       = config.lookup("warehouse")
-        self.sf_schema   = config.lookup("schema")
-        self.sf_db       = self.dbname
-
-        # Basic validation
-        missing = [k for k, v in [
-            ("account", self.sf_account),
-            ("user", self.sf_user),
-            ("authenticator|password", self.sf_auth or self.sf_password),
-            ("role", self.sf_role),
-            ("warehouse", self.sf_wh),
-            ("schema", self.sf_schema),
-        ] if not v]
-        if missing:
-            raise TokenError(
-                f"config entry '{self.dbname}' missing: {', '.join(missing)}"
-            )
-
-        self.query_field = usage.get_param('query_field')
+    # name, type, default
+    config_tuples = [
+       ("account", str, None),
+        ("user", str, None),
+        ("authenticator", str, None),
+        ("role", str, None),
+        ("warehouse", str, None),
+        ("schema", str, None),
+        ('db_name', str, None)
+    ]
+
+    def __init__(self, ptok: ParsedToken, u: Usage):
+        super().__init__(ptok, u)
+        self.sf_account  = u.get_config("account")
+        self.sf_user     = u.get_config("user")
+        self.sf_auth     = u.get_config("authenticator")
+        self.sf_role     = u.get_config("role")
+        self.sf_wh       = u.get_config("warehouse")
+        self.sf_schema   = u.get_config("schema")
+        self.sf_db       = u.get_config('db_name')
+
         self.params_field = "params"  # optional: list/tuple (positional) or dict (named)
 
     def reset(self):
@@ -174,7 +166,7 @@ class SnowflakePipe(QueryPipe, Integration):
         Build a header record with query metadata and session context.
         """
         h: Dict[str, Any] = {
-            "db": self.dbname,
+            "db_name": self.sf_db,
             "account": self.sf_account,
             "role": self.sf_role,
             "warehouse": self.sf_wh,
@@ -198,7 +190,7 @@ class SnowflakePipe(QueryPipe, Integration):
         h["result"] = "ok"
         return h
 
-    def execute_query_returning_Q_xR_iterable(self, record):
+    def execute_query_returning_S_xO_iterable(self, record):
         client = SnowflakeClient(
             account=self.sf_account,
             user=self.sf_user,

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/main.py

@@ -4,15 +4,14 @@
 #!/usr/bin/env python
 import sys
 import os
-import signal
 import shlex
+import shutil
 from typing import List
 from pjk.parser import ExpressionParser
-from pjk.base import UsageError
+from pjk.usage import UsageError
 from pjk.log import init as init_logging
 from datetime import datetime
 from pathlib import Path
-import shutil
 import traceback
 import concurrent.futures
 from pjk.registry import ComponentRegistry
@@ -70,12 +69,10 @@ def execute_threaded(sinks, stop_progress=None):
 def initialize():
     init_logging()
 
-    '''
-    src = Path("src/pjk/resources/component_configs.tmpl")
-    dst_dir = Path.home() / ".pjk"
-    dst_dir.mkdir(parents=True, exist_ok=True)
-    shutil.copy(src, dst_dir / src.name)
-    '''
+    #src = Path("src/pjk/resources/configs.tmpl")
+    #dst_dir = Path.home() / ".pjk"
+    #dst_dir.mkdir(parents=True, exist_ok=True)
+    #hutil.copy(src, dst_dir / src.name)
 
 def execute(command: str):
     tokens = shlex.split(command, comments=True, posix=True)

{python_jack_knife-0.6.4 → python_jack_knife-0.6.5}/src/pjk/man_page.py

@@ -5,11 +5,21 @@ from pjk.pipes.factory import PipeFactory
 from pjk.sources.factory import SourceFactory
 from pjk.sinks.factory import SinkFactory
 from pjk.parser import ExpressionParser
-from pjk.base import Usage
+from pjk.components import Source, Pipe, Sink
+from pjk.usage import Usage, ParsedToken
 from pjk.registry import ComponentRegistry
 from pjk.common import pager_stdout, highlight
 from contextlib import nullcontext
 
+def get_base_class(usage: Usage, as_string: bool = False):
+        if issubclass(usage.comp_class, Sink):
+            return 'sink' if as_string else Sink
+        elif issubclass(usage.comp_class, Pipe):
+            return 'pipe' if as_string else Pipe
+        elif issubclass(usage.comp_class, Source):
+            return 'source' if as_string else Source
+        raise 'improper class'
+
 def smart_print(expr_tokens: list[str], name: str):
     import re
     SAFE_UNQUOTED_RE = re.compile(r"^[a-zA-Z0-9._/:=+-]+$")
@@ -59,13 +69,13 @@ def do_all_man(registry: ComponentRegistry, no_pager: bool = True):
                 print()
 
 def print_man(registry: ComponentRegistry, name: str, usage: Usage):
-    comp_type = usage.get_base_class(as_string=True)
+    comp_type = get_base_class(usage, as_string=True)
     header = f'{name} is a {comp_type}'
+
     print('===================================')
     print('        ', highlight(header, 'bold', name))
     print('===================================')
 
-    print()
     print(usage.get_usage_text())
 
     examples = usage.get_examples()
@@ -88,8 +98,9 @@ def do_examples(token:str, registry: ComponentRegistry):
             for name, comp_class in factory.get_component_name_class_tuples():
                 usage = comp_class.usage()
 
-                comp_type = usage.get_base_class(as_string=True)
+                comp_type = get_base_class(usage, as_string=True)
                 header = f'{name} is a {comp_type}'
+
                 print('===================================')
                 print('        ', highlight(header, 'bold', name))
                 print('===================================')