python-jack-knife 0.5.1__tar.gz → 0.6.0__tar.gz

{python_jack_knife-0.5.1 → python_jack_knife-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.5.1
+Version: 0.6.0
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.5.1 → python_jack_knife-0.6.0}/src/pjk/base.py

@@ -162,7 +162,7 @@ class Usage:
         return '\n'.join(lines)
 
     def get_token_syntax(self):
-        if not self.syntax:
+        if self.syntax:
             return self.syntax # else piece it together
 
         token = f'{self.name}'
@@ -279,6 +279,11 @@ class KeyedSource(ABC):
     def deep_copy(self):
         return None
 
+# mixin 
+# just for distinguishing components for display
+class Integration(ABC):
+    pass
+
 class Source(ABC):
     @classmethod
     def usage(cls):
@@ -300,10 +305,14 @@ class Source(ABC):
 
     def deep_copy(self):
         return None  # Default: not copyable unless overridden
-
+    
+    def close(self):
+        pass
+    
+    def _get_sources(self, source_list: list):
+        pass
     
 class Pipe(Source):
-    deep_copyable: bool = False # default to false
     arity: int = 1
     
     def __init__(self, ptok: ParsedToken, usage: Usage = None):
@@ -326,20 +335,12 @@ class Pipe(Source):
         pass  # optional hook
 
     def deep_copy(self) -> Optional["Pipe"]:
-        if not self.deep_copyable:
-            return None
-        if not self.inputs:
-            raise RuntimeError(f"{self.__class__.__name__} has no inputs set")
-
-        clone = self.__class__(self.ptok, self.__class__.usage())
-
-        for input in self.inputs:
-            strand = input.deep_copy()
-            if strand is None:
-                return None
-            clone.add_source(strand)
-
-        return clone
+        return None
+    
+    def _get_sources(self, source_list: list):
+        for ix in self.inputs:
+            source_list.append(ix)
+            ix._get_sources(source_list)
 
 class DeepCopyPipe(Pipe):
     def deep_copy(self):
@@ -373,13 +374,16 @@ class Sink(ABC):
         self.process()
         self.close()
 
+        # get all inputs in the execution chain for closing
+        inputs = [self.input]
+        self.input._get_sources(inputs)
+        for input in inputs:
+            input.close()
+
     # optional
     def close(self):
         pass
 
-    def print_info(self):
-        pass
-
     def add_source(self, source: Source) -> None:
         self.input = source
         

{python_jack_knife-0.5.1 → python_jack_knife-0.6.0}/src/pjk/common.py

@@ -4,6 +4,7 @@
 import sys, shutil, subprocess, contextlib, signal
 import os
 import yaml
+from pjk.base import TokenError, Integration
 
 class SafeNamespace:
     def __init__(self, obj):
@@ -73,11 +74,12 @@ def highlight(text: str, color: str = 'bold', value: str = None) -> str:
     return text.replace(value, f"{style}{value}{RESET}")
 
 class Lookups:
-    def __init__(self):
+    def __init__(self, component_class):
         self.lookups_yaml = os.path.expanduser('~/.pjk/lookups.yaml')
+        self.class_name = type(component_class).__name__
         self._data = {}
         self._load()
-
+        
     def _load(self):
         """Load lookups from YAML file if it exists."""
         if os.path.exists(self.lookups_yaml):
@@ -93,8 +95,13 @@ class Lookups:
             yaml.safe_dump(self._data, f)
 
     def get(self, key, default=None):
-        """Retrieve a lookup value by key."""
-        return self._data.get(key, default)
+        lookup_key = f'{self.class_name}-{key}'
+        entry = self._data.get(lookup_key, default)
+        if not entry:
+            raise TokenError(
+                f"~/.pjk/lookups.yaml does not contain entry for '{lookup_key}' with required params."
+            )
+        return entry
 
     def set(self, key, value):
         """Set a lookup value and persist it."""
@@ -112,36 +119,37 @@ class Lookups:
         return dict(self._data)
 
 class ComponentFactory:
-    def __init__(self, components: dict, comp_type_name: str):
+    def __init__(self, core_components: dict):
         self.num_orig = 0
-        self.components = components # name -> component_class
-        self.comp_type_name = comp_type_name
-        self.num_orig_comps = len(components)
+        self._components = {}
+        for k, v in core_components.items():
+            if issubclass(v, Integration):
+                self.register(k, v, 'integration')
+            else:
+                self.register(k, v, 'core')
 
-    def register(self, name, comp_class):
-        self.components[name] = comp_class
+    def register(self, name, comp_class, origin: str):
+        self._components[name] = (comp_class, origin)
 
     def get_comp_type_name(self):
-        return self.comp_type_name
-
-    def print_descriptions(self):
-        header = highlight(f'{self.comp_type_name}s')
-        print(header)
-
-        i = 0
-        plugin = ''
-        for name, comp_class in self.components.items():
-            usage = comp_class.usage()
-            lines = usage.desc.split('\n')
-            if i >= self.num_orig_comps:
-                plugin = '(~/.pjk/plugin)'
-            line = f'  {name:<12} {lines[0]} {plugin}'
-            line = highlight(line, 'bold', plugin) if plugin else line
-            print(line)
-            i += 1
+        pass
+
+    def get_component_name_class_tuples(self, origin: str = None) -> list:
+        ret = []
+        for k, (v, org) in self._components.items():
+            if not origin or origin == org:
+                ret.append((k, v))
+        return ret
+
+    def get_component_class(self, name: str):
+        tuple = self._components.get(name)
+        if not tuple:
+            return None
+        component_class, origin = tuple
+        return component_class
 
     def get_usage(self, name: str):
-        comp_class = self.components.get(name)
+        comp_class = self.get_component_class(name)
         if not comp_class:
             return None
         return comp_class.usage()

python_jack_knife-0.5.1/src/pjk/sinks/ddb.py → python_jack_knife-0.6.0/src/pjk/integrations/ddb_sink.py

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-from pjk.base import Sink, Source, ParsedToken, Usage
+from pjk.base import Sink, Integration, Source, ParsedToken, Usage
 from decimal import Decimal
 
-class DDBSink(Sink):
+class DDBSink(Sink, Integration):
     @classmethod
     def usage(cls):
         usage = Usage(

{python_jack_knife-0.5.1/src/pjk/pipes → python_jack_knife-0.6.0/src/pjk/integrations}/postgres_pipe.py

@@ -9,8 +9,9 @@ import uuid
 from decimal import Decimal
 from typing import Any, Dict, Optional
 
-from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, TokenError
+from pjk.base import Integration, ParsedToken, Usage
 from pjk.common import Lookups
+from pjk.pipes.query_pipe import QueryPipe
 
 
 class DBClient:
@@ -89,49 +90,30 @@ def _row_to_dict(cursor, row) -> Dict[str, Any]:
     return {col: normalize(val) for col, val in zip(cols, row)}
 
 
-class PostgresPipe(Pipe):
-    @classmethod
-    def usage(cls):
-        usage = Usage(
-            name="pgres",
-            desc="Postgres query pipe; executes SQL from input record['query'].",
-            component_class=cls,
-        )
-        usage.def_arg(
-            "dbname",
-            "name of db. Entry in ~/.pjk/lookups.yaml containing host, user, password"
-        )
-        usage.def_param(
-            "header",
-            usage="emit header record before query results",
-            valid_values={"true", "false"}, default='true',
-        )
-
-        usage.def_example(expr_tokens=['myquery.sql', 'pgres:mydb'], expect=None)
-        usage.def_example(expr_tokens=["{'query': 'SELECT * from MY_TABLE;'}", 'pgres:mydb'], expect=None)
-        return usage
+class PostgresPipe(QueryPipe,Integration):
+    name = 'pgres'
+    desc = "Postgres query pipe; executes SQL from input."
+    arg0 = ("dbname", 'database name.')
+    examples = [
+        ['myquery.sql', 'pgres:mydb', '-'],
+        ["{'query': 'SELECT * from MY_TABLE;'}", 'pgres:mydb', '-'],
+        ["{'query': 'SELECT * FROM pg_catalog.pg_tables;'}", 'pgres:mydb']
+    ]
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        lookups = Lookups()
+        lookups = Lookups(self)
         self.dbname = usage.get_arg("dbname")
-        db_params = lookups.get(self.dbname)
-        if not db_params:
-            # f-string so dbname prints correctly
-            raise TokenError(
-                f"~/.pjk/lookups.yaml must contain entry for '{self.dbname}' with host, user, password."
-            )
-
-        self.db_host = db_params.get("host")
-        self.db_user = db_params.get("user")
-        self.db_pass = db_params.get("password")
-        self.db_port = int(db_params.get("port", 5432))
-        self.db_ssl  = bool(db_params.get("ssl", False))
-
-        self.query_field  = "query"   # SQL string
+
+        self.db_host = self.lookup_params.get("host")
+        self.db_user = self.lookup_params.get("user")
+        self.db_pass = self.lookup_params.get("password")
+        self.db_port = int(self.lookup_params.get("port", 5432))
+        self.db_ssl  = bool(self.lookup_params.get("ssl", False))
+
+        self.query_field  = usage.get_param('query_field')
         self.params_field = "params"  # optional: list/tuple (positional) or dict (named)
-        self.do_header = usage.get_param("header") == "true"
 
     def reset(self):
         # stateless across reset
@@ -143,7 +125,6 @@ class PostgresPipe(Pipe):
         Figures out result, rowcount, function automatically.
         """
         h = {
-            "query": query,
             "db": self.dbname,
             "dbhost": self.db_host,
         }
@@ -163,9 +144,9 @@ class PostgresPipe(Pipe):
             h["result"] = "ok"
             h["rowcount"] = cur.rowcount
 
-        return {"header": h}
+        return h
 
-    def __iter__(self):
+    def execute_query_returning_Q_xR_iterable(self, record):
         client = DBClient(
             host=self.db_host,
             username=self.db_user,
@@ -175,12 +156,12 @@ class PostgresPipe(Pipe):
             ssl=self.db_ssl,
         )
         try:
-            for input_record in self.left:
-                query = input_record.get(self.query_field)
-                if not query:
-                    yield {"_error": "missing query"}
-                    continue
-                params = input_record.get(self.params_field)
+            query = record.get(self.query_field)
+            if not query:
+                    record['_error'] = 'missing query'
+                    yield record
+            else:        
+                params = record.get(self.params_field)
 
                 cur = client.conn.cursor()
                 try:
@@ -194,8 +175,7 @@ class PostgresPipe(Pipe):
                             cur.execute(query, (params,))
 
                     # yield header first
-                    if self.do_header:
-                        yield self._make_header(cur, query, params)
+                    yield self._make_header(cur, query, params)
 
                     # then stream rows if it was a real SELECT with results
                     if cur.description:

python_jack_knife-0.6.0/src/pjk/integrations/snowflake_pipe.py

@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+#
+# djk/pipes/snowflake_pipe.py
+
+import base64
+import datetime as _dt
+import uuid
+from decimal import Decimal
+from typing import Any, Dict, Optional
+
+from pjk.base import ParsedToken, Usage, TokenError, Integration
+from pjk.pipes.query_pipe import QueryPipe
+
+
+# ---------- utilities ----------
+
+def _iso_dt(x: _dt.datetime) -> str:
+    """ISO 8601; normalize UTC offset to 'Z' for UTC."""
+    s = x.isoformat()
+    return s.replace("+00:00", "Z")
+
+
+def normalize(obj: Any) -> Any:
+    """
+    Make values JSON/YAML-safe and portable (schema-agnostic):
+      - Decimal -> exact string (no sci-notation)
+      - date/datetime/time -> ISO-8601 string (datetime keeps offset; UTC -> 'Z')
+      - UUID -> string
+      - bytes -> base64 string
+      - lists/tuples/sets, dicts -> normalized recursively
+      - leaves int/float/str/bool/None as-is
+    """
+    if obj is None:
+        return None
+    if isinstance(obj, Decimal):
+        return format(obj, "f")
+    if isinstance(obj, _dt.datetime):
+        return _iso_dt(obj)
+    if isinstance(obj, (_dt.date, _dt.time)):
+        return obj.isoformat()
+    if isinstance(obj, uuid.UUID):
+        return str(obj)
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return base64.b64encode(bytes(obj)).decode("ascii")
+    if isinstance(obj, dict):
+        return {k: normalize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple, set)):
+        return [normalize(v) for v in obj]
+    return obj
+
+
+def _row_to_dict(cursor, row) -> Dict[str, Any]:
+    cols = [d[0] for d in cursor.description]
+    return {col: normalize(val) for col, val in zip(cols, row)}
+
+
+def _get_any(params: Dict[str, Any], *keys: str) -> Optional[Any]:
+    """
+    Fetch a value from params using any of the provided keys,
+    trying case variants and optional SNOWFLAKE_ prefix.
+    """
+    variants = []
+    for k in keys:
+        variants.extend([
+            k, k.lower(), k.upper(),
+            f"snowflake_{k}".lower(), f"SNOWFLAKE_{k}".upper()
+        ])
+    for v in variants:
+        if v in params:
+            return params[v]
+    return None
+
+
+# ---------- client ----------
+
+class SnowflakeClient:
+    """
+    Simple connection wrapper for snowflake-connector-python.
+    One connection per client instance (safer than sharing across threads).
+    """
+    def __init__(
+        self,
+        *,
+        account: str,
+        user: str,
+        password: Optional[str] = None,
+        authenticator: Optional[str] = None,   # e.g. 'externalbrowser', 'oauth', 'snowflake'
+        role: Optional[str] = None,
+        warehouse: Optional[str] = None,
+        database: Optional[str] = None,
+        schema: Optional[str] = None
+    ):
+        import snowflake.connector  # lazy import
+
+        kwargs: Dict[str, Any] = {
+            "account": account,
+            "user": user,
+        }
+        if password:
+            kwargs["password"] = password
+        if authenticator:
+            kwargs["authenticator"] = authenticator
+        if role:
+            kwargs["role"] = role
+        if warehouse:
+            kwargs["warehouse"] = warehouse
+        if database:
+            kwargs["database"] = database
+        if schema:
+            kwargs["schema"] = schema
+
+        try:
+            self.conn = snowflake.connector.connect(**kwargs)
+            # autocommit is True by default; make explicit
+            self.conn.autocommit(True)
+            # Apply explicit USE statements as a safety net (only if provided)
+            with self.conn.cursor() as cur:
+                if role:
+                    cur.execute(f'USE ROLE "{role}"')
+                if warehouse:
+                    cur.execute(f'USE WAREHOUSE "{warehouse}"')
+                if database:
+                    cur.execute(f'USE DATABASE "{database}"')
+                if schema:
+                    cur.execute(f'USE SCHEMA "{schema}"')
+        except Exception as e:
+            print("Failed to connect to Snowflake")
+            raise e
+
+    def close(self):
+        if getattr(self, "conn", None) is not None:
+            try:
+                self.conn.close()
+            finally:
+                self.conn = None
+
+
+# ---------- pipe ----------
+
+class SnowflakePipe(QueryPipe, Integration):
+    """
+    Snowflake query pipe; executes SQL found in input record['query'] and streams rows.
+    Connection/session settings are pulled from ~/.pjk/lookups.yaml under the arg name.
+    """
+    name = 'snowflake'
+    desc = "Snowflake query pipe; executes an SQL query for each input record."
+    arg0 = ('dbname', 'database name.')
+    examples = [
+        ["{'query': 'SELECT CURRENT_ROLE();'}", "snow:EDLDB", "-"],
+        ["myquery.sql", "snow:EDLDB", "-"]
+    ]
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+
+        self.dbname = usage.get_arg(type(self).arg0[0])
+
+        # Accept both bare keys and SNOWFLAKE_* variants in lookups.yaml
+        self.sf_account  = _get_any(self.lookup_params, "account")
+        self.sf_user     = _get_any(self.lookup_params, "user")
+        self.sf_auth     = _get_any(self.lookup_params, "authenticator")
+        self.sf_role     = _get_any(self.lookup_params, "role")
+        self.sf_wh       = _get_any(self.lookup_params, "warehouse")
+        self.sf_db       = self.dbname
+        self.sf_schema   = _get_any(self.lookup_params, "schema")
+
+        # Basic validation
+        missing = [k for k, v in [
+            ("account", self.sf_account),
+            ("user", self.sf_user),
+            ("authenticator|password", self.sf_auth or self.sf_password),
+            ("role", self.sf_role),
+            ("warehouse", self.sf_wh),
+            ("schema", self.sf_schema),
+        ] if not v]
+        if missing:
+            raise TokenError(
+                f"lookups entry '{self.dbname}' missing: {', '.join(missing)}"
+            )
+
+        self.query_field = usage.get_param('query_field')
+        self.params_field = "params"  # optional: list/tuple (positional) or dict (named)
+
+    def reset(self):
+        # stateless across reset
+        pass
+
+    def _make_header(self, cur, params=None) -> Dict[str, Any]:
+        """
+        Build a header record with query metadata and session context.
+        """
+        h: Dict[str, Any] = {
+            "db": self.dbname,
+            "account": self.sf_account,
+            "role": self.sf_role,
+            "warehouse": self.sf_wh,
+        }
+        if self.sf_db:
+            h["database"] = self.sf_db
+        if self.sf_schema:
+            h["schema"] = self.sf_schema
+        if params is not None:
+            h["params"] = params
+
+        # Snowflake's cursor.rowcount is often -1 for SELECT until fully fetched.
+        # We still include it if known (for DML it may be accurate).
+        try:
+            rc = getattr(cur, "rowcount", None)
+            if isinstance(rc, int) and rc >= 0:
+                h["rowcount"] = rc
+        except Exception:
+            pass
+
+        h["result"] = "ok"
+        return h
+
+    def execute_query_returning_Q_xR_iterable(self, record):
+        client = SnowflakeClient(
+            account=self.sf_account,
+            user=self.sf_user,
+            authenticator=self.sf_auth,
+            role=self.sf_role,
+            warehouse=self.sf_wh,
+            database=self.sf_db,
+            schema=self.sf_schema,
+        )
+        try:
+            query = record.get(self.query_field)
+            if not query:
+                record['_error'] = 'missing query'
+                yield record
+
+            else:
+                params = record.get(self.params_field)
+
+                cur = client.conn.cursor()
+                try:
+                    # Execute (supports positional or named params per DB-API)
+                    if params is None:
+                        cur.execute(query)
+                    else:
+                        if isinstance(params, (list, tuple, dict)):
+                            cur.execute(query, params)
+                        else:
+                            # single scalar -> positional 1-tuple
+                            cur.execute(query, (params,))
+
+                    yield self._make_header(cur, params)
+
+                    # Stream result rows for queries that return a result set
+                    if cur.description:
+                        for row in cur:
+                            yield _row_to_dict(cur, row)
+                finally:
+                    cur.close()
+        finally:
+            client.close()