python-jack-knife 0.6.5__tar.gz → 0.6.7__tar.gz

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.5
+Version: 0.6.7
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/integrations/opensearch_client.py

@@ -1,7 +1,24 @@
 from pjk.usage import Usage
 
-class OpenSearchClient:
+ # name, type, default
+OS_CONFIG_TUPLES = [
+    ("default_index", str, None),
+    ("os_auth_use_aws", bool, "true"),
+    ("os_scheme", str, "https"),
+    ("os_verify_certs", bool, "true"),
+    ("os_ca_certs", str, None),
+    ("os_region", str, None),
+    ("os_service", str, "es"),
+    ("os_username", str, None),
+    ("os_password", str, None),
+    ("os_timeout", float, 30),
+    ("os_ssl_assert_hostname", bool, "true"),
+    ("os_ssl_show_warn", bool, "false"),
+    ("os_host", str, None),
+    ("os_port", int, None)
+]
 
+class OpenSearchClient:
     @classmethod
     def get_client(cls, u: Usage):
         aws_auth = u.get_config("os_auth_use_aws")

python_jack_knife-0.6.7/src/pjk/integrations/opensearch_index_sink.py

@@ -0,0 +1,94 @@
+# djk/sinks/opensearch_sink.py
+
+from pjk.components import Sink
+from pjk.common import Integration
+from pjk.usage import ParsedToken, Usage, CONFIG_FILE
+from pjk.integrations.opensearch_client import OpenSearchClient, OS_CONFIG_TUPLES
+from opensearchpy import helpers
+from datetime import datetime
+
+examples = [
+    ["index_docs.json", 'let:index:myidx', 'os_index:myinst'],
+    ["{'_command': 'create', 'config': {settings: {...}, mappings: {...}}, 'index': 'myidx'}", 'os_index:myinst'],
+    ["{'_command': 'delete_index', 'index': 'myidx'}", "os_index:myinst"]
+]
+
+class OpenSearchIndexSink(Sink, Integration):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name="os_index",
+            component_class=cls,
+            desc="Opensearch indexer sink. All input records must contain 'index' field.\nOptional param 'id_field' can be specified."
+        )
+        usage.def_arg("instance", f"Instance in {CONFIG_FILE} to index into.")
+        usage.def_param('id_field', usage='field to be used as unique id')
+        usage.def_param('DDHHMM_key', usage='All commands require key equal to current day, hour, minute', is_num=True, default="0")
+        usage.def_config_tuples(OS_CONFIG_TUPLES)
+        for e in examples:
+            usage.def_example(e, None)
+        return usage
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.client = OpenSearchClient.get_client(usage)
+        self.bulk_size = 500
+        self.buffer = []
+        self.total_written = 0
+        self.id_field = usage.get_param('id_field')
+        self.ddhhmm_key = usage.get_param('DDHHMM_key')
+
+    def execute_command(self, command: str, index: str, record: dict):
+        current_ddhhmm = int(datetime.now().strftime("%d%H%M"))
+        if self.ddhhmm_key != current_ddhhmm:
+            raise Exception('All os commands require the DDHHMM_key corresponding to now, e.g. 031431')
+
+        if command == 'create':
+            config = record.pop('config', None)
+            if not config:
+                raise Exception('create command missing config object.')
+            self.client.indices.create(index=index, body=config)
+
+        elif command == 'delete_index':
+            self.client.indices.delete(index=index)
+
+        else:
+            raise Exception(f'unknown os command: {command}')
+
+    def process(self):
+        for record in self.input:
+            index = record.pop('index', None)
+            if not index:
+                raise Exception("All index records must contain an 'index' field")
+
+            # Create index if config present
+            command = record.pop('_command', None)
+            if command:
+                self.execute_command(command, index, record)
+                continue
+
+            action = {
+                "_op_type": "index",     # use "create" to fail if doc already exists
+                "_index": index,
+                "_source": record        # meta fields removed; store only the actual doc
+            }
+
+            doc_id = record.get(self.id_field, None)
+            if doc_id:
+                action['_id'] = doc_id
+
+            self.buffer.append(action)
+
+            if len(self.buffer) >= self.bulk_size:
+                self.flush()
+
+        self.flush()
+
+    def flush(self):
+        if self.buffer:
+            helpers.bulk(self.client, self.buffer)
+            self.total_written += len(self.buffer)
+            self.buffer.clear()
+
+    def deep_copy(self):
+        return None  # until deep sync available

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/integrations/opensearch_query_pipe.py

@@ -7,7 +7,7 @@ from typing import Optional, Iterator, Dict, Any, Iterable
 from pjk.usage import ParsedToken, Usage
 from pjk.pipes.query_pipe import QueryPipe
 from pjk.common import Integration
-from pjk.integrations.opensearch_client import OpenSearchClient
+from pjk.integrations.opensearch_client import OpenSearchClient, OS_CONFIG_TUPLES
 
 def build_body_from_string(query_string: str) -> dict:
     if query_string == "*":
@@ -23,38 +23,24 @@ def build_body_from_string(query_string: str) -> dict:
 
 class OpenSearchQueryPipe(QueryPipe, Integration):
     name = "os_query"
-    desc = "Opensearch query pipe. Uses record['query'] or record['os_query_object'] for os query"
+    desc = ("Opensearch query pipe. Uses record['query'] or record['os_query_object'] for os query\n"
+    "An instance may define 'default_index' otherwise the query object must include an 'index' field.\n")
     arg0 = ("instance", "instance to query over.")
     examples = [
-        ["{'query': '_ping'}", 'os_query:myindex', '-'],
-        ["{'query': '*'}", 'os_query:myindex', '-'],
-        ["{'query': 'dog'}", 'os_query:myindex', '-'],
-        ["{'query': 'dog AND cat'}", 'os_query:myindex', '-'],
-        ["{'os_query_object': {query: {...}}", 'os_query:myindex', '-'],
+        ["{'query': '_ping'}", 'os_query:myinst', '-'],
+        ["{'index': 'myidx', 'query': '*'}", 'os_query:myinst', '-'],
+        ["{'index': 'myidx', 'query': 'dog AND cat'}", 'os_query:myinst', '-'],
+        ["{'index': 'myidx', 'query': 'dog'}", 'os_query:myinst', '-'],
+        ["{'os_query_object': {query: {...}}", 'os_query:myinst', '-', ' # uses instance.default_index'],
     ]
 
      # name, type, default
-    config_tuples = [
-        ("index_name", str, None),
-        ("os_auth_use_aws", bool, "true"),
-        ("os_scheme", str, "https"),
-        ("os_verify_certs", bool, "true"),
-        ("os_ca_certs", str, None),
-        ("os_region", str, None),
-        ("os_service", str, "es"),
-        ("os_username", str, None),
-        ("os_password", str, None),
-        ("os_timeout", float, 30),
-        ("os_ssl_assert_hostname", bool, "true"),
-        ("os_ssl_show_warn", bool, "false"),
-        ("os_host", str, None),
-        ("os_port", int, None)
-    ]
+    config_tuples = OS_CONFIG_TUPLES
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
 
-        self.index = usage.get_config("index_name")
+        self.index = usage.get_config("default_index")
         self.client = OpenSearchClient.get_client(usage)
 
         # Iteration state
@@ -83,6 +69,10 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
 
     def execute_query_returning_S_xO_iterable(self, query_record: dict) -> Iterator[Dict[str, Any]]:
         query_string = query_record.get('query', None)
+        query_index = query_record.get('index', None)
+        if query_index:
+            self.index = query_index # overwrite the default query
+
         query_body = None
 
         if query_string:
@@ -131,8 +121,8 @@ class OpenSearchQueryPipe(QueryPipe, Integration):
                     yield {"_type": "os_query_hit", "_hit": hit}
 
         except Exception as e:
-            print("OpenSearch query error:", e, file=sys.stderr)
-            traceback.print_exc()
+            #print("OpenSearch query error:", e, file=sys.stderr)
+            #traceback.print_exc()
             yield {
                 "_type": "os_query_error",
                 "error": str(e),

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/pipes/select.py

@@ -11,7 +11,7 @@ class SelectFields(DeepCopyPipe):
     def usage(cls):
         usage = Usage(
             name='sel',
-            desc='Keep only the specified fields from each record',
+            desc='Select specific fields from each record.',
             component_class=cls
         )
         usage.def_arg(name='fields', usage='Comma-separated list of fields to retain')

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/sinks/factory.py

@@ -14,6 +14,7 @@ from pjk.sinks.tsv_sink import TSVSink
 from pjk.sinks.expect import ExpectSink
 from pjk.sinks.format_sink import FormatSink
 from pjk.sinks.create_sink import CreateSink
+from pjk.integrations.opensearch_index_sink import OpenSearchIndexSink
 from pjk.sinks.user_sink_factory import UserSinkFactory
 
 COMPONENTS = {
@@ -23,6 +24,7 @@ COMPONENTS = {
         'json': JsonSink,
         'csv': CSVSink,
         'tsv': TSVSink,
+        'os_index': OpenSearchIndexSink,
         'create': CreateSink
         }
 

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/sources/json_source.py

@@ -6,6 +6,7 @@ from pjk.usage import NoBindUsage
 from pjk.components import Source
 from pjk.sources.lazy_file import LazyFile
 from pjk.sources.format_source import FormatSource
+from typing import Any, Dict, Iterable, Optional
 from pjk.log import logger
 
 class JsonSource(FormatSource):
@@ -15,6 +16,16 @@ class JsonSource(FormatSource):
         self.lazy_file = lazy_file
         self.num_recs = 0
 
+    def as_whole_file(self) -> Iterable[Dict[str, Any]]:
+        with self.lazy_file.open() as f:
+            string = f.read()
+            object = json.loads(string)
+            if isinstance(object, list):
+                for item in object:
+                    yield item
+            else: 
+                yield object
+
     def __iter__(self):
         with self.lazy_file.open() as f:
             for line in f:
@@ -22,11 +33,11 @@ class JsonSource(FormatSource):
                 try:
                     yield json.loads(line)
                 except json.JSONDecodeError as e:
-                    print('json decode error, see ~/.pjk/logs')
-                    snippet = line.strip()
-                    if len(snippet) > 200:
-                        snippet = snippet[:200] + "…"
-                    logger.warning(
-                        f"Skipping invalid JSON at line {self.num_recs} "
-                        f"in {self.lazy_file.path}: {e} | data: {snippet}"
-                    )
+                    break
+
+        # attempt to decode the file as a whole
+        try:
+            yield from self.as_whole_file()
+
+        except:
+            logger.error(f'cannot decode {self.lazy_file.path}')

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/usage.py

@@ -2,7 +2,7 @@ from typing import Optional, Set, List
 import os
 import yaml
 
-CONFIG_FILE = '~/pjk/configs.yaml'
+CONFIG_FILE = '~/.pjk/configs.yaml'
 
 class Config:
     def __init__(self):
@@ -41,6 +41,14 @@ class Config:
                 f"{CONFIG_FILE} does not contain entry for '{instance_key}' with required params."
             )
         
+        _alias = entry.get('_alias', None) # _alias must = another entry instance_key
+        if _alias:
+            entry = self._data.get(_alias, None)
+            if not entry:
+                raise TokenError(
+                    f"'{instance_key}:_alias' in {CONFIG_FILE} points to a non-existent entry: '{_alias}'."
+                )    
+        
         raw = entry.get(param, param_default)
 
         if not raw:

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.5"
+__version__ = "0.6.7"

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/python_jack_knife.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.5
+Version: 0.6.7
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.5 → python_jack_knife-0.6.7}/src/python_jack_knife.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ src/pjk/registry.py
 src/pjk/usage.py
 src/pjk/version.py
 src/pjk/integrations/opensearch_client.py
+src/pjk/integrations/opensearch_index_sink.py
 src/pjk/integrations/opensearch_query_pipe.py
 src/pjk/integrations/postgres_pipe.py
 src/pjk/integrations/snowflake_pipe.py