python-jack-knife 0.5.0__py3-none-any.whl

pjk/pipes/select.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/select_pipe.py
+
+from pjk.base import Pipe, Usage, ParsedToken, UsageError
+
+class SelectFields(Pipe):
+    deep_copyable: bool = True
+
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='sel',
+            desc='Keep only the specified fields from each record',
+            component_class=cls
+        )
+        usage.def_arg(name='fields', usage='Comma-separated list of fields to retain')
+        usage.def_example(expr_tokens=["{id:1, dir:'up', color:'blue'}", 'sel:id,color'], expect="id: 1, color:'blue'")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+
+        arg_string = usage.get_arg('fields')
+        if not arg_string:
+            raise UsageError("select:<f1,f2,...> requires at least one field")
+
+        self.keep_fields = {f.strip() for f in arg_string.split(',') if f.strip()}
+        if not self.keep_fields:
+            raise UsageError("select must include at least one valid field name")
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        for record in self.left:
+            keys = list(record.keys())
+            for k in keys:
+                if k not in self.keep_fields:
+                    record.pop(k)
+            yield record

pjk/pipes/sort.py

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/sort.py
+
+from pjk.base import Pipe, ParsedToken, Usage, UsageError
+
+class SortPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='sort',
+            desc="Sort records by a single field (records with missing field sort last).",
+            component_class=cls
+        )
+        usage.def_arg(name='field', usage="+name or -name for ascending or decending sort by field 'name'.")
+        usage.def_example(expr_tokens=["[{id:17}, {id:10}, {id:1}]", 'sort:+id'], expect="[{id:1}, {id:10}, {id:17}]")
+        usage.def_example(expr_tokens=["[{id:1}, {color:'blue'}, {color:'green'}]", 'sort:-color'], expect="[{color:'green'}, {color:'blue'}, {id:1}]")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+
+        arg_string = usage.get_arg('field')
+        if not arg_string:
+            raise UsageError("sort:[+-]<field> requires direction and field name")
+
+        if arg_string.startswith("-"):
+            self.field = arg_string[1:]
+            self.reverse = True
+        elif arg_string.startswith("+"):
+            self.field = arg_string[1:]
+            self.reverse = False
+        else:
+            raise UsageError("sort:[+-]<field> must start with '+' or '-'")
+
+        self._buffer = None
+        self._index = 0
+
+    def reset(self):
+        self._buffer = None
+        self._index = 0
+
+    def __iter__(self):
+        if self._buffer is None:
+            self._buffer = list(self.left)
+
+            # Partition into records with and without the sort field
+            present = [r for r in self._buffer if r.get(self.field) is not None]
+            missing = [r for r in self._buffer if r.get(self.field) is None]
+
+            present.sort(
+                key=lambda r: r.get(self.field),
+                reverse=self.reverse
+            )
+
+            self._buffer = present + missing  # always push missing to the end
+
+        while self._index < len(self._buffer):
+            yield self._buffer[self._index]
+            self._index += 1
+
+

pjk/pipes/tail.py

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/tail.py
+
+from pjk.base import Pipe, ParsedToken, Usage
+
+class TailPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='tail',
+            desc='take last records of input (when single-threaded)',
+            component_class=cls
+        )
+        usage.def_arg(name='limit', usage='number of records', is_num=True)
+        usage.def_example(expr_tokens=['[{id:1}, {id:2}]', 'tail:1'], expect="{id:2}")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.limit = usage.get_arg('limit')
+
+        self.buffer = []
+        self.ready = False
+
+    def reset(self):
+        self.buffer.clear()
+        self.ready = False
+
+    def __iter__(self):
+        if not self.ready:
+            for record in self.left:
+                self.buffer.append(record)
+                if len(self.buffer) > self.limit:
+                    self.buffer.pop(0)
+            self.ready = True
+
+        yield from self.buffer

pjk/pipes/user_pipe_factory.py

@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/user_pipe_factory.py
+
+import importlib.util
+from typing import Optional
+from pjk.base import Pipe, Sink, ParsedToken, UsageError
+
+class UserPipeFactory:
+    @staticmethod
+    def create(ptok: ParsedToken) -> Optional[Pipe]:
+        script_path = ptok.pre_colon
+
+        try:
+            # Load module dynamically from script path
+            spec = importlib.util.spec_from_file_location("user_pipe", script_path)
+            if spec is None or spec.loader is None:
+                raise UsageError(f"Could not load Python file: {script_path}")
+
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+        except Exception as e:
+            raise UsageError(f"Failed to import {script_path}: {e}")
+
+        # Look for exactly one top-level Pipe class that isn't a Sink or base Pipe
+        pipe_cls = None
+        for value in vars(module).values():
+            if (
+                isinstance(value, type)
+                and issubclass(value, Pipe)
+                and not issubclass(value, Sink)
+                and value is not Pipe
+                and value.__module__ == module.__name__
+            ):
+                if pipe_cls is not None:
+                    raise UsageError(f"Multiple Pipe classes found in {script_path}. Only one is allowed.")
+                pipe_cls = value
+
+        if pipe_cls is None:
+            return None
+
+        usage = pipe_cls.usage()
+        usage.bind(ptok)
+        return pipe_cls(ptok, usage)

pjk/pipes/where.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/where.py
+
+from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, UsageError
+from pjk.common import SafeNamespace
+
+class WherePipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = NoBindUsage(
+            name='where',
+            desc="Filter records using a Python expression over fields",
+            component_class=cls
+        )
+        usage.def_arg(name='expr', usage='Python expression using \'f.<field>\' syntax')
+        usage.def_example(expr_tokens=["[{size:1}, {size:5}, {size:10}]", "where:f.size >= 5"], expect="[{size:5}, {size:10}]")
+        usage.def_example(expr_tokens=["[{color:'blue'}, {color:'red'}, {color:'black'}]", "where:f.color.startswith('bl')"], expect="[{color:'blue'}, {color:'black'}]")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.expr = ptok.whole_token.split(':', 1)[1]
+        try:
+            self.code = compile(self.expr, '<where>', 'eval')
+        except Exception as e:
+            raise UsageError(f"Invalid where expression: {self.expr}") from e
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        for record in self.left:
+            f = SafeNamespace(record)
+            try:
+                if eval(self.code, {}, {'f': f}):
+                    yield record
+            except Exception:
+                continue  # ignore eval errors
+
+    def deep_copy(self):
+        source_clone = self.left.deep_copy()
+        if source_clone:
+            pipe = WherePipe(self.ptok, self.usage)
+            pipe.add_source(source_clone)
+            return pipe
+        else:
+            return None

pjk/registry.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+from pjk.sinks.factory import SinkFactory
+from pjk.pipes.factory import PipeFactory
+from pjk.sources.factory import SourceFactory
+import importlib.util
+import importlib
+import importlib.metadata
+from pjk.base import Pipe, Source, Sink
+
+class ComponentRegistry:
+    def __init__(self):
+        self.source_factory = SourceFactory()
+        self.pipe_factory = PipeFactory()
+        self.sink_factory = SinkFactory()
+
+        sources, pipes, sinks = load_user_components()
+        for name, comp in sources.items():
+            self.source_factory.register(name, comp)
+
+        for name, comp in pipes.items():
+            self.pipe_factory.register(name, comp)
+
+        for name, comp in sinks.items():
+            self.sink_factory.register(name, comp)
+
+        load_package_extras()
+
+    def register(self, name, comp):
+        if is_pipe(comp):
+            print('HELEELELELELELEEE')
+            if hasattr(comp, "usage"):
+                usage = comp.usage()
+                name = usage.name
+            self.pipe_factory.register(name, comp)
+        elif is_sink(comp):
+            self.sink_factory.register(name, comp)
+        elif is_source(comp):
+            self.source_factory(name, comp)
+
+    def create_source(self, token: str):
+        return self.source_factory.create(token)
+    
+    def create_pipe(self, token: str):
+        return self.pipe_factory.create(token)
+    
+    def create_sink(self, token: str):
+        return self.sink_factory.create(token)
+    
+    def get_factories(self):
+        return [self.source_factory, self.pipe_factory, self.sink_factory]
+
+    def print_usage(self):
+        print('Usage: pjk <source> [<pipe> ...] <sink>')
+        print('       pjk man <component> | --all')
+        print('       pjk examples')
+        print()
+        self.source_factory.print_descriptions()
+        print()
+        self.pipe_factory.print_descriptions()
+        print()
+        self.sink_factory.print_descriptions()
+
+        return
+    
+def is_source(obj, module):
+    return (
+        isinstance(obj, type)
+        and issubclass(obj, Source)
+        and not issubclass(obj, Pipe)
+        and not issubclass(obj, Sink)
+        and obj is not Source
+        and obj.__module__ == module.__name__  # 🧠 only user-defined classes
+        )
+
+def is_pipe(obj, module):
+    return (
+        isinstance(obj, type)
+        and issubclass(obj, Pipe)
+        and not issubclass(obj, Sink)
+        and obj is not Pipe
+        and obj.__module__ == module.__name__
+    )
+
+def is_sink(obj, module):
+     return (
+        isinstance(obj, type)
+        and issubclass(obj, Sink)
+        and obj is not Sink
+        and obj.__module__ == module.__name__
+    )
+
+def load_user_components(path=os.path.expanduser("~/.pjk/plugins")):
+    sources = {}
+    pipes = {}
+    sinks = {}
+
+    if not os.path.isdir(path):
+        return {}, {}, {}
+
+    for fname in os.listdir(path):
+        if not fname.endswith(".py"):
+            continue
+        fpath = os.path.join(path, fname)
+        modname = f"user_component_{fname[:-3]}"
+        spec = importlib.util.spec_from_file_location(modname, fpath)
+        if not spec or not spec.loader:
+            continue
+        module = importlib.util.module_from_spec(spec)
+        try:
+            spec.loader.exec_module(module)
+        except Exception as e:
+            print(f"[djk] Failed to load {fname}: {e}")
+            continue
+
+        for obj in vars(module).values():
+            if not isinstance(obj, type):
+                continue
+            if hasattr(obj, "usage"):
+                usage = obj.usage()
+                name = usage.name
+
+                if is_sink(obj, module):
+                    sinks[name] = obj
+                elif is_pipe(obj, module):
+                    pipes[name] = obj
+                elif is_source(obj, module):
+                    sources[name] = obj
+
+    return sources, pipes, sinks
+
+def load_package_extras():
+    """
+    Discover and import all installed pjk extras (via entry points).
+    """
+    for ep in importlib.metadata.entry_points(group="pjk.package_extras"):
+        try:
+            importlib.import_module(ep.value)
+            print(f"[pjk] loaded package extra: {ep.name} -> {ep.value}")
+        except Exception as e:
+            print(f"[pjk] failed to load extra {ep.name}: {e}")

pjk/sinks/csv_sink.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import csv
+from pjk.base import Sink, Source, ParsedToken, Usage
+
+class CSVSink(Sink):
+    is_format = True
+
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='csv',
+            desc='Write records to a CSV file with dynamic header from first record',
+            component_class=cls
+        )
+        usage.def_arg('path', usage='Path prefix (no extension)')
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        path_no_ext = usage.get_arg('path')
+        self.path = f"{path_no_ext}.csv"
+
+    def process(self) -> None:
+        with open(self.path, 'w', newline='') as f:
+            writer = None
+
+            for record in self.input:
+                if writer is None:
+                    writer = csv.DictWriter(f, fieldnames=record.keys())
+                    writer.writeheader()
+                writer.writerow(record)

pjk/sinks/ddb.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from pjk.base import Sink, Source, ParsedToken, Usage
+from decimal import Decimal
+
+class DDBSink(Sink):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='ddb',
+            desc='Write records to a DynamoDB table via batch_writer()',
+            component_class=cls
+        )
+        usage.def_arg('table', usage='DynamoDB table name')
+        usage.def_param('batch_size', usage='How many records to write per batch (max 25)')
+        return usage
+
+    def __init__(self, input_source: Source, ptok: ParsedToken, usage: Usage):
+        super().__init__(input_source)
+        import boto3 # lazy import
+
+        self.table_name = usage.get_arg('table')
+        self.batch_size = int(usage.get_param('batch_size', default='10'))
+        self.num_recs = 0
+        self.batch = []
+
+        dynamodb = boto3.resource('dynamodb')
+        self.table = dynamodb.Table(self.table_name)
+
+    def process_batch(self):
+        if not self.batch:
+            return
+
+        with self.table.batch_writer() as batch:
+            for item in self.batch:
+                clean_item = {
+                    k: (Decimal(str(v)) if isinstance(v, float) else v)
+                    for k, v in item.items()
+                }
+                batch.put_item(Item=clean_item)
+
+        self.batch = []
+
+    def process(self):
+        for record in self.input:
+            self.batch.append(record)
+            self.num_recs += 1
+
+            if len(self.batch) >= self.batch_size:
+                self.process_batch()
+
+        self.process_batch()
+        print(f"DDBSink wrote {self.num_recs} records.")

pjk/sinks/devnull.py

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/sinks/devnull.py
+
+from pjk.base import Sink, Source, ParsedToken, Usage
+
+class DevNullSink(Sink):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='devnull',
+            desc='Consume all input records and discard them (debug/testing)',
+            component_class=cls
+        )
+        usage.def_example(expr_tokens=['{id:1}', 'devnull'], expect=None)
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.count = 0
+
+    def process(self):
+        for record in self.input:
+            self.count += 1
+
+    def print_info(self):
+        print(f'num_recs:{self.count}')
+
+    def deep_copy(self):
+        return None  # until we implement cross-thread coordination

pjk/sinks/dir_sink.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+from pjk.base import Source, Sink, ParsedToken, Usage
+from pjk.log import logger
+
+class DirSink(Sink):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='<format>',
+            desc='Write records to a local directory in the given <format> (e.g., csv)',
+            component_class=cls
+        )
+        usage.def_arg(name='dir', usage='Path to output directory')
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage, sink_class: type, is_gz: bool, fileno: int = 0):
+        super().__init__(ptok, usage)
+        self.dir_path = usage.get_arg('dir')  # ✅ Use usage, not ptok directly
+        self.ptok = ptok
+        self.usage = usage
+        self.sink_class = sink_class
+        self.is_gz = is_gz
+        self.fileno = fileno
+        self.num_files = 1
+
+        os.makedirs(self.dir_path, exist_ok=True)
+
+    def process(self):
+        file = os.path.join(self.dir_path, f'file-{self.fileno:04d}')
+        file_ptok = ParsedToken(f'{file}:{self.is_gz}')
+        file_usage = self.sink_class.usage()
+        file_usage.bind(file_ptok)
+
+        file_sink = self.sink_class(file_ptok, file_usage)
+        file_sink.add_source(self.input)
+
+        logger.debug(f'in process sinking to: {file}')
+        file_sink.process()
+
+    def deep_copy(self):
+        source_clone = self.input.deep_copy()
+        if not source_clone:
+            return None
+
+        clone = DirSink(
+            ptok=self.ptok,
+            usage=self.usage,
+            sink_class=self.sink_class,
+            is_gz=self.is_gz,
+            fileno=self.num_files
+        )
+
+        clone.add_source(source_clone)
+
+        self.num_files += 1
+        return clone

pjk/sinks/expect.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from pjk.base import Source, Sink, ParsedToken, Usage
+from pjk.sources.inline_source import InlineSource
+import sys
+
+class ExpectSink(Sink):
+    # NOTE: ExpectSink intentionally does NOT use Usage due to raw JSON argument parsing
+    # e.g., expect:'[{a:1},{a:2}]' must preserve the entire post-colon string unparsed
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.inline = ptok.whole_token.split(':', 1)[-1]
+        self.expect_source = InlineSource(self.inline)
+        self._expect_iter = iter(self.expect_source)
+
+    def print_info(self):
+        command = ' '.join(sys.argv[1:-1])  # omit 'pjk' and 'expect'
+        print(f'{command} ==> OK!\n') # only prints on success
+
+    def process(self) -> None:
+        command = ' '.join(sys.argv[1:-1])  # omit 'pjk' and 'expect'
+
+        for test_rec in self.input:
+            try:
+                expect_rec = next(self._expect_iter)
+            except StopIteration:
+                raise ValueError(
+                    f"expect failure: {command}\n"
+                    f"expected_record:None\n"
+                    f"got_record:{test_rec}\n"
+                    f"entire_expected:{self.inline}"
+                )
+
+            if test_rec != expect_rec:
+                raise ValueError(
+                    f"expect failure: {command}\n"
+                    f"expected_record:{expect_rec}\n"
+                    f"got_record:{test_rec}\n"
+                    f"entire_expected:{self.inline}"
+                )
+
+        try:
+            expect_rec = next(self._expect_iter)
+            raise ValueError(
+                f"expect failure: {command}\n"
+                f"expected_record:{expect_rec}\n"
+                f"got_record:None\n"
+                f"entire_expected:{self.inline}"
+            )
+        except StopIteration:
+            pass