python-jack-knife 0.5.0__py3-none-any.whl

pjk/man_page.py

@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from pjk.pipes.factory import PipeFactory
+from pjk.sources.factory import SourceFactory
+from pjk.sinks.factory import SinkFactory
+from pjk.parser import ExpressionParser
+from pjk.base import Usage
+from pjk.registry import ComponentRegistry
+from pjk.common import pager_stdout, highlight
+from contextlib import nullcontext
+
+def smart_print(expr_tokens: list[str], name: str):
+    import re
+    SAFE_UNQUOTED_RE = re.compile(r"^[a-zA-Z0-9._/:=+-]+$")
+
+    def quote(token: str) -> str:
+        if SAFE_UNQUOTED_RE.fullmatch(token):
+            return token
+        elif "'" not in token:
+            return f"'{token}'"
+        elif '"' not in token:
+            return f'"{token}"'
+        else:
+            return '"' + token.replace('"', '\\"') + '"'
+
+    expr_str = ' '.join(quote(t) for t in expr_tokens)
+    expr_str = highlight(expr_str, 'bold', name)
+
+    #print("pjk", " ".join(quote(t) for t in expr_tokens))
+    print('pjk', expr_str)
+
+def do_man(name: str, registry: ComponentRegistry):
+    no_pager = name.endswith('+')
+    if '--all' in name:
+        do_all_man(registry, no_pager=no_pager)
+        return
+
+    # source and sinks have common names so go through multiple times
+    printed = False
+    for factory in registry.get_factories():
+        usage = factory.get_usage(name)
+        if usage:
+            print_man(registry, name, usage)
+            printed = True
+
+    if not printed:
+        print(f'unknown: {name}')
+
+def do_all_man(registry: ComponentRegistry, no_pager: bool = True):
+    cm = nullcontext() if no_pager else pager_stdout()
+    with cm:
+        for factory in registry.get_factories():
+            comp_type = factory.get_comp_type_name()
+            for name in factory.components.keys():
+                usage = factory.get_usage(name)
+                print_man(registry, name, usage)
+                print()
+
+def print_man(registry: ComponentRegistry, name: str, usage: Usage):
+    comp_type = usage.get_base_class(as_string=True)
+    header = f'{name} is a {comp_type}'
+    print('===================================')
+    print('        ', highlight(header, 'bold', name))
+    print('===================================')
+
+    print()
+    print(usage.get_usage_text())
+
+    examples = usage.get_examples()
+    if not examples:
+        return
+    
+    print()
+    print('examples:')
+    print()
+
+    for expr_tokens, expect in usage.get_examples(): # expect in InlineSource format
+        print_example(registry, expr_tokens, expect, name)
+
+def do_examples(token:str, registry: ComponentRegistry):
+    no_pager = token.endswith('+')
+    cm = nullcontext() if no_pager else pager_stdout()
+    with cm:
+        for factory in registry.get_factories():
+            comp_type = factory.get_comp_type_name()
+            for name, comp_class in factory.components.items():
+                usage = comp_class.usage()
+
+                comp_type = usage.get_base_class(as_string=True)
+                header = f'{name} is a {comp_type}'
+                print('===================================')
+                print('        ', highlight(header, 'bold', name))
+                print('===================================')
+
+                examples = usage.get_examples()
+                if not examples:
+                    print(f'{name} needs examples')
+                    print()
+
+                for expr_tokens, expect in examples:
+                    print_example(registry, expr_tokens, expect, name)
+
+def print_example(registry: ComponentRegistry, expr_tokens: list[str], expect:str, name: str):
+    try:
+        if not expect: # if no expect, don't run them, just print them
+            smart_print(expr_tokens, name)
+            print()
+            return
+
+        expr_tokens.append(f'expect:{expect}')
+        parser = ExpressionParser(registry)
+        sink = parser.parse(expr_tokens)
+        sink.drain() # make sure the expect is fulfilled
+
+        expr_tokens[-1] = '-' # for printing so you see simple stdout -
+        smart_print(expr_tokens, name)
+        expr_tokens[-1] = '-@less=false' # no less since man is doing less
+        parser = ExpressionParser(registry)
+        sink = parser.parse(expr_tokens)
+        sink.drain()
+        print()
+
+    except ValueError as e:
+        raise 'error executing example'

pjk/parser.py

@@ -0,0 +1,284 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from typing import Any, List, Callable
+import os
+import shlex
+from typing import Optional, Any, List
+from pjk.base import Source, Pipe, Sink, TokenError, UsageError, ParsedToken, Usage
+from pjk.pipes.user_pipe_factory import UserPipeFactory
+from pjk.pipes.let_reduce import ReducePipe
+from pjk.registry import ComponentRegistry
+
+def expand_macros(tokens: List[str]) -> List[str]:
+    expanded = []
+    for token in tokens:
+        if token.endswith(".pjk"):
+            if not os.path.isfile(token):
+                raise FileNotFoundError(f"Macro file not found: {token}")
+            with open(token, "r") as f:
+                lines = f.readlines()
+
+            # Remove comments outside quotes, then split
+            stripped = []
+            for line in lines:
+                try:
+                    parts = shlex.split(line, comments=True, posix=True)
+                    stripped.extend(parts)
+                except ValueError as e:
+                    raise UsageError(f"Error parsing {token}: {e}")
+            expanded.extend(stripped)
+        else:
+            expanded.append(token)
+    return expanded
+
+class ExpressionParser:
+    def __init__(self, registry: ComponentRegistry):
+        self.stack: List[Any] = []
+        self.registry = registry
+
+    def get_sink(self, stack_helper, token):
+        if len(self.stack) < 1:
+            raise TokenError.from_list(['expression must include source and sink.',
+                                            'pjk <source> [<pipe> ...] <sink>'])
+
+        source = self.stack.pop()
+        if len(self.stack) != 0:
+            raise TokenError.from_list(['A sink can only consume one source.',
+                                        'pjk <source> [<pipe> ...] <sink>'])
+
+        # if there's top level aggregation for reduction
+        aggregator = stack_helper.get_reducer_aggregator()
+        if aggregator:
+            aggregator.add_source(source)
+            source = aggregator
+
+        sink = self.registry.create_sink(token)
+        
+        if not sink:
+            raise TokenError.from_list(['expression must end in a sink.',
+                            'pjk <source> [<pipe> ...] <sink>'])
+
+        sink.add_source(source)
+        return sink
+
+    def parse(self, tokens: List[str]) -> Sink:
+        self.tokens = expand_macros(tokens)
+        usage_error_message = "You've got a problem here."
+        stack_helper = StackLoader()
+        pos = 0
+        
+        try:
+            if len(self.tokens) < 2:
+                raise TokenError.from_list(['expression must include source and sink.',
+                                            'pjk <source> [<pipe> ...] <sink>'])
+
+            for pos, token in enumerate(self.tokens):
+                if pos == len(self.tokens) - 1: # token should be THE sink
+                    return self.get_sink(stack_helper, token)
+                    
+                source = self.registry.create_source(token)
+                if source:
+                    stack_helper.add_operator(source, self.stack)
+                    continue
+
+                subexp = SubExpression.create(token)
+                if subexp:
+                    stack_helper.add_operator(subexp, self.stack)
+                    continue
+
+                pipe = self.registry.create_pipe(token)
+                if pipe:
+                    stack_helper.add_operator(pipe, self.stack)
+                    continue
+
+                else: # unrecognized token
+                    # could be sink in WRONG position, let's see for better error message
+                    sink = self.registry.create_sink(token, None) 
+                    if sink:
+                        raise TokenError.from_list(['sink may only occur in final position.',
+                                            'pjk <source> [<pipe> ...] <sink>'])
+                    raise TokenError.from_list([token, 'unrecognized token'])
+        
+        except TokenError as e:
+            raise UsageError(usage_error_message, self.tokens, pos, e)
+    
+class ReducerAggregatorPipe(Pipe):
+    def __init__(self, top_level_reducers: List[Any]):
+        super().__init__(None)
+        self.top_level_reducers = top_level_reducers
+        self.reduction = {}
+        self.done = False
+
+    def reset(self):
+        self.done = False
+        self.reduction.clear()
+
+    def __iter__(self):
+        if not self.done:
+            for _ in self.left:
+                pass  # consume all input
+            for reducer in self.top_level_reducers:
+                name, value = reducer.get_subexp_result()
+                self.reduction[name] = value
+            self.done = True
+            yield self.reduction
+
+class StackLoader:
+    def __init__(self):
+        self.top_level_reducers = []
+
+    def get_reducer_aggregator(self) -> ReducerAggregatorPipe:
+        if not self.top_level_reducers:
+            return None
+        
+        return ReducerAggregatorPipe(top_level_reducers=self.top_level_reducers)
+
+    def add_operator(self, op, stack):
+        if len(stack) > 0 and isinstance(stack[-1], Pipe):
+            target = stack[-1]
+
+            if isinstance(target, SubExpression):
+                if isinstance(op, SubExpressionOver):
+                    subexp_begin = stack.pop()
+                    subexp_begin.set_over_arg(op.get_over_arg())
+                    op.add_source(subexp_begin)
+                    stack.append(op)
+                    return
+                else: # an operator within the subexpression
+                    target.add_subop(op)
+                    return
+
+        # order matters, sources are pipes
+        if isinstance(op, Pipe):
+            arity = op.arity # class level attribute
+            if len(stack) < arity:
+                raise UsageError(f"'{op}' requires {arity} input(s)")
+            for _ in range(arity):
+                op.add_source(stack.pop())
+            stack.append(op)
+
+            if isinstance(op, ReducePipe):
+                self.top_level_reducers.append(op)
+
+            return
+
+        elif isinstance(op, Source):
+            stack.append(op)
+            return
+            
+# special upstream source put in subexp stack for flexibility
+# when we don't know what that upstream source will be.
+class UpstreamSource(Source):
+    def __init__(self):
+        self.data = []
+        self.inner_source = None
+
+    def set_source(self, source: Source):
+        self.inner_source = source
+
+    def set_list(self, items):
+        self.data = items if items else []
+
+    def add_item(self, rec):
+        self.data.append(rec)
+
+    def reset(self):
+        # nothing needed in generator model
+        pass
+
+    def __iter__(self):
+        if self.inner_source:
+            yield from self.inner_source
+        else:
+            for item in self.data:
+                yield item
+    
+class SubExpression(Pipe):
+    @classmethod
+    def create(cls, token: str) -> Pipe:
+        ptok = ParsedToken(token)
+        if ptok.pre_colon == '[':
+            return SubExpression(ptok, None)
+        if ptok.pre_colon == 'over':
+            return SubExpressionOver(ptok, None)
+        return None
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.upstream_source = UpstreamSource()
+        self.over_arg = None
+        self.over_field = None
+        self.subexp_stack = [self.upstream_source]
+        self.subexp_ops = []
+        self.over_pipe = None
+        self.stack_helper = StackLoader()
+
+    def add_subop(self, op):
+        self.subexp_ops.append(op)
+        self.stack_helper.add_operator(op, self.subexp_stack)
+
+    def set_over_arg(self, over_arg):
+        self.over_arg = over_arg
+        if over_arg.endswith('.py'):
+            self.over_field = 'child'
+            self.over_pipe = UserPipeFactory.create(over_arg)
+            self.upstream_source.set_source(self.over_pipe)
+            self.subexp_ops.append(self.over_pipe)
+        else:
+            self.over_field = over_arg
+
+    def reset(self):
+        for op in self.subexp_ops:
+            if isinstance(op, Pipe):
+                op.reset()
+
+    def __iter__(self):
+        for record in self.left:
+            if self.over_pipe:
+                one = UpstreamSource()
+                one.add_item(record)
+                self.over_pipe.set_sources([one])
+            else:
+                field_data = record.pop(self.over_field, None)
+                if not field_data:
+                    yield record
+                    continue
+                if isinstance(field_data, list):
+                    self.upstream_source.set_list(field_data)
+                else:
+                    self.upstream_source.set_list([field_data])
+
+            # Reset sub-pipe stack
+            for op in self.subexp_ops:
+                op.reset()
+
+            out_recs = []
+            for rec in self.subexp_stack[-1]:
+                out_recs.append(rec)
+
+            record[self.over_field] = out_recs
+
+            for op in self.subexp_ops:
+                get_subexp = getattr(op, "get_subexp_result", None)
+                if get_subexp:
+                    name, value = get_subexp()
+                    if name:
+                        record[name] = value
+
+            yield record
+
+class SubExpressionOver(Pipe):
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.over_arg = ptok.get_arg(0)
+
+    def get_over_arg(self):
+        return self.over_arg
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        yield from self.left
+

pjk/pipes/denorm.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/denorm.py
+
+from pjk.base import Pipe, ParsedToken, Usage, UsageError
+from typing import Iterator
+
+class Denormer:
+    def __init__(self, record, field):
+        self.field = field
+        data = record.pop(field, None)
+
+        if not data:
+            self.subrec_list = [record]
+            self.base_record = {}
+            return
+
+        self.base_record = record
+
+        if isinstance(data, list):
+            self.subrec_list = data
+        elif isinstance(data, dict):
+            self.subrec_list = [data]
+        else:
+            raise UsageError("can only denorm sub-records")
+
+    def __iter__(self) -> Iterator[dict]:
+        for subrec in self.subrec_list:
+            if not isinstance(subrec, dict):
+                subrec = {self.field: subrec}
+            out = self.base_record.copy()
+            out.update(subrec)
+            yield out
+
+
+class DenormPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='explode',
+            desc='Explode a nested list/dict field into separate flattened records',
+            component_class=cls
+        )
+        usage.def_arg(name='field', usage='Field to explode')
+        usage.def_example(expr_tokens=["{ferry:'orca', cars:[{make: 'ford', size:9}, {make:'bmw', size:4}]}",
+                                       'explode:cars'
+                                       ],
+                        expect="[{ferry:'orca', make: 'ford', size:9}, {ferry:'orca', make:'bmw', size:4}]")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+
+        self.field = usage.get_arg('field')
+        if not self.field:
+            raise UsageError("denorm must include a field name")
+
+        self._pending_iter = None
+
+    def reset(self):
+        self._pending_iter = None
+
+    def __iter__(self):
+        for record in self.left:
+            denormer = Denormer(record, self.field)
+            for out in denormer:
+                yield out

pjk/pipes/factory.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/factory.py
+from pjk.base import Usage, Pipe, ParsedToken
+from pjk.common import ComponentFactory
+from pjk.pipes.move_field import MoveField
+from pjk.pipes.remove_field import RemoveField
+from pjk.pipes.let_reduce import LetPipe
+from pjk.pipes.let_reduce import ReducePipe
+from pjk.pipes.head import HeadPipe
+from pjk.pipes.tail import TailPipe
+from pjk.pipes.sort import SortPipe
+from pjk.pipes.where import WherePipe
+from pjk.pipes.map import MapPipe
+from pjk.pipes.join import JoinPipe
+from pjk.pipes.filter import FilterPipe
+from pjk.pipes.select import SelectFields
+from pjk.pipes.denorm import DenormPipe
+from pjk.pipes.postgres_pipe import PostgresPipe
+from pjk.pipes.user_pipe_factory import UserPipeFactory
+
+COMPONENTS = {
+        'head': HeadPipe,
+        'tail': TailPipe,
+        'join': JoinPipe,
+        'filter': FilterPipe,
+        'map': MapPipe,            
+        'as': MoveField,
+        'drop': RemoveField,        
+        'let': LetPipe,
+        'reduce': ReducePipe,        
+        'sort': SortPipe,
+        'where': WherePipe,
+        'sel': SelectFields,
+        'explode': DenormPipe,
+        'pgres': PostgresPipe,
+    }
+
+class PipeFactory(ComponentFactory):
+    def __init__(self):
+        super().__init__(COMPONENTS, 'pipe')
+
+    def create(self, token: str) -> Pipe:
+
+        ptok = ParsedToken(token)
+        if ptok.pre_colon.endswith('.py'):
+            pipe = UserPipeFactory.create(ptok)
+            if pipe:
+                return pipe # else keep looking
+
+        pipe_cls = self.components.get(ptok.pre_colon)
+
+        if not pipe_cls:
+            return None
+        
+        usage = pipe_cls.usage()
+        usage.bind(ptok)
+        
+        pipe = pipe_cls(ptok, usage)
+        return pipe
+

pjk/pipes/filter.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from pjk.base import Pipe, Usage, UsageError, ParsedToken, KeyedSource
+
+class FilterPipe(Pipe):
+    arity = 2  # left = record stream, right = keyed source
+
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name="filter",
+            desc="Filters left records based on presence in right keyed source",
+            component_class=cls
+        )
+        usage.def_arg("mode", "'+' to include matches, '-' to exclude matches",
+                      valid_values={'+', '-'})
+        usage.def_syntax("pjk <left_source> <map_source> map:<how>:<key> filter:<mode> <sink>")
+
+        usage.def_example(expr_tokens=
+        [
+            "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
+            "[{id:1}, {id:3}, {id:5}]",
+            'map:o:id',
+            "filter:+"
+        ],
+        expect="[{id:1}, {id:3}, {id:5}]")
+
+        usage.def_example(expr_tokens=
+        [
+            "[{id:1}, {id:2}, {id:3}, {id:4}, {id:5}]",
+            "[{id:1}, {id:3}, {id:5}]",
+            'map:o:id',
+            "filter:-"
+        ],
+        expect="[{id:2}, {id:4}]")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.mode = usage.get_arg('mode')
+        self.left = None
+        self.right = None
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        if not isinstance(self.right, KeyedSource):
+            raise UsageError("Right input to filter must be a KeyedSource")
+
+        for record in self.left:
+            match = self.right.lookup(record)
+            exists = match is not None
+
+            if (self.mode == "+" and exists) or (self.mode == "-" and not exists):
+                yield record

pjk/pipes/head.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+# djk/pipes/head.py
+
+from typing import Optional
+from pjk.base import Pipe, ParsedToken, Usage
+
+class HeadPipe(Pipe):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='head',
+            desc='take first records of input (when single-threaded)',
+            component_class=cls
+        )
+        usage.def_arg(name='limit', usage='number of records', is_num=True)
+        usage.def_example(expr_tokens=['[{id:1}, {id:2}]', 'head:1'], expect="{id:1}")
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok)
+        self.limit = usage.get_arg('limit')
+        self.count = 0
+
+    def __iter__(self):
+        for record in self.left:
+            if self.count >= self.limit:
+                break
+            self.count += 1
+            yield record
+    
+    def reset(self):
+        self.count = 0