python-jack-knife 0.6.0__tar.gz → 0.6.2__tar.gz

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.0
+Version: 0.6.2
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/common.py

@@ -3,6 +3,7 @@
 
 import sys, shutil, subprocess, contextlib, signal
 import os
+import re
 import yaml
 from pjk.base import TokenError, Integration
 
@@ -156,3 +157,6 @@ class ComponentFactory:
 
     def create(self, token: str):
         pass
+
+def is_valid_field_name(name: str):
+    return re.fullmatch(r'^[A-Za-z_][A-Za-z0-9_]*$', name)

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/parser.py

@@ -55,8 +55,20 @@ class OperandStack:
             return None
         return self.stack[-1]
     
+    def clear(self):
+        self.stack.clear()
+    
     def empty(self):
         return len(self.stack) == 0
+    
+    def print(self, toadd):
+        print('---------')
+        if toadd:
+            print(f'{type(toadd).__name__}={id(toadd)}')
+        if len(self.stack) == 0:
+            print(f'Stack={id(self)} StackEmpty')
+        for op in self.stack:
+            print(f'Stack={id(self)} {type(op).__name__}={id(op)}')
 
 class ExpressionParser:
     def __init__(self, registry: ComponentRegistry):
@@ -69,6 +81,9 @@ class ExpressionParser:
                                             'pjk <source> [<pipe> ...] <sink>'])
 
         source = self.stack.pop()
+        if isinstance(source, SubExpression):
+            raise TokenError("Poorly formed sub-expression.  Begin token '[' without matching 'over' keyword." )
+
         if not self.stack.empty():
             raise TokenError.from_list(['A sink can only consume one source.',
                                         'pjk <source> [<pipe> ...] <sink>'])
@@ -167,29 +182,38 @@ class StackLoader:
         return ReducerAggregatorPipe(top_level_reducers=self.top_level_reducers)
 
     def add_operator(self, op, stack: OperandStack):
+        #stack.print(op)
+
         if not stack.empty() and isinstance(stack.peek(), SubExpression):
-            top = stack.peek()
+            subexp = stack.peek()
 
-            if isinstance(op, SubExpressionOver):
-                subexp_begin = stack.pop()
-                subexp_begin.set_over_arg(op.get_over_arg())
-                op.add_source(subexp_begin)
+            if isinstance(op, SubExpressionOver) and subexp.recursion_depth() == 0:
+                subexp = stack.pop()
+                op.add_source(subexp)
                 stack.push(op)
 
                 global stack_level
-                # SEEMS LIKE A HACK! FIXME.  The stack should handle this but its off by one
-                stack_level-=1 
+                stack_level -=1 # not sure why this can't be handled exclusively by the stack
                 return
+            
             else: # an operator within the subexpression
-                top.add_subop(op)
+                subexp = stack.peek()
+                subexp.add_subop(op)
                 return
 
+        if isinstance(op, SubExpressionOver):
+            if stack.empty or not isinstance(stack.peek(), SubExpression):
+                raise TokenError("Poorly formed sub-expression.  'over' keyword without matching begin token '['.")
+            op.add_source(stack.pop())
+            stack.push(op)
+
         # order matters, because sources are pipes
         if isinstance(op, Pipe):
             arity = op.arity # class level attribute
             for _ in range(arity):
                 if stack.empty():
-                    raise UsageError(f"'{op}' requires {arity} input(s)")
+                    name = type(op).usage().name
+                    raise TokenError(f"'{name}' requires {arity} input(s)")
                 op.add_source(stack.pop())
             stack.push(op)
 
@@ -243,6 +267,31 @@ class UpstreamSource(Source):
                 self.sub_recs_in.increment()
                 yield item
     
+class SubExpressionOver(Pipe):
+    @classmethod
+    def usage(cls) -> Usage:
+        u = Usage(
+            name="over",
+            desc="sub-expression over.",
+            component_class=cls,
+        )
+        return u
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.over_arg = ptok.get_arg(0)
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        if not isinstance(self.left, SubExpression):
+            raise Exception('this actually cannot happen, but did')
+
+        for record in self.left:
+            self.left.subexp_process(record, self.over_arg)
+            yield record
+
 class SubExpression(Pipe, ProgressIgnore):
     @classmethod
     def create(cls, token: str) -> Pipe:
@@ -255,28 +304,34 @@ class SubExpression(Pipe, ProgressIgnore):
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok)
-        self.over_arg = None
-        self.over_field = None
         self.subexp_ops = []
-        self.over_pipe = None
         self.stack_helper = StackLoader()
         self.subexp_stack = OperandStack() 
         self.upstream_source = UpstreamSource()
         self.subexp_stack.push(self.upstream_source)
+        self.recursions = 0 # number of subexpression within
+        self.subexp_left = None
 
     def add_subop(self, op):
         self.subexp_ops.append(op)
+        if isinstance(op, SubExpression):
+            self.recursions += 1
+        elif isinstance(op, SubExpressionOver):
+            self.recursions -= 1
         self.stack_helper.add_operator(op, self.subexp_stack)
 
-    def set_over_arg(self, over_arg):  #FIXME, this should take QueryPipe
-        self.over_arg = over_arg
-        if over_arg.endswith('.py'):
-            self.over_field = 'child'
-            self.over_pipe = UserPipeFactory.create(over_arg)
-            self.upstream_source.set_source(self.over_pipe)
-            self.subexp_ops.append(self.over_pipe)
-        else:
-            self.over_field = over_arg
+    def recursion_depth(self):
+        return self.recursions
+    
+    #def bind(self, subex_over: SubExpressionOver):
+    #    self.over_arg = subex_over.get_over_arg()
+    #    if self.over_arg.endswith('.py'):
+    #        self.over_field = 'child'
+    #        self.over_pipe = UserPipeFactory.create(self.over_arg)
+    #        self.upstream_source.set_source(self.over_pipe)
+    #        self.subexp_ops.append(self.over_pipe)
+    #    else:
+    #        self.over_field = self.over_arg
 
     def reset(self):
         for op in self.subexp_ops:
@@ -284,60 +339,40 @@ class SubExpression(Pipe, ProgressIgnore):
                 op.reset()
 
     def __iter__(self):
-        for record in self.left:
-            if self.over_pipe:
-                one = UpstreamSource()
-                one.add_item(record)
-                self.over_pipe.set_sources([one])
-            else:
-                field_data = record.pop(self.over_field, None)
-                if not field_data:
-                    yield record
-                    continue
-                if isinstance(field_data, list):
-                    self.upstream_source.set_list(field_data)
-                else:
-                    self.upstream_source.set_list([field_data])
+        yield from self.left # pass thru to subexp_over which then calls process
 
-            # Reset sub-pipe stack
-            for op in self.subexp_ops:
-                op.reset()
+    def subexp_process(self, record: dict, over_field: str):
+        #for record in self.left:
+        #    if self.over_pipe:
+        #        one = UpstreamSource()
+        #        one.add_item(record)
+        #        self.over_pipe.set_sources([one])
 
-            out_recs = []
-            top = self.subexp_stack.peek()
-            for rec in top:
-                out_recs.append(rec)
+        if not self.subexp_left:
+            self.subexp_left = self.subexp_stack.pop()
 
-            record[self.over_field] = out_recs
+        field_data = record.pop(over_field, None)
+        if not field_data:
+            return
 
-            for op in self.subexp_ops:
-                get_subexp = getattr(op, "get_subexp_result", None)
-                if get_subexp:
-                    name, value = get_subexp()
-                    if name:
-                        record[name] = value
+        if isinstance(field_data, list):
+            self.upstream_source.set_list(field_data)
+        else:
+            self.upstream_source.set_list([field_data])
 
-            yield record
+        # Reset sub-pipe stack
+        for op in self.subexp_ops:
+            op.reset()
 
-class SubExpressionOver(Pipe):
-    @classmethod
-    def usage(cls) -> Usage:
-        u = Usage(
-            name="over",
-            desc="sub-expression over.",
-            component_class=cls,
-        )
-        return u
-    
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
-        self.over_arg = ptok.get_arg(0)
+        out_recs = []
 
-    def get_over_arg(self):
-        return self.over_arg
+        for rec in self.subexp_left:
+            out_recs.append(rec)
 
-    def reset(self):
-        pass  # stateless
+        record[over_field] = out_recs
 
-    def __iter__(self):
-        yield from self.left
+        for op in self.subexp_ops:
+            if isinstance(op, ReducePipe):
+                name, value = op.get_subexp_result()
+                if name:
+                    record[name] = value

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/pipes/denorm.py

@@ -5,6 +5,7 @@
 
 from pjk.base import Pipe, ParsedToken, Usage, UsageError
 from typing import Iterator
+from pjk.progress import papi
 
 class Denormer:
     def __init__(self, record, field):
@@ -23,7 +24,7 @@ class Denormer:
         elif isinstance(data, dict):
             self.subrec_list = [data]
         else:
-            raise UsageError("can only denorm sub-records")
+            raise UsageError("can only explode sub-records")
 
     def __iter__(self) -> Iterator[dict]:
         for subrec in self.subrec_list:
@@ -53,8 +54,8 @@ class DenormPipe(Pipe):
         super().__init__(ptok)
 
         self.field = usage.get_arg('field')
-        if not self.field:
-            raise UsageError("denorm must include a field name")
+        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_out = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 
         self._pending_iter = None
 
@@ -63,6 +64,8 @@ class DenormPipe(Pipe):
 
     def __iter__(self):
         for record in self.left:
+            self.recs_in.increment()
             denormer = Denormer(record, self.field)
             for out in denormer:
+                self.recs_out.increment()
                 yield out

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/pipes/move_field.py

@@ -3,34 +3,33 @@
 
 # djk/pipes/move_field.py
 
-from pjk.base import Pipe, ParsedToken, Usage
+from pjk.base import Pipe, ParsedToken, Usage, TokenError
+from pjk.common import is_valid_field_name
 
 class MoveField(Pipe):
     @classmethod
     def usage(cls):
-        usage = Usage(
+        u = Usage(
             name='as',
             desc='rename a field in the record',
             component_class=cls
         )
-        usage.def_arg(name='src', usage='Source field name')
-        usage.def_arg(name='dst', usage='Destination field name')
-        usage.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
+        u.def_arg(name='src', usage='Source field name')
+        u.def_arg(name='dst', usage='Destination field name')
+        u.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
 
-        return usage
+        return u
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
         self.src = usage.get_arg('src')
         self.dst = usage.get_arg('dst')
-        self.count = 0
 
-    def reset(self):
-        self.count = 0
+        if not is_valid_field_name(self.dst) or not is_valid_field_name(self.src):
+            raise TokenError('field names only allow letters, numbers (non-initially) and underbar')
 
     def __iter__(self):
         for record in self.left:
-            self.count += 1
             if self.src in record:
                 record[self.dst] = record.pop(self.src)
             yield record

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/pipes/query_pipe.py

@@ -18,7 +18,7 @@ class QueryPipe(Pipe):
             desc=cls.desc,
             component_class=cls
         )
-        u.def_arg(name=cls.arg0[0], usage=f'{cls.arg0[1]} ~/.pjk/lookups.yaml must containing entry {cls.__name__}-<{cls.arg0[0]}>\n   containing necessary parameters.')
+        u.def_arg(name=cls.arg0[0], usage=f"{cls.arg0[1]} ~/.pjk/lookups.yaml must contain entry '{cls.__name__}-<{cls.arg0[0]}'>\n  with necessary parameters.")
         u.def_param("query_field", usage="field of query.", default="query")
         u.def_param("count", usage="Number of search results, (databases may ignore)", is_num=True, default="10")
         u.def_param("shape", usage='the shape of ouput records', is_num=False,

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.0"
+__version__ = "0.6.2"

{python_jack_knife-0.6.0 → python_jack_knife-0.6.2}/src/python_jack_knife.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.0
+Version: 0.6.2
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: