python-jack-knife 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

pjk/common.py

@@ -3,6 +3,7 @@
 
 import sys, shutil, subprocess, contextlib, signal
 import os
+import re
 import yaml
 from pjk.base import TokenError, Integration
 
@@ -156,3 +157,6 @@ class ComponentFactory:
 
     def create(self, token: str):
         pass
+
+def is_valid_field_name(name: str):
+    return re.fullmatch(r'^[A-Za-z_][A-Za-z0-9_]*$', name)

pjk/parser.py

@@ -55,8 +55,20 @@ class OperandStack:
             return None
         return self.stack[-1]
     
+    def clear(self):
+        self.stack.clear()
+    
     def empty(self):
         return len(self.stack) == 0
+    
+    def print(self, toadd):
+        print('---------')
+        if toadd:
+            print(f'{type(toadd).__name__}={id(toadd)}')
+        if len(self.stack) == 0:
+            print(f'Stack={id(self)} StackEmpty')
+        for op in self.stack:
+            print(f'Stack={id(self)} {type(op).__name__}={id(op)}')
 
 class ExpressionParser:
     def __init__(self, registry: ComponentRegistry):
@@ -69,6 +81,9 @@ class ExpressionParser:
                                             'pjk <source> [<pipe> ...] <sink>'])
 
         source = self.stack.pop()
+        if isinstance(source, SubExpression):
+            raise TokenError("Poorly formed sub-expression.  Begin token '[' without matching 'over' keyword." )
+
         if not self.stack.empty():
             raise TokenError.from_list(['A sink can only consume one source.',
                                         'pjk <source> [<pipe> ...] <sink>'])
@@ -167,29 +182,38 @@ class StackLoader:
         return ReducerAggregatorPipe(top_level_reducers=self.top_level_reducers)
 
     def add_operator(self, op, stack: OperandStack):
+        #stack.print(op)
+
         if not stack.empty() and isinstance(stack.peek(), SubExpression):
-            top = stack.peek()
+            subexp = stack.peek()
 
-            if isinstance(op, SubExpressionOver):
-                subexp_begin = stack.pop()
-                subexp_begin.set_over_arg(op.get_over_arg())
-                op.add_source(subexp_begin)
+            if isinstance(op, SubExpressionOver) and subexp.recursion_depth() == 0:
+                subexp = stack.pop()
+                op.add_source(subexp)
                 stack.push(op)
 
                 global stack_level
-                # SEEMS LIKE A HACK! FIXME.  The stack should handle this but its off by one
-                stack_level-=1 
+                stack_level -=1 # not sure why this can't be handled exclusively by the stack
                 return
+            
             else: # an operator within the subexpression
-                top.add_subop(op)
+                subexp = stack.peek()
+                subexp.add_subop(op)
                 return
 
+        if isinstance(op, SubExpressionOver):
+            if stack.empty or not isinstance(stack.peek(), SubExpression):
+                raise TokenError("Poorly formed sub-expression.  'over' keyword without matching begin token '['.")
+            op.add_source(stack.pop())
+            stack.push(op)
+
         # order matters, because sources are pipes
         if isinstance(op, Pipe):
             arity = op.arity # class level attribute
             for _ in range(arity):
                 if stack.empty():
-                    raise UsageError(f"'{op}' requires {arity} input(s)")
+                    name = type(op).usage().name
+                    raise TokenError(f"'{name}' requires {arity} input(s)")
                 op.add_source(stack.pop())
             stack.push(op)
 
@@ -243,6 +267,31 @@ class UpstreamSource(Source):
                 self.sub_recs_in.increment()
                 yield item
     
+class SubExpressionOver(Pipe):
+    @classmethod
+    def usage(cls) -> Usage:
+        u = Usage(
+            name="over",
+            desc="sub-expression over.",
+            component_class=cls,
+        )
+        return u
+    
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.over_arg = ptok.get_arg(0)
+
+    def reset(self):
+        pass  # stateless
+
+    def __iter__(self):
+        if not isinstance(self.left, SubExpression):
+            raise Exception('this actually cannot happen, but did')
+
+        for record in self.left:
+            self.left.subexp_process(record, self.over_arg)
+            yield record
+
 class SubExpression(Pipe, ProgressIgnore):
     @classmethod
     def create(cls, token: str) -> Pipe:
@@ -255,28 +304,34 @@ class SubExpression(Pipe, ProgressIgnore):
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok)
-        self.over_arg = None
-        self.over_field = None
         self.subexp_ops = []
-        self.over_pipe = None
         self.stack_helper = StackLoader()
         self.subexp_stack = OperandStack() 
         self.upstream_source = UpstreamSource()
         self.subexp_stack.push(self.upstream_source)
+        self.recursions = 0 # number of subexpression within
+        self.subexp_left = None
 
     def add_subop(self, op):
         self.subexp_ops.append(op)
+        if isinstance(op, SubExpression):
+            self.recursions += 1
+        elif isinstance(op, SubExpressionOver):
+            self.recursions -= 1
         self.stack_helper.add_operator(op, self.subexp_stack)
 
-    def set_over_arg(self, over_arg):  #FIXME, this should take QueryPipe
-        self.over_arg = over_arg
-        if over_arg.endswith('.py'):
-            self.over_field = 'child'
-            self.over_pipe = UserPipeFactory.create(over_arg)
-            self.upstream_source.set_source(self.over_pipe)
-            self.subexp_ops.append(self.over_pipe)
-        else:
-            self.over_field = over_arg
+    def recursion_depth(self):
+        return self.recursions
+    
+    #def bind(self, subex_over: SubExpressionOver):
+    #    self.over_arg = subex_over.get_over_arg()
+    #    if self.over_arg.endswith('.py'):
+    #        self.over_field = 'child'
+    #        self.over_pipe = UserPipeFactory.create(self.over_arg)
+    #        self.upstream_source.set_source(self.over_pipe)
+    #        self.subexp_ops.append(self.over_pipe)
+    #    else:
+    #        self.over_field = self.over_arg
 
     def reset(self):
         for op in self.subexp_ops:
@@ -284,60 +339,40 @@ class SubExpression(Pipe, ProgressIgnore):
                 op.reset()
 
     def __iter__(self):
-        for record in self.left:
-            if self.over_pipe:
-                one = UpstreamSource()
-                one.add_item(record)
-                self.over_pipe.set_sources([one])
-            else:
-                field_data = record.pop(self.over_field, None)
-                if not field_data:
-                    yield record
-                    continue
-                if isinstance(field_data, list):
-                    self.upstream_source.set_list(field_data)
-                else:
-                    self.upstream_source.set_list([field_data])
+        yield from self.left # pass thru to subexp_over which then calls process
 
-            # Reset sub-pipe stack
-            for op in self.subexp_ops:
-                op.reset()
+    def subexp_process(self, record: dict, over_field: str):
+        #for record in self.left:
+        #    if self.over_pipe:
+        #        one = UpstreamSource()
+        #        one.add_item(record)
+        #        self.over_pipe.set_sources([one])
 
-            out_recs = []
-            top = self.subexp_stack.peek()
-            for rec in top:
-                out_recs.append(rec)
+        if not self.subexp_left:
+            self.subexp_left = self.subexp_stack.pop()
 
-            record[self.over_field] = out_recs
+        field_data = record.pop(over_field, None)
+        if not field_data:
+            return
 
-            for op in self.subexp_ops:
-                get_subexp = getattr(op, "get_subexp_result", None)
-                if get_subexp:
-                    name, value = get_subexp()
-                    if name:
-                        record[name] = value
+        if isinstance(field_data, list):
+            self.upstream_source.set_list(field_data)
+        else:
+            self.upstream_source.set_list([field_data])
 
-            yield record
+        # Reset sub-pipe stack
+        for op in self.subexp_ops:
+            op.reset()
 
-class SubExpressionOver(Pipe):
-    @classmethod
-    def usage(cls) -> Usage:
-        u = Usage(
-            name="over",
-            desc="sub-expression over.",
-            component_class=cls,
-        )
-        return u
-    
-    def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok, usage)
-        self.over_arg = ptok.get_arg(0)
+        out_recs = []
 
-    def get_over_arg(self):
-        return self.over_arg
+        for rec in self.subexp_left:
+            out_recs.append(rec)
 
-    def reset(self):
-        pass  # stateless
+        record[over_field] = out_recs
 
-    def __iter__(self):
-        yield from self.left
+        for op in self.subexp_ops:
+            if isinstance(op, ReducePipe):
+                name, value = op.get_subexp_result()
+                if name:
+                    record[name] = value

pjk/pipes/denorm.py

@@ -5,6 +5,7 @@
 
 from pjk.base import Pipe, ParsedToken, Usage, UsageError
 from typing import Iterator
+from pjk.progress import papi
 
 class Denormer:
     def __init__(self, record, field):
@@ -23,7 +24,7 @@ class Denormer:
         elif isinstance(data, dict):
             self.subrec_list = [data]
         else:
-            raise UsageError("can only denorm sub-records")
+            raise UsageError("can only explode sub-records")
 
     def __iter__(self) -> Iterator[dict]:
         for subrec in self.subrec_list:
@@ -53,8 +54,8 @@ class DenormPipe(Pipe):
         super().__init__(ptok)
 
         self.field = usage.get_arg('field')
-        if not self.field:
-            raise UsageError("denorm must include a field name")
+        self.recs_in = papi.get_counter(self, None) # don't display
+        self.recs_out = papi.get_percentage_counter(self, 'recs_out', self.recs_in)
 
         self._pending_iter = None
 
@@ -63,6 +64,8 @@ class DenormPipe(Pipe):
 
     def __iter__(self):
         for record in self.left:
+            self.recs_in.increment()
             denormer = Denormer(record, self.field)
             for out in denormer:
+                self.recs_out.increment()
                 yield out

pjk/pipes/move_field.py

@@ -3,34 +3,33 @@
 
 # djk/pipes/move_field.py
 
-from pjk.base import Pipe, ParsedToken, Usage
+from pjk.base import Pipe, ParsedToken, Usage, TokenError
+from pjk.common import is_valid_field_name
 
 class MoveField(Pipe):
     @classmethod
     def usage(cls):
-        usage = Usage(
+        u = Usage(
             name='as',
             desc='rename a field in the record',
             component_class=cls
         )
-        usage.def_arg(name='src', usage='Source field name')
-        usage.def_arg(name='dst', usage='Destination field name')
-        usage.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
+        u.def_arg(name='src', usage='Source field name')
+        u.def_arg(name='dst', usage='Destination field name')
+        u.def_example(expr_tokens=['{up:1}', 'as:up:down'], expect="{down:1}")
 
-        return usage
+        return u
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
         super().__init__(ptok, usage)
         self.src = usage.get_arg('src')
         self.dst = usage.get_arg('dst')
-        self.count = 0
 
-    def reset(self):
-        self.count = 0
+        if not is_valid_field_name(self.dst) or not is_valid_field_name(self.src):
+            raise TokenError('field names only allow letters, numbers (non-initially) and underbar')
 
     def __iter__(self):
         for record in self.left:
-            self.count += 1
             if self.src in record:
                 record[self.dst] = record.pop(self.src)
             yield record

pjk/pipes/query_pipe.py

@@ -18,7 +18,7 @@ class QueryPipe(Pipe):
             desc=cls.desc,
             component_class=cls
         )
-        u.def_arg(name=cls.arg0[0], usage=f'{cls.arg0[1]} ~/.pjk/lookups.yaml must containing entry {cls.__name__}-<{cls.arg0[0]}>\n   containing necessary parameters.')
+        u.def_arg(name=cls.arg0[0], usage=f"{cls.arg0[1]} ~/.pjk/lookups.yaml must contain entry '{cls.__name__}-<{cls.arg0[0]}'>\n  with necessary parameters.")
         u.def_param("query_field", usage="field of query.", default="query")
         u.def_param("count", usage="Number of search results, (databases may ignore)", is_num=True, default="10")
         u.def_param("shape", usage='the shape of ouput records', is_num=False,

pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.6.0"
+__version__ = "0.6.2"

{python_jack_knife-0.6.0.dist-info → python_jack_knife-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.0
+Version: 0.6.2
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.6.0.dist-info → python_jack_knife-0.6.2.dist-info}/RECORD

@@ -1,27 +1,27 @@
 pjk/__init__.py,sha256=6HGDVcFOFv6VPSNjxVnusm9wHqy01pELX3AyCWFzqWg,128
 pjk/base.py,sha256=ObBsoTkfWp5qigr-QdNGZBs6tNH41P18kTnSMSJnntA,12775
-pjk/common.py,sha256=0buZgs3dBgNaxUugrFkzBR3qEr_IFfC1-PhvTP9FnCs,4809
+pjk/common.py,sha256=c2Y9TTS1Lz8KNGSaKqN3DMfb7fOGkndZon_d0V0iwm0,4914
 pjk/log.py,sha256=LjBboWhUrr2Cz-ygpftCIK17ee2-zNHKZjeJyoW0PlA,2163
 pjk/main.py,sha256=fM2xuofSv_lzSBk4rD7ZCWvR1beRMgHiezL4rIBD0dE,3924
 pjk/man_page.py,sha256=BNxRbzXfvR_5FoPkpCwxApyMsfSSrSdDu2PM-IVblA4,4447
-pjk/parser.py,sha256=C0rWqb_v-HvV7TTcacn6eU4w5o2fConPYVadSLwMha4,11404
+pjk/parser.py,sha256=2aZiW0SY9KPKb2h4xPasOhxehcxn_l_BEx2swmN_7_E,12673
 pjk/progress.py,sha256=scacaLRbSMBJEEEDeOTYt8sKwXP-CYEGoANG7IXVoQ4,9597
 pjk/registry.py,sha256=0_exY3sivaOHyRtf-v88OSugl2LGAbbFjyFS4cPbGO8,6864
-pjk/version.py,sha256=YIm7tlVfTl0uD8veTyKIyKgY-0nK0BdMVCC3KdWugN4,91
+pjk/version.py,sha256=pUQpi34dOjfho08lZGT_KTgebMk9IucMzzRKSwCd6Bc,91
 pjk/integrations/ddb_sink.py,sha256=qFuh0ZkZ1oGrx576-226eu75sauGqa8BF_u-FLs4yW4,1704
 pjk/integrations/postgres_pipe.py,sha256=MZut7dikqbXr0XIrL31yrXM_0ZrgExD6suVA9jd_xmA,6192
 pjk/integrations/snowflake_pipe.py,sha256=z22E4KBfZHnAi6Bb8e9PXJwvwVYPeJjF02FI5o0rRus,8549
 pjk/pipes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pjk/pipes/denorm.py,sha256=3zY8E34e5ORiddP3zEXRg1_iLYV6ivsdEBilNBjjHTo,2065
+pjk/pipes/denorm.py,sha256=tL8v3lTDCZPXDlffrx71PIojC_JWjN3PCJriNP6mw0o,2235
 pjk/pipes/factory.py,sha256=rwCWneXB1gosk9I24yr1Z0RDUYHrjHx8swFLrDR1k44,2119
 pjk/pipes/filter.py,sha256=Fo5-MJaNAfTabRZxw9fQ1n4amhgUUCncC_GoOWswdRw,2053
 pjk/pipes/head.py,sha256=5ZAacwzpVWuR_xWCnlHqyJUwbxfEcF8vIogdKIJBTRs,903
 pjk/pipes/join.py,sha256=zE9JrrGnOjDB7OuBT2_FxleptFfbBJJLdFak9FbsOjc,3149
 pjk/pipes/let_reduce.py,sha256=QfCs-omZq-a2hMFr5Nnt1hhQuiXol0IMA2diXwesVUA,7153
 pjk/pipes/map.py,sha256=mOBKjx-AHDmOa3mBMnFfb-PYqFbKnqGVjBj3hx2fcys,5107
-pjk/pipes/move_field.py,sha256=H494BdPsoYVbASGO-EUjEqxuClQF3a3R2OjRyeAQyYk,1018
+pjk/pipes/move_field.py,sha256=ZgJZZSD7irWVnOdHyhNUq9Mw3oLuGVUr1WKjdjBbwgo,1142
 pjk/pipes/progress_pipe.py,sha256=GfnQFmeAOyeSHl69TXNWWz7sBd6MVpzDvYH2mZBp1Hg,1239
-pjk/pipes/query_pipe.py,sha256=3rO6pblw741QGfuTMlKv95sppdCstnDE3sT8VZe7knI,2978
+pjk/pipes/query_pipe.py,sha256=nGcQh0qOcHHzCyVcwfrac5HmT9pPzyGlk--zJtmxVnc,2970
 pjk/pipes/remove_field.py,sha256=QjEO6-phRngM2emBJ6xv8UA2d_iA44tYN8Crx4lhqQ0,1169
 pjk/pipes/sample.py,sha256=_YezoC3hZbG0qpkg9C6AuHLjuWseeAqsYYOXXAMvbOQ,2269
 pjk/pipes/select.py,sha256=OuVAk8x7rCkb0neroBmUUuovxHc-z2q7VaUNNWrEdZY,1331
@@ -66,9 +66,9 @@ pjk/sources/source_list.py,sha256=5L2vFrtVSl9rKf2NjfpUFOOAb-iypVDKYCw1-3xgcEo,64
 pjk/sources/sql_source.py,sha256=2CpOuikd2BVmMW_UbhVGPfXUf3BJmOcK5yMtmbTMYvw,743
 pjk/sources/tsv_source.py,sha256=37nhEblCZ8XeTNHVo-WcdJ8HbIbiwKgsDES_yzz6EdU,306
 pjk/sources/user_source_factory.py,sha256=0XIz6NkiBhGAteGDP8cNy7MALWsUF3smtLj4Qnj41Q8,1326
-python_jack_knife-0.6.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-python_jack_knife-0.6.0.dist-info/METADATA,sha256=lEQvJQufcy-g2Zq4MGV6FSk7XX3XXtyQojWrP9sXrmM,14641
-python_jack_knife-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-python_jack_knife-0.6.0.dist-info/entry_points.txt,sha256=kzZ10zEisvEaG2xYqqw7xRpuV62rAO_dPEHnM6USelk,38
-python_jack_knife-0.6.0.dist-info/top_level.txt,sha256=r-Ef_I9SbVDL9jD-W0WtshstLos_7guWbpItYxxSllQ,4
-python_jack_knife-0.6.0.dist-info/RECORD,,
+python_jack_knife-0.6.2.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+python_jack_knife-0.6.2.dist-info/METADATA,sha256=6GiGT2ccWGYUKbaGPIQXPMSKJsHfM7_K8xatC6eGn1w,14641
+python_jack_knife-0.6.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+python_jack_knife-0.6.2.dist-info/entry_points.txt,sha256=kzZ10zEisvEaG2xYqqw7xRpuV62rAO_dPEHnM6USelk,38
+python_jack_knife-0.6.2.dist-info/top_level.txt,sha256=r-Ef_I9SbVDL9jD-W0WtshstLos_7guWbpItYxxSllQ,4
+python_jack_knife-0.6.2.dist-info/RECORD,,