python-jack-knife 0.6.11__tar.gz → 0.6.12__tar.gz

{python_jack_knife-0.6.11 → python_jack_knife-0.6.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.6.11
+Version: 0.6.12
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

python_jack_knife-0.6.12/src/pjk/common.py

@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import contextlib, io, os, subprocess, sys
+import os
+import re
+from abc import ABC
+from enum import Enum
+from pjk.sources.format_source import FormatSource
+from pjk.sinks.format_sink import FormatSink
+from typing import List, Type
+
+class SafeNamespace:
+    def __init__(self, obj):
+        for k, v in obj.items():
+            if isinstance(v, dict):
+                v = SafeNamespace(v)
+            elif isinstance(v, list):
+                v = [SafeNamespace(x) if isinstance(x, dict) else x for x in v]
+            setattr(self, k, v)
+
+    def __getattr__(self, key):
+        return None  # gracefully handle missing keys
+
+class ReducingNamespace:
+    def __init__(self, record):
+        self._record = record
+
+    def __getattr__(self, name):
+        value = self._record[name]
+        if isinstance(value, (list, tuple, set)):
+            return value
+        return [value]  # promote scalars to singleton lists
+
+# pjk/common.py
+import contextlib, io, os, subprocess, sys
+
+@contextlib.contextmanager
+def pager_stdout(use_pager: bool = True):
+    """
+    Stream stdout into `less` via a pipe.
+    - If stdout is not a TTY or use_pager is False → write directly to sys.stdout.
+    - Otherwise spawn `less` and replace sys.stdout with less.stdin.
+    """
+    # If not a TTY, paging makes no sense
+    if not use_pager or not sys.stdout.isatty():
+        yield
+        return
+
+    env = os.environ.copy()
+    # -R: pass ANSI; -S: chop long lines; you can add -F/-X to taste
+    env.setdefault("LESS", "-R")
+    # Ensure UTF-8
+    env.setdefault("LESSCHARSET", "utf-8")
+
+    stdout_orig = sys.stdout
+    stderr_orig = sys.stderr
+
+    # Start less with a *pipe* for stdin and inherit the real terminal for out/err
+    pager = subprocess.Popen(
+        ["less"],
+        stdin=subprocess.PIPE,
+        stdout=stdout_orig,   # keep interactivity
+        stderr=stderr_orig,
+        env=env,
+        close_fds=True,
+        bufsize=0,            # unbuffered pipe
+    )
+
+    # Wrap less.stdin as a text writer and swap sys.stdout
+    assert pager.stdin is not None
+    pager_bin = pager.stdin
+    pager_txt = io.TextIOWrapper(pager_bin, encoding="utf-8", write_through=True)
+
+    sys.stdout = pager_txt
+    try:
+        yield
+    except BrokenPipeError:
+        pass
+    finally:
+        try:
+            sys.stdout.flush()
+        except Exception:
+            pass
+        # Restore first, then close pager stdin to send EOF
+        sys.stdout = stdout_orig
+        try:
+            pager_txt.flush()
+        except Exception:
+            pass
+        try:
+            pager_bin.close()   # EOF → lets less exit
+        except Exception:
+            pass
+        try:
+            pager.wait()
+        except Exception:
+            pass
+
+COLOR_CODES = {
+        'bold': '\033[1m',
+        'underline': '\033[4m',
+        'red': '\033[31m',
+        'green': '\033[32m',
+        'yellow': '\033[33m',
+        'blue': '\033[34m',
+        'magenta': '\033[35m',
+        'cyan': '\033[36m',
+        'gray': '\033[90m',
+    }
+
+RESET = '\033[0m'
+
+def highlight(text: str, color: str = 'bold', value: str = None) -> str:
+    value = text if not value else value
+    style = COLOR_CODES.get(color.lower(), COLOR_CODES['bold'])
+    return text.replace(value, f"{style}{value}{RESET}")
+
+# mixin
+class Integration(ABC):
+    pass
+
+class ComponentOrigin(Enum):
+    CORE = 0 # core components defined in python-jack-knife
+    EXTERNAL = 1 # component loaded via load_package_extras (displayed in either 'integrations' or 'applications')
+    USER = 2 # components loaded via load_user_components (always displayed in user_components)
+
+class ComponentWrapper:
+    def __init__(self, name: str, comp_class, origin: ComponentOrigin):
+        self.name = name
+        self.comp_class = comp_class
+        self.origin = origin
+        self.is_integration = issubclass(comp_class, Integration)
+
+class ComponentFactory:
+    def __init__(self, core_components: dict):
+        self.wrappers = {}
+        for k, v in core_components.items():
+            self.register(k, v, origin=ComponentOrigin.CORE)
+
+    def register(self, name, comp_class, origin: ComponentOrigin):
+        self.wrappers[name] = ComponentWrapper(name, comp_class=comp_class, origin=origin)
+
+    # is_integration True|False|None=don't care
+    def get_components(self, origin_list: List[ComponentOrigin], is_integration: bool) -> dict:
+        all = {}
+        for wrapper in self.wrappers.values():
+            if is_integration is not None:
+                if wrapper.is_integration != is_integration:
+                    continue
+
+            for o in origin_list:
+                if wrapper.origin == o:
+                    all[wrapper.name] = wrapper.comp_class
+
+        return all
+
+    def get_component_class(self, name: str):
+        wrapper = self.wrappers.get(name, None)
+        if not wrapper:
+            return None
+        return wrapper.comp_class
+
+    #def get_usage(self, name: str):
+    #    comp_class = self.get_component_class(name)
+    #   if not comp_class:
+    #       return None
+    #   return comp_class.usage()
+
+    def create(self, token: str):
+        pass
+
+def is_valid_field_name(name: str):
+    return re.fullmatch(r'^[A-Za-z_][A-Za-z0-9_]*$', name)

{python_jack_knife-0.6.11 → python_jack_knife-0.6.12}/src/pjk/main.py

@@ -5,7 +5,6 @@
 import sys
 import os
 import shlex
-import shutil
 from typing import List
 from pjk.parser import ExpressionParser
 from pjk.usage import UsageError
@@ -80,7 +79,7 @@ def execute(command: str):
 
 def execute_tokens(tokens: List[str]):
     initialize()
-
+    
     if '--version' in tokens:
         print(f"pjk version {__version__}")
         sys.exit(0)

{python_jack_knife-0.6.11 → python_jack_knife-0.6.12}/src/pjk/man_page.py

@@ -8,7 +8,7 @@ from pjk.parser import ExpressionParser
 from pjk.components import Source, Pipe, Sink
 from pjk.usage import Usage, ParsedToken
 from pjk.registry import ComponentRegistry
-from pjk.common import pager_stdout, highlight
+from pjk.common import pager_stdout, highlight, ComponentOrigin
 from contextlib import nullcontext
 
 def get_base_class(usage: Usage, as_string: bool = False):
@@ -49,9 +49,9 @@ def do_man(name: str, registry: ComponentRegistry):
     # source and sinks have common names so go through multiple times
     printed = False
     for factory in registry.get_factories():
-        usage = factory.get_usage(name)
-        if usage:
-            print_man(registry, name, usage)
+        comp_class = factory.get_component_class(name)
+        if comp_class:
+            print_man(registry, name, comp_class.usage())
             printed = True
 
     if not printed:
@@ -61,11 +61,9 @@ def do_all_man(registry: ComponentRegistry, no_pager: bool = True):
     cm = nullcontext() if no_pager else pager_stdout()
     with cm:
         for factory in registry.get_factories():
-            #comp_type = factory.get_comp_type_name()
-            component_tuples = factory.get_component_name_class_tuples() # all of them
-            for name, comp_class in component_tuples:
-                usage = factory.get_usage(name)
-                print_man(registry, name, usage)
+            component_dict = factory.get_components([ComponentOrigin.CORE, ComponentOrigin.EXTERNAL, ComponentOrigin.USER], is_integration=None)
+            for name, comp_class in component_dict.items():
+                print_man(registry, name, comp_class.usage())
                 print()
 
 def print_man(registry: ComponentRegistry, name: str, usage: Usage):
@@ -94,8 +92,8 @@ def do_examples(token:str, registry: ComponentRegistry):
     cm = nullcontext() if no_pager else pager_stdout()
     with cm:
         for factory in registry.get_factories():
-            comp_type = factory.get_comp_type_name()
-            for name, comp_class in factory.get_component_name_class_tuples():
+            component_dict = factory.get_components([ComponentOrigin.CORE, ComponentOrigin.EXTERNAL, ComponentOrigin.USER], is_integration=None)
+            for name, comp_class in component_dict.items():
                 usage = comp_class.usage()
 
                 comp_type = get_base_class(usage, as_string=True)

{python_jack_knife-0.6.11 → python_jack_knife-0.6.12}/src/pjk/registry.py

@@ -3,6 +3,7 @@
 
 import os
 import sys
+import time
 from pjk.sinks.factory import SinkFactory
 from pjk.pipes.factory import PipeFactory
 from pjk.sources.factory import SourceFactory
@@ -11,24 +12,33 @@ from pjk.sources.format_source import FormatSource
 import importlib.util
 import importlib
 import importlib.metadata
+import sysconfig, pathlib, importlib
 from pjk.components import Pipe, Source, Sink
-from pjk.common import ComponentFactory, highlight
-from typing import List
+from pjk.common import ComponentFactory, highlight, ComponentOrigin
+from typing import List, Type
 
-class DisplayHolder:
-    def __init__(self, factories: List[ComponentFactory]):
-        for factory in factories:
-            pass            
-                 
+class ExternalRegistrar:
+    def __init__(self, sources: SourceFactory, pipes: PipeFactory, sinks: SinkFactory) -> None:
+        self._sources: SourceFactory = sources
+        self._pipes: PipeFactory = pipes
+        self._sinks: SinkFactory = sinks
+
+    def source(self, name: str, cls: Type[Source]) -> None:
+        self._sources.register(name, cls, origin=ComponentOrigin.EXTERNAL)
+
+    def pipe(self, name: str, cls: Type[Pipe]) -> None:
+        self._pipes.register(name, cls, origin=ComponentOrigin.EXTERNAL)
+
+    def sink(self, name: str, cls: Type[Sink]) -> None:
+        self._sinks.register(name, cls, origin=ComponentOrigin.EXTERNAL)
 
 class ComponentRegistry:
     def __init__(self):
         self.source_factory = SourceFactory()
         self.pipe_factory = PipeFactory()
         self.sink_factory = SinkFactory()
-
         self.load_user_components()
-        load_package_extras()
+        self.load_package_extras()
 
     def create_source(self, token: str):
         return self.source_factory.create(token)
@@ -56,9 +66,31 @@ class ComponentRegistry:
         print()
         print_factory_core(self.sink_factory, header='sinks')
 
-        self.print_origin_components('integration', 'integrations')
-        self.print_origin_components('user', 'user components (~/.pjk/plugins)')
+        self.print_non_core([ComponentOrigin.CORE,ComponentOrigin.EXTERNAL], is_integration=True, header='integrations')
+        self.print_non_core([ComponentOrigin.EXTERNAL], is_integration=False, header='apps')
+        self.print_non_core([ComponentOrigin.USER], is_integration=None, header='user components (~/.pjk/plugins)')        
+
+    # is_integration = True|False|None  None=don't care
+    def print_non_core(self, origin_list: List[ComponentOrigin], is_integration: bool, header:str):
+        all = {}
+        for factory in [self.source_factory, self.pipe_factory, self.sink_factory]:
+            component_dict = factory.get_components(origin_list=origin_list, is_integration=is_integration)
+            all.update(component_dict)
+
+        if not all:
+            return
         
+        print()        
+        print(highlight(header))
+
+        for name, comp_class in all.items():
+            usage = comp_class.usage()
+            comp_class_type_str = get_component_type(comp_class)
+            lines = usage.desc.split('\n')
+            temp = highlight(comp_class_type_str)
+            line = f'  {name:<17} {temp:<15} {lines[0]}'
+            print(line)
+
     def load_user_components(self, path=os.path.expanduser("~/.pjk/plugins")):
         if not os.path.isdir(path):
             return
@@ -87,36 +119,55 @@ class ComponentRegistry:
                     name = usage.name
 
                     if is_sink(obj, module):
-                        self.sink_factory.register(name, obj, 'user')
+                        self.sink_factory.register(name, obj, ComponentOrigin.USER)
                     elif is_pipe(obj, module):
-                        self.pipe_factory.register(name, obj, 'user')
+                        self.pipe_factory.register(name, obj, ComponentOrigin.USER)
                     elif is_source(obj, module):
-                        self.source_factory.register(name, obj, 'user')
+                        self.source_factory.register(name, obj, ComponentOrigin.USER)
 
-    def print_origin_components(self, origin: str, header:str):
-        component_tuples = []
-        for factory in [self.source_factory, self.pipe_factory, self.sink_factory]:
-            component_tuples.extend(factory.get_component_name_class_tuples(origin))
+    def iter_entry_points(self, group: str):
+        """
+        Return entry points in the given group across Python 3.9–3.12.
+        """
+        eps = importlib.metadata.entry_points()
+        return eps.select(group=group) if hasattr(eps, "select") else eps.get(group, [])
 
-        if not component_tuples:
-            return
-        print()        
-        print(highlight(header))
+    def load_package_extras(self, group: str = "pjk.package_extras") -> None:
+        """
+        Load pip-installed extras and register their components into THIS registry.
 
-        for name, comp_class in component_tuples:
-            usage = comp_class.usage()
-            comp_class_type_str = get_component_type(comp_class)
-            lines = usage.desc.split('\n')
-            temp = highlight(comp_class_type_str)
-            line = f'  {name:<17} {temp:<15} {lines[0]}'
-            print(line)
+        Preferred contract:
+        entry point -> callable register(registrar)
+
+        Fallback (legacy):
+        entry point -> module path; we import the MODULE PART ONLY for side-effects.
+        """
+        registrar = ExternalRegistrar(self.source_factory, self.pipe_factory, self.sink_factory)
+
+        for ep in self.iter_entry_points(group):
+            try:
+                # Try the modern, explicit path first.
+                loader = getattr(ep, "load", None)
+                if callable(loader):
+                    target = ep.load()  # resolves "module:object" to the actual object
+                    if callable(target):
+                        target(registrar)  # plugin registers into your live factories
+                        #print(f"[pjk] loaded extra (callable): {ep.name} -> {ep.value}")
+                        continue
+                    # Not callable -> fall through to legacy import
+                # Legacy path: import ONLY the module portion before ':' for side-effects
+                mod = ep.value.split(":", 1)[0]
+                importlib.import_module(mod)
+                #print(f"[pjk] loaded extra (import): {ep.name} -> {ep.value}")
+            except Exception as e:
+                print(f"[pjk] failed to load extra {ep.name}: {e}")
 
 def print_core_formats(factories: List[ComponentFactory]):
     print(highlight('formats'))
     formats = set()
     for factory in factories:
-        tuples = factory.get_component_name_class_tuples('core')
-        for name, comp_class in tuples:
+        component_dict = factory.get_components([ComponentOrigin.CORE], is_integration=False)
+        for name, comp_class in component_dict.items():
             if issubclass(comp_class, FormatSink|FormatSource):
                 formats.add(name)
     
@@ -124,23 +175,21 @@ def print_core_formats(factories: List[ComponentFactory]):
     lst = ', '.join(list(formats))
     print(f'{space:<15}{lst}. (sources/sinks in local files, dirs and s3)')
 
-def print_factory_core(factory: ComponentFactory, header: str, include_formats: bool=False, include_integrations=False):
-        components:list = factory.get_component_name_class_tuples('core')
-        header = highlight(header)
-        print(header)
+def print_factory_core(factory: ComponentFactory, header: str):
+    component_dict = factory.get_components([ComponentOrigin.CORE], is_integration=False)
+    header = highlight(header)
+    print(header)
 
-        i = 0
-        # user and outside package components are also here, but printed from registry class
-        for name, comp_class in components:
-            if issubclass(comp_class, FormatSink|FormatSource) and not include_formats:
-                continue
+    # user and outside package components are also here, but printed from registry class
+    for name, comp_class in component_dict.items():
+        if issubclass(comp_class, FormatSink|FormatSource):
+            continue
 
-            usage = comp_class.usage()
-            lines = usage.desc.split('\n')
+        usage = comp_class.usage()
+        lines = usage.desc.split('\n')
 
-            line = f'  {name:<12} {lines[0]}'
-            print(line)
-            i += 1
+        line = f'  {name:<12} {lines[0]}'
+        print(line)
     
 def get_component_type(component_class) -> str:
     if issubclass(component_class, Sink):
@@ -178,22 +227,3 @@ def is_sink(obj, module):
         and obj.__module__ == module.__name__
     )
 
-
-def iter_entry_points(group: str):
-    eps = importlib.metadata.entry_points()
-    if hasattr(eps, "select"):
-        # Python 3.10+ (importlib.metadata.EntryPoints)
-        return eps.select(group=group)
-    # Python 3.9 and older
-    return eps.get(group, [])
-
-def load_package_extras():
-    """
-    Discover and import all installed pjk extras (via entry points).
-    """
-    for ep in iter_entry_points("pjk.package_extras"):
-        try:
-            importlib.import_module(ep.value)
-            print(f"[pjk] loaded package extra: {ep.name} -> {ep.value}")
-        except Exception as e:
-            print(f"[pjk] failed to load extra {ep.name}: {e}")

{python_jack_knife-0.6.11 → python_jack_knife-0.6.12}/src/pjk/sinks/graph.py

@@ -18,8 +18,8 @@ class GraphSink(Sink):
             component_class=cls
         )
         usage.def_arg(name='kind', usage='hist|scatter|bar|line|cumulative')
-        usage.def_param(name='x', usage='Name of x-axis field', default='x')
-        usage.def_param(name='y', usage='Name of y-axis field', default='y')
+        usage.def_param(name='x', usage='x-axis field', default='x')
+        usage.def_param(name='y', usage='comma separated list of y-axis fields', default='y')
         usage.def_param(name='pause', usage='Seconds to show graph', is_num=True, default='-1')
         return usage