python-jack-knife 0.5.0__tar.gz → 0.5.1__tar.gz

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.5.0
+Version: 0.5.1
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/base.py

@@ -152,16 +152,17 @@ class Usage:
         lines.append(self.desc)
 
         syntax_str = self.get_token_syntax() # might be ''
-        if len(syntax_str) > 0:
-            lines.append('')
-            lines.append(f'syntax:')
-            lines.append(f'  {self.get_token_syntax()}')
-
+        if not syntax_str:
+            return '\n'.join(lines)
+        
+        lines.append('')
+        lines.append(f'syntax:')
+        lines.append(f'  {self.get_token_syntax()}')
         lines.extend(f"{line}" for line in self.get_arg_param_desc())
         return '\n'.join(lines)
 
     def get_token_syntax(self):
-        if self.syntax != None:
+        if not self.syntax:
             return self.syntax # else piece it together
 
         token = f'{self.name}'
@@ -216,7 +217,10 @@ class Usage:
                 self.args[name] = self._get_val(val_str, is_num, valid_values)
             except (ValueError, TypeError) as e:
                 raise TokenError.from_list([f"wrong value for '{name}' arg.", '', self.get_usage_text()])
-            
+
+        self.bind_params(ptok)
+        
+    def bind_params(self, ptok: ParsedToken):
         for name, str_val in ptok.get_params().items():
             usage = self.param_usages.get(name, None)
             if not usage:
@@ -276,8 +280,6 @@ class KeyedSource(ABC):
         return None
 
 class Source(ABC):
-    is_format = False
-
     @classmethod
     def usage(cls):
         return NoBindUsage(
@@ -296,7 +298,6 @@ class Source(ABC):
             self._iter = iter(self)
         return next(self._iter)
 
-
     def deep_copy(self):
         return None  # Default: not copyable unless overridden
 
@@ -307,6 +308,7 @@ class Pipe(Source):
     
     def __init__(self, ptok: ParsedToken, usage: Usage = None):
         self.ptok = ptok
+        self.usage = usage
         self.left = None  # left source for convience
         self.right = None # right source for convience
         self.inputs: List[Source] = []
@@ -339,9 +341,22 @@ class Pipe(Source):
 
         return clone
 
-class Sink(ABC):
-    is_format = False
+class DeepCopyPipe(Pipe):
+    def deep_copy(self):
+        """
+        Generic deep_copy: clone left source, re-instantiate
+        this pipe class with the same ptok/usage, and attach.
+        """
+        source_clone = self.left.deep_copy()
+        if not source_clone:
+            return None
 
+        # re-instantiate using the actual subclass
+        pipe = type(self)(self.ptok, self.usage)
+        pipe.add_source(source_clone)
+        return pipe
+
+class Sink(ABC):
     @classmethod
     def usage(cls):
         return NoBindUsage(
@@ -356,6 +371,11 @@ class Sink(ABC):
 
     def drain(self):
         self.process()
+        self.close()
+
+    # optional
+    def close(self):
+        pass
 
     def print_info(self):
         pass

python_jack_knife-0.5.1/src/pjk/log.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import logging, os, tempfile
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("pjk")
+
+def _truthy(v: Optional[str]) -> bool:
+    return str(v).lower() in ("1", "true", "yes", "on")
+
+def init(force: bool = False, level: Optional[int] = None):
+    """
+    Initialize 'pjk' logging.
+
+    - Rotates at DJK_LOG_MAX_MB (default 2 MB), keeps DJK_LOG_BACKUPS (default 3).
+    - Files under ~/.pjk/logs by default; override with DJK_LOG_DIR / DJK_LOG_FILE.
+    - Set DJK_DEBUG=1|true|yes for DEBUG, else INFO (or pass explicit level).
+    - If the log directory is not writable, fall back to console logging
+      (stderr → CloudWatch in AWS).
+    - Set force=True to replace existing handlers.
+    """
+    if logger.handlers and not force:
+        return
+    logger.handlers.clear()
+
+    level = level or (logging.DEBUG if _truthy(os.getenv("DJK_DEBUG")) else logging.INFO)
+    fmt = "[%(levelname)s] [%(threadName)s] %(message)s"
+    formatter = logging.Formatter(fmt)
+
+    try:
+        # Preferred: rotating file handler under ~/.pjk/logs
+        log_dir = Path(os.getenv("DJK_LOG_DIR", Path.home() / ".pjk" / "logs"))
+        log_dir.mkdir(parents=True, exist_ok=True)
+
+        log_file = log_dir / os.getenv("DJK_LOG_FILE", "pjk.log")
+        max_bytes = int(float(os.getenv("DJK_LOG_MAX_MB", "2")) * 1024 * 1024)  # 2 MB
+        backups = int(os.getenv("DJK_LOG_BACKUPS", "3"))
+
+        fh = RotatingFileHandler(
+            log_file,
+            maxBytes=max_bytes,
+            backupCount=backups,
+            encoding="utf-8",
+            delay=False,
+        )
+        fh.setLevel(level)
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+    except Exception:
+        # Fallback: console handler
+        ch = logging.StreamHandler()
+        ch.setLevel(level)
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+        logger.warning("Falling back to console logging (log file not writable)")
+
+    logger.setLevel(level)
+    # Do not propagate to root
+    logger.propagate = False

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/main.py

@@ -24,8 +24,12 @@ def write_history(tokens):
     log_path = ".pjk-history.txt"
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
     command = " ".join(tokens)
-    with open(log_path, "a") as f:
-        f.write(f"{timestamp}\tpjk {command}\n")
+
+    try:
+        with open(log_path, "a") as f:
+            f.write(f"{timestamp}\tpjk {command}\n")
+    except (PermissionError, OSError):
+        pass
 
 def execute_threaded(sinks):
     # Choose a max thread limit (explicitly)

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/parser.py

@@ -94,7 +94,7 @@ class ExpressionParser:
 
                 else: # unrecognized token
                     # could be sink in WRONG position, let's see for better error message
-                    sink = self.registry.create_sink(token, None) 
+                    sink = self.registry.create_sink(token) 
                     if sink:
                         raise TokenError.from_list(['sink may only occur in final position.',
                                             'pjk <source> [<pipe> ...] <sink>'])

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/pipes/head.py

@@ -4,9 +4,9 @@
 # djk/pipes/head.py
 
 from typing import Optional
-from pjk.base import Pipe, ParsedToken, Usage
+from pjk.base import Pipe, ParsedToken, Usage, DeepCopyPipe
 
-class HeadPipe(Pipe):
+class HeadPipe(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = Usage(
@@ -19,7 +19,7 @@ class HeadPipe(Pipe):
         return usage
 
     def __init__(self, ptok: ParsedToken, usage: Usage):
-        super().__init__(ptok)
+        super().__init__(ptok, usage)
         self.limit = usage.get_arg('limit')
         self.count = 0
 

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/pipes/where.py

@@ -3,10 +3,10 @@
 
 # djk/pipes/where.py
 
-from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, UsageError
+from pjk.base import Pipe, ParsedToken, NoBindUsage, Usage, UsageError, DeepCopyPipe
 from pjk.common import SafeNamespace
 
-class WherePipe(Pipe):
+class WherePipe(DeepCopyPipe):
     @classmethod
     def usage(cls):
         usage = NoBindUsage(
@@ -39,11 +39,3 @@ class WherePipe(Pipe):
             except Exception:
                 continue  # ignore eval errors
 
-    def deep_copy(self):
-        source_clone = self.left.deep_copy()
-        if source_clone:
-            pipe = WherePipe(self.ptok, self.usage)
-            pipe.add_source(source_clone)
-            return pipe
-        else:
-            return None

{python_jack_knife-0.5.0 → python_jack_knife-0.5.1}/src/pjk/registry.py

@@ -30,7 +30,6 @@ class ComponentRegistry:
 
     def register(self, name, comp):
         if is_pipe(comp):
-            print('HELEELELELELELEEE')
             if hasattr(comp, "usage"):
                 usage = comp.usage()
                 name = usage.name
@@ -131,11 +130,19 @@ def load_user_components(path=os.path.expanduser("~/.pjk/plugins")):
 
     return sources, pipes, sinks
 
+def iter_entry_points(group: str):
+    eps = importlib.metadata.entry_points()
+    if hasattr(eps, "select"):
+        # Python 3.10+ (importlib.metadata.EntryPoints)
+        return eps.select(group=group)
+    # Python 3.9 and older
+    return eps.get(group, [])
+
 def load_package_extras():
     """
     Discover and import all installed pjk extras (via entry points).
     """
-    for ep in importlib.metadata.entry_points(group="pjk.package_extras"):
+    for ep in iter_entry_points("pjk.package_extras"):
         try:
             importlib.import_module(ep.value)
             print(f"[pjk] loaded package extra: {ep.name} -> {ep.value}")

python_jack_knife-0.5.1/src/pjk/sinks/csv_sink.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import csv
+from typing import IO, Dict, Any
+from .format_sink import FormatSink
+
+class CSVSink(FormatSink):
+    extension = "csv"
+
+    def __init__(self, outfile: IO[str], delimiter:str = ','):
+        super().__init__(outfile=outfile)
+        self.delimiter = delimiter
+
+    def process(self) -> None:
+        writer = None
+        for record in self.input:
+            if writer is None:
+                # Initialize DictWriter with dynamic fieldnames from first record
+                writer = csv.DictWriter(self.outfile, fieldnames=record.keys(), delimiter=self.delimiter)
+                writer.writeheader()
+            writer.writerow(record)

python_jack_knife-0.5.1/src/pjk/sinks/dir_sink.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os, gzip, shutil
+from pjk.base import Sink, ParsedToken, Usage
+from typing import Optional, Type
+from .format_sink import Sink
+from pjk.log import logger
+import gzip
+
+class DirSink(Sink):
+    def __init__(self, sink_class: Type[Sink], path_no_ext: str, is_gz: bool, fileno: int = 0):
+        super().__init__(None, None)
+        self.sink_class = sink_class
+        self.path_no_ext = path_no_ext
+        self.is_gz = is_gz
+        self.fileno = fileno
+        self.num_files = 1
+
+        if os.path.isdir(self.path_no_ext):
+            # remove everything inside
+            for entry in os.listdir(self.path_no_ext):
+                full = os.path.join(self.path_no_ext, entry)
+            if os.path.isfile(full) or os.path.islink(full):
+                os.unlink(full)
+            elif os.path.isdir(full):
+                shutil.rmtree(full)
+        else:
+            os.makedirs(self.path_no_ext, exist_ok=True)
+
+    def process(self):
+        # build the base filename
+        base = os.path.join(self.path_no_ext, f"file-{self.fileno:04d}")
+
+        # include extension here (format sink name + gz logic)
+        filename = f"{base}.{self.sink_class.extension}"
+        if self.is_gz:
+            filename += ".gz"
+
+        # open output file handle
+        outfile = gzip.open(filename, "wt", encoding="utf-8") if self.is_gz else open(filename, "wt", encoding="utf-8")
+
+        # create the format-specific sink with the open handle
+        file_sink = self.sink_class(outfile)
+        file_sink.add_source(self.input)
+
+        logger.debug(f"in process sinking to local file: {filename}")
+        file_sink.process()
+        outfile.close()
+
+    def deep_copy(self):
+        # Ask the upstream source to duplicate itself
+        source_clone = self.input.deep_copy()
+        if source_clone is None:
+            return None
+
+        # Create a new DirSink with the next file index
+        clone = DirSink(
+            sink_class=self.sink_class,
+            path_no_ext=self.path_no_ext,
+            is_gz=self.is_gz,
+            fileno=self.num_files,
+        )
+
+        # Wire up the cloned source to the new sink
+        clone.add_source(source_clone)
+
+        # Increment file counter for the next clone
+        self.num_files += 1
+        return clone
+

python_jack_knife-0.5.1/src/pjk/sinks/factory.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from typing import Callable
+import os
+import gzip
+from pjk.base import Source, Sink, ParsedToken
+from pjk.common import ComponentFactory
+from pjk.sinks.stdout import StdoutSink
+from pjk.sinks.json_sink import JsonSink
+from pjk.sinks.devnull import DevNullSink
+from pjk.sinks.graph import GraphSink
+from pjk.sinks.csv_sink import CSVSink
+from pjk.sinks.tsv_sink import TSVSink
+from pjk.sinks.ddb import DDBSink
+from pjk.sinks.dir_sink import DirSink
+from pjk.sinks.s3_sink import S3Sink
+from pjk.sinks.expect import ExpectSink
+from pjk.sinks.format_sink import FormatSink
+from pjk.sinks.user_sink_factory import UserSinkFactory
+
+COMPONENTS = {
+        '-': StdoutSink,
+        'devnull': DevNullSink,
+        'graph': GraphSink,
+        'ddb': DDBSink,
+        'json': JsonSink,
+        'csv': CSVSink,
+        'tsv': TSVSink,
+        }
+
+class SinkFactory(ComponentFactory):
+    def __init__(self):
+        super().__init__(COMPONENTS, 'sink')   
+
+    def create(self, token: str) -> Callable[[Source], Sink]:
+        token = token.strip()
+        ptok = ParsedToken(token)
+
+        # non-usage sink (bind incompatible)
+        if ptok.pre_colon == 'expect':
+            return ExpectSink(ptok, None)
+
+        if ptok.pre_colon.endswith('.py'):
+            sink = UserSinkFactory.create(ptok)
+            if sink:
+                return sink
+        
+        sink_cls = self.components.get(ptok.pre_colon)
+        if sink_cls and not issubclass(sink_cls, FormatSink):
+            usage = sink_cls.usage()
+            usage.bind(ptok)
+            return sink_cls(ptok, usage)
+    
+        return FormatSink.create(ptok, COMPONENTS)

python_jack_knife-0.5.1/src/pjk/sinks/format_sink.py

@@ -0,0 +1,126 @@
+from pjk.base import Sink, ParsedToken, NoBindUsage
+from pjk.sinks.s3_sink import S3Sink
+from pjk.sinks.dir_sink import DirSink
+from typing import IO
+import re
+import gzip
+
+class SinkFormatUsage(NoBindUsage):
+    def __init__(self, name: str, component_class: type, desc_override: str = None):
+        desc = f'{name} source for s3 and local files/directories.\ns3 defaults to \'json.gz\', others require format param' if desc_override == None else desc_override
+        super().__init__(name, desc, component_class)
+
+        self.def_syntax("") # don't use generated syntax for these, rely on examples
+        self.def_param('format', 'file format', is_num=False, valid_values={'json', 'csv', 'tsv', 'json.gz', 'tsv.gz', 'csv.gz'}, default='json.gz')
+        self.def_example(expr_tokens=["{hello: 'world'}", f"myfile.{name}"], expect=None)
+        self.def_example(expr_tokens=["{hello: 'world}", f"{name}:mydir"], expect=None)
+        self.def_example(expr_tokens=["{hello: 'world'}", f"s3://mybucket/myfile.{name}"], expect=None)
+        self.def_example(expr_tokens=["{hello: 'world'}", f"s3://mybucket/myfiles@format={name}"], expect=None)
+
+class FormatSink(Sink):
+    extension: str = None
+    desc_override = None
+
+    @classmethod
+    def usage(cls):
+        return SinkFormatUsage(name=cls.extension,
+                               component_class=cls,
+                               desc_override=cls.desc_override)
+
+    def __init__(self, outfile: IO[str]):
+        super().__init__(None, None)
+        self.outfile = outfile
+
+    def close(self):
+        self.outfile.close()
+
+    @classmethod
+    def get_format_gz(cls, input:str):
+        is_gz = False
+        format = input
+        if input.endswith('.gz'):
+            is_gz = True
+            format = input[:-3]
+        return format, is_gz
+
+    @classmethod
+    def create(cls, ptok: ParsedToken, sinks):
+        """
+        use cases covered:
+        1) foo.<format>                 # local single file
+        2) <format>:foo                 # local directory
+        3) s3://bucket/prefix.<format>  # s3 single file
+        4) s3://bucket/prefix           # s3 directory (@format=<format parameter with default = json)
+
+        format = json, csv, tsv, and also json.gz etc.
+        """
+
+        pattern = re.compile(
+            r'^(?:(?P<pre_colon>[^:]+):)?'            # optional precolon
+            r'(?P<path>[^:]+?)'                      # main path
+            r'(?:\.(?P<ext>\w+(?:\.gz)?))?$'         # optional extension, e.g. json, csv, json.gz
+        )
+
+        # we don't use framework token parsing (except for params) cuz too complicated
+        input = ptok.all_but_params
+        
+        # Example usage
+        match = pattern.match(input)
+        if not match:
+            return None
+        
+        gd = match.groupdict()
+        pre_colon = gd.get('pre_colon', None)
+        path_no_ext = gd.get('path', None)
+        ext = gd.get('ext', None)
+
+        usage = cls.usage()
+        usage.bind_params(ptok) # only bind params
+
+        is_gz = False
+        format = None
+
+        if pre_colon and pre_colon != 's3': # local dir case
+            format, is_gz = cls.get_format_gz(pre_colon)
+            sink_class = sinks.get(format)
+            if not sink_class or not issubclass(sink_class, FormatSink):
+                return None
+            if ext:
+                raise Exception('fix this exception message, extensions not allowed for local directory sinks')
+            return DirSink(sink_class, path_no_ext, is_gz, fileno=0)
+
+        if ext and not pre_colon: # single local file case
+            format, is_gz = cls.get_format_gz(ext)
+            sink_class = sinks.get(format)
+            if not sink_class:
+                raise Exception('fix this exception message, extension for single file must be recognized format')
+
+            filename = f'{path_no_ext}.{format}'
+
+            # open the output file stream
+            if is_gz:
+                outfile = gzip.open(f'{filename}.gz', "wt", encoding="utf-8", newline="")
+            else:
+                outfile = open(filename, "wt", encoding="utf-8", newline="")
+
+            # instantiate the sink with the prepared stream
+            sink = sink_class(outfile)
+            return sink
+
+        if pre_colon == 's3':
+            if ext: # single file
+                format, is_gz = cls.get_format_gz(ext)
+                sink_class = sinks.get(format)
+                if not sink_class:
+                    raise Exception('fix this exception message, extension for single file must be recognized format')    
+            else:
+                format, is_gz = cls.get_format_gz(usage.get_param('format'))
+                sink_class = sinks.get(format)
+
+            fileno = -1 if ext else 0 # -1 tells s3 single file, no threading
+            return S3Sink(sink_class, path_no_ext, is_gz, fileno)
+        
+        return None
+        
+        
+        

python_jack_knife-0.5.1/src/pjk/sinks/json_sink.py

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import json
+from typing import IO
+from .format_sink import FormatSink
+
+class JsonSink(FormatSink):
+    extension = 'json'
+
+    def process(self) -> None:
+        for record in self.input:
+            self.outfile.write(json.dumps(record) + "\n")
+        # Caller (DirSink/S3Sink) owns closing the outfile

python_jack_knife-0.5.1/src/pjk/sinks/s3_sink.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2025 Mike Schultz
+
+import io
+import gzip
+from typing import Optional, Type
+from pjk.base import Source, Sink
+from pjk.log import logger
+from pjk.sinks.s3_stream import S3MultipartWriter
+
+
+class S3Sink(Sink):
+    """
+    Write records to S3 in the given <format>.
+
+    - Folder mode (path without extension):
+        s3:bucket/prefix/ → file-0000.ext, file-0001.ext, ...
+    - Single-file mode (path ends with .ext or .ext.gz):
+        s3:bucket/prefix/output.csv[.gz]
+
+    Args (via Usage):
+      - path: 'bucket/path/to/files' (bucket required, prefix optional)
+    """
+
+    _FILENAME_BASE: str = "file"
+    _FILENAME_DIGITS: int = 4
+    
+    def __init__(self, sink_class: Type[Sink], path_no_ext: str, is_gz: bool, fileno: int):
+        self.path_no_ext = path_no_ext if not path_no_ext.startswith('//') else path_no_ext[2:] # strip leading //
+        self.sink_class = sink_class
+        self.is_gz = is_gz
+        self.fileno = fileno
+        self.is_single_file = fileno == -1
+        if self.path_no_ext.endswith('/') and not self.is_single_file:
+            self.path_no_ext = self.path_no_ext[:-1]
+
+        self.num_files = 1
+
+    def _build_object_key(self, index: int) -> str:
+        if self.is_single_file:
+            file_name = f'{self.path_no_ext}.{self.sink_class.extension}'
+        else:
+            file_name = f"{self.path_no_ext}/{self._FILENAME_BASE}-{index:0{self._FILENAME_DIGITS}d}.{self.sink_class.extension}"
+
+        if self.is_gz:
+            file_name += ".gz"
+
+        return file_name
+
+    def process(self):
+        object_key = self._build_object_key(self.fileno)
+        bucket, key = object_key.split("/", 1)
+
+        with S3MultipartWriter(bucket, key) as writer:
+            if self.is_gz:
+                # gzip needs a binary sink → use writer directly
+                with gzip.GzipFile(fileobj=writer, mode="wb") as gz:
+                    with io.TextIOWrapper(gz, encoding="utf-8", newline="") as outfile:
+                        file_sink = self.sink_class(outfile)
+                        file_sink.add_source(self.input)
+                        logger.debug(f"S3Sink streaming GZ to s3://{bucket}/{key}")
+                        file_sink.process()
+            else:
+                # plain text path
+                with io.TextIOWrapper(writer, encoding="utf-8", newline="") as outfile:
+                    file_sink = self.sink_class(outfile)
+                    file_sink.add_source(self.input)
+                    logger.debug(f"S3Sink streaming to s3://{bucket}/{key}")
+                    file_sink.process()
+
+
+    def deep_copy(self):
+        if self.is_single_file:
+            # Single-file mode: no fanout allowed
+            return None
+
+        source_clone: Optional[Source] = self.input.deep_copy()
+        if not source_clone:
+            return None
+
+        clone = S3Sink(
+            sink_class=self.sink_class,
+            path_no_ext=self.path_no_ext,
+            is_gz=self.is_gz,
+            fileno=self.num_files,
+        )
+        clone.add_source(source_clone)
+
+        self.num_files += 1
+        return clone