python-jack-knife 0.7.5__tar.gz → 0.7.7__tar.gz

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.7.5
+Version: 0.7.7
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 
@@ -226,6 +226,9 @@ Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: opensearch-py; extra == "dev"
 Provides-Extra: all
 Requires-Dist: boto3>=1.34; extra == "all"
 Requires-Dist: pg8000>=1.30.0; extra == "all"

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/pyproject.toml

@@ -25,7 +25,7 @@ aws = ["boto3>=1.34"]
 postgres = ["pg8000>=1.30.0"]
 parquet = ["pyarrow>=15.0.0"]
 plot = ["matplotlib>=3.9.0", "pandas>=2.2.0"]
-dev = ["pytest", "black", "ruff"]
+dev = ["pytest", "black", "ruff", "build", "twine", "opensearch-py"]
 all = [
   "boto3>=1.34",
   "pg8000>=1.30.0",

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/__init__.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 from .version import __version__
-from .engine import PjkEngine
+from .engine import PjkStream
 
-__all__ = ["__version__", "PjkEngine"]
+__all__ = ["__version__", "PjkStream"]

python_jack_knife-0.7.7/src/pjk/engine.py

@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import shlex
+from pathlib import Path
+from typing import Iterator, List, Optional, Sequence, Union
+
+from pjk.log import init_stream
+from pjk.parser import ExpressionParser, expand_macros
+from pjk.registry import ComponentRegistry
+from pjk.runtime import enter_pjk_stream, exit_pjk_stream
+from pjk.sources.dict_list_source import DictListSource
+
+
+class PjkStream:
+    """
+    Lazy record stream from a pjk pipeline.
+
+    Build with a named factory, then iterate:
+
+        for rec in PjkStream.expression('{foo: 1} select:foo -'):
+            ...
+
+    The trailing sink token is required for parsing; records are taken from the
+    pipe chain (sink.drain is not called).
+
+    With inrecs supplied, the pipeline source is replaced by those records.
+    The expression may include a source (which is overridden) or be pipes-only.
+
+    PjkStream never writes CLI history and uses stderr logging (CloudWatch on AWS).
+    """
+
+    def __init__(self, tokens: Sequence[str], inrecs: Optional[List[dict]] = None):
+        self._tokens = list(tokens)
+        self._inrecs = inrecs
+
+    @classmethod
+    def expression(cls, expr: str, *, inrecs: Optional[List[dict]] = None) -> "PjkStream":
+        """From a CLI-style expression string (shlex-split)."""
+        try:
+            tokens = shlex.split(expr, comments=True, posix=True)
+        except ValueError as e:
+            raise ValueError(f"Invalid pjk expression: {e}") from e
+        return cls(tokens, inrecs)
+
+    @classmethod
+    def tokens(cls, tokens: Sequence[str], *, inrecs: Optional[List[dict]] = None) -> "PjkStream":
+        """From an explicit token list."""
+        return cls(list(tokens), inrecs)
+
+    @classmethod
+    def file(cls, path: Union[Path, str], *, inrecs: Optional[List[dict]] = None) -> "PjkStream":
+        """From a .pjk file path (also supports m: macros as a single token via tokens())."""
+        path = Path(path)
+        if path.suffix != ".pjk":
+            raise ValueError(f"PjkStream.file() expects a .pjk path, got: {path}")
+        return cls([str(path)], inrecs)
+
+    def __iter__(self) -> Iterator[dict]:
+        stream_token = enter_pjk_stream()
+        init_stream()
+        try:
+            yield from self._iter_records()
+        finally:
+            exit_pjk_stream(stream_token)
+
+    def _iter_records(self) -> Iterator[dict]:
+        registry = ComponentRegistry()
+        parser = ExpressionParser(registry)
+        expanded = expand_macros(self._tokens)
+
+        if self._inrecs is not None:
+            source_override = DictListSource(self._inrecs)
+            try:
+                first_is_source = registry.create_source(expanded[0]) is not None
+            except Exception:
+                first_is_source = False
+            if first_is_source:
+                expanded = ["{to_override: 'true'}"] + expanded[1:]
+            else:
+                expanded = ["{to_override: 'true'}"] + expanded
+        else:
+            source_override = None
+
+        sink = parser.parse(expanded, source_override=source_override)
+
+        inputs = [sink.input]
+        sink.input._get_sources(inputs)
+        try:
+            for record in sink.input:
+                yield record
+        finally:
+            for inp in inputs:
+                inp.close()

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/history.py

@@ -3,6 +3,7 @@ import os
 import shlex
 from typing import List, Set
 from pjk.common import pager_stdout, highlight
+from pjk.runtime import pjk_stream_active
 
 LOG_FILE = '.pjk-history.txt'
 
@@ -50,6 +51,9 @@ def read_history(log_path: str) -> List[int]:
     return clist, cset
 
 def write_history(tokens: list):
+    if pjk_stream_active():
+        return
+
     if os.environ.get("PJK_NO_HISTORY") == "1":
         return
     

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/integrations/opensearch_client.py

@@ -2,20 +2,20 @@ from pjk.usage import Usage
 
  # name, type, default
 OS_CONFIG_TUPLES = [
-    ("default_index", str, None),
+    ("default_index", str, None, True),
     ("os_auth_use_aws", bool, "true"),
     ("os_scheme", str, "https"),
     ("os_verify_certs", bool, "true"),
-    ("os_ca_certs", str, None),
-    ("os_region", str, None),
+    ("os_ca_certs", str, None, True),
+    ("os_region", str, None, True),
     ("os_service", str, "es"),
-    ("os_username", str, None),
-    ("os_password", str, None),
+    ("os_username", str, None, True),
+    ("os_password", str, None, True),
     ("os_timeout", float, 30),
     ("os_ssl_assert_hostname", bool, "true"),
     ("os_ssl_show_warn", bool, "false"),
     ("os_host", str, None),
-    ("os_port", int, None)
+    ("os_port", int, None, True),
 ]
 
 class OpenSearchClient:

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/integrations/snowflake_pipe.py

@@ -124,7 +124,7 @@ class SnowflakeClient:
 class SnowflakePipe(QueryPipe, Integration):
     """
     Snowflake query pipe; executes SQL found in input record['query'] and streams rows.
-    Connection/session settings are pulled from ~/.pjk/component_configs.yaml under the arg name.
+    Connection/session settings are pulled from configs.yaml under PJK_HOME (or PJK_CONFIG_FILE).
     """
     name = 'snowflake'
     desc = "Snowflake query pipe; executes SQL over input record['query']."

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/log.py

@@ -1,22 +1,37 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-import logging, os, tempfile
+import logging, os, sys
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
 from typing import Optional
 
+from pjk.paths import logs_dir_path
+
 logger = logging.getLogger("pjk")
 
 def _truthy(v: Optional[str]) -> bool:
     return str(v).lower() in ("1", "true", "yes", "on")
 
+def _level_from_env(explicit: Optional[int]) -> int:
+    if explicit is not None:
+        return explicit
+    return logging.DEBUG if _truthy(os.getenv("DJK_DEBUG")) else logging.INFO
+
+def _configure_logger(logger_obj, handlers, level: int) -> None:
+    logger_obj.handlers.clear()
+    for handler in handlers:
+        handler.setLevel(level)
+        logger_obj.addHandler(handler)
+    logger_obj.setLevel(level)
+    logger_obj.propagate = False
+
 def init(force: bool = False, level: Optional[int] = None):
     """
     Initialize 'pjk' logging.
 
     - Rotates at DJK_LOG_MAX_MB (default 2 MB), keeps DJK_LOG_BACKUPS (default 3).
-    - Files under ~/.pjk/logs by default; override with DJK_LOG_DIR / DJK_LOG_FILE.
+    - Files under PJK_HOME/logs by default; override with DJK_LOG_DIR / DJK_LOG_FILE.
     - Set DJK_DEBUG=1|true|yes for DEBUG, else INFO (or pass explicit level).
     - If the log directory is not writable, fall back to console logging
       (stderr → CloudWatch in AWS).
@@ -26,13 +41,13 @@ def init(force: bool = False, level: Optional[int] = None):
         return
     logger.handlers.clear()
 
-    level = level or (logging.DEBUG if _truthy(os.getenv("DJK_DEBUG")) else logging.INFO)
+    level = _level_from_env(level)
     fmt = "[%(levelname)s] [%(threadName)s] %(message)s"
     formatter = logging.Formatter(fmt)
 
     try:
-        # Preferred: rotating file handler under ~/.pjk/logs
-        log_dir = Path(os.getenv("DJK_LOG_DIR", Path.home() / ".pjk" / "logs"))
+        # Preferred: rotating file handler under PJK_HOME/logs
+        log_dir = Path(os.getenv("DJK_LOG_DIR", logs_dir_path()))
         log_dir.mkdir(parents=True, exist_ok=True)
 
         log_file = log_dir / os.getenv("DJK_LOG_FILE", "pjk.log")
@@ -46,17 +61,27 @@ def init(force: bool = False, level: Optional[int] = None):
             encoding="utf-8",
             delay=False,
         )
-        fh.setLevel(level)
         fh.setFormatter(formatter)
-        logger.addHandler(fh)
+        _configure_logger(logger, [fh], level)
     except Exception:
         # Fallback: console handler
         ch = logging.StreamHandler()
-        ch.setLevel(level)
         ch.setFormatter(formatter)
-        logger.addHandler(ch)
+        _configure_logger(logger, [ch], level)
         logger.warning("Falling back to console logging (log file not writable)")
 
-    logger.setLevel(level)
-    # Do not propagate to root
-    logger.propagate = False
+def init_stream(force: bool = False, level: Optional[int] = None):
+    """
+    Console-only logging for PjkStream and other embedded/library use.
+
+    Writes to stderr so AWS Lambda, ECS, and Fargate ship logs to CloudWatch
+    without relying on PJK_HOME/logs or a writable home directory.
+    """
+    if logger.handlers and not force:
+        return
+
+    level = _level_from_env(level)
+    fmt = "[%(levelname)s] [%(threadName)s] %(message)s"
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter(fmt))
+    _configure_logger(logger, [handler], level)

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/man_page.py

@@ -1,9 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-from pjk.parser import ExpressionParser, MACRO_PREFIX, MACROS_FILE, read_macros
+from pjk.parser import ExpressionParser, MACRO_PREFIX, read_macros
 from pjk.components import Source, Pipe, Sink
-from pjk.usage import Usage, CONFIG_FILE
+from pjk.paths import config_file_path, macros_file_path
+from pjk.usage import Usage
 from pjk.registry import ComponentRegistry
 from pjk.common import pager_stdout, highlight, ComponentOrigin
 from contextlib import nullcontext
@@ -88,7 +89,7 @@ def print_man(registry: ComponentRegistry, name: str, usage: Usage):
         print_example(registry, expr_tokens, expect, name)
 
 def display_configs():
-    path = Path(CONFIG_FILE).expanduser()
+    path = config_file_path()
 
     with pager_stdout():
         with path.open("r", encoding="utf-8") as f:
@@ -96,7 +97,7 @@ def display_configs():
         if not isinstance(data, dict):
             raise ValueError("Top-level YAML must be a mapping of records")
         
-        print(f'Component configs defined in {CONFIG_FILE}')
+        print(f'Component configs defined in {path}')
         print()
         for name, body_dict in data.items():
             print('=========================================')
@@ -123,7 +124,7 @@ def display_macros():
     macros = read_macros()
 
     with pager_stdout():
-        print(f"Macros defined in '{MACROS_FILE}'")
+        print(f"Macros defined in '{macros_file_path()}'")
 
         print(f"Usage: pjk [...] {MACRO_PREFIX}:<macro_name> [...]")
         print()

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/parser.py

@@ -16,11 +16,13 @@ from pjk.progress import ProgressIgnore
 from pjk.parse_pjk_file import handle_pjk_file
 from pjk.common import SafeNamespace
 
-MACROS_FILE = '~/.pjk/macros.txt'
+from pjk.paths import macros_file_path
+
 MACRO_PREFIX = 'm'
-def read_macros(file_name: str = MACROS_FILE) -> Dict[str, str]:
+
+def read_macros(file_name: str | None = None) -> Dict[str, str]:
     out: Dict[str, str] = {}
-    path = Path(file_name).expanduser()
+    path = Path(file_name) if file_name else macros_file_path()
     with path.open(encoding="utf-8") as f:
         for raw in f:
             line = raw.split("#", 1)[0].strip()

python_jack_knife-0.7.7/src/pjk/paths.py

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+from pathlib import Path
+
+PJK_HOME_ENV = "PJK_HOME"
+PJK_CONFIG_FILE_ENV = "PJK_CONFIG_FILE"
+PJK_MACROS_FILE_ENV = "PJK_MACROS_FILE"
+PJK_PLUGINS_DIR_ENV = "PJK_PLUGINS_DIR"
+
+
+def pjk_home() -> Path:
+    """
+    Root directory for pjk runtime files.
+
+    PJK_HOME if set, otherwise ~/.pjk (dev default).
+    """
+    override = os.environ.get(PJK_HOME_ENV)
+    if override:
+        return Path(override).expanduser().resolve()
+    return Path.home() / ".pjk"
+
+
+def config_file_path() -> Path:
+    """Component instance configs (OpenSearchQueryPipe-products, etc.)."""
+    override = os.environ.get(PJK_CONFIG_FILE_ENV)
+    if override:
+        return Path(override).expanduser().resolve()
+    return pjk_home() / "configs.yaml"
+
+
+def macros_file_path() -> Path:
+    override = os.environ.get(PJK_MACROS_FILE_ENV)
+    if override:
+        return Path(override).expanduser().resolve()
+    return pjk_home() / "macros.txt"
+
+
+def plugins_dir_path() -> Path:
+    override = os.environ.get(PJK_PLUGINS_DIR_ENV)
+    if override:
+        return Path(override).expanduser().resolve()
+    return pjk_home() / "plugins"
+
+
+def logs_dir_path() -> Path:
+    return pjk_home() / "logs"
+
+
+def config_file_display() -> str:
+    """Human-readable config path for errors and man pages."""
+    return str(config_file_path())

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/registry.py

@@ -11,6 +11,7 @@ from pjk.sources.format_source import FormatSource
 import importlib.util
 import importlib
 from pjk.components import Pipe, Source, Sink
+from pjk.paths import plugins_dir_path
 from pjk.common import ComponentFactory, highlight, ComponentOrigin, pager_stdout
 from typing import List, Type
 
@@ -73,7 +74,7 @@ class ComponentRegistry:
 
             self.print_non_core([ComponentOrigin.CORE,ComponentOrigin.EXTERNAL], is_integration=True, header='integrations')
             self.print_non_core([ComponentOrigin.EXTERNAL], is_integration=False, header='apps')
-            self.print_non_core([ComponentOrigin.USER], is_integration=None, header='user components (~/.pjk/plugins)')        
+            self.print_non_core([ComponentOrigin.USER], is_integration=None, header=f'user components ({plugins_dir_path()})')        
 
     # is_integration = True|False|None  None=don't care
     def print_non_core(self, origin_list: List[ComponentOrigin], is_integration: bool, header:str):
@@ -96,7 +97,9 @@ class ComponentRegistry:
             line = f'  {name:<17} {temp:<15} {lines[0]}'
             print(line)
 
-    def load_user_components(self, path=os.path.expanduser("~/.pjk/plugins")):
+    def load_user_components(self, path=None):
+        if path is None:
+            path = str(plugins_dir_path())
         if not os.path.isdir(path):
             return
 
@@ -113,7 +116,7 @@ class ComponentRegistry:
                 sys.modules[spec.name] = module
                 spec.loader.exec_module(module)
             except Exception as e:
-                print(f"[pjk] Failed to load {fname} from ~/.pjk/plugins: {e}")
+                print(f"[pjk] Failed to load {fname} from {path}: {e}")
                 continue
 
             for obj in vars(module).values():

python_jack_knife-0.7.7/src/pjk/runtime.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import contextvars
+
+_pjk_stream = contextvars.ContextVar("pjk_stream", default=False)
+
+
+def pjk_stream_active() -> bool:
+    return _pjk_stream.get()
+
+
+def enter_pjk_stream():
+    return _pjk_stream.set(True)
+
+
+def exit_pjk_stream(token) -> None:
+    _pjk_stream.reset(token)

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/sinks/create_sink.py

@@ -2,6 +2,7 @@
 # Copyright 2024 Mike Schultz
 
 from pjk.components import Sink
+from pjk.paths import plugins_dir_path
 from pjk.usage import ParsedToken, Usage
 from importlib.resources import files
 from pathlib import Path
@@ -45,7 +46,7 @@ class CreateSink(Sink):
             desc=(
                 "Write a bare bones source, pipe or sink python file.\n"
                 "Requires input record of the form: \"{<type>: '<name>'}\". E.g. \"{pipe: 'mypipe'}\"\n"
-                "User components can be used inline or deposited in ~/.pjk/plugins"
+                f"User components can be used inline or deposited in {plugins_dir_path()}"
             ),
             component_class=cls
         )

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/sources/dict_list_source.py

@@ -5,7 +5,7 @@ from pjk.components import Source
 
 
 class DictListSource(Source):
-    """Source that yields records from a list of dicts. Used by PjkEngine."""
+    """Source that yields records from a list of dicts. Used by PjkStream."""
 
     def __init__(self, records: list):
         super().__init__(root=None)

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/sources/favorite_source.py

@@ -1,28 +1,15 @@
+from pjk.parser import read_macros
+from pjk.paths import macros_file_path
 from pjk.usage import Usage, ParsedToken
 from pjk.components import Source
-from pathlib import Path
 from typing import Dict
 
-MACRO_FILE = '~/.pjk/macros.txt'
-
-def read_macros(file_name: str = MACRO_FILE) -> Dict[str, str]:
-    out: Dict[str, str] = {}
-    path = Path(file_name).expanduser()
-    with path.open(encoding="utf-8") as f:
-        for raw in f:
-            line = raw.split("#", 1)[0].strip()
-            if not line or ":" not in line:
-                continue
-            key, val = line.split(":", 1)
-            out[key.strip()] = val.strip()
-    return out
-
 class MacroSource(Source):
     @classmethod
     def usage(cls):
         u = Usage(
             name='macros',
-            desc=f"Source to list the macro expressions stored in {MACRO_FILE}.\n"
+            desc=f"Source to list the macro expressions stored in {macros_file_path()}.\n"
                   "A specific macro is referenced using 'm:<instance>, e.g. pjk m:hw -",
             component_class=cls
     	)

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/usage.py

@@ -2,26 +2,74 @@ from typing import Optional, Set, List
 import os
 import yaml
 
-CONFIG_FILE = '~/.pjk/configs.yaml'
+from pjk.paths import config_file_display, config_file_path
+
+EXTENDS_KEY = '_extends'
+
+# Display path for errors; resolved via PJK_HOME or PJK_CONFIG_FILE.
+CONFIG_FILE = config_file_display()
 
 class Config:
     def __init__(self):
-        self.configs_yaml = os.path.expanduser(CONFIG_FILE)
+        self._loaded_path: Optional[str] = None
         self._data = {}
-        self._load()
-        
-    def _load(self):
-        if os.path.exists(self.configs_yaml):
-            with open(self.configs_yaml, 'r') as f:
+
+    def _ensure_loaded(self):
+        path = str(config_file_path())
+        if path != self._loaded_path:
+            self._loaded_path = path
+            self._load(path)
+
+    def _load(self, path: str):
+        if os.path.exists(path):
+            with open(path, 'r') as f:
                 self._data = yaml.safe_load(f) or {}
         else:
             self._data = {}
 
+    def _extends_hint(self, instance_key: str) -> str:
+        entry = self._data.get(instance_key) or {}
+        base_key = entry.get(EXTENDS_KEY)
+        if not base_key:
+            return ""
+        return f" ('{instance_key}' {EXTENDS_KEY}: '{base_key}')"
+
+    def _resolve_entry(self, instance_key: str, visiting: Optional[Set[str]] = None) -> dict:
+        entry = self._data.get(instance_key)
+        if entry is None:
+            return None
+
+        base_key = entry.get(EXTENDS_KEY)
+        if not base_key:
+            return entry
+
+        if visiting is None:
+            visiting = set()
+        if instance_key in visiting:
+            raise TokenError(
+                f"Cycle in {config_file_display()} config inheritance involving '{instance_key}'."
+            )
+        visiting.add(instance_key)
+
+        base_entry = self._resolve_entry(base_key, visiting)
+        if base_entry is None:
+            raise TokenError(
+                f"'{instance_key}:{EXTENDS_KEY}' in {config_file_display()} points to a non-existent entry: '{base_key}'. "
+                f"Add '{base_key}' or define params directly on '{instance_key}'."
+            )
+
+        merged = dict(base_entry)
+        for key, value in entry.items():
+            if key != EXTENDS_KEY:
+                merged[key] = value
+        return merged
+
     def lookup(self, usage: "Usage", param: str):
+        self._ensure_loaded()
          # this should be advertised as a well-known requirement: usage must define a 'instance' arg
         instance = usage.get_arg("instance")
         if not instance:
-            raise TokenError(f"'instance' arg must be defined when using configs in {CONFIG_FILE}")
+            raise TokenError(f"'instance' arg must be defined when using configs in {config_file_display()}")
 
         component_class = usage.get_component_class()
         class_name = component_class.__name__
@@ -32,24 +80,27 @@ class Config:
         if not type_default:
             raise TokenError(f"{class_name} does not define '{param}' in config_tuples")
 
-        (param_type, param_default) = type_default
+        (param_type, param_default, optional) = type_default
 
         instance_key = f'{class_name}-{instance}'
         entry = self._data.get(instance_key, None)
         if not entry:
             raise TokenError(
-                f"{CONFIG_FILE} does not contain entry for '{instance_key}' with required params."
+                f"{config_file_display()} does not contain entry for '{instance_key}' with required params."
             )
         
-        _alias = entry.get('_alias', None) # _alias must = another entry instance_key
-        if _alias:
-            entry = self._data.get(_alias, None)
-            if not entry:
+        resolved = self._resolve_entry(instance_key)
+
+        if param not in resolved and not optional and param_default is None:
+            hint = self._extends_hint(instance_key)
+            if hint:
+                base_key = entry.get(EXTENDS_KEY)
                 raise TokenError(
-                    f"'{instance_key}:_alias' in {CONFIG_FILE} points to a non-existent entry: '{_alias}'."
-                )    
-        
-        raw = entry.get(param, param_default)
+                    f"'{param}' missing from '{instance_key}' in {config_file_display()}{hint}. "
+                    f"Set it on '{base_key}' or override it on '{instance_key}'."
+                )
+
+        raw = resolved.get(param, param_default)
 
         if not raw:
             return None
@@ -171,7 +222,15 @@ class Usage:
         self.config_tuples = tuples
 
     def get_config_tuples(self) -> dict:
-        return {n: (t, d) for n, t, d in self.config_tuples}
+        result = {}
+        for t in self.config_tuples:
+            if len(t) == 3:
+                name, param_type, default = t
+                optional = False
+            else:
+                name, param_type, default, optional = t
+            result[name] = (param_type, default, optional)
+        return result
 
     # args and param values default as str
     def def_arg(self, name: str, usage: str, is_num: bool = False, valid_values: Optional[Set[str]] = None):

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/pjk/version.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2024 Mike Schultz
 
-__version__ = "0.7.5"
+__version__ = "0.7.7"

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/python_jack_knife.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-jack-knife
-Version: 0.7.5
+Version: 0.7.7
 Summary: Python Jack Knife – a command line data processor
 Author-email: Mike Schultz <mike.schultz@gmail.com>
 License: 
@@ -226,6 +226,9 @@ Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: opensearch-py; extra == "dev"
 Provides-Extra: all
 Requires-Dist: boto3>=1.34; extra == "all"
 Requires-Dist: pg8000>=1.30.0; extra == "all"

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/python_jack_knife.egg-info/SOURCES.txt

@@ -11,8 +11,10 @@ src/pjk/main.py
 src/pjk/man_page.py
 src/pjk/parse_pjk_file.py
 src/pjk/parser.py
+src/pjk/paths.py
 src/pjk/progress.py
 src/pjk/registry.py
+src/pjk/runtime.py
 src/pjk/usage.py
 src/pjk/version.py
 src/pjk/integrations/opensearch_client.py

{python_jack_knife-0.7.5 → python_jack_knife-0.7.7}/src/python_jack_knife.egg-info/requires.txt

@@ -20,6 +20,9 @@ boto3>=1.34
 pytest
 black
 ruff
+build
+twine
+opensearch-py
 
 [parquet]
 pyarrow>=15.0.0

python_jack_knife-0.7.5/src/pjk/engine.py

@@ -1,51 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# Copyright 2024 Mike Schultz
-
-from typing import Iterator, List, Optional
-
-from pjk.parser import ExpressionParser, expand_macros
-from pjk.registry import ComponentRegistry
-from pjk.sources.dict_list_source import DictListSource
-
-
-class PjkEngine:
-    """
-    Run a pjk pipeline from a .pjk file, optionally with supplied input records.
-
-    - inrecs supplied: the source in the .pjk file is replaced with inrecs.
-      Expression may be full (source + pipes + sink) or pipes-only.
-    - inrecs=None: expression.pjk is fully self-contained (source, pipes, sink)
-    """
-
-    def __init__(self, inrecs: Optional[List[dict]] = None, pjk_file: str = ""):
-        self.inrecs = inrecs
-        self.pjk_file = pjk_file
-
-    def __iter__(self) -> Iterator[dict]:
-        registry = ComponentRegistry()
-        parser = ExpressionParser(registry)
-        expanded = expand_macros([self.pjk_file])
-
-        if self.inrecs is not None:
-            source_override = DictListSource(self.inrecs)
-            try:
-                first_is_source = registry.create_source(expanded[0]) is not None
-            except Exception:
-                first_is_source = False
-            if first_is_source:
-                expanded = ["{to_override: 'true'}"] + expanded[1:]
-            else:
-                expanded = ["{to_override: 'true'}"] + expanded
-        else:
-            source_override = None
-
-        sink = parser.parse(expanded, source_override=source_override)
-
-        inputs = [sink.input]
-        sink.input._get_sources(inputs)
-        try:
-            for record in sink.input:
-                yield record
-        finally:
-            for inp in inputs:
-                inp.close()