streamwright 0.1.12__tar.gz → 0.1.13__tar.gz

{streamwright-0.1.12/src/streamwright.egg-info → streamwright-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.12
+Version: 0.1.13
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
 License: Apache-2.0

{streamwright-0.1.12 → streamwright-0.1.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.12"
+version = "0.1.13"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = {text = "Apache-2.0"}

streamwright-0.1.13/src/streamwright/core/cli/commands/jobs.py

@@ -0,0 +1,31 @@
+"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
+
+import argparse
+import sys
+
+from streamwright.core.jobs.engine import load_job_engine
+from streamwright.core.runtime import plugins
+
+
+def register(subparsers):
+    parser = subparsers.add_parser("duckdb", help="validate, plan and run DuckDBJob manifests and manage their "
+                                                  "batches (the explicit form of run|validate|plan JOB)",
+                                   add_help=False)
+    parser.add_argument("-h", "--help", action="store_true", help="show the streamwright duckdb help")
+    parser.add_argument("arguments", nargs=argparse.REMAINDER)
+    parser.set_defaults(command_handler=lambda args: execute(["--help"] if args.help else args.arguments))
+
+
+def execute(arguments):
+    return run_job_kind("DuckDBJob", arguments, prog="streamwright duckdb")
+
+
+def run_job_kind(kind, argv, prog=None):
+    """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
+    try:
+        job_engine = load_job_engine(kind)
+    except plugins.PluginError as exc:
+        sys.stderr.write("streamwright: %s\n" % exc)
+        return 2
+    # `prog`: the command's name in usage lines (`streamwright duckdb`), for engines whose main takes it
+    return (job_engine.main(argv, prog=prog) if prog else job_engine.main(argv)) or 0

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/cli/jobs.py

@@ -38,10 +38,14 @@ def add_job_arguments(parser):
     """Add common job manifest and execution options to an argparse parser."""
     parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
     parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
-    parser.add_argument("--overlay", action="append", default=[], metavar="FILE")
-    parser.add_argument("--config", metavar="FILE")
-    parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE")
-    parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME")
-    parser.add_argument("--strict", action="store_true")
+    parser.add_argument("--overlay", action="append", default=[], metavar="FILE",
+                        help="a YAML overlay merged onto the job, in order (repeatable; a null deletes a key, a list "
+                             "replaces the job's)")
+    parser.add_argument("--config", metavar="FILE", help="a YAML/JSON file of spec.config values")
+    parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE",
+                        help="a spec.config value (repeatable; lists as a,b,c); overrides --config")
+    parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME",
+                        help="allow only this connector (repeatable); by default any installed connector")
+    parser.add_argument("--strict", action="store_true", help="treat warnings as errors")
     add_log_options(parser)
     return parser

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/cli/main.py

@@ -27,7 +27,7 @@ from streamwright.core.cli.commands.connectors import run_connectors_command
 from streamwright.core.cli.commands.source import run
 from streamwright.core.cli.jobs import JOB_OPTION
 from streamwright.core.cli.log_options import log_options, log_setup
-from streamwright.core.cli.parser import build_parser, build_validate_parser, command_parsers
+from streamwright.core.cli.parser import build_parser, build_plan_parser, build_validate_parser, command_parsers
 from streamwright.core.validation import engine
 from streamwright.core.validation.engine import NON_STRICT_CODES, check_components
 
@@ -147,7 +147,11 @@ def main(argv=None):
     if command == "validate":
         return _validate_command(argv)
     if command == "plan":
-        sys.stderr.write("streamwright: 'plan' needs an installed job manifest kind and path\n")
+        if any(argument in ("-h", "--help") for argument in argv[1:]):
+            build_plan_parser().print_help()
+            return 0
+        sys.stderr.write("streamwright: 'plan' needs a job manifest (JOB or --job JOB) of an installed engine's "
+                         "kind; see streamwright plan -h\n")
         return 2
     args = build_parser().parse_args(argv)
     if args.command == "connectors":

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/cli/parser.py

@@ -30,19 +30,36 @@ from streamwright.core.validation import engine
 def build_validate_parser():
     """`streamwright validate --help`: validation options, then the logging options (as streamwright run's)."""
     parser = engine._parser("streamwright validate")
+    parser.epilog = ("A job manifest (kind: DuckDBJob, SparkJob) is checked by its engine, with that engine's options: "
+                     "streamwright duckdb validate -h, streamwright spark validate -h.")
     add_log_options(parser.add_argument_group("logging options"))
     return parser
 
 
+def build_plan_parser():
+    """`streamwright plan --help`: the command takes a job manifest; its engine's plan command takes the options."""
+    parser = argparse.ArgumentParser(
+        prog="streamwright plan",
+        description="Show a job's execution plan - its steps, outputs and the order they run in - without running it "
+                    "or needing secrets. The manifest's kind picks the engine, and its options are that engine's: "
+                    "streamwright duckdb plan -h (kind: DuckDBJob), streamwright spark plan -h (kind: SparkJob).")
+    parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
+    parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
+    return parser
+
+
 def build_parser():
     parser = argparse.ArgumentParser(prog="streamwright",
-                                     description="Run and check StreamWright sources (`kind: source`).")
+                                     description="Run and check StreamWright sources (`kind: source`) and jobs "
+                                                 "(`kind: DuckDBJob`, `kind: SparkJob`).")
     commands = parser.add_subparsers(dest="command", metavar="COMMAND")
     commands.required = True
 
-    run = commands.add_parser("run", help="run a source and write its records",
+    run = commands.add_parser("run", help="run a source and write its records, or a job (JOB or --job JOB)",
                               description="Run a source. Records go to stdout as Singer messages unless --output "
-                                          "names a folder for files or a warehouse; logs go to stderr.")
+                                          "names a folder for files or a warehouse; logs go to stderr. A job "
+                                          "manifest (kind: DuckDBJob, SparkJob) runs on its engine, with that "
+                                          "engine's options: streamwright duckdb run -h, streamwright spark run -h.")
     run.add_argument("source", nargs="?", metavar="SOURCE",
                      help="a source folder (source.yaml and one file per stream in streams/) or a source file; or a "
                           "job manifest (kind: DuckDBJob, SparkJob), which runs on its engine")
@@ -114,7 +131,8 @@ def build_parser():
                                    add_help=False)
     validate.add_argument("arguments", nargs=argparse.REMAINDER)
 
-    plan_cmd = commands.add_parser("plan", help="show execution plan DAG for a DuckDB job", add_help=False)
+    plan_cmd = commands.add_parser("plan", help="show a job's execution plan: its steps, outputs and their order "
+                                                 "(no secrets needed)", add_help=False)
     plan_cmd.add_argument("arguments", nargs=argparse.REMAINDER)
     installed_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)}
     if "duckdb" not in installed_commands:  # editable installs have stale entry-point metadata

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/cli.py

@@ -197,7 +197,8 @@ def _cmd_state(args):
 def _inputs(parser, secrets=False):
     add_job_arguments(parser)
     if secrets:
-        parser.add_argument("--secrets", help="a YAML file of secret values (or STREAMWRIGHT_SECRET_<NAME>)")
+        parser.add_argument("--secrets", metavar="FILE",
+                            help="a YAML file of spec.secrets values (or STREAMWRIGHT_SECRET_<NAME> variables)")
 
 
 def _build_parser(prog):
@@ -220,9 +221,11 @@ def _build_parser(prog):
     _inputs(run, secrets=True)
     run.set_defaults(func=_cmd_run, parser=run)
     state = sub.add_parser("state", allow_abbrev=False, help="list, retry or abandon a job's batches (runtime.state)")
-    state.add_argument("action", choices=["list", "retry", "abandon"])
-    state.add_argument("--batch", help="the batch id (retry, abandon)")
-    state.add_argument("--limit", type=int, default=20, help="batches to list")
+    state.add_argument("action", choices=["list", "retry", "abandon"],
+                       help="list: the job's recent batches; retry: run a failed batch again from where it failed; "
+                            "abandon: give a failed batch up so the job can run again")
+    state.add_argument("--batch", metavar="ID", help="the batch id (retry, abandon)")
+    state.add_argument("--limit", type=int, default=20, metavar="N", help="list: the number of batches (default 20)")
     _inputs(state, secrets=True)
     state.set_defaults(func=_cmd_state, parser=state)
     return parser

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/jobs/duckdb/plugin.py

@@ -31,8 +31,8 @@ class DuckDBJobEngine:
 
     kind = KIND
 
-    def main(self, argv=None):
-        return cli.main(argv)
+    def main(self, argv=None, prog="streamwright"):
+        return cli.main(argv, prog=prog)
 
     def validate_job(self, job, *, overlays=(), config=None, strict=False, allowed=None):
         return engine.validate_job(job, overlays=overlays, config=config, strict=strict, allowed=allowed)

{streamwright-0.1.12 → streamwright-0.1.13}/src/streamwright/core/jobs/engine.py

@@ -41,7 +41,10 @@ class JobEngine(Protocol):
     kind: str  # the manifest `kind` it runs, e.g. "DuckDBJob" (= the entry-point name)
 
     def main(self, argv: Sequence[str] | None = None) -> int:
-        """Runs the engine's command line on the original arguments (`validate|plan|run JOB ...`); the exit code."""
+        """
+        Runs the engine's command line on the original arguments (`validate|plan|run JOB ...`); the exit code. An engine
+        may also take `prog`, the command's name in its usage lines (`streamwright duckdb` for the explicit form).
+        """
 
     def validate_job(self, job: str | Mapping[str, Any], *, overlays: Sequence[str] = (),
                      config: Mapping[str, Any] | None = None, strict: bool = False) -> JobReport:

{streamwright-0.1.12 → streamwright-0.1.13/src/streamwright.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.12
+Version: 0.1.13
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
 License: Apache-2.0

{streamwright-0.1.12 → streamwright-0.1.13}/tests/test_cli_subcommands.py

@@ -6,8 +6,12 @@ import subprocess
 import sys
 from types import SimpleNamespace
 
+import pytest
+
 from streamwright.core import cli
 from streamwright.core.cli.jobs import parse_set
+from streamwright.core.cli.parser import build_parser, build_validate_parser
+from streamwright.core.jobs.duckdb import cli as duckdb_cli
 from streamwright.core.runtime import plugins
 
 
@@ -126,9 +130,10 @@ def test_command_entry_is_registered_from_metadata(monkeypatch, capsys):
 def test_builtin_job_shorthand_uses_plugin_route(monkeypatch):
     jobs_module = importlib.import_module("streamwright.core.cli.commands.jobs")
     calls = []
-    monkeypatch.setattr(jobs_module, "run_job_kind", lambda kind, argv: calls.append((kind, argv)) or 0)
+    monkeypatch.setattr(jobs_module, "run_job_kind",
+                        lambda kind, argv, prog=None: calls.append((kind, argv, prog)) or 0)
     assert cli.main(["duckdb", "run", "job.yaml"]) == 0
-    assert calls == [("DuckDBJob", ["run", "job.yaml"])]
+    assert calls == [("DuckDBJob", ["run", "job.yaml"], "streamwright duckdb")]  # its usage lines name the command
 
 
 def test_shared_job_set_parser_uses_jobs_input_rules():
@@ -139,3 +144,33 @@ def test_shared_job_set_parser_uses_jobs_input_rules():
         assert "NAME=VALUE" in str(exc)
     else:
         raise AssertionError("an invalid --set must be rejected")
+
+
+def _without_help(parser, path):
+    missing = []
+    for action in parser._actions:
+        if isinstance(action, argparse._HelpAction) or action.help == argparse.SUPPRESS:
+            continue
+        if isinstance(action, argparse._SubParsersAction):
+            helps = {choice.dest: choice.help for choice in action._choices_actions}
+            for name, sub in action.choices.items():
+                missing += [] if helps.get(name) else ["%s %s" % (path, name)]
+                missing += _without_help(sub, "%s %s" % (path, name))
+        elif not action.help and action.nargs != argparse.REMAINDER:  # (REMAINDER: another parser's arguments)
+            missing.append("%s %s" % (path, "/".join(action.option_strings) or action.dest))
+    return missing
+
+
+def test_every_command_and_option_has_help(capsys):
+    plugin_commands = {item["name"] for item in plugins.discover(plugins.COMMANDS)} - {"duckdb"}
+    missing = [item for item in _without_help(build_parser(), "streamwright")
+               if item.split()[1] not in plugin_commands]  # (installed plug-ins check their own)
+    assert missing == []
+    assert _without_help(build_validate_parser(), "streamwright validate") == []
+    assert _without_help(duckdb_cli._build_parser("streamwright duckdb"), "streamwright duckdb") == []
+    assert cli.main(["plan", "-h"]) == 0
+    with pytest.raises(SystemExit) as shown:
+        cli.main(["duckdb", "-h"])  # forwarded to the engine's own parser
+    assert shown.value.code == 0
+    out = capsys.readouterr().out
+    assert "usage: streamwright plan" in out and "usage: streamwright duckdb" in out

streamwright-0.1.12/src/streamwright/core/cli/commands/jobs.py

@@ -1,27 +0,0 @@
-"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
-
-import argparse
-import sys
-
-from streamwright.core.jobs.engine import load_job_engine
-from streamwright.core.runtime import plugins
-
-
-def register(subparsers):
-    parser = subparsers.add_parser("duckdb", help="run an installed DuckDBJob engine", add_help=False)
-    parser.add_argument("arguments", nargs=argparse.REMAINDER)
-    parser.set_defaults(command_handler=lambda args: execute(args.arguments))
-
-
-def execute(arguments):
-    return run_job_kind("DuckDBJob", arguments)
-
-
-def run_job_kind(kind, argv):
-    """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
-    try:
-        job_engine = load_job_engine(kind)
-    except plugins.PluginError as exc:
-        sys.stderr.write("streamwright: %s\n" % exc)
-        return 2
-    return job_engine.main(argv) or 0