streamwright 0.1.9__tar.gz → 0.1.11__tar.gz

{streamwright-0.1.9/src/streamwright.egg-info → streamwright-0.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.9
+Version: 0.1.11
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
 License: Apache-2.0

{streamwright-0.1.9 → streamwright-0.1.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.9"
+version = "0.1.11"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = {text = "Apache-2.0"}

streamwright-0.1.11/src/streamwright/core/cli/commands/connectors.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# /*************************************************************************
+# * Copyright 2025 Karthick Jaganathan
+# *
+# * Licensed under the Apache License, Version 2.0 (the "License");
+# * you may not use this file except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# * https://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# **************************************************************************/
+
+"""`streamwright connectors`: the installed connectors, and the catalog's (list, search, install)."""
+
+import re
+import sys
+from importlib import metadata
+
+import duckdb
+
+from streamwright.core import catalog
+from streamwright.core.runtime import components, plugins
+from streamwright.core.runtime.logs.lines import NETWORK
+
+_REQUIREMENT_NAME = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
+# a connector that logs on streamwright.network calls through StreamWright's HTTP client, which is built on requests
+HTTP_CLIENT_LIBRARY = "requests"
+
+
+def _installed(name):
+    try:
+        return "%s==%s" % (name, metadata.version(name))
+    except metadata.PackageNotFoundError:
+        return "%s (not installed)" % name
+
+
+def libraries(distribution, loggers=()):
+    """
+    The third-party libraries a connector uses, as "name==version" (or "name (not installed)"): what its distribution
+    requires (not its extras', not StreamWright's), and requests when it calls through StreamWright's HTTP client.
+    """
+    try:
+        requires = (metadata.requires(distribution) or []) if distribution else []
+    except metadata.PackageNotFoundError:
+        requires = []
+    names = []
+    for requirement in requires:
+        spec, _, marker = requirement.partition(";")
+        match = _REQUIREMENT_NAME.match(spec)
+        if "extra" not in marker and match and not match.group(1).lower().startswith("streamwright"):
+            names.append(match.group(1))
+    if NETWORK.name in loggers and HTTP_CLIENT_LIBRARY not in names:
+        names.append(HTTP_CLIENT_LIBRARY)
+    return [_installed(name) for name in names]
+
+
+class DuckDBExtensions:
+    """The installed DuckDB's extensions (one in-memory query, made when a connector names one): name@version."""
+
+    def __init__(self):
+        self._installed = None
+
+    def _load(self):
+        self._installed = {}
+        rows = duckdb.connect().execute(
+            "SELECT extension_name, aliases, installed, extension_version FROM duckdb_extensions()").fetchall()
+        for name, aliases, installed, version in rows:
+            for key in (name, *(aliases or ())):
+                self._installed[key] = (name, version if installed else None)
+        self._installed[None] = ("duckdb", duckdb.__version__)
+
+    def describe(self, names):
+        """("duckdb==VERSION", ["EXTENSION@VERSION" or "EXTENSION (not installed)", ...]); (None, []) for no names."""
+        if not names:
+            return None, []
+        if self._installed is None:
+            self._load()
+        found = []
+        for name in names:
+            extension, version = self._installed.get(name, (name, None))
+            found.append("%s@%s" % (extension, version) if version else "%s (not installed)" % extension)
+        return "duckdb==%s" % self._installed[None][1], found
+
+
+def _fields(name, definition, entry, extensions):
+    """A connector's `pip show`-style fields."""
+    used = libraries(entry.get("distribution"), definition.loggers)
+    duckdb_library, loaded = extensions.describe(definition.duckdb_extensions)
+    return (("Name", name), ("Title", definition.title), ("Family", definition.family),
+            ("Summary", definition.summary), ("Package", entry.get("distribution") or "(registered in-process)"),
+            ("Version", entry.get("version") or ""), ("Libraries", ", ".join(used + [duckdb_library] * bool(loaded))),
+            ("DuckDB-Extensions", ", ".join(loaded)), ("Loggers", ", ".join(definition.loggers)))
+
+
+def _connector_lines(names=None):
+    """
+    The installed (and registered) connectors, sorted by family and name, as `pip show` shows packages (`---` between
+    them): name, title, family, summary, distribution and version, the third-party libraries and DuckDB extensions it
+    uses (installed versions) and its loggers (for --log). `names`: only these. Only specs are imported.
+    """
+    metadata_of = {entry["name"]: entry for entry in plugins.discover(plugins.CONNECTORS)}
+    blocks, extensions = [], DuckDBExtensions()
+    for name in components.available():
+        if names is not None and name not in names:
+            continue
+        try:
+            definition = components.load(name)
+        except components.ComponentLoadError as exc:
+            blocks.append(("~", name, (("Name", name), ("Error", str(exc)))))  # (after the families)
+            continue
+        blocks.append((definition.family, name, _fields(name, definition, metadata_of.get(name, {}), extensions)))
+    lines = []
+    for _, _, fields in sorted(blocks, key=lambda block: block[:2]):
+        if lines:
+            lines.append("---")
+        lines.extend(("%s: %s" % (key, value)).rstrip() for key, value in fields)
+    return lines
+
+
+def run_connectors_command(args):
+    """`streamwright connectors`: bare or installed names = installed connectors; ls/i/find use the catalog (hub)."""
+    if getattr(args, "flag_list", False):
+        action = "ls"
+        query = []
+    elif getattr(args, "flag_install", None):
+        action = "install"
+        query = args.flag_install
+    elif getattr(args, "flag_search", None):
+        action = "find"
+        query = args.flag_search
+    else:
+        action = getattr(args, "action", None)
+        query = getattr(args, "query", [])
+
+    if action is None:
+        lines = _connector_lines()
+        print("\n".join(lines) if lines else "no connectors installed (use streamwright connectors list to find one)")
+        return 0
+
+    norm_action = action.lower()
+    if norm_action in ("ls", "list", "-ls", "--list"):
+        lines = catalog.listing(args.hub_url)
+        print("\n".join(lines) if lines else "the catalog is empty")
+        return 0
+
+    if norm_action in ("i", "install", "-i", "--install"):
+        if not query:
+            sys.stderr.write("streamwright: 'connectors -i' needs at least one connector KEY\n")
+            return 2
+        return catalog.install_many(query, assume_yes=args.yes, hub_url=args.hub_url)
+
+    words = [action, *query]
+    if norm_action in ("find", "search", "s", "-s", "-f", "--find", "--search"):
+        words = query
+    elif set(words) <= set(components.available()):
+        # installed connector names show those connectors, as `pip show NAME` does (`sw connectors google_ads s3`)
+        print("\n".join(_connector_lines(set(words))))
+        return 0
+    return _search(" ".join(words), args.hub_url)  # any other words search the catalog (`sw connectors ads`)
+
+
+def _search(text, hub_url):
+    hits = catalog.search(text, hub_url)
+    if not hits:
+        print("no connectors match %r" % text)
+    for key, entry in sorted(hits.items()):
+        print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
+    return 0

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/cli/jobs.py

@@ -2,6 +2,7 @@
 
 import argparse
 
+from streamwright.core.cli.log_options import add_log_options
 from streamwright.core.jobs.inputs import InputError
 from streamwright.core.jobs.inputs import parse_set as parse_job_set
 
@@ -19,13 +20,28 @@ def parse_set(text):
         raise argparse.ArgumentTypeError(str(exc)) from exc
 
 
+JOB_OPTION = "job_option"
+
+
+def resolve_job(parser, args):
+    """The job path of JOB or --job JOB (one of them, or both the same); a usage error otherwise."""
+    option = getattr(args, JOB_OPTION, None)
+    if args.job and option and args.job != option:
+        parser.error("pass the job once: JOB or --job JOB")
+    args.job = args.job or option
+    if not args.job:
+        parser.error("a job manifest is required: JOB or --job JOB")
+    return args.job
+
+
 def add_job_arguments(parser):
     """Add common job manifest and execution options to an argparse parser."""
-    parser.add_argument("job", metavar="JOB")
+    parser.add_argument("job", nargs="?", metavar="JOB", help="the job manifest (a YAML file or its folder)")
+    parser.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB", help="the job manifest, as an option")
     parser.add_argument("--overlay", action="append", default=[], metavar="FILE")
     parser.add_argument("--config", metavar="FILE")
     parser.add_argument("--set", action="append", default=[], type=parse_set, metavar="NAME=VALUE")
     parser.add_argument("--allow-connector", action="append", default=[], metavar="NAME")
-    parser.add_argument("--log-level", metavar="LEVEL")
     parser.add_argument("--strict", action="store_true")
+    add_log_options(parser)
     return parser

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/cli/main.py

@@ -25,6 +25,7 @@ import yaml
 from streamwright.core.cli.commands import jobs
 from streamwright.core.cli.commands.connectors import run_connectors_command
 from streamwright.core.cli.commands.source import run
+from streamwright.core.cli.jobs import JOB_OPTION
 from streamwright.core.cli.log_options import log_options, log_setup
 from streamwright.core.cli.parser import build_parser, build_validate_parser, command_parsers
 from streamwright.core.validation import engine
@@ -43,9 +44,14 @@ def _extract_all_paths(args, command="run"):
     index = 0
     while index < len(args):
         item = args[index]
-        option, equals, _ = item.partition("=")
+        option, equals, value = item.partition("=")
         action = options.get(option)
         if action is not None:
+            if action.dest == JOB_OPTION:  # --job JOB: a manifest path too
+                if not equals and index + 1 < len(args):
+                    value = args[index + 1]
+                if value:
+                    paths.append(value)
             if not equals and action.nargs != 0:
                 index += 1
                 if action.nargs in ("+", "*"):
@@ -118,7 +124,7 @@ def _job_kind(argv):
     """The manifest kind of a job command's path (None: a source, or not a job command); raises ValueError."""
     if argv[0] not in ("run", "validate", "plan"):
         return None
-    paths = _extract_all_paths(argv[1:], argv[0])
+    paths = list(dict.fromkeys(_extract_all_paths(argv[1:], argv[0])))  # JOB and --job JOB may name the same
     kinds = {detect_kind(path) for path in paths} - {None, "source"}
     if len(kinds) > 1 or (kinds and len(paths) > 1):
         raise ValueError("a job command requires one manifest kind and path")
@@ -145,6 +151,18 @@ def main(argv=None):
         return run_connectors_command(args)
     if hasattr(args, "command_handler"):
         return args.command_handler(args)
+    return _run_source(args)
+
+
+def _run_source(args):
+    """`streamwright run SOURCE`: a job manifest (JOB or --job JOB) was routed to its engine before."""
+    if getattr(args, JOB_OPTION, None):
+        sys.stderr.write("streamwright: --job %s is not a job manifest (kind: DuckDBJob, SparkJob); run a source "
+                         "as: streamwright run SOURCE\n" % getattr(args, JOB_OPTION))
+        return 2
+    if not args.source:
+        sys.stderr.write("streamwright: run needs a SOURCE, or --job JOB\n")
+        return 2
     setup = log_setup(args)
     if setup is None:
         return 2

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/cli/parser.py

@@ -20,7 +20,7 @@
 import argparse
 
 from streamwright.core.cli.commands import jobs
-from streamwright.core.cli.jobs import add_job_arguments
+from streamwright.core.cli.jobs import JOB_OPTION, add_job_arguments
 from streamwright.core.cli.log_options import add_log_options, key_value
 from streamwright.core.config.inputs import SECRET_ENV_PREFIX
 from streamwright.core.runtime import plugins
@@ -43,8 +43,13 @@ def build_parser():
     run = commands.add_parser("run", help="run a source and write its records",
                               description="Run a source. Records go to stdout as Singer messages unless --output "
                                           "names a folder for files or a warehouse; logs go to stderr.")
-    run.add_argument("source", metavar="SOURCE",
-                     help="a source folder (source.yaml and one file per stream in streams/) or a source file")
+    run.add_argument("source", nargs="?", metavar="SOURCE",
+                     help="a source folder (source.yaml and one file per stream in streams/) or a source file; or a "
+                          "job manifest (kind: DuckDBJob, SparkJob), which runs on its engine")
+    run.add_argument("--job", "-j", dest=JOB_OPTION, metavar="JOB",
+                     help="a job manifest (kind: DuckDBJob, SparkJob) to run on its engine, e.g. "
+                          "sw run --job jobs/campaigns.yaml (the same as passing it as the path); the job engine's "
+                          "options apply: sw duckdb run --help, sw spark run --help")
     run.add_argument("--config", metavar="FILE",
                      help="YAML/JSON file with `config` (values for spec.config) and `streams` (streams/exports to "
                           "run), e.g. one client's settings for the source")
@@ -85,8 +90,8 @@ def build_parser():
     add_log_options(run)
 
     conn = commands.add_parser("connectors",
-                               help="list, find or install connectors (bare: the installed connectors and their "
-                                    "SDK loggers)")
+                               help="show, list, find or install connectors (bare or NAME ...: the installed "
+                                    "connectors, their libraries, DuckDB extensions and loggers)")
     conn.add_argument("-ls", "--list", action="store_true", dest="flag_list",
                       help="list catalog and installed connectors")
     conn.add_argument("-i", "--install", nargs="+", dest="flag_install", metavar="KEY",
@@ -94,9 +99,10 @@ def build_parser():
     conn.add_argument("-s", "-f", "--search", "--find", nargs="+", dest="flag_search", metavar="TEXT",
                       help="find/search connectors in the catalog by keyword")
     conn.add_argument("action", nargs="?", metavar="{ls,i,find}",
-                      help="ls (catalog + installed), -i KEY (install), or find TEXT (search catalog)")
+                      help="ls (catalog + installed), -i KEY (install), find TEXT (search catalog), or installed "
+                           "connector names (show them, like pip show)")
     conn.add_argument("query", nargs="*", metavar="KEY|TEXT",
-                      help="connector keys (install: one or more) or search words")
+                      help="connector keys (install, show: one or more) or search words")
     conn.add_argument("--yes", action="store_true", help="skip the confirmation prompt when installing")
     conn.add_argument("--hub-url", metavar="URL",
                       help="the connector hub (default: $STREAMWRIGHT_HUB_URL, else the live StreamWright hub; when "

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/jobs/duckdb/cli.py

@@ -22,7 +22,8 @@ The DuckDBJob command line (registered for `kind: DuckDBJob` in the `streamwrigh
   plan JOB
   run JOB [--secrets FILE] [--dry-run]
   state list|retry|abandon JOB [--batch ID] [--secrets FILE]
-  (every command: --config FILE, --set NAME=VALUE, --overlay FILE, --allow-connector NAME, --strict, --log-level)
+  (every command: --config FILE, --set NAME=VALUE, --overlay FILE, --allow-connector NAME, --strict, and the logging
+  options of `streamwright run`: --log-level, --log NAME=LEVEL, --log-format, --log-config, --log-max-chars)
 
 The job options, exit codes and logging are the shared ones of `streamwright.core.cli.jobs`: 0 success, 1 invalid
 job or failed run, 2 usage or input error. Validation and planning never need secret values.
@@ -31,14 +32,14 @@ job or failed run, 2 usage or input error. Validation and planning never need se
 import argparse
 import sys
 
-from streamwright.core.cli.jobs import FAILURE, INVALID_INPUT, SUCCESS, add_job_arguments
+from streamwright.core.cli.jobs import FAILURE, INVALID_INPUT, SUCCESS, add_job_arguments, resolve_job
+from streamwright.core.cli.log_options import log_setup
 from streamwright.core.config.inputs import InputError, read_values_file
 from streamwright.core.jobs.duckdb.engine import JobInvalidError, job_state, redactor, run_job, validate_job
 from streamwright.core.jobs.inputs import read_config_values
 from streamwright.core.jobs.manifest import JobError
 from streamwright.core.jobs.runner import owner_id
 from streamwright.core.jobs.state import StateError
-from streamwright.core.runtime.logs import LogSetup
 
 __all__ = ["main"]
 
@@ -100,8 +101,16 @@ def _cmd_run(args):
     for name, rows in sorted(result.outputs.items()):
         if name in result.skipped:
             sys.stdout.write("  %-28s already committed in an earlier attempt\n" % name)
-        else:
-            sys.stdout.write("  %-28s %s rows\n" % (name, "?" if rows is None else rows))
+            continue
+        locations = [location for location in result.locations.get(name, ()) if location]
+        written = "%s rows" % ("?" if rows is None else rows)
+        if len(locations) == 1:
+            written += " -> %s" % locations[0]
+        elif locations:
+            written += " -> %d files" % len(locations)
+        sys.stdout.write("  %-28s %s\n" % (name, written))
+        if len(locations) > 1:
+            sys.stdout.writelines("      %s\n" % location for location in locations)
     if result.exit_code:
         _err("Run failed: %s" % (result.error or "see the log"))
     return result.exit_code
@@ -146,25 +155,26 @@ def _inputs(parser, secrets=False):
 
 
 def _build_parser(prog):
-    parser = argparse.ArgumentParser(prog=prog, description="StreamWright DuckDBJob engine.")
+    # no prefix matching: --log must not be taken for --log-level
+    parser = argparse.ArgumentParser(prog=prog, description="StreamWright DuckDBJob engine.", allow_abbrev=False)
     sub = parser.add_subparsers(dest="subcommand")
-    validate = sub.add_parser("validate", help="check a DuckDBJob manifest (no secrets needed)")
+    validate = sub.add_parser("validate", allow_abbrev=False, help="check a DuckDBJob manifest (no secrets needed)")
     validate.add_argument("--graph", action="store_true", help="print the plan of a valid job")
     _inputs(validate)
-    validate.set_defaults(func=_cmd_validate)
-    plan = sub.add_parser("plan", help="print the execution plan (no secrets needed)")
+    validate.set_defaults(func=_cmd_validate, parser=validate)
+    plan = sub.add_parser("plan", allow_abbrev=False, help="print the execution plan (no secrets needed)")
     _inputs(plan)
-    plan.set_defaults(func=_cmd_plan)
-    run = sub.add_parser("run", help="run a DuckDBJob (unfinished batches resume first)")
+    plan.set_defaults(func=_cmd_plan, parser=plan)
+    run = sub.add_parser("run", allow_abbrev=False, help="run a DuckDBJob (unfinished batches resume first)")
     run.add_argument("--dry-run", action="store_true", help="check and plan, write nothing")
     _inputs(run, secrets=True)
-    run.set_defaults(func=_cmd_run)
-    state = sub.add_parser("state", help="list, retry or abandon a job's batches (runtime.state)")
+    run.set_defaults(func=_cmd_run, parser=run)
+    state = sub.add_parser("state", allow_abbrev=False, help="list, retry or abandon a job's batches (runtime.state)")
     state.add_argument("action", choices=["list", "retry", "abandon"])
     state.add_argument("--batch", help="the batch id (retry, abandon)")
     state.add_argument("--limit", type=int, default=20, help="batches to list")
     _inputs(state, secrets=True)
-    state.set_defaults(func=_cmd_state)
+    state.set_defaults(func=_cmd_state, parser=state)
     return parser
 
 
@@ -174,10 +184,9 @@ def main(argv=None, prog="streamwright"):
     if not getattr(args, "func", None):
         parser.print_help()
         return INVALID_INPUT
-    try:
-        args.log_setup = LogSetup(args.log_level)
-    except ValueError as exc:
-        _err("Error: %s" % exc)
+    resolve_job(args.parser, args)
+    args.log_setup = log_setup(args)
+    if args.log_setup is None:
         return INVALID_INPUT
     try:
         return args.func(args)

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/jobs/duckdb/engine.py

@@ -42,6 +42,7 @@ from streamwright.core.jobs.refs import resolve
 from streamwright.core.jobs.runner import Runner
 from streamwright.core.jobs.schema import json_schema as generate_schema
 from streamwright.core.jobs.state import DEFAULT_SCHEMA, open_sqlite, open_store
+from streamwright.core.jobs.types import written_files
 from streamwright.core.runtime.redact import Redactor
 
 __all__ = [
@@ -78,6 +79,8 @@ class RunResult:
     outputs: dict[str, int | None] = field(default_factory=dict)      # output -> rows, from the receipts
     skipped: set[str] = field(default_factory=set)                    # outputs already committed in an earlier attempt
     error: str | None = None
+    # output -> where it wrote: its files (a file_name template's, split_by's), else its destination
+    locations: dict[str, list[str]] = field(default_factory=dict)
 
 
 def _providers(job):
@@ -236,14 +239,16 @@ def _run(raw, *, config, secret_map, allowed_connectors, dry_run, base_dir, reda
                         engine_metadata={"engine": "duckdb"})
         session.start_clock()
         code = runner.run(mode="once", run_key=state.get("run_key"), max_messages=0 if resume_only else 500)
-        outputs = {}
+        outputs, locations = {}, {}
         skipped = set()
         for batch_id in runner.processed:
             for name, receipt in store.get_receipts(job["name"], batch_id).items():
                 outputs[name] = receipt.get("rows")
+                locations[name] = written_files(receipt) or [receipt.get("destination")]
                 if receipt.get("already_committed"):
                     skipped.add(name)
-        return RunResult(code, job["name"], plan.describe(), list(runner.processed), outputs, skipped, runner.error)
+        return RunResult(code, job["name"], plan.describe(), list(runner.processed), outputs, skipped, runner.error,
+                         locations)
     finally:
         session.close()
 

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/jobs/duckdb/executor.py

@@ -26,8 +26,9 @@ from streamwright.core.jobs.duckdb.adapter import ReadContext, WriteContext
 from streamwright.core.jobs.duckdb.steps import build_step
 from streamwright.core.jobs.policy import SKIPPED, ExecutionPolicy, classify_transient, guard
 from streamwright.core.jobs.refs import RefError, decode_value, encode_value
-from streamwright.core.jobs.types import ExecutionResult, WriteReceipt
+from streamwright.core.jobs.types import ExecutionResult, WriteReceipt, written_files
 from streamwright.core.jobs.write_rules import OutputSpec, check_output
+from streamwright.core.runtime.logs.lines import fields
 from streamwright.core.sql.quote import quote_ident
 
 __all__ = ["DuckDBExecutor", "ExecutionError"]
@@ -131,7 +132,11 @@ class DuckDBExecutor:
             if receipt.already_committed:
                 LOG.info("output %s: already committed in an earlier attempt", node.name)
             else:
-                LOG.info("output %s: %s rows to %s", node.name, receipt.rows, receipt.destination)
+                LOG.info("output %s: %s rows to %s", node.name, receipt.rows, receipt.destination,
+                         extra=fields(event="output_end", export=node.name, records=receipt.rows))
+                for path in written_files(receipt):
+                    LOG.info("output %s: wrote %s", node.name, path,
+                             extra=fields(event="output_file", export=node.name, path=path))
         return ExecutionResult(schema_signatures=tuple(run.schema_signatures),
                                schema_reports=tuple(run.schema_reports))
 

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/jobs/duckdb/guide.md

@@ -25,12 +25,14 @@ secret values. Exit codes: 0 success, 1 invalid job or failed run, 2 usage or in
     `row_index`, `columns`, `column_groups`, `path_pattern`, `vocabulary`, `on_unmapped_column`; databases:
     `table` or `query`). `path` may be `{{ step.column }}`. Planned reads pin their file list in the batch.
   - call: a connection and a connector call (`service`/`method`/`arguments`, or `request`), optional
-    `partitions: [{from, field, name}]` (crossed) and `sql` over the table `records` (column `record`).
+    `partitions: [{from, field, name}]` (crossed) and `sql` over the table `records` (column `record`); see Call
+    steps.
   - `source` + `stream`: a StreamWright source folder (relative to the job's folder) and one of its streams.
 - `outputs`: `connection`, `step` or `sql`, and a required `mode`. What else is allowed depends on the writer:
   tables (`append | overwrite | merge`, `table`, `primary_key`), files (`overwrite | error`, `path`, `format`:
   `csv | tsv | json | jsonl | parquet`; `path` is the file, or with a `file_name` template the folder: `file_name`
-  (and `layout`, its folders, e.g. `{today:%Y/%m}`) names one file, or with `split_by` one file per split; or
+  (and `layout`, its folders, e.g. `{today:%Y/%m}`) names one file, or with `split_by` one file per split (see File
+  names); or
   `partition_by` for Hive folders), Delta
   (`append | overwrite | merge` with `replace_where` or `whole_table`, `condition`, `when_*`, `target_predicate`,
   `schema_evolution`). Modes are the same words on every engine. A key the writer does not read is an error;
@@ -39,6 +41,75 @@ secret values. Exit codes: 0 success, 1 invalid job or failed run, 2 usage or in
   `read_groups`) and `state`: `{connection: NAME, schema: streamwright}` (a `duckdb` or `postgres` connection) or
   `{sqlite: PATH}`, plus `ttl_seconds` and `run_key` (a batch with that key runs once).
 
+## Call steps
+A call step runs a connector call on the driver, once per partition, and loads the records into a temp table in
+batches, as JSON. A `source` + `stream` step loads its records the same way.
+
+- **The `records` table.** With `sql`, the step is that query over the table `records`, which has **one column,
+  `record` (JSON)**: one row per record, as the connector returned it. Without `sql`, the records' top-level keys
+  become the step's columns. `records` exists only inside the step's own `sql`.
+- **Partitions.** `partitions: [{from, field, name}]` calls once per distinct, non-null value of `field` in the
+  earlier step `from` (several partitions are crossed); the call's arguments read the value as
+  `{{ partition.NAME }}` (`name` defaults to `field`). The values are pinned in the batch, so a replay makes the same
+  calls. **Partition values are not columns of `records`**: to keep one, request it in the call (e.g. select
+  `campaign.id` in the query) and extract it from `record`.
+- **Extract scalars with `->>`.** `record->>'$.campaign.id'` returns VARCHAR; cast it for other types
+  (`(record->>'$.metrics.clicks')::BIGINT`). Dot notation (`record.campaign.id`) and `->` keep the value as JSON,
+  so a CSV or TSV output writes it as a quoted JSON string (`"""18497872503"""`), while Parquet hides the issue.
+
+```yaml
+- name: ad_groups
+  connection: google_ads_api
+  service: GoogleAdsService
+  method: search_stream
+  partitions: [{from: campaigns, field: campaign_id}]
+  arguments:
+    customer_id: "{{ config.customer_id }}"
+    query: {gaql: {select: [campaign.id, ad_group.id, ad_group.name], from: ad_group,
+                   where: [{field: campaign.id, op: "=", type: int, value: "{{ partition.campaign_id }}"}]}}
+  sql: |
+    SELECT record->>'$.campaign.id'               AS campaign_id,
+           record->>'$.ad_group.id'               AS ad_group_id,
+           record->>'$.ad_group.name'             AS ad_group_name
+    FROM records
+```
+
+## File names
+A file output's `path` is the file; with `file_name` it is the folder, and `file_name` names the file inside it
+(`layout` adds subfolders). With `split_by: [col_a, col_b]` the output writes one file per distinct combination,
+and both templates may use `{col_a}` and `{col_b}`; without `split_by` they name the one file.
+
+| Placeholder | Renders |
+|---|---|
+| `{today}`, `{today:%Y-%m-%d}` | the batch's UTC date (default format `%Y%m%d`) |
+| `{now}`, `{batch_ts}`, `{now:%Y%m%d%H%M%S}` | the batch's UTC time (default format `%Y%m%dT%H%M%SZ`) |
+| `{today-1d:%Y%m%d}`, `{now+2h:%H}` | a time with offsets: `+`/`-` N then `s m h d w`, combinable (`-1d+6h`) |
+| `{batch_id}` | the batch id (no offset or format) |
+| `{seq}` | the split's 5-digit number, `00001`, splits ordered by their values (no offset or format) |
+| `{col}`, `{col:%Y%m}` | a `split_by` column's value, reduced to `[A-Za-z0-9._-]`; with a format (or offset), a date, timestamp or ISO text column |
+
+- Formats are strftime directives, rendered in UTC and English: `Y y m d e H M S I j u w U G V a A b B p Z z s F %`;
+  `%-d` drops zero padding. `%T` is not supported (colons are unsafe in names): use `%H%M%S`.
+- `today`, `now` and `batch_ts` are the batch's time, pinned when the batch starts, so a replay renders the same
+  names and overwrites the same files.
+- `file_name` may not contain `/`; `layout` may (`{today:%Y/%m}`, `client={client_id}`). Literal text may use letters,
+  digits and `. _ - = , @ +`; `..` is never allowed.
+- `file_name` needs at least one placeholder; for a fixed name, set the file as `path` and leave `file_name` out.
+  With `split_by`, two splits that render the same name fail the run: use a split column or `{seq}`.
+
+```yaml
+outputs:
+  - name: daily_export
+    step: campaign_stats
+    connection: exports
+    mode: overwrite
+    path: reports
+    format: tsv
+    split_by: [customer_id]
+    layout: "{today:%Y/%m}"
+    file_name: "campaigns_{customer_id}_{today-1d:%Y%m%d}.tsv"
+```
+
 ## Delivery
 Every run takes the job's lease (renewed while it runs; every output write is fenced by it), resumes unfinished
 batches first and skips outputs that already have a receipt. Delta writes commit an application transaction, so a

{streamwright-0.1.9 → streamwright-0.1.11}/src/streamwright/core/jobs/duckdb/rules.py

@@ -34,6 +34,12 @@ __all__ = ["KIND", "DuckDBRules"]
 
 KIND = "DuckDBJob"
 _TEXT = Field("string")
+FILE_NAME_HELP = (
+    "file outputs: the file's name inside `path` (then a folder). Placeholders: {today} and {now}/{batch_ts}, the "
+    "batch's UTC date and time, with offsets and strftime formats ({today-1d:%Y%m%d}, {now+2h:%Y%m%d%H%M%S}; "
+    "offsets +/-N with s m h d w); {batch_id}; {seq}, the split's 5-digit number; and the split_by columns ({col}). "
+    "No '/', and at least one placeholder"
+)
 # the execution-policy vocabulary (contract v1.2 §6a), accepted structurally on steps and outputs
 _POLICY_FIELDS = MappingProxyType({
     "retry": Field("object"), "rate_limit": Field("object"), "paginator": Field("object"),
@@ -51,7 +57,9 @@ _COLUMN_GROUP = Field("object", fields={
     "keep_nulls": Field("boolean")})
 _STEP_FIELDS = MappingProxyType({
     "name": Field("string", required=True, pattern=NAME_PATTERN), "description": _TEXT,
-    "sql": Field("string", description="DuckDB SQL over earlier steps (in call/source steps: over `records`)"),
+    "sql": Field("string", description="DuckDB SQL over earlier steps; in a call or source step, over the table "
+                                       "`records`, whose one column `record` (JSON) holds each record: extract "
+                                       "scalars with ->> (record->>'$.campaign.id'), which returns VARCHAR, not JSON"),
     "materialize": Field("string", enum=("view", "table")),
     "connection": _TEXT,
     "path": Field("string", description="a file, glob or {{ step.column }} (relative to the connection's roots)"),
@@ -71,7 +79,11 @@ _STEP_FIELDS = MappingProxyType({
     "paginator": Field("object"), "retry": Field("object"), "rate_limit": Field("object"),
     "page_size": _POLICY_FIELDS["page_size"], "on_error": _TEXT, "timeout": _POLICY_FIELDS["timeout"],
     "partitions": Field("array", items=_named(**{"from": Field("string", required=True),
-                                                  "field": Field("string", required=True), "name": _TEXT})),
+                                                  "field": Field("string", required=True), "name": _TEXT}),
+                        description="call steps: one call per distinct value of `field` in the step `from` (crossed "
+                                    "when several), read as {{ partition.NAME }} in the call's arguments (`name` "
+                                    "defaults to `field`). The values are not columns of `records`: request the key "
+                                    "in the call and extract it from `record`"),
     "source": Field("string", description="a StreamWright source folder (relative to the job's folder)"),
     "stream": _TEXT, "config": Field("object"),
 })
@@ -84,8 +96,14 @@ _OUTPUT_FIELDS = MappingProxyType({
     "when_matched": Field("array", items=Field("object")), "when_not_matched": Field("array", items=Field("object")),
     "when_not_matched_by_source": Field("array", items=Field("object")),
     "schema_evolution": Field("string", enum=("strict", "add_columns")),
-    "partition_by": Field("array", items=_TEXT), "split_by": Field("array", items=_TEXT),
-    "file_name": _TEXT, "layout": _TEXT, "options": Field("object"),
+    "partition_by": Field("array", items=_TEXT, description="file outputs: Hive folders (col=value/) under `path`"),
+    "split_by": Field("array", items=_TEXT,
+                      description="file outputs: one file per distinct combination of these columns; file_name and "
+                                  "layout may use {col} (with a format, {col:%Y%m%d}, for a date or timestamp column)"),
+    "file_name": Field("string", description=FILE_NAME_HELP),
+    "layout": Field("string", description="the subfolders of file_name's file under `path`; the placeholders of "
+                                          "file_name, and '/', e.g. {today:%Y/%m} or client={client_id}"),
+    "options": Field("object"),
     **_POLICY_FIELDS,
 })
 _RUNTIME = _named(