continuo-python-runtime 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

continuo_python_runtime/contract/loader.py

@@ -18,9 +18,11 @@ from sqlglot.errors import TokenError
 from continuo_python_runtime.contract.model import (
     CRITICALITIES,
     EXTRA_COLUMNS_POLICIES,
+    KINDS,
     Column,
     Node,
 )
+from continuo_python_runtime.csv_source import parse_csv_uri
 from continuo_python_runtime.errors import ContractError
 from continuo_python_runtime.types import parse_sql_type
 
@@ -31,6 +33,7 @@ _ALLOWED_KEYS = {
     "owner",
     "schedule",
     "criticality",
+    "kind",
     "script",
     "extra_columns",
     "reads",
@@ -39,7 +42,7 @@ _ALLOWED_KEYS = {
     "content_hash",
 }
 
-_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule", "script")
+_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
 
 _ALLOWED_OUTPUT_COLUMN_KEYS = {"name", "type", "nullable"}
 
@@ -170,6 +173,12 @@ def parse_node(
     if unknown:
         raise ContractError(f"{label}: unknown key(s) {sorted(unknown)}")
 
+    kind = raw.get("kind", "python-model")
+    if not isinstance(kind, str) or kind not in KINDS:
+        raise ContractError(
+            f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}"
+        )
+
     for field in _REQUIRED_STRING_FIELDS:
         value = raw.get(field)
         if not isinstance(value, str) or not value.strip():
@@ -181,7 +190,21 @@ def parse_node(
     table = raw["table"]
     owner = raw["owner"]
     schedule = raw["schedule"]
-    script = raw["script"]
+
+    if kind == "python-csv":
+        if "script" in raw:
+            raise ContractError(
+                f"{label}: 'script' is forbidden for kind python-csv "
+                "(csv nodes are contract-only)"
+            )
+        script = ""
+    else:
+        raw_script = raw.get("script")
+        if not isinstance(raw_script, str) or not raw_script.strip():
+            raise ContractError(
+                f"{label}: required field 'script' must be a non-empty string"
+            )
+        script = raw_script
 
     criticality = raw.get("criticality")
     if not isinstance(criticality, str) or criticality not in CRITICALITIES:
@@ -200,39 +223,50 @@ def parse_node(
         )
 
     reads = raw.get("reads")
-    if not isinstance(reads, dict) or not reads:
-        raise ContractError(
-            f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
-        )
-    for name, sql in reads.items():
-        if not isinstance(name, str) or not name.strip():
-            raise ContractError(
-                f"{label}: 'reads' name {name!r} must be a non-empty string"
-            )
-        if not isinstance(sql, str) or not sql.strip():
+    if kind == "python-csv":
+        if not isinstance(reads, dict) or set(reads) != {"csv"}:
             raise ContractError(
-                f"{label}: 'reads.{name}' must be a non-empty SQL string"
+                f"{label}: a python-csv node's 'reads' must be exactly {{csv: <uri>}}"
             )
-        if not check_reads:
-            continue
         try:
-            ensure_single_read(sql, dialect)
-        except (ValueError, TokenError) as exc:
-            # ensure_single_read's own message is phrased for check_binds
-            # (its only other caller today), so it's wrapped rather than
-            # surfaced bare here. TokenError is also caught: an unterminated
-            # string literal or comment fails sqlglot's tokenizer with a
-            # TokenError, a SqlglotError sibling of ParseError and not a
-            # subclass of ValueError -- despite ensure_single_read's
-            # docstring promising every rejection is a ValueError. Only
-            # TokenError, not the broader SqlglotError, is caught here: by
-            # the time control reaches this point `dialect` has already been
-            # validated once in load_contract_dir, so any other SqlglotError
-            # a future sqlglot version might raise from this call should
-            # surface as itself, not get relabeled as a rejected read.
+            parse_csv_uri(reads["csv"])
+        except (ValueError, TypeError) as exc:
+            raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
+    else:
+        if not isinstance(reads, dict) or not reads:
             raise ContractError(
-                f"{label}: 'reads.{name}' must be a single read query ({exc})"
-            ) from exc
+                f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
+            )
+        for name, sql in reads.items():
+            if not isinstance(name, str) or not name.strip():
+                raise ContractError(
+                    f"{label}: 'reads' name {name!r} must be a non-empty string"
+                )
+            if not isinstance(sql, str) or not sql.strip():
+                raise ContractError(
+                    f"{label}: 'reads.{name}' must be a non-empty SQL string"
+                )
+            if not check_reads:
+                continue
+            try:
+                ensure_single_read(sql, dialect)
+            except (ValueError, TokenError) as exc:
+                # ensure_single_read's own message is phrased for check_binds
+                # (its only other caller today), so it's wrapped rather than
+                # surfaced bare here. TokenError is also caught: an unterminated
+                # string literal or comment fails sqlglot's tokenizer with a
+                # TokenError, a SqlglotError sibling of ParseError and not a
+                # subclass of ValueError -- despite ensure_single_read's
+                # docstring promising every rejection is a ValueError. Only
+                # TokenError, not the broader SqlglotError, is caught here: by
+                # the time control reaches this point `dialect` has already
+                # been validated once in load_contract_dir, so any other
+                # SqlglotError a future sqlglot version might raise from this
+                # call should surface as itself, not get relabeled as a
+                # rejected read.
+                raise ContractError(
+                    f"{label}: 'reads.{name}' must be a single read query ({exc})"
+                ) from exc
 
     raw_columns = raw.get("output_columns")
     if not isinstance(raw_columns, list) or not raw_columns:
@@ -297,6 +331,7 @@ def parse_node(
         extra_columns=extra_columns,
         config=config,
         content_hash=content_hash,
+        kind=kind,
     )
 
 

continuo_python_runtime/contract/merge.py

@@ -27,6 +27,7 @@ def node_entry(node: Node) -> dict:
         "owner": node.owner,
         "schedule": node.schedule,
         "criticality": node.criticality,
+        "kind": node.kind,
         "script": node.script,
         "reads": node.reads,
         "output_columns": [
@@ -128,12 +129,19 @@ def build_wire_contract(
     for node in nodes:
         entry = node_entry(node)
 
-        script_path = resolve_script_path(node.script, repo_root, context=node.relation)
-        script_bytes = script_path.read_bytes()
-        closure = resolve_closure(script_path, repo_root)
-        member_bytes = [member.read_bytes() for member in closure]
-        _lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
-        entry.update(hash_parts(entry, script_bytes, member_bytes))
+        if node.kind == "python-csv":
+            # A csv node has no script and no import closure: its source IS
+            # the uri -- new file content at the same uri is new data, not a
+            # new node version.
+            uri_bytes = node.reads["csv"].encode()
+            entry.update(hash_parts(entry, uri_bytes, []))
+        else:
+            script_path = resolve_script_path(node.script, repo_root, context=node.relation)
+            script_bytes = script_path.read_bytes()
+            closure = resolve_closure(script_path, repo_root)
+            member_bytes = [member.read_bytes() for member in closure]
+            _lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
+            entry.update(hash_parts(entry, script_bytes, member_bytes))
 
         wire_nodes.append(entry)
 

continuo_python_runtime/contract/model.py

@@ -6,6 +6,7 @@ from typing import Any
 # Module-level constants
 CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
 EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
+KINDS = frozenset({"python-model", "python-csv"})
 CONTRACT_VERSION = 1
 
 
@@ -34,6 +35,7 @@ class Node:
     extra_columns: str = "raise"
     config: dict[str, Any] = field(default_factory=dict)
     content_hash: str | None = None
+    kind: str = "python-model"
 
     @property
     def relation(self) -> str:

continuo_python_runtime/csv_loader.py

@@ -0,0 +1,68 @@
+"""continuo_python_runtime/csv_loader.py
+
+Producer for python-csv nodes: materialize the declared table from the csv
+source alone. Everything from conform() down (type coercion, extra_columns
+policy, ensure_table, transactional load) is the existing harness path —
+this module only turns the contract entry into a pyarrow Table.
+"""
+import logging
+import tempfile
+from pathlib import Path
+
+import pyarrow.csv  # type: ignore[import-untyped]
+
+from continuo_python_runtime.contract.model import Node
+from continuo_python_runtime.csv_readers import reader_for
+from continuo_python_runtime.csv_source import CsvSourceReader, parse_csv_uri
+from continuo_python_runtime.errors import LoadError
+from continuo_python_runtime.types import arrow_type, parse_sql_type
+
+logger = logging.getLogger("continuo_python_runtime.csv_loader")
+
+
+def produce_csv(node: Node, reader: CsvSourceReader | None = None) -> "pyarrow.Table":
+    """Fetch node.reads['csv'] and parse it (RFC4180 defaults) into a Table.
+
+    The caller conforms the result to output_columns exactly as for a script
+    node, so declared types — not csv inference — decide the warehouse schema.
+
+    ``output_columns`` names/types are also passed to ``read_csv`` itself as
+    its convert schema (via ``ConvertOptions.column_types``): pyarrow's default
+    type inference is otherwise the *first* place a value gets interpreted,
+    and it can destroy the very lexical value ``conform()`` is supposed to
+    preserve -- a VARCHAR column holding ``00123`` infers as int64 and
+    conform() writes back ``"123"``, and a NUMERIC column holding a valid
+    decimal like ``10.50`` infers as float64, which conform()'s own
+    lossy-cast guard then rejects outright. Reading every declared column
+    directly as its target Arrow type sidesteps both: the value is parsed
+    once, as the type it is actually declared to be.
+    """
+    uri = parse_csv_uri(node.reads["csv"])
+    active_reader = reader if reader is not None else reader_for(uri)
+    column_types = {
+        col.name: arrow_type(parse_sql_type(col.type)) for col in node.output_columns
+    }
+    convert_options = pyarrow.csv.ConvertOptions(column_types=column_types)
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            dest = active_reader.fetch(uri, Path(tmp) / "source.csv")
+            table = pyarrow.csv.read_csv(dest, convert_options=convert_options)
+    except LoadError:
+        raise
+    except Exception as exc:
+        raise LoadError(f"csv fetch failed for {node.relation}: {exc}") from exc
+    declared = {col.name for col in node.output_columns}
+    extras = set(table.column_names) - declared
+    if extras:
+        # Spec parity with the validation runner's csv_source header check
+        # (continuo_python_runtime/validation/runner.py): extra_columns: drop
+        # silently discards these at conform() time, so this structured
+        # warning is the only place the RUN path surfaces which columns were
+        # dropped.
+        logger.warning(
+            "csv_header_extra_columns node=%s columns=%s — columns present in the "
+            "csv but not declared in output_columns; they will not be loaded",
+            node.relation, sorted(extras))
+    logger.info("csv source %s: %d rows, columns=%s",
+                uri.raw, table.num_rows, table.column_names)
+    return table

continuo_python_runtime/csv_readers/__init__.py

@@ -0,0 +1,17 @@
+"""continuo_python_runtime/csv_readers/__init__.py"""
+from continuo_python_runtime.csv_source import CsvSourceReader, CsvUri
+from continuo_python_runtime.csv_readers.https import HttpsCsvSourceReader
+from continuo_python_runtime.csv_readers.s3 import S3CsvSourceReader
+
+
+def reader_for(uri: CsvUri) -> CsvSourceReader:
+    """Composition edge: pick the adapter for the parsed scheme.
+
+    parse_csv_uri already constrains uri.scheme to "s3" or "https", but the
+    dispatch stays explicit (rather than an s3/else fallback) so a scheme
+    added to the parser without a matching adapter fails loudly here too."""
+    if uri.scheme == "s3":
+        return S3CsvSourceReader()
+    if uri.scheme == "https":
+        return HttpsCsvSourceReader()
+    raise ValueError(f"unsupported csv scheme: {uri.scheme!r}")

continuo_python_runtime/csv_readers/https.py

@@ -0,0 +1,118 @@
+"""continuo_python_runtime/csv_readers/https.py"""
+import shutil
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from continuo_python_runtime.csv_source import (
+    HEADER_PROBE_BYTES,
+    MAX_HEADER_BYTES,
+    CsvSourceReader,
+    CsvUri,
+    extract_header_line,
+)
+
+# Both urlopen calls below must never hang forever: a source that accepts the
+# TCP connection but stalls on headers or body would otherwise wedge a
+# validation run or a scheduled node run indefinitely.
+_TIMEOUT_SECONDS = 30
+
+# Chunk size for the bounded read used when a server ignores our Range header
+# and answers 200: reading in chunks this small lets fetch_header_line stop
+# at the first newline (or MAX_HEADER_BYTES) without ever buffering a
+# multi-gigabyte body just to inspect its first line.
+_READ_CHUNK_BYTES = 65_536
+
+
+class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """Refuses to follow a redirect whose target is not itself https://.
+
+    urllib follows redirects automatically, and by default does not care
+    what scheme the target uses -- an https:// source that redirects to
+    http:// (or any other scheme) would otherwise silently downgrade both
+    the header probe and the full fetch to plaintext, defeating
+    parse_csv_uri's https-only restriction.
+    """
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):  # noqa: D102
+        if not newurl.lower().startswith("https://"):
+            raise urllib.error.HTTPError(
+                newurl, code,
+                f"refusing to follow redirect to non-https URL: {newurl}",
+                headers, fp,
+            )
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+_opener = urllib.request.build_opener(_HttpsOnlyRedirectHandler)
+
+
+def _read_bounded(resp, limit: int) -> bytes:
+    """Read ``resp`` in small chunks, stopping at the first newline or once
+    more than ``limit`` bytes have been buffered, whichever comes first.
+
+    Used for the range-ignored (status 200) path, where the server's
+    response is the whole object: an unbounded ``resp.read()`` there would
+    buffer a multi-gigabyte body in full just to read its header line.
+    """
+    buf = b""
+    while True:
+        chunk = resp.read(_READ_CHUNK_BYTES)
+        if not chunk:
+            return buf
+        buf += chunk
+        if b"\n" in buf or len(buf) > limit:
+            return buf
+
+
+class HttpsCsvSourceReader(CsvSourceReader):
+    """Reads a csv source over HTTPS (public URLs; no auth in v1). Mirrors
+    S3CsvSourceReader's probe-and-extend strategy: each ranged request is
+    independent, so a server that ignores Range and answers 200 (not 206)
+    is handled too -- its response is the whole object, so it is treated
+    as terminal on the first pass regardless of whether it contains a
+    newline or how large it is, rather than being re-fetched and
+    re-appended pass after pass."""
+
+    def fetch_header_line(self, uri: CsvUri) -> str:
+        start = 0
+        buf = b""
+        while True:
+            end = start + HEADER_PROBE_BYTES - 1
+            req = urllib.request.Request(
+                uri.raw, headers={"Range": f"bytes={start}-{end}"})
+            with _opener.open(req, timeout=_TIMEOUT_SECONDS) as resp:  # noqa: S310 — scheme gated by parse_csv_uri and _HttpsOnlyRedirectHandler
+                range_honoured = resp.status == 206
+                if range_honoured:
+                    body = resp.read()
+                else:
+                    # The server ignored our Range header and returned the
+                    # entire object (status 200): bound the read itself so a
+                    # multi-gigabyte body is never buffered in full, and
+                    # this response is terminal regardless of its size --
+                    # every retry would re-fetch the identical full body, so
+                    # looping would only re-append it pass after pass and
+                    # eventually trip a false MAX_HEADER_BYTES overflow.
+                    body = _read_bounded(resp, MAX_HEADER_BYTES)
+            if not range_honoured:
+                return extract_header_line(body, uri.raw)
+            buf += body
+            if b"\n" in buf:
+                return extract_header_line(buf, uri.raw)
+            if len(body) < HEADER_PROBE_BYTES:  # whole object read, no newline
+                return buf.rstrip(b"\r").decode("utf-8-sig")
+            if len(buf) > MAX_HEADER_BYTES:
+                raise ValueError(
+                    f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {uri.raw}")
+            start += HEADER_PROBE_BYTES
+
+    def fetch(self, uri: CsvUri, dest: Path) -> Path:
+        with (
+            _opener.open(uri.raw, timeout=_TIMEOUT_SECONDS) as resp,  # noqa: S310 — scheme gated by parse_csv_uri and _HttpsOnlyRedirectHandler
+            open(dest, "wb") as f,
+        ):
+            shutil.copyfileobj(resp, f)
+        return dest
+
+
+assert issubclass(HttpsCsvSourceReader, CsvSourceReader)

continuo_python_runtime/csv_readers/s3.py

@@ -0,0 +1,57 @@
+"""continuo_python_runtime/csv_readers/s3.py"""
+from pathlib import Path
+
+from botocore.exceptions import ClientError  # type: ignore[import-untyped]
+
+from continuo_python_runtime.csv_source import (
+    HEADER_PROBE_BYTES,
+    MAX_HEADER_BYTES,
+    CsvSourceReader,
+    CsvUri,
+    extract_header_line,
+)
+from continuo_python_runtime.validation.s3 import make_s3_client
+
+
+class S3CsvSourceReader(CsvSourceReader):
+    """Reads a csv source from S3. Reuses make_s3_client so S3_ENDPOINT_URL
+    (minio, localstack) and boto3's own credential chain behave identically
+    to the validation runner's existing S3 access."""
+
+    def fetch_header_line(self, uri: CsvUri) -> str:
+        client = make_s3_client()
+        start = 0
+        buf = b""
+        while True:
+            end = start + HEADER_PROBE_BYTES - 1
+            try:
+                body = client.get_object(
+                    Bucket=uri.bucket, Key=uri.key, Range=f"bytes={start}-{end}"
+                )["Body"].read()
+            except ClientError as exc:
+                if start == 0 and exc.response.get("Error", {}).get("Code") == "InvalidRange":
+                    # A Range request on byte 0 is unsatisfiable only when the
+                    # object itself is 0 bytes long: treat a 0-byte csv source
+                    # as an empty header line rather than a hard failure --
+                    # the caller (validation/runner.py) raises a clear error
+                    # for an empty header line.
+                    return ""
+                raise
+            buf += body
+            if b"\n" in buf:
+                return extract_header_line(buf, uri.raw)
+            if len(body) < HEADER_PROBE_BYTES:  # whole object read, no newline
+                return buf.rstrip(b"\r").decode("utf-8-sig")
+            if len(buf) > MAX_HEADER_BYTES:
+                raise ValueError(
+                    f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {uri.raw}")
+            start += HEADER_PROBE_BYTES
+
+    def fetch(self, uri: CsvUri, dest: Path) -> Path:
+        client = make_s3_client()
+        with open(dest, "wb") as f:
+            client.download_fileobj(uri.bucket, uri.key, f)
+        return dest
+
+
+assert issubclass(S3CsvSourceReader, CsvSourceReader)

continuo_python_runtime/csv_source.py

@@ -0,0 +1,96 @@
+"""CSV-source rules shared by the run harness and the validation runner:
+the URI grammar, the header-conformance rule, and the reader port both
+consumers depend on. Dependency-free — adapters that do I/O live in
+continuo_python_runtime/csv_readers/ and implement CsvSourceReader.
+"""
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+
+HEADER_PROBE_BYTES = 65_536   # first ranged fetch when probing for the header line
+MAX_HEADER_BYTES = 1_048_576  # a header line longer than this is a failure
+
+
+@dataclass(frozen=True)
+class CsvUri:
+    scheme: str  # "s3" | "https"
+    raw: str
+    bucket: str = ""  # s3 only
+    key: str = ""     # s3 only
+
+
+def parse_csv_uri(uri: str) -> CsvUri:
+    """Parse a csv source URI. Accepts exactly s3://bucket/key and https://...
+
+    Raises ValueError for anything else (http:// included), and for a
+    non-string ``uri`` (e.g. a contract's `reads: {csv: 123}`) rather than
+    letting `.startswith()` raise a bare AttributeError/TypeError -- callers
+    (the contract loader) catch ValueError to turn this into a ContractError,
+    so a malformed uri fails at lint/parse time, never at run time.
+    """
+    if not isinstance(uri, str):
+        raise ValueError(f"csv uri must be a string, got {type(uri).__name__}: {uri!r}")
+    if uri.startswith("s3://"):
+        bucket, _, key = uri[len("s3://"):].partition("/")
+        if not bucket or not key:
+            raise ValueError(f"invalid s3 csv uri (missing bucket or key): {uri!r}")
+        return CsvUri(scheme="s3", raw=uri, bucket=bucket, key=key)
+    if uri.startswith("https://"):
+        remainder = uri[len("https://"):]
+        host, _, _ = remainder.partition("/")
+        if host:
+            return CsvUri(scheme="https", raw=uri)
+    raise ValueError(
+        f"invalid csv uri {uri!r}: must be s3://bucket/key or https://..."
+    )
+
+
+def extract_header_line(data: bytes, source_desc: str) -> str:
+    """Return the first line of ``data`` (no trailing newline), decoded as
+    utf-8-sig so a UTF-8 byte-order mark on the CSV's first byte does not end
+    up prepended to the first column name.
+
+    Shared by every :class:`CsvSourceReader` adapter's ``fetch_header_line``:
+    the check must run on the *resolved line itself* (the bytes up to, and
+    including the absence of, the first newline), not merely on an
+    intermediate buffer length -- a newline that only arrives after the
+    accumulated buffer has already grown past ``MAX_HEADER_BYTES`` must still
+    be rejected as oversized, not returned as a successful (if enormous)
+    header line.
+
+    Raises:
+        ValueError: If the line exceeds ``MAX_HEADER_BYTES``.
+    """
+    line = data.split(b"\n", 1)[0] if b"\n" in data else data
+    if len(line) > MAX_HEADER_BYTES:
+        raise ValueError(f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {source_desc}")
+    return line.rstrip(b"\r").decode("utf-8-sig")
+
+
+def check_header(header_cols: list[str], declared_cols: list[str]) -> set[str]:
+    """Presence-only header conformance: every declared column must appear in
+    the CSV header, in any order. Returns the set of header columns NOT
+    declared (extras) so callers can surface them as a warning.
+
+    Raises ValueError naming every missing declared column.
+    """
+    header = set(header_cols)
+    missing = [c for c in declared_cols if c not in header]
+    if missing:
+        raise ValueError(f"csv header missing declared column(s): {sorted(missing)}")
+    return header - set(declared_cols)
+
+
+class CsvSourceReader(ABC):
+    """Port for reading a csv source. Implemented by csv_readers adapters;
+    consumed by the run harness (full fetch) and the validation runner
+    (header line only). The dependency arrow runs adapter -> this port."""
+
+    @abstractmethod
+    def fetch_header_line(self, uri: CsvUri) -> str:
+        """Return the CSV's first line (no trailing newline). Raises on an
+        unreachable source or a header longer than MAX_HEADER_BYTES."""
+
+    @abstractmethod
+    def fetch(self, uri: CsvUri, dest: Path) -> Path:
+        """Stream the full object to dest and return dest."""

continuo_python_runtime/harness.py

@@ -31,6 +31,8 @@ from continuo_python_runtime.context import RunContext
 from continuo_python_runtime.contract.loader import load_contract_dir
 from continuo_python_runtime.contract.model import Node
 from continuo_python_runtime.contract.paths import resolve_script_path
+from continuo_python_runtime.csv_loader import produce_csv
+from continuo_python_runtime.csv_source import CsvSourceReader
 from continuo_python_runtime.errors import ContractError, HarnessError, LoadError, ScriptError
 
 logger = logging.getLogger("continuo_python_runtime.harness")
@@ -206,9 +208,16 @@ def _validate_config_early(adapter: Any, node: Node) -> None:
         ) from exc
 
 
-def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
+def run_node(
+    env: Mapping[str, str], adapter: Any = None, reader: CsvSourceReader | None = None
+) -> int:
     """Run a single node end-to-end and print exactly one sentinel result block.
 
+    ``reader`` mirrors the ``adapter`` injection seam: when given, it is
+    threaded into :func:`produce_csv` for a python-csv node instead of
+    letting that function pick a reader via ``reader_for``. Ignored for a
+    python-model node.
+
     Returns 0 on success, 1 on any :class:`HarnessError`.
     """
     node_id = env.get("NODE_ID") or ""
@@ -246,13 +255,15 @@ def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
 
         _validate_config_early(active_adapter, node)
 
-        with contextlib.redirect_stdout(sys.stderr):
-            module = load_script(node, app_root)
-
-        ctx = RunContext(node, active_adapter)
-        raw_result = _execute_script(module, ctx)
+        if node.kind == "python-csv":
+            table = produce_csv(node, reader=reader)
+        else:
+            with contextlib.redirect_stdout(sys.stderr):
+                module = load_script(node, app_root)
+            ctx = RunContext(node, active_adapter)
+            raw_result = _execute_script(module, ctx)
+            table = to_arrow(raw_result)
 
-        table = to_arrow(raw_result)
         conformed = conform(table, node.output_columns, node.extra_columns)
 
         columns = [

continuo_python_runtime/validation/runner.py

@@ -8,10 +8,16 @@ Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
 - ``build_from_columns``: for python nodes, which have no SELECT to shape their output
   from. Fetch the node's validation spec JSON from S3 (``CANDIDATE_SPEC_URI`` —
   ``{"reads": [sql, ...], "output_columns": [{"name","type","nullable"}, ...],
-  "config": {...}}``; ``config`` is optional and defaults to ``{}``), bind-check every
-  declared read against the candidate schema so an upstream that
-  dropped a column the script reads fails the release gate, then materialize the
-  output table empty from the declared typed columns and the declared physical layout.
+  "config": {...}, "csv_source": "s3://..." | "https://..."}``; ``config`` and
+  ``csv_source`` are both optional. When ``csv_source`` is set, its header row is
+  fetched (without downloading the full object) and checked against the declared
+  output columns: a declared column missing from the header fails the release gate,
+  while a header column not declared only logs a ``csv_header_extra_columns``
+  warning (it is silently dropped at load time). ``config`` defaults to ``{}``.
+  Every declared read is bind-checked against the candidate schema so an upstream
+  that dropped a column the script reads fails the release gate, then the output
+  table is materialized empty from the declared typed columns and the declared
+  physical layout.
 
 The engine adapter is discovered from the single installed
 ``continuo_engine.adapters`` entry point — each runner image installs exactly one.
@@ -19,6 +25,7 @@ stdout is reserved exclusively for the runner's one structured ``result_block``,
 printed as its last line; all diagnostics go to stderr via the ``logging`` module.
 A non-zero exit marks the node failed.
 """
+import csv
 import json
 import logging
 import os
@@ -29,6 +36,8 @@ from continuo_engine_contract.port import (  # type: ignore[import-untyped]
     AdapterDiscoveryError,
     discover_adapter,
 )
+from continuo_python_runtime.csv_readers import reader_for
+from continuo_python_runtime.csv_source import check_header, parse_csv_uri
 from continuo_python_runtime.validation import s3
 
 logger = logging.getLogger("validation_runner")
@@ -126,6 +135,7 @@ def main() -> None:
     prod_schema = None
     spec: dict | None = None
     config: dict = {}
+    csv_source: str = ""
     if op in _NODE_OPS:
         table = _require("TABLE_NAME")
         unique_id = _node_id() or f"model.{table}"
@@ -175,6 +185,16 @@ def main() -> None:
                 print(result.result_block("error", msg, unique_id=unique_id), flush=True)
                 sys.exit(2)
             config = raw_config or {}
+            csv_source = spec.get("csv_source", "")
+            if "csv_source" in spec and not isinstance(csv_source, str):
+                # Checked by presence, not truthiness: `csv_source: 0` / `false` /
+                # `[]` / `{}` are all non-string values a malformed spec could
+                # carry, and a truthiness guard would silently skip both this
+                # type check and the header check below for every one of them.
+                msg = f"candidate spec 'csv_source' must be a string, got {type(csv_source).__name__}"
+                logger.error("%s", msg)
+                print(result.result_block("error", msg, unique_id=unique_id), flush=True)
+                sys.exit(2)
         else:
             prod_schema = _require("PROD_SCHEMA")
     elif op in _SCHEMA_OPS:
@@ -220,6 +240,21 @@ def main() -> None:
                 adapter.build_empty_from_sql(schema, table, candidate_sql)
             elif op == "build_from_columns":
                 assert spec is not None, "spec must be set for build_from_columns"
+                if csv_source:
+                    csv_uri = parse_csv_uri(csv_source)
+                    header_line = reader_for(csv_uri).fetch_header_line(csv_uri)
+                    if not header_line:
+                        raise ValueError(
+                            "csv source has no header line (empty or unreadable): "
+                            f"{csv_source}"
+                        )
+                    declared = [c["name"] for c in spec["output_columns"]]
+                    extras = check_header(next(csv.reader([header_line])), declared)
+                    if extras:
+                        logger.warning(
+                            "csv_header_extra_columns node=%s columns=%s — columns present in the "
+                            "csv but not declared in output_columns; they will not be loaded",
+                            unique_id, sorted(extras))
                 for read_sql in spec.get("reads", []):
                     adapter.check_binds(read_sql)
                 adapter.build_empty_from_columns(schema, table, spec["output_columns"], config)

{continuo_python_runtime-0.3.1.dist-info → continuo_python_runtime-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: continuo-python-runtime
-Version: 0.3.1
+Version: 0.4.0
 Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
 Author: Simone Carolini
 Maintainer: Simone Carolini
@@ -12,7 +12,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
 Requires-Dist: boto3==1.43.59
-Requires-Dist: continuo-engine-contract==0.7.1
+Requires-Dist: continuo-engine-contract==0.7.2
 Requires-Dist: pyarrow==25.0.0
 Requires-Dist: pyyaml==6.0.3
 Requires-Dist: sqlglot==30.15.0
@@ -114,7 +114,9 @@ the Go parser has not been taught is a production outage, not a refactor.
    login` step to `release.yml`.
 5. Write a contract file under `contracts/` (see
    `template/contracts/example.yml`) and a script under `scripts/` that
-   implements `run(ctx)` (see `template/scripts/example.py`).
+   implements `run(ctx)` (see `template/scripts/example.py`). A node that
+   only needs to land a csv file needs no script at all — see
+   `template/contracts/example_csv.yml` and "Node kinds" below.
 6. Push to `main`. The `release.yml` workflow lints the scripts, validates
    and merges the contracts, builds and pushes the image, uploads the merged
    contract to S3, and POSTs the release.
@@ -141,6 +143,30 @@ The runtime image does not re-run this gate, so a read that passes here is
 not re-judged under a different grammar in production. See
 `docs/boundary-contract.md` §13.1.
 
+## Node kinds
+
+A contract node's `kind:` field selects how the node produces its rows.
+Every rule below (`extra_columns`, `output_columns`, "Conform rules") applies
+to both kinds identically — `kind` only changes how the pre-conform table is
+produced, never how it is checked or written.
+
+- **`python-model`** (the default; the field may be omitted) — a script node.
+  It requires `script:` and a `reads:` map of one or more named SQL queries,
+  as described in "The script API" below.
+- **`python-csv`** — a contract-only node: it has no script and its `reads:`
+  map must be exactly `{csv: <uri>}`, where the uri is `s3://bucket/key` or
+  an `https://` url (`http://` is rejected at validate time, not run time).
+  The harness fetches the file, parses it with RFC 4180 defaults, and feeds
+  the result straight into `conform()` — declared `output_columns` types
+  decide the warehouse schema, not whatever pyarrow infers from the csv.
+  Because there is no script, `script:` is a forbidden key for this kind;
+  `continuo-runtime validate`/`merge`/`lint` reject one that sets it. The
+  csv's header row must contain every declared output column (checked again,
+  independently, at release time before promotion); columns present in the
+  header but not declared are governed by the same `extra_columns` policy as
+  a script node's output — `raise` (default) fails the run, `warn` drops
+  them and logs a warning. See `template/contracts/example_csv.yml`.
+
 ## The script API
 
 A node script is a Python file with exactly one required entry point:
@@ -236,9 +262,9 @@ A domain repo picks its warehouse engine by which base image it builds
 `FROM`:
 
 ```dockerfile
-FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.3.1
+FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.4.0
 # or
-FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.3.1
+FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.4.0
 ```
 
 The engine is part of the image **name**; the tag is the bare version, so

continuo_python_runtime-0.4.0.dist-info/RECORD

@@ -0,0 +1,29 @@
+continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
+continuo_python_runtime/cli.py,sha256=X4JBC4FDpgFOeDdtBbeX9GTq1XTuJCghDpemlR_CPvE,5925
+continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
+continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
+continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7DLZfVg,1856
+continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
+continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
+continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
+continuo_python_runtime/harness.py,sha256=BBJBmmiQ922WVmPi0Xh7CFnXjNxKefPey0YOEHercjs,12634
+continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
+continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
+continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
+continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
+continuo_python_runtime/contract/loader.py,sha256=oUer1sbaxhUMNQVUuqO5j5vbQCrgwuHOWJzGe0MHSLg,16973
+continuo_python_runtime/contract/merge.py,sha256=36awQV8cHGcQJv6XJGC_mC5MuyRS5-7XTVTLw0sjMgE,6903
+continuo_python_runtime/contract/model.py,sha256=L7GwSYKrh6Z0J1c1zWFHQw8GzYUUzzWycuBw5Qq5Dfg,1017
+continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
+continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
+continuo_python_runtime/csv_readers/https.py,sha256=tMJ9k5F0b9SAydpon0ZlOznfjvjhF2BBLltyVmvRneE,4998
+continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nBiAsHRXXe1mE0,2246
+continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
+continuo_python_runtime/validation/runner.py,sha256=AwYbiFrQubxSm1vfxJJpnp7xfekviUW2anw4LRQQ4Rc,13062
+continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
+continuo_python_runtime-0.4.0.dist-info/METADATA,sha256=JtdvcXJZtH27JKMGd5m1Rm6wvch7MzzrsvvAawOQkCU,17023
+continuo_python_runtime-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+continuo_python_runtime-0.4.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
+continuo_python_runtime-0.4.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
+continuo_python_runtime-0.4.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
+continuo_python_runtime-0.4.0.dist-info/RECORD,,

continuo_python_runtime-0.3.1.dist-info/RECORD

@@ -1,24 +0,0 @@
-continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
-continuo_python_runtime/cli.py,sha256=X4JBC4FDpgFOeDdtBbeX9GTq1XTuJCghDpemlR_CPvE,5925
-continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
-continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
-continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7DLZfVg,1856
-continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
-continuo_python_runtime/harness.py,sha256=1b_ze2_AJ9vncY-yj88TgJylVzKwGw1-idxHoaYAymE,12101
-continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
-continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
-continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
-continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
-continuo_python_runtime/contract/loader.py,sha256=WeKngAe6VH-WxgRTVdCERPMdsGBszJM9RkanUTTs9-w,15611
-continuo_python_runtime/contract/merge.py,sha256=XCCeE2vJUm0KA60d2XsHpjRh5fIdVfFu_HsBzEyUWow,6505
-continuo_python_runtime/contract/model.py,sha256=2hKI04M0WwvTkOy9eDGl2dRZgY3bAuJYktAG-ICI-AU,936
-continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
-continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
-continuo_python_runtime/validation/runner.py,sha256=zHs5CR3Wr9512sh6eY9hpN8NG3weJPYUFRDPJf2BNnc,10875
-continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
-continuo_python_runtime-0.3.1.dist-info/METADATA,sha256=CbkAD-RaAJiq31POHm89uDUFgQEYQMN2DCCfwhAJqVo,15416
-continuo_python_runtime-0.3.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-continuo_python_runtime-0.3.1.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
-continuo_python_runtime-0.3.1.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
-continuo_python_runtime-0.3.1.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
-continuo_python_runtime-0.3.1.dist-info/RECORD,,