dirigent-plugin 0.27.0__tar.gz → 0.28.0__tar.gz

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: dirigent-plugin
-Version: 0.27.0
+Version: 0.28.0
 Summary: The dirigent plugin contract: block specs, protocols, and markers.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: dirigent-common==0.27.0
+Requires-Dist: dirigent-common==0.28.0
 Requires-Dist: httpx2>=2.12.0
 Requires-Dist: pluginkit>=0.5.0
 Requires-Dist: pydantic>=2.13.5

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-plugin"
-version = "0.27.0"
+version = "0.28.0"
 description = "The dirigent plugin contract: block specs, protocols, and markers."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,7 +11,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-common==0.27.0",
+    "dirigent-common==0.28.0",
     "httpx2>=2.12.0",
     "pluginkit>=0.5.0",
     "pydantic>=2.13.5",

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-plugin"
-version = "0.27.0"
+version = "0.28.0"
 description = "The dirigent plugin contract: block specs, protocols, and markers."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -11,7 +11,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "dirigent-common==0.27.0",
+    "dirigent-common==0.28.0",
     "httpx2>=2.12.0",
     "pluginkit>=0.5.0",
     "pydantic>=2.13.5",

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/__init__.py

@@ -44,8 +44,13 @@ from dirigent_plugin.blocks import (
     StorageBackend,
     classify_default,
     mark_refusal,
+    media_fields,
+    media_fields_of,
+    published_config_schema,
     reference_fields,
+    shell_fields_of,
     shell_string_fields,
+    sql_values_of,
 )
 from dirigent_plugin.contribution import Contribution, merge_contributions
 from dirigent_plugin.markers import (
@@ -70,6 +75,9 @@ from dirigent_plugin.transforms import (
     Transformer,
     TransformError,
     TransformOutput,
+    csv_header,
+    hold_csv_columns,
+    hold_csv_empty_cells,
 )
 
 __all__ = [
@@ -128,6 +136,9 @@ __all__ = [
     "StorageBackend",
     "TransformConfig",
     "TransformError",
+    "csv_header",
+    "hold_csv_columns",
+    "hold_csv_empty_cells",
     "TransformOutput",
     "Transformer",
     "classify_default",
@@ -140,4 +151,9 @@ __all__ = [
     "merge_contributions",
     "reference_fields",
     "shell_string_fields",
+    "media_fields",
+    "media_fields_of",
+    "shell_fields_of",
+    "published_config_schema",
+    "sql_values_of",
 ]

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/blocks.py

@@ -1,5 +1,6 @@
 """The dirigent block contract: the only module a third-party plugin package needs to import."""
 
+import functools
 import re
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
@@ -18,8 +19,10 @@ from pydantic_core import CoreSchema
 
 from dirigent_common import (
     SHELL_MEDIA_TYPE,
+    SQL_MEDIA_TYPE,
     BlockModel,
     HealthReport,
+    HumaneJsonSchema,
     Issue,
     JsonMap,
     Message,
@@ -237,6 +240,9 @@ def reference_fields(schema: JsonMap, kind: str) -> set[str]:
     return named
 
 
+#: The schema keywords that say a value takes one or all of several shapes.
+_UNION_KEYWORDS: Final = ("anyOf", "oneOf", "allOf")
+
 #: What the engine names the variables it substitutes a shell string's references out into.
 SHELL_VARIABLE_PREFIX: Final = "DIRIGENT_V"
 
@@ -275,6 +281,148 @@ def shell_string_fields(model: type[BaseModel]) -> frozenset[str]:
     )
 
 
+def shell_fields_of(schema: JsonMap) -> frozenset[str]:
+    """Name the fields a config schema publishes as a shell string, or an optional one."""
+    properties = schema.get("properties")
+    if not isinstance(properties, dict):
+        return frozenset()
+    return frozenset(
+        str(name)
+        for name, declared in cast("dict[str, Any]", properties).items()
+        if isinstance(declared, dict) and _publishes_shell(cast("dict[str, Any]", declared))
+    )
+
+
+def _publishes_shell(shape: dict[str, Any]) -> bool:
+    """Whether a field's schema or a branch of its union carries the shell media type."""
+    options = cast("list[dict[str, Any]]", shape.get("anyOf") or [])
+    return shape.get("contentMediaType") == SHELL_MEDIA_TYPE or any(_publishes_shell(option) for option in options)
+
+
+def sql_values_of(config: JsonMap, schema: JsonMap) -> dict[str, JsonValue]:
+    """Find what a config holds where its schema publishes SQL, by dotted location.
+
+    A reference in one of these is refused, so a value only ever reaches SQL as a bound
+    parameter. The walk follows the config through the schema's objects, lists, maps, unions and
+    named definitions, so a pack's nested field is found wherever it sits. A string standing in
+    for a shape with SQL anywhere under it counts too, because a whole reference there would
+    resolve to the object or the list holding the statement.
+    """
+    held = schema.get("$defs")
+    pool = cast("dict[str, Any]", held) if isinstance(held, dict) else {}
+    found: dict[str, JsonValue] = {}
+    _collect_sql(config, schema, pool, "", found)
+    return found
+
+
+def _collect_sql(value: Any, shape: dict[str, Any], pool: dict[str, Any], where: str, found: dict[str, Any]) -> None:
+    """Record every value under ``value`` at a place ``shape`` publishes as SQL."""
+    published = shape.get("contentMediaType") == SQL_MEDIA_TYPE
+    if where and (published or (isinstance(value, str) and _holds_sql(shape, pool))):
+        found[where] = value
+    target = _pointed_at(shape, pool)
+    if target is not None:
+        _collect_sql(value, target, pool, where, found)
+    for keyword in _UNION_KEYWORDS:
+        for option in cast("list[Any]", shape.get(keyword) or []):
+            if isinstance(option, dict):
+                _collect_sql(value, cast("dict[str, Any]", option), pool, where, found)
+    if isinstance(value, dict):
+        for key, element in cast("dict[str, Any]", value).items():
+            child = _property_shape(shape, str(key))
+            if child is not None:
+                _collect_sql(element, child, pool, _extended(where, str(key)), found)
+    elif isinstance(value, list):
+        elements: list[Any] = value  # pyright: ignore[reportUnknownVariableType]
+        for index, element in enumerate(elements):
+            child = _item_shape(shape, index)
+            if child is not None:
+                _collect_sql(element, child, pool, _extended(where, str(index)), found)
+
+
+def _extended(where: str, part: str) -> str:
+    """Add one step to a dotted location."""
+    return f"{where}.{part}" if where else part
+
+
+def _property_shape(shape: dict[str, Any], key: str) -> dict[str, Any] | None:
+    """The shape one key of an object takes, from its own property or the rest of the map."""
+    properties = shape.get("properties")
+    declared = cast("dict[str, Any]", properties).get(key) if isinstance(properties, dict) else None
+    if isinstance(declared, dict):
+        return cast("dict[str, Any]", declared)
+    patterns = shape.get("patternProperties")
+    if isinstance(patterns, dict):
+        for pattern, matched in cast("dict[str, Any]", patterns).items():
+            if isinstance(matched, dict) and re.search(pattern, key):
+                return cast("dict[str, Any]", matched)
+    rest = shape.get("additionalProperties")
+    return cast("dict[str, Any]", rest) if isinstance(rest, dict) else None
+
+
+def _item_shape(shape: dict[str, Any], index: int) -> dict[str, Any] | None:
+    """The shape one position of a list takes, from its own place in a tuple or the items."""
+    positions = shape.get("prefixItems")
+    if isinstance(positions, list):
+        places: list[Any] = positions  # pyright: ignore[reportUnknownVariableType]
+        if index < len(places):
+            return cast("dict[str, Any]", places[index]) if isinstance(places[index], dict) else None
+    items = shape.get("items")
+    return cast("dict[str, Any]", items) if isinstance(items, dict) else None
+
+
+def _holds_sql(shape: dict[str, Any], pool: dict[str, Any], seen: frozenset[int] = frozenset()) -> bool:
+    """Whether a shape publishes SQL itself or anywhere under it, across definitions that recurse."""
+    if id(shape) in seen:
+        return False
+    if shape.get("contentMediaType") == SQL_MEDIA_TYPE:
+        return True
+    walked = seen | {id(shape)}
+    below: list[Any] = []
+    target = _pointed_at(shape, pool)
+    if target is not None:
+        below.append(target)
+    for keyword in (*_UNION_KEYWORDS, "prefixItems"):
+        below.extend(cast("list[Any]", shape.get(keyword) or []))
+    for keyword in ("properties", "patternProperties"):
+        held = shape.get(keyword)
+        if isinstance(held, dict):
+            below.extend(cast("dict[str, Any]", held).values())
+    for keyword in ("items", "additionalProperties"):
+        below.append(shape.get(keyword))
+    return any(isinstance(one, dict) and _holds_sql(cast("dict[str, Any]", one), pool, walked) for one in below)
+
+
+@functools.cache
+def published_config_schema(model: type[BaseModel]) -> JsonMap:
+    """The schema the catalog publishes for a config model, which SQL fields are read from."""
+    return model.model_json_schema(mode="serialization", schema_generator=HumaneJsonSchema)
+
+
+def media_fields_of(schema: JsonMap, media_type: str) -> frozenset[str]:
+    """Name the fields a config schema publishes as one media type: a string, or an optional one."""
+    properties = schema.get("properties")
+    if not isinstance(properties, dict):
+        return frozenset()
+    return frozenset(
+        str(name)
+        for name, declared in cast("dict[str, Any]", properties).items()
+        if isinstance(declared, dict) and _publishes(cast("dict[str, Any]", declared), media_type)
+    )
+
+
+def _publishes(shape: dict[str, Any], media_type: str) -> bool:
+    """Whether a field's schema or a branch of its union carries a media type."""
+    options = cast("list[dict[str, Any]]", shape.get("anyOf") or [])
+    return shape.get("contentMediaType") == media_type or any(_publishes(option, media_type) for option in options)
+
+
+@functools.cache
+def media_fields(model: type[BaseModel], media_type: str) -> frozenset[str]:
+    """List the config fields a model publishes as one media type, by the schema the catalog publishes."""
+    return media_fields_of(model.model_json_schema(mode="serialization", schema_generator=HumaneJsonSchema), media_type)
+
+
 class RemoteHandle(BaseModel):
     """Serializable claim on a job running somewhere other than this block call.
 
@@ -526,6 +674,9 @@ class RunState(StrEnum):
 
     QUEUED = "queued"
     RUNNING = "running"
+    PAUSED = "paused"
+    """Held by an operator; it goes on once resumed."""
+
     SUCCEEDED = "succeeded"
     COMPLETED_WITH_ERRORS = "completed_with_errors"
     FAILED = "failed"

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/messages.py

@@ -15,6 +15,19 @@ FILTER_ANSWER = PLUGIN.define(
     "{block} answered {answer} for element {index}, and a filter's answer is true or false",
 )
 
+CSV_TOO_MANY_COLUMNS = PLUGIN.define(
+    "csv_too_many_columns",
+    "the records name more than {limit} distinct keys, and each is one csv column; "
+    "keep the keys you need with a transform first",
+)
+
+CSV_TOO_SPARSE = PLUGIN.define(
+    "csv_too_sparse",
+    "{rows} rows by {columns} columns leave {empty} cells empty, more than the {limit} a csv here may hold; "
+    "records that each carry keys of their own have no table worth writing, "
+    "so give them shared keys with a transform first",
+)
+
 UNSUPPORTED_PAIR = PLUGIN.define(
     "unsupported_pair",
     "{block} does not convert {source_format} to {target_format} ({supported})",

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/runners.py

@@ -24,6 +24,7 @@ pipe closing is how the parent says it is done.
 import asyncio
 import atexit
 import json
+import os
 import subprocess
 from collections.abc import Callable, Sequence
 from contextvars import ContextVar
@@ -40,6 +41,10 @@ from dirigent_plugin.transforms import Engine, ProgramConfig, TransformError
 #: TransformError, because nothing about the program says it will happen again.
 RUNNER_STOPPED = "the process running the program stopped before it answered"
 
+#: All of the worker's environment a runner starts with. A program runs behind no allowlist,
+#: and the worker's environment holds the instance's secrets.
+RUNNER_ENVIRONMENT = ("PATH", "LANG", "LC_ALL", "TZ")
+
 
 class ProgramRunner:
     """One runner process, and the two pipes a program and its outputs cross."""
@@ -51,6 +56,7 @@ class ProgramRunner:
             self.command,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
+            env={name: os.environ[name] for name in RUNNER_ENVIRONMENT if name in os.environ},
         )
         # Popen types both pipes as optional; both were asked for.
         self._to = cast("IO[bytes]", self.process.stdin)

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/sources.py

@@ -5,6 +5,8 @@ already settled and its output is the input. A kind is hosted one of three ways:
 receives it, a worker polls it, or a worker holds it open.
 """
 
+import hashlib
+import hmac
 import re
 from abc import ABC, abstractmethod
 from contextlib import AbstractAsyncContextManager
@@ -76,8 +78,17 @@ class SourceAuth(BaseModel):
     signature of the body, made with the source's signing secret; ``public`` admits anyone."""
 
     header: str = Field(default="X-Dirigent-Signature", min_length=1)
-    """For ``hmac``, the header the signature arrives in: the hex HMAC-SHA256 of the raw body,
-    with or without a leading ``sha256=``."""
+    """For ``hmac``, the header the signature arrives in: the hex HMAC-SHA256 of the timestamp
+    in ``X-Dirigent-Timestamp``, a dot, and the raw body, with or without a leading ``sha256=``."""
+
+
+#: The header a signed call carries the moment it was signed in, as whole Unix seconds.
+SIGNATURE_TIMESTAMP_HEADER: Final = "X-Dirigent-Timestamp"
+
+
+def sign_body(secret: bytes, timestamp: str, body: bytes) -> str:
+    """Compute the hex signature a signed source verifies: HMAC-SHA256 over ``<timestamp>.<body>``."""
+    return hmac.new(secret, timestamp.encode("ascii") + b"." + body, hashlib.sha256).hexdigest()
 
 
 class SourceRate(BaseModel):

{dirigent_plugin-0.27.0 → dirigent_plugin-0.28.0}/src/dirigent_plugin/transforms.py

@@ -25,7 +25,7 @@ misses both. Every frame hands a step's engine work to ``Engine.offload`` and aw
 
 import asyncio
 from abc import ABC, abstractmethod
-from collections.abc import Callable
+from collections.abc import Callable, Mapping, Sequence
 from typing import Any, ClassVar, Final, cast
 
 from pydantic import BaseModel, Field, JsonValue
@@ -40,6 +40,8 @@ from dirigent_plugin.blocks import (
     StepContext,
 )
 from dirigent_plugin.messages import (
+    CSV_TOO_MANY_COLUMNS,
+    CSV_TOO_SPARSE,
     ELEMENT_FAILED,
     FILTER_ANSWER,
     NOT_AN_ARRAY,
@@ -59,6 +61,14 @@ MAP_PROMISE: Final = "replaces every element of a list"
 FILTER_PROMISE: Final = "keeps some of the elements of a list"
 
 
+#: The most columns a csv a converter writes may have.
+CSV_COLUMN_LIMIT: Final = 10_000
+
+#: The most empty cells a csv a converter writes may hold, each of them a separator with
+#: nothing in the source behind it.
+CSV_EMPTY_CELL_LIMIT: Final = 10_000_000
+
+
 class TransformError(Exception):
     """An engine refused what it was given: a program it cannot compile, or a value it cannot reshape.
 
@@ -385,6 +395,41 @@ class Filterer(Engine, Operator[ProgramConfig, TransformOutput], ABC):
         return []
 
 
+def hold_csv_columns(columns: int) -> None:
+    """Refuse a csv with more columns than a converter writes."""
+    if columns > CSV_COLUMN_LIMIT:
+        raise TransformError(CSV_TOO_MANY_COLUMNS.render(limit=f"{CSV_COLUMN_LIMIT:,}"))
+
+
+def hold_csv_empty_cells(*, rows: int, columns: int, empty: int) -> None:
+    """Refuse a csv holding more empty cells than a converter writes."""
+    if empty > CSV_EMPTY_CELL_LIMIT:
+        raise TransformError(
+            CSV_TOO_SPARSE.render(
+                rows=f"{rows:,}",
+                columns=f"{columns:,}",
+                empty=f"{empty:,}",
+                limit=f"{CSV_EMPTY_CELL_LIMIT:,}",
+            )
+        )
+
+
+def csv_header(records: Sequence[Mapping[str, Any]]) -> list[str]:
+    """The header of a csv these records write: every key any has, in first-seen order.
+
+    Every row spells a cell for every column, so records that each carry a key of their own
+    write rows times columns cells from a source the size of one of those. That is refused
+    before a row is written, while a dense table of any size is not.
+    """
+    columns: dict[str, None] = {}
+    for record in records:
+        columns.update(dict.fromkeys(record))
+        hold_csv_columns(len(columns))
+    filled = sum(value is not None for record in records for value in record.values())
+    hold_csv_empty_cells(rows=len(records), columns=len(columns), empty=len(records) * len(columns) - filled)
+    return list(columns)
+
+
 class Converter(Engine, Operator[ConvertConfig, ConvertOutput], ABC):
     """The ``convert`` verb: re-encode bytes from one format into another, content preserved.