dirigent-core 0.23.3__tar.gz → 0.24.0__tar.gz

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirigent-core
-Version: 0.23.3
+Version: 0.24.0
 Summary: Dirigent engine core: schema, configuration, and plugin host.
 License-Expression: LicenseRef-Proprietary
 License-File: LICENSE
@@ -12,9 +12,9 @@ Requires-Dist: argon2-cffi>=25.1.0
 Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: cronsim>=2.7
 Requires-Dist: cryptography>=46.0.5
-Requires-Dist: dirigent-client==0.23.3
-Requires-Dist: dirigent-common==0.23.3
-Requires-Dist: dirigent-plugin==0.23.3
+Requires-Dist: dirigent-client==0.24.0
+Requires-Dist: dirigent-common==0.24.0
+Requires-Dist: dirigent-plugin==0.24.0
 Requires-Dist: jsonschema>=4.26.0
 Requires-Dist: opentelemetry-api>=1.44.0
 Requires-Dist: pydantic-settings>=2.15.0

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.23.3"
+version = "0.24.0"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -17,9 +17,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.23.3",
-    "dirigent-common==0.23.3",
-    "dirigent-plugin==0.23.3",
+    "dirigent-client==0.24.0",
+    "dirigent-common==0.24.0",
+    "dirigent-plugin==0.24.0",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "dirigent-core"
-version = "0.23.3"
+version = "0.24.0"
 description = "Dirigent engine core: schema, configuration, and plugin host."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -17,9 +17,9 @@ dependencies = [
     "asyncpg>=0.31.0",
     "cronsim>=2.7",
     "cryptography>=46.0.5",
-    "dirigent-client==0.23.3",
-    "dirigent-common==0.23.3",
-    "dirigent-plugin==0.23.3",
+    "dirigent-client==0.24.0",
+    "dirigent-common==0.24.0",
+    "dirigent-plugin==0.24.0",
     "jsonschema>=4.26.0",
     "opentelemetry-api>=1.44.0",
     "pydantic-settings>=2.15.0",

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/src/dirigent_core/alembic/versions/0001_baseline_schema.py

@@ -545,6 +545,71 @@ def upgrade() -> None:
         batch_op.create_index(batch_op.f("ix_watches_pipeline_id"), ["pipeline_id"], unique=False)
         batch_op.create_index(batch_op.f("ix_watches_trigger_document_id"), ["trigger_document_id"], unique=False)
 
+    op.create_table(
+        "sources",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("pipeline_id", sa.Uuid(), nullable=False),
+        sa.Column("code", sa.String(length=200), nullable=False),
+        sa.Column("name", sa.String(length=200), nullable=True),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("kind", sa.String(length=200), nullable=False),
+        sa.Column("config", sa.JSON().with_variant(postgresql.JSONB(astext_type=Text()), "postgresql"), nullable=False),
+        sa.Column("token_hash", sa.String(length=128), nullable=True),
+        sa.Column("token_prefix", sa.String(length=16), nullable=True),
+        sa.Column("paused", sa.Boolean(), server_default=sa.false(), nullable=False),
+        sa.Column("accepted", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("refused", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("last_input_at", dirigent_core.types.UtcDateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            dirigent_core.types.UtcDateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            dirigent_core.types.UtcDateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["pipeline_id"], ["pipelines.id"], name=op.f("fk_sources_pipeline_id_pipelines"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")),
+        sa.UniqueConstraint("pipeline_id", "code", name=op.f("uq_sources_pipeline_id_code")),
+        sa.UniqueConstraint("token_hash", name=op.f("uq_sources_token_hash")),
+    )
+    with op.batch_alter_table("sources", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_sources_pipeline_id"), ["pipeline_id"], unique=False)
+
+    op.create_table(
+        "source_routes",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("source_id", sa.Uuid(), nullable=False),
+        sa.Column("method", sa.String(length=16), nullable=False),
+        sa.Column("template", sa.String(length=1000), nullable=False),
+        sa.Column("normalised", sa.String(length=1000), nullable=False),
+        sa.Column(
+            "created_at",
+            dirigent_core.types.UtcDateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            dirigent_core.types.UtcDateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_id"], ["sources.id"], name=op.f("fk_source_routes_source_id_sources"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_source_routes")),
+        sa.UniqueConstraint("method", "normalised", name=op.f("uq_source_routes_method_normalised")),
+    )
+    with op.batch_alter_table("source_routes", schema=None) as batch_op:
+        batch_op.create_index(batch_op.f("ix_source_routes_source_id"), ["source_id"], unique=False)
+
     op.create_table(
         "runs",
         sa.Column("id", sa.Uuid(), nullable=False),
@@ -591,6 +656,7 @@ def upgrade() -> None:
         ),
         sa.Column("triggered_by_id", sa.Uuid(), nullable=True),
         sa.Column("triggered_by_label", sa.String(length=200), nullable=True),
+        sa.Column("source_step", sa.String(length=200), nullable=True),
         sa.Column("traceparent", sa.String(length=64), nullable=True),
         sa.Column("error", sa.Text(), nullable=True),
         sa.Column("error_code", sa.String(length=128), nullable=True),
@@ -639,6 +705,7 @@ def upgrade() -> None:
             "ix_runs_pipeline_id_status_created_at", ["pipeline_id", "status", "created_at"], unique=False
         )
         batch_op.create_index(batch_op.f("ix_runs_status"), ["status"], unique=False)
+        batch_op.create_index("ix_runs_triggered_by_id_status", ["triggered_by_id", "status"], unique=False)
 
     op.create_table(
         "notifications",
@@ -838,6 +905,43 @@ def upgrade() -> None:
         batch_op.create_index("ix_webhook_deliveries_created_at", ["created_at"], unique=False)
         batch_op.create_index("ix_webhook_deliveries_webhook_id_id", ["webhook_id", "id"], unique=False)
 
+    op.create_table(
+        "source_inputs",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), autoincrement=True, nullable=False),
+        sa.Column("source_id", sa.Uuid(), nullable=False),
+        sa.Column("run_id", sa.Uuid(), nullable=True),
+        sa.Column(
+            "outcome",
+            sa.Enum("accepted", "refused", name="input_outcome", native_enum=False, length=32),
+            nullable=False,
+        ),
+        sa.Column("status", sa.Integer(), nullable=False),
+        sa.Column("code", sa.String(length=128), nullable=True),
+        sa.Column("reason", sa.Text(), nullable=True),
+        sa.Column("params", sa.JSON().with_variant(postgresql.JSONB(astext_type=Text()), "postgresql"), nullable=True),
+        sa.Column("method", sa.String(length=16), nullable=False),
+        sa.Column("path", sa.String(length=2000), nullable=False),
+        sa.Column("body_bytes", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("duration_ms", sa.Integer(), server_default=sa.text("0"), nullable=False),
+        sa.Column("caller", sa.String(length=64), nullable=True),
+        sa.Column(
+            "created_at",
+            dirigent_core.types.UtcDateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["run_id"], ["runs.id"], name=op.f("fk_source_inputs_run_id_runs"), ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(
+            ["source_id"], ["sources.id"], name=op.f("fk_source_inputs_source_id_sources"), ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_source_inputs")),
+    )
+    with op.batch_alter_table("source_inputs", schema=None) as batch_op:
+        batch_op.create_index("ix_source_inputs_created_at", ["created_at"], unique=False)
+        batch_op.create_index("ix_source_inputs_source_id_id", ["source_id", "id"], unique=False)
+
     op.create_table(
         "step_attempts",
         sa.Column("id", sa.Uuid(), nullable=False),
@@ -1034,6 +1138,11 @@ def downgrade() -> None:
         batch_op.drop_index("ix_webhook_deliveries_created_at")
 
     op.drop_table("webhook_deliveries")
+    with op.batch_alter_table("source_inputs", schema=None) as batch_op:
+        batch_op.drop_index("ix_source_inputs_source_id_id")
+        batch_op.drop_index("ix_source_inputs_created_at")
+
+    op.drop_table("source_inputs")
     with op.batch_alter_table("schedule_firings", schema=None) as batch_op:
         batch_op.drop_index("ix_schedule_firings_schedule_id_id")
         batch_op.drop_index("ix_schedule_firings_created_at")
@@ -1052,6 +1161,7 @@ def downgrade() -> None:
 
     op.drop_table("notifications")
     with op.batch_alter_table("runs", schema=None) as batch_op:
+        batch_op.drop_index("ix_runs_triggered_by_id_status")
         batch_op.drop_index(batch_op.f("ix_runs_status"))
         batch_op.drop_index("ix_runs_pipeline_id_status_created_at")
         batch_op.drop_index("ix_runs_finished_at")
@@ -1062,6 +1172,14 @@ def downgrade() -> None:
         batch_op.drop_index(batch_op.f("ix_watches_pipeline_id"))
 
     op.drop_table("watches")
+    with op.batch_alter_table("source_routes", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_source_routes_source_id"))
+
+    op.drop_table("source_routes")
+    with op.batch_alter_table("sources", schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f("ix_sources_pipeline_id"))
+
+    op.drop_table("sources")
     with op.batch_alter_table("webhook_triggers", schema=None) as batch_op:
         batch_op.drop_index(batch_op.f("ix_webhook_triggers_trigger_document_id"))
         batch_op.drop_index(batch_op.f("ix_webhook_triggers_pipeline_id"))

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/src/dirigent_core/blockdocs.py

@@ -7,7 +7,7 @@ catalog disagree.
 import json
 from typing import Final, cast
 
-from dirigent_client.schemas import BlockEntry, BlockKind, Catalog, SurfaceEntry
+from dirigent_client.schemas import BlockEntry, BlockKind, Catalog, SourceEntry, SurfaceEntry
 from dirigent_common import JsonMap
 from dirigent_core.configdocs import first_paragraph, one_line
 
@@ -53,7 +53,8 @@ SURFACES_PREAMBLE: Final = """## Other surfaces
 Blocks are two of the six surfaces a plugin contributes to. Three of the other four -- storage
 backends, notifiers and connection kinds -- are listed here by id, since a document references
 them by code and never by id of any other kind. The sixth, a JSON Schema format checker, is in
-[JSON Schema](json-schema.md).
+[JSON Schema](json-schema.md). Source kinds, which a source node names as ``source.kind``, are
+listed after them; see [Source and reply](concepts.md#source-and-reply).
 """
 
 NO_DESCRIPTION: Final = "--"
@@ -73,6 +74,7 @@ def render(catalog: Catalog) -> str:
     parts.append(_surfaces("Storage schemes", "scheme", catalog.storage_schemes))
     parts.append(_surfaces("Notifiers", "id", catalog.notifiers))
     parts.append(_surfaces("Connection kinds", "id", catalog.connection_kinds))
+    parts.append(_sources(catalog.sources))
     return "\n".join(parts).rstrip() + "\n"
 
 
@@ -210,6 +212,45 @@ def _surfaces(title: str, label: str, entries: list[SurfaceEntry]) -> str:
     )
 
 
+def _sources(entries: list[SourceEntry]) -> str:
+    """Render the source kinds as a short table, and the input each one hands on."""
+    if not entries:
+        return "### Source kinds\n\n_None installed._\n"
+    rows = [
+        f"| `{entry.kind}` | {entry.hosting} | {_yes(entry.replies)} | {entry.summary} | `{entry.plugin}` |"
+        for entry in entries
+    ]
+    lines = [
+        "### Source kinds",
+        "",
+        "| Kind | Hosting | Replies | Summary | Contributed by |",
+        "| --- | --- | --- | --- | --- |",
+        *rows,
+        "",
+    ]
+    for entry in entries:
+        lines.extend(
+            [
+                f"#### `{entry.kind}`",
+                "",
+                "**Config**",
+                "",
+                _schema_table(entry.config_schema),
+                "",
+                "**Input it hands on**",
+                "",
+                _schema_table(entry.output_schema),
+                "",
+            ]
+        )
+    return "\n".join(lines)
+
+
+def _yes(flag: bool) -> str:
+    """Render a flag as a table cell reads it."""
+    return "yes" if flag else "no"
+
+
 def _anchor(block_id: str) -> str:
     """Render the anchor mkdocs gives a heading whose text is a backticked block id."""
     return block_id.replace(".", "")

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/src/dirigent_core/config.py

@@ -290,6 +290,9 @@ class Settings(BaseSettings):
     retention_deliveries: Duration | None = None
     """How long a webhook delivery is kept."""
 
+    retention_inputs: Duration | None = None
+    """How long the record of an input that reached a source is kept."""
+
     retention_firings: Duration | None = None
     """How long a schedule firing is kept."""
 
@@ -355,6 +358,31 @@ class Settings(BaseSettings):
 
     A real webhook's own ``rate_limit_per_minute`` applies on top of this."""
 
+    endpoint_prefix: str = "/endpoints"
+    """Where endpoint sources are served, outside the versioned API.
+
+    A proxy in front of an instance publishes this prefix and ``/hooks`` and keeps
+    ``/api/v1`` inside, so a source's route is public without the API being so."""
+
+    source_max_body: Size = Field(default=1_048_576, ge=1)
+    """The largest body any endpoint reads, whatever its own ``max_body`` says; reading stops one byte past it."""
+
+    source_intake_rate_per_minute: int = Field(default=600, ge=1)
+    """How fast calls to one method and path are taken, before any source is looked up.
+
+    A source's own ``rate`` applies on top of this."""
+
+    source_reply_max: Duration = Field(default=timedelta(seconds=60), gt=timedelta(0))
+    """The longest a caller is held for a reply, whatever a reply node's ``timeout`` says."""
+
+    source_max_waiting: int = Field(default=256, ge=1)
+    """How many callers one server process holds open for their replies at once.
+
+    A call past it is answered 503 before its run is created."""
+
+    source_reply_poll: Duration = Field(default=timedelta(milliseconds=50), gt=timedelta(0))
+    """How often one server process reads the replies its held callers wait for, in one query."""
+
     login_rate_per_minute: int = Field(default=10, ge=1)
     """How many login attempts one address, and one username, get a minute.
 
@@ -383,10 +411,10 @@ class Settings(BaseSettings):
         """Accept a comma-separated ``key=value`` string for a mapping setting."""
         return parse_string_mapping(value)
 
-    @field_validator("api_prefix")
+    @field_validator("api_prefix", "endpoint_prefix")
     @classmethod
     def _normalise_prefix(cls, value: str) -> str:
-        """Keep the API prefix rooted and free of a trailing slash."""
+        """Keep a prefix rooted and free of a trailing slash."""
         prefix = "/" + value.strip("/")
         return prefix.rstrip("/") or "/"
 

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/src/dirigent_core/documents.py

@@ -45,8 +45,10 @@ from dirigent_core.messages import (
     NOT_A_MAPPING,
     NOT_YAML,
     PARAMS_SCHEMA_INVALID,
+    REFERENCE_ERROR_NOT_UPSTREAM,
     REFERENCE_GRID_IN_CONFIG,
     REFERENCE_MALFORMED_ARTIFACTS,
+    REFERENCE_MALFORMED_ERROR,
     REFERENCE_MALFORMED_ITEM,
     REFERENCE_MALFORMED_RUN,
     REFERENCE_MALFORMED_STEP,
@@ -366,6 +368,7 @@ def validate_against_catalog(
     blocks: Mapping[str, Any] | None = None,
     format_checker: FormatChecker | None = None,
     target: TriggerTarget | None = None,
+    source_kinds: Mapping[str, Any] | None = None,
 ) -> list[ValidationIssue]:
     """Check a document against what this instance actually has, reporting everything at once.
 
@@ -379,8 +382,13 @@ def validate_against_catalog(
     against; unset, they are checked against the base formats alone, which is all a caller
     holding no host can know. ``target`` is the pipeline a triggers document names, as the
     instance holds it now; on the offline path there is none to read, and only the clocks and
-    the codes are checked.
+    the codes are checked. ``source_kinds`` is the source kinds the instance knows; unset, the
+    built-in ones.
     """
+    from dirigent_core.sources.kinds import built_in_kinds
+    from dirigent_core.sources.rules import source_issues
+
+    kinds = source_kinds if source_kinds is not None else built_in_kinds()
     checker = format_checker if format_checker is not None else base_format_checker()
     sensors = _sensors(catalog) if check_blocks else None
     if isinstance(definition, TriggersDefinition):
@@ -406,6 +414,7 @@ def validate_against_catalog(
     issues.extend(_carried_schema_issues(definition))
     issues.extend(_reference_issues(definition))
     issues.extend(_trigger_issues(definition, checker, check_params=not malformed_params, sensors=sensors))
+    issues.extend(source_issues(definition, kinds))
     if check_blocks:
         issues.extend(
             _step_block_issues(
@@ -607,6 +616,10 @@ def _triggers_document_issues(
         return [ValidationIssue.of(TARGET_PIPELINE_MISSING, location="pipeline", code=repr(definition.pipeline))]
     if not target.active:
         return [ValidationIssue.of(TARGET_PIPELINE_INACTIVE, location="pipeline", code=repr(definition.pipeline))]
+    if target.definition is not None and target.definition.sources:
+        from dirigent_core.sources.rules import triggers_issues
+
+        return triggers_issues(target.code, definition.triggers)
     return _trigger_issues(definition, format_checker, against=target.definition, sensors=sensors)
 
 
@@ -672,6 +685,8 @@ def _step_block_issues(
     issues: list[ValidationIssue] = []
     for name in sorted(definition.steps):
         step = definition.steps[name]
+        if step.block is None:
+            continue
         entry = catalog.block(step.block)
         if entry is None:
             issues.append(
@@ -851,6 +866,12 @@ def _reference_issues(definition: PipelineDefinition) -> list[ValidationIssue]:
             problem = _reference_problem(reference.strip(), definition, name, upstream, declared, family)
             if problem is not None:
                 issues.append(_at(f"steps.{name}.config", problem))
+        if step.reply is not None:
+            written = cast("JsonValue", step.reply.model_dump(mode="json", exclude={"timeout"}))
+            for reference in sorted(set(references_in(written))):
+                problem = _reference_problem(reference.strip(), definition, name, upstream, declared, family)
+                if problem is not None:
+                    issues.append(_at(f"steps.{name}.reply", problem))
         issues.extend(_fan_out_literal_issues(name, step.for_each))
         issues.extend(_adoption_issues(definition, name))
         for reference in sorted(set(references_in(cast("JsonValue", step.for_each)))):
@@ -952,6 +973,14 @@ def _reference_problem(
             if target not in upstream:
                 return Issue.of(REFERENCE_NOT_UPSTREAM, reference=reference, target=repr(target), step=repr(step))
             return None
+        case ["steps", target, "error", *path]:
+            if target not in definition.steps:
+                return Issue.of(REFERENCE_UNKNOWN_STEP, reference=reference, step=repr(target))
+            if target not in upstream:
+                return Issue.of(REFERENCE_ERROR_NOT_UPSTREAM, reference=reference, target=repr(target), step=repr(step))
+            if path[:1] not in (["code"], ["message"], ["params"]) or (path[:1] != ["params"] and len(path) > 1):
+                return Issue.of(REFERENCE_MALFORMED_ERROR, reference=reference)
+            return None
         case ["steps", target, "items"]:
             return Issue.of(REFERENCE_GRID_IN_CONFIG, reference=reference, target=repr(target), bare_target=target)
         case ["steps", target, "item", "output", *_]:
@@ -1032,7 +1061,9 @@ def summarize(definition: Document) -> DocumentSummary:
     pipeline = definition.pipeline if isinstance(definition, TriggersDefinition) else None
     steps = sorted(definition.steps) if isinstance(definition, PipelineDefinition) else []
     blocks = (
-        sorted({step.block for step in definition.steps.values()}) if isinstance(definition, PipelineDefinition) else []
+        sorted({step.block for step in definition.steps.values() if step.block is not None})
+        if isinstance(definition, PipelineDefinition)
+        else []
     )
     return DocumentSummary(
         kind=definition.kind,

{dirigent_core-0.23.3 → dirigent_core-0.24.0}/src/dirigent_core/engine/__init__.py

@@ -36,6 +36,7 @@ from dirigent_core.engine.references import ReferenceScope, UnknownReference, re
 from dirigent_core.engine.runs import (
     Attribution,
     EngineRuns,
+    Entry,
     FanOutError,
     Provenance,
     RunCreationError,
@@ -62,6 +63,7 @@ __all__ = [
     "KIND_PIPELINE",
     "KIND_TRIGGERS",
     "Attribution",
+    "Entry",
     "BufferedLogger",
     "ClaimedUnit",
     "ConcurrencyPolicy",