dirigent-core 0.28.0__tar.gz → 0.29.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/PKG-INFO +4 -4
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/pyproject.toml +4 -4
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/pyproject.toml.orig +4 -4
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/alerting.py +2 -1
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/definition.py +23 -4
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/executor.py +23 -8
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/runs.py +20 -4
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/messages.py +12 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/pipelines.py +35 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/reporting.py +2 -1
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/endpoint.py +2 -16
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/telemetry.py +32 -11
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/triggers/backfill.py +1 -1
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/LICENSE +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/README.md +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/__init__.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/alembic/env.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/alembic/script.py.mako +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/alembic/versions/0001_baseline_schema.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/artifacts.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/auth.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/blockdocs.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/config.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/configdocs.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/connections.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/database.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/directory.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/documents.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/documentschema.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/__init__.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/claim.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/context.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/expansion.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/failure.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/quota.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/recovery.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/references.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/replies.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/services.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/shell.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/engine/state.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/errors.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/examples.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/ids.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/logging.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/migrations.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/models.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/plugins.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/protocol.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/py.typed +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/ratelimit.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/registry.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/retention.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/scheduler.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/schemas.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/secrets.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/__init__.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/entries.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/holding.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/intake.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/kinds.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/polling.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/rows.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/rules.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/sensor.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/storage.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/sources/waiter.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/statedir.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/stats.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/storage.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/trigger_documents.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/triggers/__init__.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/triggers/materialize.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/triggers/schedules.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/types.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/wakeups.py +0 -0
- {dirigent_core-0.28.0 → dirigent_core-0.29.0}/src/dirigent_core/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dirigent-core
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.29.0
|
|
4
4
|
Summary: Dirigent engine core: schema, configuration, and plugin host.
|
|
5
5
|
License-Expression: LicenseRef-Proprietary
|
|
6
6
|
License-File: LICENSE
|
|
@@ -12,9 +12,9 @@ Requires-Dist: argon2-cffi>=25.1.0
|
|
|
12
12
|
Requires-Dist: asyncpg>=0.31.0
|
|
13
13
|
Requires-Dist: cronsim>=2.7
|
|
14
14
|
Requires-Dist: cryptography>=46.0.5
|
|
15
|
-
Requires-Dist: dirigent-client==0.
|
|
16
|
-
Requires-Dist: dirigent-common==0.
|
|
17
|
-
Requires-Dist: dirigent-plugin==0.
|
|
15
|
+
Requires-Dist: dirigent-client==0.29.0
|
|
16
|
+
Requires-Dist: dirigent-common==0.29.0
|
|
17
|
+
Requires-Dist: dirigent-plugin==0.29.0
|
|
18
18
|
Requires-Dist: jsonschema>=4.26.0
|
|
19
19
|
Requires-Dist: opentelemetry-api>=1.44.0
|
|
20
20
|
Requires-Dist: pydantic-settings>=2.15.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-core"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.29.0"
|
|
4
4
|
description = "Dirigent engine core: schema, configuration, and plugin host."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -17,9 +17,9 @@ dependencies = [
|
|
|
17
17
|
"asyncpg>=0.31.0",
|
|
18
18
|
"cronsim>=2.7",
|
|
19
19
|
"cryptography>=46.0.5",
|
|
20
|
-
"dirigent-client==0.
|
|
21
|
-
"dirigent-common==0.
|
|
22
|
-
"dirigent-plugin==0.
|
|
20
|
+
"dirigent-client==0.29.0",
|
|
21
|
+
"dirigent-common==0.29.0",
|
|
22
|
+
"dirigent-plugin==0.29.0",
|
|
23
23
|
"jsonschema>=4.26.0",
|
|
24
24
|
"opentelemetry-api>=1.44.0",
|
|
25
25
|
"pydantic-settings>=2.15.0",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dirigent-core"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.29.0"
|
|
4
4
|
description = "Dirigent engine core: schema, configuration, and plugin host."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -17,9 +17,9 @@ dependencies = [
|
|
|
17
17
|
"asyncpg>=0.31.0",
|
|
18
18
|
"cronsim>=2.7",
|
|
19
19
|
"cryptography>=46.0.5",
|
|
20
|
-
"dirigent-client==0.
|
|
21
|
-
"dirigent-common==0.
|
|
22
|
-
"dirigent-plugin==0.
|
|
20
|
+
"dirigent-client==0.29.0",
|
|
21
|
+
"dirigent-common==0.29.0",
|
|
22
|
+
"dirigent-plugin==0.29.0",
|
|
23
23
|
"jsonschema>=4.26.0",
|
|
24
24
|
"opentelemetry-api>=1.44.0",
|
|
25
25
|
"pydantic-settings>=2.15.0",
|
|
@@ -38,6 +38,7 @@ from dirigent_common import (
|
|
|
38
38
|
scrub,
|
|
39
39
|
secrets_of,
|
|
40
40
|
)
|
|
41
|
+
from dirigent_common.depth import shown
|
|
41
42
|
from dirigent_core.database import is_deadlock, session_scope
|
|
42
43
|
from dirigent_core.engine.definition import load_definition
|
|
43
44
|
from dirigent_core.engine.services import EngineServices
|
|
@@ -146,7 +147,7 @@ def build_context(
|
|
|
146
147
|
"error": run.error,
|
|
147
148
|
"error_code": run.error_code,
|
|
148
149
|
"error_params": dict(run.error_params or {}),
|
|
149
|
-
"params": dict(run.params),
|
|
150
|
+
"params": shown(dict(run.params)),
|
|
150
151
|
"trigger": run.triggered_by_label or run.triggered_by_kind.value,
|
|
151
152
|
"started_at": run.started_at.isoformat() if run.started_at else None,
|
|
152
153
|
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""The pipeline definition model: one immutable document per version, shared by everything."""
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
import re
|
|
5
|
+
import threading
|
|
4
6
|
from datetime import datetime, timedelta
|
|
5
7
|
from enum import StrEnum
|
|
6
8
|
from typing import Annotated, Any, Final, Literal, Self, cast
|
|
@@ -23,6 +25,7 @@ from dirigent_common import (
|
|
|
23
25
|
bounded_evaluation,
|
|
24
26
|
compile_template,
|
|
25
27
|
)
|
|
28
|
+
from dirigent_common.depth import MAX_STORED_DEPTH, nests_past
|
|
26
29
|
from dirigent_common.durations import Duration, format_duration
|
|
27
30
|
from dirigent_core.engine.references import has_reference, references_in
|
|
28
31
|
from dirigent_core.errors import DomainError
|
|
@@ -33,6 +36,7 @@ from dirigent_core.messages import (
|
|
|
33
36
|
PARAMETER_INVALID,
|
|
34
37
|
PARAMETER_SCHEMA_INVALID,
|
|
35
38
|
PARAMETER_UNCHECKED,
|
|
39
|
+
PARAMETERS_TOO_DEEP,
|
|
36
40
|
REPLY_BAD_HEADER,
|
|
37
41
|
REPLY_BAD_STATUS,
|
|
38
42
|
REPLY_BODY_TWICE,
|
|
@@ -659,7 +663,20 @@ class PipelineDefinition(BaseModel):
|
|
|
659
663
|
pending.difference_update(ready)
|
|
660
664
|
return ordered
|
|
661
665
|
|
|
662
|
-
def
|
|
666
|
+
async def check_params(self, params: JsonMap, format_checker: FormatChecker) -> JsonMap:
|
|
667
|
+
"""Validate parameters as :meth:`validate_params` does, on a thread rather than the event loop.
|
|
668
|
+
|
|
669
|
+
The check stops at ``PARAMETERS_CHECKED_WITHIN`` or as soon as the caller is cancelled.
|
|
670
|
+
"""
|
|
671
|
+
stopped = threading.Event()
|
|
672
|
+
try:
|
|
673
|
+
return await asyncio.to_thread(self.validate_params, params, format_checker, stopped)
|
|
674
|
+
finally:
|
|
675
|
+
stopped.set()
|
|
676
|
+
|
|
677
|
+
def validate_params(
|
|
678
|
+
self, params: JsonMap, format_checker: FormatChecker, stopped: threading.Event | None = None
|
|
679
|
+
) -> JsonMap:
|
|
663
680
|
"""Validate supplied parameters against the pipeline's own schema, filling defaults.
|
|
664
681
|
|
|
665
682
|
``format_checker`` is the instance's assembled checker, so a ``format`` a parameter
|
|
@@ -669,14 +686,16 @@ class PipelineDefinition(BaseModel):
|
|
|
669
686
|
existed can hold a schema that is not one, and jsonschema then raises something no
|
|
670
687
|
caller of this method catches.
|
|
671
688
|
|
|
672
|
-
The check runs where it is called
|
|
673
|
-
``
|
|
689
|
+
The check runs where it is called and stops at ``PARAMETERS_CHECKED_WITHIN``, or once
|
|
690
|
+
``stopped`` is set. A caller on the event loop uses :meth:`check_params` instead.
|
|
674
691
|
"""
|
|
675
692
|
resolved = _with_defaults(self.params, params)
|
|
693
|
+
if nests_past(cast("JsonValue", resolved)):
|
|
694
|
+
raise ParameterError(PARAMETERS_TOO_DEEP, limit=MAX_STORED_DEPTH)
|
|
676
695
|
try:
|
|
677
696
|
Draft202012Validator.check_schema(self.params)
|
|
678
697
|
validator = ValueValidator(self.params, format_checker=format_checker, registry=LOCAL_REFERENCES)
|
|
679
|
-
with bounded_evaluation(PARAMETERS_CHECKED_WITHIN):
|
|
698
|
+
with bounded_evaluation(PARAMETERS_CHECKED_WITHIN, stopped):
|
|
680
699
|
validator.validate(resolved)
|
|
681
700
|
except SchemaValidationError as error:
|
|
682
701
|
location = "/".join(str(part) for part in error.absolute_path) or "(root)"
|
|
@@ -30,6 +30,7 @@ from dirigent_common import (
|
|
|
30
30
|
raised_detail,
|
|
31
31
|
scrub,
|
|
32
32
|
)
|
|
33
|
+
from dirigent_common.depth import MAX_STORED_DEPTH, nests_past
|
|
33
34
|
from dirigent_core import telemetry, wakeups
|
|
34
35
|
from dirigent_core.artifacts import canonical_json, persist_output
|
|
35
36
|
from dirigent_core.database import session_scope, with_deadlock_retry
|
|
@@ -79,6 +80,7 @@ from dirigent_core.messages import (
|
|
|
79
80
|
GONE_PROBES,
|
|
80
81
|
ITEM_UNPAIRED,
|
|
81
82
|
LEASE_RECOVERIES,
|
|
83
|
+
OUTPUT_TOO_DEEP,
|
|
82
84
|
REMOTE_JOB_FAILED,
|
|
83
85
|
REMOTE_JOB_GONE,
|
|
84
86
|
REMOTE_JOB_SAID,
|
|
@@ -468,7 +470,7 @@ class Engine:
|
|
|
468
470
|
stop = asyncio.Event()
|
|
469
471
|
flusher = asyncio.create_task(self._flush_while_running(logger, stop))
|
|
470
472
|
try:
|
|
471
|
-
result = await self._counted(await self._call_block(unit, context), meter)
|
|
473
|
+
result = await self._counted(_bounded(await self._call_block(unit, context)), meter)
|
|
472
474
|
finally:
|
|
473
475
|
stop.set()
|
|
474
476
|
with contextlib.suppress(asyncio.CancelledError):
|
|
@@ -490,6 +492,9 @@ class Engine:
|
|
|
490
492
|
storage=context.uncounted_storage,
|
|
491
493
|
seconds=time.monotonic() - started,
|
|
492
494
|
)
|
|
495
|
+
except Exception as error:
|
|
496
|
+
telemetry.record_failure(span, error, logger.secrets)
|
|
497
|
+
raise
|
|
493
498
|
finally:
|
|
494
499
|
await meter.settle()
|
|
495
500
|
|
|
@@ -547,7 +552,7 @@ class Engine:
|
|
|
547
552
|
try:
|
|
548
553
|
async with asyncio.timeout(unit.step.timeout.total_seconds() if unit.step.timeout else None):
|
|
549
554
|
if sensor is not None:
|
|
550
|
-
with self._timed(unit.block_id, "poke"):
|
|
555
|
+
with self._timed(unit.block_id, "poke", context):
|
|
551
556
|
return _read_poke(await sensor.poke(config, context))
|
|
552
557
|
if operator is None: # pragma: no cover - the host resolved it above
|
|
553
558
|
return Errored(failure=Failure.rejected(BLOCK_NOT_INSTALLED, block=repr(unit.block_id)))
|
|
@@ -576,12 +581,12 @@ class Engine:
|
|
|
576
581
|
if unit.fetched_output is not None:
|
|
577
582
|
return Produced(output=unit.fetched_output)
|
|
578
583
|
if not unit.is_probe or unit.remote_handle is None:
|
|
579
|
-
with self._timed(unit.block_id, "execute"):
|
|
584
|
+
with self._timed(unit.block_id, "execute", context):
|
|
580
585
|
produced = await operator.execute(config, context)
|
|
581
586
|
if isinstance(produced, RemoteHandle):
|
|
582
587
|
return Submitted(handle=produced)
|
|
583
588
|
return Produced(output=produced.model_dump(mode="json"))
|
|
584
|
-
with self._timed(unit.block_id, "probe"):
|
|
589
|
+
with self._timed(unit.block_id, "probe", context):
|
|
585
590
|
probe = await operator.probe(unit.remote_handle, config, context)
|
|
586
591
|
advanced = None if probe.meta is None else unit.remote_handle.model_copy(update={"meta": probe.meta})
|
|
587
592
|
handle = advanced or unit.remote_handle
|
|
@@ -594,7 +599,7 @@ class Engine:
|
|
|
594
599
|
handle=advanced,
|
|
595
600
|
)
|
|
596
601
|
case ProbeStatus.SUCCEEDED:
|
|
597
|
-
with self._timed(unit.block_id, "fetch"):
|
|
602
|
+
with self._timed(unit.block_id, "fetch", context):
|
|
598
603
|
retrieved = await operator.fetch(handle, config, context)
|
|
599
604
|
return Produced(output=retrieved.model_dump(mode="json"), fetched=True)
|
|
600
605
|
case ProbeStatus.FAILED:
|
|
@@ -608,10 +613,13 @@ class Engine:
|
|
|
608
613
|
return Vanished(message=probe.message or REMOTE_JOB_GONE.render())
|
|
609
614
|
|
|
610
615
|
@contextlib.contextmanager
|
|
611
|
-
def _timed(self, block_id: str, call: str) -> Generator[None]:
|
|
612
|
-
"""Wrap one block call in its own span and record how long it took.
|
|
616
|
+
def _timed(self, block_id: str, call: str, context: EngineStepContext) -> Generator[None]:
|
|
617
|
+
"""Wrap one block call in its own span and record how long it took.
|
|
618
|
+
|
|
619
|
+
A failure lands on the span with every secret the call opened scrubbed from it.
|
|
620
|
+
"""
|
|
613
621
|
started = time.monotonic()
|
|
614
|
-
with telemetry.block_span(block_id, call):
|
|
622
|
+
with telemetry.block_span(block_id, call, context.opened_secrets):
|
|
615
623
|
try:
|
|
616
624
|
yield
|
|
617
625
|
finally:
|
|
@@ -1289,3 +1297,10 @@ async def _pair_refusal(session: AsyncSession, run_id: UUID, parent: str, item_i
|
|
|
1289
1297
|
return None
|
|
1290
1298
|
key = pair[1].item_key if pair is not None else str(item_index)
|
|
1291
1299
|
return Issue.of(ITEM_UNPAIRED, index=item_index, key=repr(key), step=repr(parent))
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
def _bounded(result: CallResult) -> CallResult:
|
|
1303
|
+
"""Fail a call whose output nests deeper than a run stores."""
|
|
1304
|
+
if isinstance(result, Produced) and nests_past(cast("JsonValue", result.output)):
|
|
1305
|
+
return Errored(failure=Failure.rejected(OUTPUT_TOO_DEEP, limit=MAX_STORED_DEPTH))
|
|
1306
|
+
return result
|
|
@@ -21,7 +21,8 @@ from dirigent_client.enums import (
|
|
|
21
21
|
RunStatus,
|
|
22
22
|
TriggerKind,
|
|
23
23
|
)
|
|
24
|
-
from dirigent_common import JsonMap, Message
|
|
24
|
+
from dirigent_common import JsonMap, Message, raised_detail
|
|
25
|
+
from dirigent_common.depth import MAX_STORED_DEPTH, nests_past
|
|
25
26
|
from dirigent_core import telemetry, wakeups
|
|
26
27
|
from dirigent_core.artifacts import persist_output
|
|
27
28
|
from dirigent_core.database import session_scope
|
|
@@ -41,6 +42,7 @@ from dirigent_core.engine.definition import (
|
|
|
41
42
|
schemas_named,
|
|
42
43
|
)
|
|
43
44
|
from dirigent_core.engine.expansion import is_gate, new_attempt, write_items
|
|
45
|
+
from dirigent_core.engine.failure import Failure
|
|
44
46
|
from dirigent_core.engine.references import ReferenceScope, resolve
|
|
45
47
|
from dirigent_core.engine.services import EngineServices
|
|
46
48
|
from dirigent_core.engine.state import advance, lock_pipeline, lock_run
|
|
@@ -60,6 +62,7 @@ from dirigent_core.messages import (
|
|
|
60
62
|
FAN_OUT_TOO_WIDE,
|
|
61
63
|
GATE_NOT_RETRYABLE,
|
|
62
64
|
GIVEN_NO_ENTRY,
|
|
65
|
+
INPUT_TOO_DEEP,
|
|
63
66
|
NO_ATTEMPT_TO_RETRY,
|
|
64
67
|
NOT_ENTERED,
|
|
65
68
|
NOT_PAUSABLE,
|
|
@@ -280,7 +283,7 @@ async def create_run(
|
|
|
280
283
|
moment = now or utcnow()
|
|
281
284
|
definition = load_definition(version.document)
|
|
282
285
|
_check_entry(definition, entry)
|
|
283
|
-
resolved_params = definition.
|
|
286
|
+
resolved_params = await definition.check_params(params or {}, services.format_checker)
|
|
284
287
|
_refuse_disabled_blocks(definition, services)
|
|
285
288
|
pinned_schemas = await _pin_schemas(session, services, version)
|
|
286
289
|
who = attribution or Attribution()
|
|
@@ -444,8 +447,17 @@ async def _write_source(
|
|
|
444
447
|
session.add(attempt)
|
|
445
448
|
return
|
|
446
449
|
attempt.kind = AttemptKind.RECEIVED
|
|
447
|
-
attempt.status = AttemptStatus.SUCCEEDED
|
|
448
450
|
attempt.input = {"via": entry.via}
|
|
451
|
+
if nests_past(cast("JsonValue", entry.output)):
|
|
452
|
+
refused = Failure.rejected(INPUT_TOO_DEEP, step=repr(name), limit=MAX_STORED_DEPTH)
|
|
453
|
+
attempt.status = AttemptStatus.FAILED
|
|
454
|
+
attempt.error = refused.message
|
|
455
|
+
attempt.error_code = refused.code
|
|
456
|
+
attempt.error_params = refused.params or None
|
|
457
|
+
attempt.error_class = refused.error_class.value
|
|
458
|
+
session.add(attempt)
|
|
459
|
+
return
|
|
460
|
+
attempt.status = AttemptStatus.SUCCEEDED
|
|
449
461
|
session.add(attempt)
|
|
450
462
|
await session.flush()
|
|
451
463
|
reference = await persist_output(
|
|
@@ -707,7 +719,11 @@ async def cancel_remote(
|
|
|
707
719
|
config = block.config_model.model_validate((attempt.input or {}).get("config", {}))
|
|
708
720
|
accepted = await block.cancel(RemoteHandle.model_validate(attempt.remote_handle), config, context)
|
|
709
721
|
except Exception as error: # best effort by contract: a refusal must not block the cancel
|
|
710
|
-
_logger.warning(
|
|
722
|
+
_logger.warning(
|
|
723
|
+
"remote cancellation failed",
|
|
724
|
+
attempt_id=str(attempt.id),
|
|
725
|
+
error=raised_detail(error, redact=context.opened_secrets),
|
|
726
|
+
)
|
|
711
727
|
return
|
|
712
728
|
_logger.info("remote cancellation requested", attempt_id=str(attempt.id), accepted=accepted)
|
|
713
729
|
|
|
@@ -208,6 +208,10 @@ PARAMETER_UNCHECKED = PARAMETER.define(
|
|
|
208
208
|
"the parameters were not checked against the pipeline's parameter schema within {timeout}",
|
|
209
209
|
)
|
|
210
210
|
|
|
211
|
+
PARAMETERS_TOO_DEEP = PARAMETER.define(
|
|
212
|
+
"too_deep", "the parameters nest deeper than the {limit} levels of lists and objects a run stores"
|
|
213
|
+
)
|
|
214
|
+
|
|
211
215
|
DOCUMENT = Catalogue("document")
|
|
212
216
|
|
|
213
217
|
NOT_YAML = DOCUMENT.define("not_yaml", "the document is not valid YAML or JSON: {detail}")
|
|
@@ -873,6 +877,14 @@ SERVER_SETTLED = RUN.define("server_settled", "step {step} is a {kind}, which th
|
|
|
873
877
|
|
|
874
878
|
NOT_ENTERED = RUN.define("not_entered", "the input arrived at {entered}, so {step} was not entered")
|
|
875
879
|
|
|
880
|
+
INPUT_TOO_DEEP = RUN.define(
|
|
881
|
+
"input_too_deep", "the input at {step} nests deeper than the {limit} levels of lists and objects a run stores"
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
OUTPUT_TOO_DEEP = RUN.define(
|
|
885
|
+
"output_too_deep", "the step's output nests deeper than the {limit} levels of lists and objects a run stores"
|
|
886
|
+
)
|
|
887
|
+
|
|
876
888
|
REPLY_REFUSED = RUN.define("reply_refused", "reply {step} could not be rendered: {detail}")
|
|
877
889
|
|
|
878
890
|
REPLY_STATUS = RUN.define(
|
|
@@ -9,6 +9,7 @@ from typing import Final, NamedTuple
|
|
|
9
9
|
from uuid import UUID
|
|
10
10
|
|
|
11
11
|
import sqlalchemy as sa
|
|
12
|
+
from pydantic import ValidationError
|
|
12
13
|
from sqlalchemy.dialects import postgresql
|
|
13
14
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
14
15
|
from sqlalchemy.orm import InstrumentedAttribute
|
|
@@ -22,6 +23,7 @@ from dirigent_client.schemas import (
|
|
|
22
23
|
PipelinePlan,
|
|
23
24
|
PlanAction,
|
|
24
25
|
SourceChanges,
|
|
26
|
+
StepShape,
|
|
25
27
|
ValidationIssue,
|
|
26
28
|
)
|
|
27
29
|
from dirigent_common import JsonMap
|
|
@@ -523,6 +525,39 @@ async def last_runs(session: AsyncSession, pipeline_ids: Sequence[UUID]) -> dict
|
|
|
523
525
|
}
|
|
524
526
|
|
|
525
527
|
|
|
528
|
+
async def listing_shapes(session: AsyncSession, pipeline_ids: Sequence[UUID]) -> dict[UUID, list[StepShape]]:
|
|
529
|
+
"""Read the steps of each of a page of pipelines' current versions, as a miniature draws them.
|
|
530
|
+
|
|
531
|
+
One statement for the whole page: each pipeline's current version, joined on the version
|
|
532
|
+
number the pipeline row holds. A stored document this build can no longer read draws no shape.
|
|
533
|
+
"""
|
|
534
|
+
if not pipeline_ids:
|
|
535
|
+
return {}
|
|
536
|
+
current = (
|
|
537
|
+
sa.select(PipelineVersion)
|
|
538
|
+
.join(Pipeline, Pipeline.id == PipelineVersion.pipeline_id)
|
|
539
|
+
.where(Pipeline.id.in_(pipeline_ids), PipelineVersion.version == Pipeline.current_version)
|
|
540
|
+
)
|
|
541
|
+
shapes: dict[UUID, list[StepShape]] = {}
|
|
542
|
+
for version in (await session.execute(current)).scalars():
|
|
543
|
+
try:
|
|
544
|
+
definition = load_definition(version.ordered_document)
|
|
545
|
+
except ValidationError:
|
|
546
|
+
continue
|
|
547
|
+
shapes[version.pipeline_id] = [
|
|
548
|
+
StepShape(
|
|
549
|
+
code=code,
|
|
550
|
+
name=step.name,
|
|
551
|
+
kind=step.kind,
|
|
552
|
+
block=step.block_id,
|
|
553
|
+
depends_on=list(step.depends_on),
|
|
554
|
+
fan_out=step.is_fan_out,
|
|
555
|
+
)
|
|
556
|
+
for code, step in definition.steps.items()
|
|
557
|
+
]
|
|
558
|
+
return shapes
|
|
559
|
+
|
|
560
|
+
|
|
526
561
|
class FailedStep(NamedTuple):
|
|
527
562
|
"""A run's first failed attempt, as a listing names it: which step, and what it said."""
|
|
528
563
|
|
|
@@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
12
12
|
|
|
13
13
|
from dirigent_client.enums import AttemptStatus, LogLevel, RunItemStatus
|
|
14
14
|
from dirigent_common import JsonMap, render
|
|
15
|
+
from dirigent_common.depth import shown
|
|
15
16
|
from dirigent_core import alerting, telemetry
|
|
16
17
|
from dirigent_core.artifacts import MARKDOWN_CONTENT_TYPE, canonical_json, persist_document
|
|
17
18
|
from dirigent_core.engine.definition import PipelineDefinition
|
|
@@ -210,7 +211,7 @@ async def run_facts(
|
|
|
210
211
|
# The refusal the step ended on, taken whole: its sentence, its code and its params
|
|
211
212
|
# come off one attempt, so a reader re-rendering it is not mixing two failures.
|
|
212
213
|
failed = next((one for one in of_step if one.status is AttemptStatus.FAILED and one.error), None)
|
|
213
|
-
output = dict(last.output) if last is not None and last.output is not None else None
|
|
214
|
+
output = shown(dict(last.output)) if last is not None and last.output is not None else None
|
|
214
215
|
uri, size = stored.get(last.id, (None, None)) if last is not None else (None, None)
|
|
215
216
|
steps.append(
|
|
216
217
|
StepFacts(
|
|
@@ -16,6 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
|
16
16
|
from dirigent_client.enums import BodyKind
|
|
17
17
|
from dirigent_client.schemas import EndpointInput
|
|
18
18
|
from dirigent_common import Issue, JsonMap, Size
|
|
19
|
+
from dirigent_common.depth import nests_past
|
|
19
20
|
from dirigent_common.requests import form_values, gather_values, media_type
|
|
20
21
|
from dirigent_core.messages import (
|
|
21
22
|
BODY_WITHOUT_TYPE,
|
|
@@ -290,7 +291,7 @@ def _read_body(raw: bytes, family: BodyKind, charset: str) -> JsonValue | Refusa
|
|
|
290
291
|
return Refusal.of(ENDPOINT_BODY_NOT_JSON, status=400, detail=str(error))
|
|
291
292
|
except RecursionError:
|
|
292
293
|
return Refusal.of(ENDPOINT_BODY_TOO_DEEP, status=400, limit=MAX_BODY_DEPTH)
|
|
293
|
-
if
|
|
294
|
+
if nests_past(value, MAX_BODY_DEPTH):
|
|
294
295
|
return Refusal.of(ENDPOINT_BODY_TOO_DEEP, status=400, limit=MAX_BODY_DEPTH)
|
|
295
296
|
return value
|
|
296
297
|
codec = CHARSETS.get(charset.strip().lower()) if family is BodyKind.TEXT else "ascii"
|
|
@@ -303,21 +304,6 @@ def _read_body(raw: bytes, family: BodyKind, charset: str) -> JsonValue | Refusa
|
|
|
303
304
|
return form_values(text) if family is BodyKind.FORM else text
|
|
304
305
|
|
|
305
306
|
|
|
306
|
-
def _nests_past(value: JsonValue, limit: int) -> bool:
|
|
307
|
-
"""Say whether a parsed body holds a list or object more than ``limit`` levels down."""
|
|
308
|
-
level = [value]
|
|
309
|
-
for _ in range(limit):
|
|
310
|
-
level = [
|
|
311
|
-
child
|
|
312
|
-
for held in level
|
|
313
|
-
if isinstance(held, dict | list)
|
|
314
|
-
for child in (held.values() if isinstance(held, dict) else held)
|
|
315
|
-
]
|
|
316
|
-
if not level:
|
|
317
|
-
return False
|
|
318
|
-
return any(isinstance(held, dict | list) for held in level)
|
|
319
|
-
|
|
320
|
-
|
|
321
307
|
def _holds_a_lone_surrogate(value: JsonValue) -> bool:
|
|
322
308
|
"""Say whether a parsed body holds half a surrogate pair, which JSON parses and nothing can store."""
|
|
323
309
|
try:
|
|
@@ -7,7 +7,7 @@ says where to send data.
|
|
|
7
7
|
|
|
8
8
|
import os
|
|
9
9
|
import socket
|
|
10
|
-
from collections.abc import Callable, Generator, Iterable, Mapping
|
|
10
|
+
from collections.abc import Callable, Collection, Generator, Iterable, Mapping
|
|
11
11
|
from contextlib import contextmanager
|
|
12
12
|
from typing import Any, Final
|
|
13
13
|
|
|
@@ -17,6 +17,7 @@ from opentelemetry.metrics import CallbackOptions, Observation
|
|
|
17
17
|
from opentelemetry.trace import Span, SpanKind, Status, StatusCode
|
|
18
18
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
19
19
|
|
|
20
|
+
from dirigent_common import raised_detail, scrub
|
|
20
21
|
from dirigent_core.config import Settings
|
|
21
22
|
from dirigent_core.logging import get_logger
|
|
22
23
|
|
|
@@ -316,7 +317,9 @@ def attempt_span(*, run_id: str, step: str, block: str, attempt: int, parent: Co
|
|
|
316
317
|
"""Open the span covering one step attempt, from claim to settled outcome.
|
|
317
318
|
|
|
318
319
|
``parent`` is the run's own trace context, read back from the run row, which is what
|
|
319
|
-
puts an attempt run on a worker into the trace of whatever asked for the run.
|
|
320
|
+
puts an attempt run on a worker into the trace of whatever asked for the run. An
|
|
321
|
+
exception leaving the span is not recorded on it: the attempt records its failure
|
|
322
|
+
through :func:`record_failure`, scrubbed.
|
|
320
323
|
"""
|
|
321
324
|
with _tracer.start_as_current_span(
|
|
322
325
|
f"step {step}",
|
|
@@ -328,28 +331,46 @@ def attempt_span(*, run_id: str, step: str, block: str, attempt: int, parent: Co
|
|
|
328
331
|
"dirigent.block": block,
|
|
329
332
|
"dirigent.attempt": attempt,
|
|
330
333
|
},
|
|
334
|
+
record_exception=False,
|
|
335
|
+
set_status_on_exception=False,
|
|
331
336
|
) as span:
|
|
332
337
|
yield span
|
|
333
338
|
|
|
334
339
|
|
|
335
340
|
@contextmanager
|
|
336
|
-
def block_span(block: str, call: str) -> Generator[Span]:
|
|
337
|
-
"""Open the span covering one block call: execute, probe, fetch, or cancel.
|
|
341
|
+
def block_span(block: str, call: str, secrets: Collection[str] = ()) -> Generator[Span]:
|
|
342
|
+
"""Open the span covering one block call: execute, probe, fetch, or cancel.
|
|
343
|
+
|
|
344
|
+
An exception the call raises is recorded with every one of ``secrets`` scrubbed from it,
|
|
345
|
+
and raised on unchanged. ``secrets`` is read when the call fails, so a live set the call
|
|
346
|
+
adds to covers what it opened.
|
|
347
|
+
"""
|
|
338
348
|
with _tracer.start_as_current_span(
|
|
339
349
|
f"{block}.{call}",
|
|
340
350
|
kind=SpanKind.CLIENT,
|
|
341
351
|
attributes={"dirigent.block": block, "dirigent.block.call": call},
|
|
352
|
+
record_exception=False,
|
|
353
|
+
set_status_on_exception=False,
|
|
342
354
|
) as span:
|
|
343
|
-
|
|
355
|
+
try:
|
|
356
|
+
yield span
|
|
357
|
+
except Exception as error:
|
|
358
|
+
record_failure(span, error, secrets)
|
|
359
|
+
raise
|
|
344
360
|
|
|
345
361
|
|
|
346
|
-
def record_failure(span: Span, error:
|
|
347
|
-
"""Mark a span as failed, with the message an operator would want in the trace.
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
362
|
+
def record_failure(span: Span, error: Exception | str, secrets: Collection[str] = ()) -> None:
|
|
363
|
+
"""Mark a span as failed, with the message an operator would want in the trace.
|
|
364
|
+
|
|
365
|
+
Every one of ``secrets`` is scrubbed from what is recorded. An exception is recorded by its
|
|
366
|
+
class and its scrubbed text, never its traceback.
|
|
367
|
+
"""
|
|
368
|
+
if isinstance(error, Exception):
|
|
369
|
+
said = raised_detail(error, redact=secrets)
|
|
370
|
+
span.add_event("exception", {"exception.type": type(error).__qualname__, "exception.message": said})
|
|
371
|
+
span.set_status(Status(StatusCode.ERROR, f"{type(error).__name__}: {said}"))
|
|
351
372
|
else:
|
|
352
|
-
span.set_status(Status(StatusCode.ERROR, error))
|
|
373
|
+
span.set_status(Status(StatusCode.ERROR, scrub(error, secrets)))
|
|
353
374
|
|
|
354
375
|
|
|
355
376
|
def current_traceparent() -> str | None:
|
|
@@ -97,7 +97,7 @@ async def backfill(
|
|
|
97
97
|
resolved = dict(schedule.params) if params is None else dict(params)
|
|
98
98
|
if dry_run:
|
|
99
99
|
return [Filled(window=window) for window in windows]
|
|
100
|
-
load_definition(version.document).
|
|
100
|
+
await load_definition(version.document).check_params(resolved, services.format_checker)
|
|
101
101
|
filled: list[Filled] = []
|
|
102
102
|
for window in windows:
|
|
103
103
|
run = await create_run(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|