dirigent-blocks 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_blocks-0.9.0/LICENSE +18 -0
- dirigent_blocks-0.9.0/PKG-INFO +26 -0
- dirigent_blocks-0.9.0/README.md +3 -0
- dirigent_blocks-0.9.0/pyproject.toml +37 -0
- dirigent_blocks-0.9.0/pyproject.toml.orig +36 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/__init__.py +251 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/build.py +320 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/capture.py +289 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/clock.py +220 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/compose.py +668 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/connections.py +36 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/convert_std.py +554 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/docker.py +1218 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/environment.py +47 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/git.py +507 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/http.py +437 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/kafka.py +853 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/notifiers.py +492 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/pipelines.py +164 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/py.typed +0 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/rabbitmq.py +377 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/reap.py +174 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/secrets.py +42 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/shell.py +144 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/sql.py +962 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/storage.py +104 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/subprocess.py +239 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/transform_jq.py +153 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/validate.py +78 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/values.py +40 -0
- dirigent_blocks-0.9.0/src/dirigent_blocks/webhooks.py +150 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Copyright (c) 2026 Morten Olav Hansen <morten@winterop.com>. All rights reserved.
|
|
2
|
+
|
|
3
|
+
This source code and accompanying documentation are the property of
|
|
4
|
+
Morten Olav Hansen. No license, express or implied, is granted to use, copy,
|
|
5
|
+
modify, merge, publish, distribute, sublicense, or sell copies of this
|
|
6
|
+
software or its derivatives.
|
|
7
|
+
|
|
8
|
+
The source is published for reference only. Any use beyond reading
|
|
9
|
+
requires written permission from the copyright holder.
|
|
10
|
+
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
12
|
+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
13
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT.
|
|
14
|
+
IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES,
|
|
15
|
+
OR OTHER LIABILITY ARISING FROM THE USE OF THE SOFTWARE.
|
|
16
|
+
|
|
17
|
+
Third-party components redistributed with this software, and the licences they
|
|
18
|
+
carry, are listed in THIRD_PARTY_NOTICES.md.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dirigent-blocks
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Built-in generic operators and sensors for dirigent.
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: aio-pika>=9.5.7
|
|
8
|
+
Requires-Dist: aiokafka>=0.12.0
|
|
9
|
+
Requires-Dist: aiosmtplib>=5.1.2
|
|
10
|
+
Requires-Dist: dirigent-common
|
|
11
|
+
Requires-Dist: dirigent-plugin
|
|
12
|
+
Requires-Dist: httpx2>=2.12.0
|
|
13
|
+
Requires-Dist: jq>=1.8
|
|
14
|
+
Requires-Dist: jsonschema>=4.26.0
|
|
15
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
16
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.52
|
|
17
|
+
Requires-Dist: structlog>=25.5.0
|
|
18
|
+
Requires-Dist: duckdb>=1.4.0 ; extra == 'duckdb'
|
|
19
|
+
Requires-Dist: duckdb-engine>=0.17.0 ; extra == 'duckdb'
|
|
20
|
+
Requires-Python: >=3.13
|
|
21
|
+
Provides-Extra: duckdb
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# dirigent-blocks
|
|
25
|
+
|
|
26
|
+
Built-in generic operators and sensors for dirigent.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dirigent-blocks"
|
|
3
|
+
version = "0.9.0"
|
|
4
|
+
description = "Built-in generic operators and sensors for dirigent."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "LicenseRef-Proprietary"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
dependencies = [
|
|
10
|
+
"aio-pika>=9.5.7",
|
|
11
|
+
"aiokafka>=0.12.0",
|
|
12
|
+
"aiosmtplib>=5.1.2",
|
|
13
|
+
"dirigent-common",
|
|
14
|
+
"dirigent-plugin",
|
|
15
|
+
"httpx2>=2.12.0",
|
|
16
|
+
"jq>=1.8",
|
|
17
|
+
"jsonschema>=4.26.0",
|
|
18
|
+
"pyyaml>=6.0.3",
|
|
19
|
+
"sqlalchemy[asyncio]>=2.0.52",
|
|
20
|
+
"structlog>=25.5.0",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
duckdb = [
|
|
25
|
+
"duckdb>=1.4.0",
|
|
26
|
+
"duckdb-engine>=0.17.0",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.entry-points."dirigent.plugins.v1"]
|
|
30
|
+
builtin = "dirigent_blocks:plugin"
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["uv_build>=0.12.0,<0.13.0"]
|
|
34
|
+
build-backend = "uv_build"
|
|
35
|
+
|
|
36
|
+
[tool.uv.sources.dirigent-plugin]
|
|
37
|
+
workspace = true
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dirigent-blocks"
|
|
3
|
+
version = "0.9.0"
|
|
4
|
+
description = "Built-in generic operators and sensors for dirigent."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "LicenseRef-Proprietary"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
dependencies = [
|
|
10
|
+
"aio-pika>=9.5.7",
|
|
11
|
+
"aiokafka>=0.12.0",
|
|
12
|
+
"aiosmtplib>=5.1.2",
|
|
13
|
+
"dirigent-common",
|
|
14
|
+
"dirigent-plugin",
|
|
15
|
+
"httpx2>=2.12.0",
|
|
16
|
+
"jq>=1.8",
|
|
17
|
+
"jsonschema>=4.26.0",
|
|
18
|
+
"pyyaml>=6.0.3",
|
|
19
|
+
"sqlalchemy[asyncio]>=2.0.52",
|
|
20
|
+
"structlog>=25.5.0",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
# DuckDB is a 45MB engine binary, which is more than every other driver the family speaks to
|
|
25
|
+
# put together; a worker that never runs a duckdb url should not carry it.
|
|
26
|
+
duckdb = ["duckdb>=1.4.0", "duckdb-engine>=0.17.0"]
|
|
27
|
+
|
|
28
|
+
[project.entry-points."dirigent.plugins.v1"]
|
|
29
|
+
builtin = "dirigent_blocks:plugin"
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["uv_build>=0.12.0,<0.13.0"]
|
|
33
|
+
build-backend = "uv_build"
|
|
34
|
+
|
|
35
|
+
[tool.uv.sources]
|
|
36
|
+
dirigent-plugin = { workspace = true }
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""The built-in generic block pack: what a fresh install can already do."""
|
|
2
|
+
|
|
3
|
+
from dirigent_blocks.build import DockerBuildConfig, DockerBuildOperator, DockerBuildOutput
|
|
4
|
+
from dirigent_blocks.clock import (
|
|
5
|
+
TimeSleepConfig,
|
|
6
|
+
TimeSleepOutput,
|
|
7
|
+
TimeSleepSensor,
|
|
8
|
+
TimeWindowConfig,
|
|
9
|
+
TimeWindowOutput,
|
|
10
|
+
TimeWindowSensor,
|
|
11
|
+
)
|
|
12
|
+
from dirigent_blocks.compose import (
|
|
13
|
+
DockerComposeDownConfig,
|
|
14
|
+
DockerComposeDownOperator,
|
|
15
|
+
DockerComposeDownOutput,
|
|
16
|
+
DockerComposeUpConfig,
|
|
17
|
+
DockerComposeUpOperator,
|
|
18
|
+
DockerComposeUpOutput,
|
|
19
|
+
)
|
|
20
|
+
from dirigent_blocks.connections import HttpConnectionConfig, HttpConnectionKind
|
|
21
|
+
from dirigent_blocks.convert_std import StdConverter
|
|
22
|
+
from dirigent_blocks.docker import (
|
|
23
|
+
DockerConnectionConfig,
|
|
24
|
+
DockerConnectionKind,
|
|
25
|
+
DockerRunConfig,
|
|
26
|
+
DockerRunOperator,
|
|
27
|
+
DockerRunOutput,
|
|
28
|
+
)
|
|
29
|
+
from dirigent_blocks.git import (
|
|
30
|
+
GitCheckoutConfig,
|
|
31
|
+
GitCheckoutOperator,
|
|
32
|
+
GitCheckoutOutput,
|
|
33
|
+
GitConnectionConfig,
|
|
34
|
+
GitConnectionKind,
|
|
35
|
+
)
|
|
36
|
+
from dirigent_blocks.http import (
|
|
37
|
+
HttpReadyConfig,
|
|
38
|
+
HttpReadyOutput,
|
|
39
|
+
HttpReadySensor,
|
|
40
|
+
HttpRequestConfig,
|
|
41
|
+
HttpRequestOperator,
|
|
42
|
+
HttpRequestOutput,
|
|
43
|
+
)
|
|
44
|
+
from dirigent_blocks.kafka import (
|
|
45
|
+
KafkaConnectionConfig,
|
|
46
|
+
KafkaConnectionKind,
|
|
47
|
+
KafkaConsumeConfig,
|
|
48
|
+
KafkaConsumeOutput,
|
|
49
|
+
KafkaConsumeSensor,
|
|
50
|
+
KafkaMessage,
|
|
51
|
+
KafkaProduceConfig,
|
|
52
|
+
KafkaProduceOperator,
|
|
53
|
+
KafkaProduceOutput,
|
|
54
|
+
)
|
|
55
|
+
from dirigent_blocks.notifiers import (
|
|
56
|
+
EmailConnectionKind,
|
|
57
|
+
EmailNotifier,
|
|
58
|
+
EmailNotifierConfig,
|
|
59
|
+
LogNotifier,
|
|
60
|
+
LogNotifierConfig,
|
|
61
|
+
SlackConnectionKind,
|
|
62
|
+
SlackNotifier,
|
|
63
|
+
SlackNotifierConfig,
|
|
64
|
+
WebhookConnectionKind,
|
|
65
|
+
WebhookNotifier,
|
|
66
|
+
WebhookNotifierConfig,
|
|
67
|
+
)
|
|
68
|
+
from dirigent_blocks.pipelines import PipelineRunConfig, PipelineRunOperator, PipelineRunOutput
|
|
69
|
+
from dirigent_blocks.rabbitmq import (
|
|
70
|
+
RabbitConnectionConfig,
|
|
71
|
+
RabbitConnectionKind,
|
|
72
|
+
RabbitConsumeConfig,
|
|
73
|
+
RabbitConsumeOutput,
|
|
74
|
+
RabbitConsumeSensor,
|
|
75
|
+
RabbitMessage,
|
|
76
|
+
)
|
|
77
|
+
from dirigent_blocks.shell import ShellRunConfig, ShellRunOperator, ShellRunOutput
|
|
78
|
+
from dirigent_blocks.sql import (
|
|
79
|
+
SqlConnectionConfig,
|
|
80
|
+
SqlConnectionKind,
|
|
81
|
+
SqlExecuteConfig,
|
|
82
|
+
SqlExecuteOperator,
|
|
83
|
+
SqlExecuteOutput,
|
|
84
|
+
SqlQueryConfig,
|
|
85
|
+
SqlQueryOperator,
|
|
86
|
+
SqlQueryOutput,
|
|
87
|
+
)
|
|
88
|
+
from dirigent_blocks.storage import (
|
|
89
|
+
StorageCopyConfig,
|
|
90
|
+
StorageCopyOperator,
|
|
91
|
+
StorageCopyOutput,
|
|
92
|
+
StorageExistsConfig,
|
|
93
|
+
StorageExistsOutput,
|
|
94
|
+
StorageExistsSensor,
|
|
95
|
+
)
|
|
96
|
+
from dirigent_blocks.transform_jq import JqFilterer, JqMapper, JqTransformer
|
|
97
|
+
from dirigent_blocks.validate import ValidateSchemaConfig, ValidateSchemaOperator, ValidateSchemaOutput
|
|
98
|
+
from dirigent_blocks.values import ValueConstConfig, ValueConstOperator, ValueConstOutput
|
|
99
|
+
from dirigent_blocks.webhooks import WebhookPostConfig, WebhookPostOperator, WebhookPostOutput
|
|
100
|
+
from dirigent_plugin import Contribution, extension
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class BuiltinBlocks:
|
|
104
|
+
"""The plugin object the host discovers under the dirigent.plugins.v1 entry-point group."""
|
|
105
|
+
|
|
106
|
+
@extension
|
|
107
|
+
def contribute(self) -> Contribution:
|
|
108
|
+
"""Contribute the built-in operators, sensors, notifiers, and their connection kinds."""
|
|
109
|
+
return Contribution(
|
|
110
|
+
operators=[
|
|
111
|
+
HttpRequestOperator(),
|
|
112
|
+
StorageCopyOperator(),
|
|
113
|
+
ShellRunOperator(),
|
|
114
|
+
DockerRunOperator(),
|
|
115
|
+
DockerComposeUpOperator(),
|
|
116
|
+
DockerComposeDownOperator(),
|
|
117
|
+
DockerBuildOperator(),
|
|
118
|
+
GitCheckoutOperator(),
|
|
119
|
+
SqlQueryOperator(),
|
|
120
|
+
SqlExecuteOperator(),
|
|
121
|
+
PipelineRunOperator(),
|
|
122
|
+
KafkaProduceOperator(),
|
|
123
|
+
WebhookPostOperator(),
|
|
124
|
+
JqTransformer(),
|
|
125
|
+
JqMapper(),
|
|
126
|
+
JqFilterer(),
|
|
127
|
+
StdConverter(),
|
|
128
|
+
ValueConstOperator(),
|
|
129
|
+
ValidateSchemaOperator(),
|
|
130
|
+
],
|
|
131
|
+
sensors=[
|
|
132
|
+
HttpReadySensor(),
|
|
133
|
+
KafkaConsumeSensor(),
|
|
134
|
+
RabbitConsumeSensor(),
|
|
135
|
+
StorageExistsSensor(),
|
|
136
|
+
TimeSleepSensor(),
|
|
137
|
+
TimeWindowSensor(),
|
|
138
|
+
],
|
|
139
|
+
notifiers=[LogNotifier(), WebhookNotifier(), SlackNotifier(), EmailNotifier()],
|
|
140
|
+
connection_kinds=[
|
|
141
|
+
HttpConnectionKind(),
|
|
142
|
+
DockerConnectionKind(),
|
|
143
|
+
GitConnectionKind(),
|
|
144
|
+
KafkaConnectionKind(),
|
|
145
|
+
RabbitConnectionKind(),
|
|
146
|
+
SqlConnectionKind(),
|
|
147
|
+
SlackConnectionKind(),
|
|
148
|
+
EmailConnectionKind(),
|
|
149
|
+
WebhookConnectionKind(),
|
|
150
|
+
],
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
plugin = BuiltinBlocks()
|
|
155
|
+
|
|
156
|
+
__all__ = [
|
|
157
|
+
"BuiltinBlocks",
|
|
158
|
+
"DockerBuildConfig",
|
|
159
|
+
"DockerBuildOperator",
|
|
160
|
+
"DockerBuildOutput",
|
|
161
|
+
"DockerComposeDownConfig",
|
|
162
|
+
"DockerComposeDownOperator",
|
|
163
|
+
"DockerComposeDownOutput",
|
|
164
|
+
"DockerComposeUpConfig",
|
|
165
|
+
"DockerComposeUpOperator",
|
|
166
|
+
"DockerComposeUpOutput",
|
|
167
|
+
"DockerConnectionConfig",
|
|
168
|
+
"DockerConnectionKind",
|
|
169
|
+
"DockerRunConfig",
|
|
170
|
+
"DockerRunOperator",
|
|
171
|
+
"DockerRunOutput",
|
|
172
|
+
"EmailConnectionKind",
|
|
173
|
+
"EmailNotifier",
|
|
174
|
+
"EmailNotifierConfig",
|
|
175
|
+
"GitCheckoutConfig",
|
|
176
|
+
"GitCheckoutOperator",
|
|
177
|
+
"GitCheckoutOutput",
|
|
178
|
+
"GitConnectionConfig",
|
|
179
|
+
"GitConnectionKind",
|
|
180
|
+
"HttpConnectionConfig",
|
|
181
|
+
"HttpConnectionKind",
|
|
182
|
+
"HttpReadyConfig",
|
|
183
|
+
"HttpReadyOutput",
|
|
184
|
+
"HttpReadySensor",
|
|
185
|
+
"HttpRequestConfig",
|
|
186
|
+
"HttpRequestOperator",
|
|
187
|
+
"HttpRequestOutput",
|
|
188
|
+
"JqFilterer",
|
|
189
|
+
"JqMapper",
|
|
190
|
+
"JqTransformer",
|
|
191
|
+
"KafkaConnectionConfig",
|
|
192
|
+
"KafkaConnectionKind",
|
|
193
|
+
"KafkaConsumeConfig",
|
|
194
|
+
"KafkaConsumeOutput",
|
|
195
|
+
"KafkaConsumeSensor",
|
|
196
|
+
"KafkaMessage",
|
|
197
|
+
"KafkaProduceConfig",
|
|
198
|
+
"KafkaProduceOperator",
|
|
199
|
+
"KafkaProduceOutput",
|
|
200
|
+
"LogNotifier",
|
|
201
|
+
"LogNotifierConfig",
|
|
202
|
+
"PipelineRunConfig",
|
|
203
|
+
"PipelineRunOperator",
|
|
204
|
+
"PipelineRunOutput",
|
|
205
|
+
"RabbitConnectionConfig",
|
|
206
|
+
"RabbitConnectionKind",
|
|
207
|
+
"RabbitConsumeConfig",
|
|
208
|
+
"RabbitConsumeOutput",
|
|
209
|
+
"RabbitConsumeSensor",
|
|
210
|
+
"RabbitMessage",
|
|
211
|
+
"ShellRunConfig",
|
|
212
|
+
"ShellRunOperator",
|
|
213
|
+
"ShellRunOutput",
|
|
214
|
+
"SlackConnectionKind",
|
|
215
|
+
"SlackNotifier",
|
|
216
|
+
"SlackNotifierConfig",
|
|
217
|
+
"SqlConnectionConfig",
|
|
218
|
+
"SqlConnectionKind",
|
|
219
|
+
"SqlExecuteConfig",
|
|
220
|
+
"SqlExecuteOperator",
|
|
221
|
+
"SqlExecuteOutput",
|
|
222
|
+
"SqlQueryConfig",
|
|
223
|
+
"SqlQueryOperator",
|
|
224
|
+
"SqlQueryOutput",
|
|
225
|
+
"StdConverter",
|
|
226
|
+
"StorageCopyConfig",
|
|
227
|
+
"StorageCopyOperator",
|
|
228
|
+
"StorageCopyOutput",
|
|
229
|
+
"StorageExistsConfig",
|
|
230
|
+
"StorageExistsOutput",
|
|
231
|
+
"StorageExistsSensor",
|
|
232
|
+
"TimeSleepConfig",
|
|
233
|
+
"TimeSleepOutput",
|
|
234
|
+
"TimeSleepSensor",
|
|
235
|
+
"TimeWindowConfig",
|
|
236
|
+
"TimeWindowOutput",
|
|
237
|
+
"TimeWindowSensor",
|
|
238
|
+
"ValidateSchemaConfig",
|
|
239
|
+
"ValidateSchemaOperator",
|
|
240
|
+
"ValidateSchemaOutput",
|
|
241
|
+
"ValueConstConfig",
|
|
242
|
+
"ValueConstOperator",
|
|
243
|
+
"ValueConstOutput",
|
|
244
|
+
"WebhookConnectionKind",
|
|
245
|
+
"WebhookNotifier",
|
|
246
|
+
"WebhookNotifierConfig",
|
|
247
|
+
"WebhookPostConfig",
|
|
248
|
+
"WebhookPostOperator",
|
|
249
|
+
"WebhookPostOutput",
|
|
250
|
+
"plugin",
|
|
251
|
+
]
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
"""``docker.build``: build an image from a context on the worker, with buildx.
|
|
2
|
+
|
|
3
|
+
The block shells out to ``docker buildx build`` -- BuildKit, its cache and multi-stage builds
|
|
4
|
+
come for free -- and reads the built image's id from an ``--iidfile`` rather than scraping the
|
|
5
|
+
log. The image lands in the worker's own daemon store, so a later ``docker.run`` or
|
|
6
|
+
``docker.compose`` step on the same worker references it by tag.
|
|
7
|
+
|
|
8
|
+
Reaching the Docker daemon is reaching root on the host when the daemon is the host's own, so
|
|
9
|
+
the block declares ``local_execution`` and the engine refuses it unless the instance allowlists
|
|
10
|
+
its id.
|
|
11
|
+
|
|
12
|
+
``push`` needs a ``docker`` connection carrying a registry credential, and is refused without
|
|
13
|
+
one. The login goes to a ``DOCKER_CONFIG`` directory of its own under scratch -- never the
|
|
14
|
+
worker's own config -- with the password on stdin rather than in an argument, and the
|
|
15
|
+
directory and the session in it go when the step leaves.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from datetime import timedelta
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import ClassVar
|
|
22
|
+
|
|
23
|
+
from pydantic import BaseModel, Field, model_validator
|
|
24
|
+
|
|
25
|
+
from dirigent_blocks import subprocess
|
|
26
|
+
from dirigent_blocks.capture import log_stream, scrub, tail
|
|
27
|
+
from dirigent_blocks.docker import (
|
|
28
|
+
DAEMON_ENV,
|
|
29
|
+
DockerConnectionConfig,
|
|
30
|
+
Sealed,
|
|
31
|
+
daemon_environment,
|
|
32
|
+
login,
|
|
33
|
+
logout,
|
|
34
|
+
sealed,
|
|
35
|
+
)
|
|
36
|
+
from dirigent_blocks.environment import reject_reserved
|
|
37
|
+
from dirigent_common import BlockModel, Duration
|
|
38
|
+
from dirigent_plugin import (
|
|
39
|
+
BlockFailure,
|
|
40
|
+
ConnectionRef,
|
|
41
|
+
ErrorClass,
|
|
42
|
+
Operator,
|
|
43
|
+
OperatorSpec,
|
|
44
|
+
RemoteHandle,
|
|
45
|
+
StepContext,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
#: What a daemon that is not there or not ours says, a transient condition rather than a broken build.
|
|
49
|
+
DAEMON_UNREACHABLE = ("cannot connect to the docker daemon", "is the docker daemon running", "permission denied")
|
|
50
|
+
|
|
51
|
+
#: What ``docker push`` prints once the registry has taken a tag, which is where its digest is.
|
|
52
|
+
PUSHED_DIGEST = re.compile(r"digest:\s*(sha256:[0-9a-f]{64})")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DockerBuildConfig(BlockModel):
|
|
56
|
+
"""Which context to build, with what Dockerfile, tags, and build arguments."""
|
|
57
|
+
|
|
58
|
+
context: str
|
|
59
|
+
"""The build context, as a directory inside the run's scratch space; never absolute, never climbing out."""
|
|
60
|
+
|
|
61
|
+
dockerfile: str = "Dockerfile"
|
|
62
|
+
"""The Dockerfile, as a path relative to the context."""
|
|
63
|
+
|
|
64
|
+
tags: list[str] = Field(default_factory=list[str])
|
|
65
|
+
"""Tags to give the built image (``--tag``); a later step references the image by one of them."""
|
|
66
|
+
|
|
67
|
+
build_args: dict[str, str] = Field(default_factory=dict[str, str])
|
|
68
|
+
"""Build arguments the Dockerfile reads (``--build-arg``)."""
|
|
69
|
+
|
|
70
|
+
target: str | None = None
|
|
71
|
+
"""The stage to stop at in a multi-stage build (``--target``)."""
|
|
72
|
+
|
|
73
|
+
platform: str | None = None
|
|
74
|
+
"""The platform to build for, such as ``linux/amd64`` (``--platform``)."""
|
|
75
|
+
|
|
76
|
+
pull: bool = False
|
|
77
|
+
"""Always attempt to pull a newer version of the base image (``--pull``)."""
|
|
78
|
+
|
|
79
|
+
no_cache: bool = False
|
|
80
|
+
"""Build every layer from scratch, ignoring the cache (``--no-cache``)."""
|
|
81
|
+
|
|
82
|
+
push: bool = False
|
|
83
|
+
"""Push every tag the build produced to the registry the connection names.
|
|
84
|
+
|
|
85
|
+
Needs a ``connection`` carrying a registry username and password: an anonymous push is
|
|
86
|
+
refused rather than attempted."""
|
|
87
|
+
|
|
88
|
+
connection: ConnectionRef | None = None
|
|
89
|
+
"""A ``docker`` connection naming the daemon to build on, the registry to push to, or both.
|
|
90
|
+
|
|
91
|
+
Absent, the build runs against whatever daemon the worker's own environment names and
|
|
92
|
+
nothing is pushed."""
|
|
93
|
+
|
|
94
|
+
env: dict[str, str] = Field(default_factory=dict[str, str])
|
|
95
|
+
"""Variables set for the CLI itself, such as BuildKit's own toggles."""
|
|
96
|
+
|
|
97
|
+
env_allowlist: list[str] = Field(default_factory=list[str])
|
|
98
|
+
"""Worker environment variables the CLI is allowed to inherit; never the instance's ``DIRIGENT_*``."""
|
|
99
|
+
|
|
100
|
+
command_path: list[str] = Field(default_factory=lambda: ["docker", "buildx", "build"])
|
|
101
|
+
"""The CLI to invoke, for a host that wraps buildx."""
|
|
102
|
+
|
|
103
|
+
timeout: Duration = timedelta(minutes=30)
|
|
104
|
+
"""The overall deadline on the build, after which it is killed as transient."""
|
|
105
|
+
|
|
106
|
+
@model_validator(mode="after")
|
|
107
|
+
def _check_shape(self) -> "DockerBuildConfig":
|
|
108
|
+
"""Refuse an unbacked push, a context or Dockerfile that climbs out, and a reserved env inheritance."""
|
|
109
|
+
if self.push and self.connection is None:
|
|
110
|
+
raise ValueError(
|
|
111
|
+
"docker.build cannot push without a connection: set connection to a docker connection "
|
|
112
|
+
"holding registry, username and password"
|
|
113
|
+
)
|
|
114
|
+
if self.push and not self.tags:
|
|
115
|
+
raise ValueError("docker.build pushes the tags it built, so a push needs at least one tag")
|
|
116
|
+
for path in (self.context, self.dockerfile):
|
|
117
|
+
if Path(path).is_absolute() or ".." in Path(path).parts:
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"the context and Dockerfile are paths inside the run's scratch space, so they "
|
|
120
|
+
"cannot be absolute or climb out"
|
|
121
|
+
)
|
|
122
|
+
reject_reserved(self.env_allowlist)
|
|
123
|
+
return self
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class DockerBuildOutput(BlockModel):
|
|
127
|
+
"""The image the build produced, with its full log addressable as an artifact."""
|
|
128
|
+
|
|
129
|
+
image_id: str
|
|
130
|
+
"""The built image's id, read from the ``--iidfile`` the daemon wrote, not scraped from the log."""
|
|
131
|
+
|
|
132
|
+
tags: list[str] = Field(default_factory=list[str])
|
|
133
|
+
"""The tags the image was given, by which a later step on the same worker references it."""
|
|
134
|
+
|
|
135
|
+
size_bytes: int = 0
|
|
136
|
+
"""The image's size on disk, from the daemon; zero when it could not be read."""
|
|
137
|
+
|
|
138
|
+
pushed: list[str] = Field(default_factory=list[str])
|
|
139
|
+
"""The tags that reached the registry, empty when the step pushed nothing."""
|
|
140
|
+
|
|
141
|
+
digests: dict[str, str] = Field(default_factory=dict[str, str])
|
|
142
|
+
"""The registry's digest for each pushed tag, for the tags the CLI reported one for."""
|
|
143
|
+
|
|
144
|
+
build_log_uri: str
|
|
145
|
+
"""Where the whole of the build log (buildx's progress on stderr) was written."""
|
|
146
|
+
|
|
147
|
+
stdout_uri: str
|
|
148
|
+
"""Where the whole of the CLI's stdout was written."""
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class DockerBuildOperator(Operator[DockerBuildConfig, DockerBuildOutput]):
|
|
152
|
+
"""Builds an image from a scratch context with buildx, behind the allowlist."""
|
|
153
|
+
|
|
154
|
+
spec = OperatorSpec(
|
|
155
|
+
id="docker.build",
|
|
156
|
+
group="execute",
|
|
157
|
+
summary="Build a container image on the worker.",
|
|
158
|
+
idempotent=False,
|
|
159
|
+
local_execution=True,
|
|
160
|
+
)
|
|
161
|
+
config_model: ClassVar[type[BaseModel]] = DockerBuildConfig
|
|
162
|
+
output_model: ClassVar[type[BaseModel]] = DockerBuildOutput
|
|
163
|
+
|
|
164
|
+
async def execute(self, config: DockerBuildConfig, ctx: StepContext) -> DockerBuildOutput | RemoteHandle:
|
|
165
|
+
"""Build the image, read its id from the iidfile, and report its size and tags."""
|
|
166
|
+
root = subprocess.local_root(ctx)
|
|
167
|
+
context = root / config.context
|
|
168
|
+
iidfile = Path(f"{root / subprocess.segment(ctx, 'build')}.iid")
|
|
169
|
+
iidfile.parent.mkdir(parents=True, exist_ok=True)
|
|
170
|
+
settings = ctx.connection(config.connection, DockerConnectionConfig) if config.connection else None
|
|
171
|
+
if config.push and (settings is None or not settings.authenticates):
|
|
172
|
+
raise BlockFailure(
|
|
173
|
+
"docker.build cannot push through a connection with no registry credential: set "
|
|
174
|
+
"username and password on the docker connection, and registry unless it is Docker Hub",
|
|
175
|
+
error_class=ErrorClass.REJECTED,
|
|
176
|
+
)
|
|
177
|
+
timeout = config.timeout.total_seconds()
|
|
178
|
+
artifacts = subprocess.prefix(ctx, "build")
|
|
179
|
+
stdout_uri = f"{artifacts}-stdout.txt"
|
|
180
|
+
build_log_uri = f"{artifacts}-build.log"
|
|
181
|
+
|
|
182
|
+
with sealed(settings, root, isolate_config=config.push) as material:
|
|
183
|
+
environ = daemon_environment(
|
|
184
|
+
subprocess.environment([*DAEMON_ENV, *config.env_allowlist], config.env, root), material
|
|
185
|
+
)
|
|
186
|
+
code, out, err = await subprocess.run(
|
|
187
|
+
directory=root,
|
|
188
|
+
ctx=ctx,
|
|
189
|
+
stdout_uri=stdout_uri,
|
|
190
|
+
stderr_uri=build_log_uri,
|
|
191
|
+
timeout_seconds=timeout,
|
|
192
|
+
environ=environ,
|
|
193
|
+
argv=build_argv(config, context, iidfile),
|
|
194
|
+
what="docker buildx build",
|
|
195
|
+
redact=material.secrets,
|
|
196
|
+
)
|
|
197
|
+
log_stream(ctx, "stdout", out, material.secrets)
|
|
198
|
+
log_stream(ctx, "stderr", err, material.secrets)
|
|
199
|
+
ctx.log.info("docker build finished", exit_code=code, tags=", ".join(config.tags))
|
|
200
|
+
if code != 0:
|
|
201
|
+
detail = (
|
|
202
|
+
tail(err.tail, redact=material.secrets) or tail(out.tail, redact=material.secrets) or "no output"
|
|
203
|
+
)
|
|
204
|
+
raise BlockFailure(f"docker build exited {code}: {detail}", error_class=_classify(err.tail))
|
|
205
|
+
|
|
206
|
+
image_id = iidfile.read_text().strip() if iidfile.exists() else ""
|
|
207
|
+
if not image_id:
|
|
208
|
+
raise BlockFailure(
|
|
209
|
+
"docker build reported success but wrote no image id to the iidfile",
|
|
210
|
+
error_class=ErrorClass.UNKNOWN,
|
|
211
|
+
)
|
|
212
|
+
size_bytes = await _image_size(config, image_id, root, environ, timeout)
|
|
213
|
+
pushed, digests = await _push(config, settings, root, environ, material, timeout, ctx)
|
|
214
|
+
ctx.log.info("image built", image_id=image_id[:19], size_bytes=size_bytes, tags=", ".join(config.tags))
|
|
215
|
+
return DockerBuildOutput(
|
|
216
|
+
image_id=image_id,
|
|
217
|
+
tags=config.tags,
|
|
218
|
+
size_bytes=size_bytes,
|
|
219
|
+
pushed=pushed,
|
|
220
|
+
digests=digests,
|
|
221
|
+
build_log_uri=build_log_uri,
|
|
222
|
+
stdout_uri=stdout_uri,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def build_argv(config: DockerBuildConfig, context: Path, iidfile: Path) -> list[str]:
|
|
227
|
+
"""Assemble the ``buildx build`` argv, flag by flag, with the context last."""
|
|
228
|
+
argv = [*config.command_path, "--file", str(context / config.dockerfile), "--iidfile", str(iidfile)]
|
|
229
|
+
for tag in config.tags:
|
|
230
|
+
argv += ["--tag", tag]
|
|
231
|
+
for name, value in config.build_args.items():
|
|
232
|
+
argv += ["--build-arg", f"{name}={value}"]
|
|
233
|
+
if config.target is not None:
|
|
234
|
+
argv += ["--target", config.target]
|
|
235
|
+
if config.platform is not None:
|
|
236
|
+
argv += ["--platform", config.platform]
|
|
237
|
+
if config.pull:
|
|
238
|
+
argv.append("--pull")
|
|
239
|
+
if config.no_cache:
|
|
240
|
+
argv.append("--no-cache")
|
|
241
|
+
# Load the result into the worker's daemon store, so a later step can run it by tag; the
|
|
242
|
+
# container-driver default would otherwise leave the image only in the builder's cache.
|
|
243
|
+
argv.append("--load")
|
|
244
|
+
argv.append(str(context))
|
|
245
|
+
return argv
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
async def _push(
|
|
249
|
+
config: DockerBuildConfig,
|
|
250
|
+
settings: DockerConnectionConfig | None,
|
|
251
|
+
root: Path,
|
|
252
|
+
environ: dict[str, str],
|
|
253
|
+
material: Sealed,
|
|
254
|
+
timeout: float,
|
|
255
|
+
ctx: StepContext,
|
|
256
|
+
) -> tuple[list[str], dict[str, str]]:
|
|
257
|
+
"""Log in, push every tag the build produced, and log out again.
|
|
258
|
+
|
|
259
|
+
The login lives in the ``DOCKER_CONFIG`` directory the sealed material made, so it reaches
|
|
260
|
+
neither the worker's own config nor any later step, and the logout takes even that away.
|
|
261
|
+
"""
|
|
262
|
+
if not config.push:
|
|
263
|
+
return [], {}
|
|
264
|
+
assert settings is not None
|
|
265
|
+
docker = config.command_path[0] if config.command_path else "docker"
|
|
266
|
+
code, err = await login(docker, root, environ, settings)
|
|
267
|
+
if code != 0:
|
|
268
|
+
detail = tail(err, redact=material.secrets) or f"docker login exited {code}"
|
|
269
|
+
raise BlockFailure(
|
|
270
|
+
f"docker login to {settings.registry_name} failed: {detail}", error_class=ErrorClass.REJECTED
|
|
271
|
+
)
|
|
272
|
+
pushed: list[str] = []
|
|
273
|
+
digests: dict[str, str] = {}
|
|
274
|
+
try:
|
|
275
|
+
for tag in config.tags:
|
|
276
|
+
code, out, err = await subprocess.output(
|
|
277
|
+
argv=[docker, "push", tag],
|
|
278
|
+
directory=root,
|
|
279
|
+
environ=environ,
|
|
280
|
+
timeout_seconds=timeout,
|
|
281
|
+
what="docker push",
|
|
282
|
+
)
|
|
283
|
+
if code != 0:
|
|
284
|
+
detail = tail(err, redact=material.secrets) or tail(out, redact=material.secrets) or "no output"
|
|
285
|
+
raise BlockFailure(f"docker push {tag} exited {code}: {detail}", error_class=_classify(err))
|
|
286
|
+
found = PUSHED_DIGEST.search(scrub(out.decode("utf-8", errors="replace"), material.secrets))
|
|
287
|
+
if found is not None:
|
|
288
|
+
digests[tag] = found.group(1)
|
|
289
|
+
pushed.append(tag)
|
|
290
|
+
ctx.log.info("image pushed", tag=tag, registry=settings.registry_name, digest=digests.get(tag, ""))
|
|
291
|
+
finally:
|
|
292
|
+
await logout(docker, root, environ, settings)
|
|
293
|
+
return pushed, digests
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
async def _image_size(
|
|
297
|
+
config: DockerBuildConfig, image_id: str, root: Path, environ: dict[str, str], timeout: float
|
|
298
|
+
) -> int:
|
|
299
|
+
"""Read the built image's size from the daemon, best effort; zero when it cannot be read."""
|
|
300
|
+
docker = config.command_path[0] if config.command_path else "docker"
|
|
301
|
+
try:
|
|
302
|
+
_code, out, _err = await subprocess.output(
|
|
303
|
+
argv=[docker, "image", "inspect", "--format", "{{.Size}}", image_id],
|
|
304
|
+
directory=root,
|
|
305
|
+
environ=environ,
|
|
306
|
+
timeout_seconds=timeout,
|
|
307
|
+
what="docker image inspect",
|
|
308
|
+
)
|
|
309
|
+
except BlockFailure:
|
|
310
|
+
return 0
|
|
311
|
+
text = out.decode("utf-8", errors="replace").strip()
|
|
312
|
+
return int(text) if text.isdigit() else 0
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _classify(stderr_tail: bytes) -> ErrorClass:
|
|
316
|
+
"""A daemon that cannot be reached is transient; a build that would not compile is not."""
|
|
317
|
+
text = stderr_tail.decode("utf-8", errors="replace").lower()
|
|
318
|
+
if any(marker in text for marker in DAEMON_UNREACHABLE):
|
|
319
|
+
return ErrorClass.TRANSIENT
|
|
320
|
+
return ErrorClass.UNKNOWN
|