dirigent-cli 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_cli/__init__.py +5 -0
- dirigent_cli/aliases.py +41 -0
- dirigent_cli/commands.py +2525 -0
- dirigent_cli/context.py +136 -0
- dirigent_cli/formatters.py +158 -0
- dirigent_cli/graph.py +109 -0
- dirigent_cli/health.py +294 -0
- dirigent_cli/local.py +790 -0
- dirigent_cli/main.py +1169 -0
- dirigent_cli/output.py +543 -0
- dirigent_cli/params.py +389 -0
- dirigent_cli/profiles.py +221 -0
- dirigent_cli/project.py +643 -0
- dirigent_cli/py.typed +0 -0
- dirigent_cli/reaper.py +115 -0
- dirigent_cli/scaffold.py +63 -0
- dirigent_cli/schemas.py +85 -0
- dirigent_cli/sources.py +76 -0
- dirigent_cli/stream.py +180 -0
- dirigent_cli/summaries.py +420 -0
- dirigent_cli/templates/pack/README.md.tmpl +23 -0
- dirigent_cli/templates/pack/__init__.py.tmpl +24 -0
- dirigent_cli/templates/pack/operator.py.tmpl +34 -0
- dirigent_cli/templates/pack/pyproject.toml.tmpl +21 -0
- dirigent_cli/templates/pack/test_plugin.py.tmpl +21 -0
- dirigent_cli/timing.py +322 -0
- dirigent_cli/triggers.py +631 -0
- dirigent_cli-0.9.0.dist-info/METADATA +24 -0
- dirigent_cli-0.9.0.dist-info/RECORD +32 -0
- dirigent_cli-0.9.0.dist-info/WHEEL +4 -0
- dirigent_cli-0.9.0.dist-info/entry_points.txt +4 -0
- dirigent_cli-0.9.0.dist-info/licenses/LICENSE +18 -0
dirigent_cli/project.py
ADDED
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""Pipeline projects: a uv project of documents, and the scaffolding that creates one."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Final, cast
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
11
|
+
|
|
12
|
+
from dirigent_core.configdocs import example_document, project_document
|
|
13
|
+
|
|
14
|
+
#: The one file a project hand-edits: where its documents are, and what this instance sets.
|
|
15
|
+
PROJECT_FILE: Final = "dirigent.yaml"
|
|
16
|
+
|
|
17
|
+
#: Every setting, commented out, beside it. Read, never loaded.
|
|
18
|
+
EXAMPLE_CONFIG_FILE: Final = "dirigent.example.yaml"
|
|
19
|
+
DEFAULT_PIPELINES_DIR: Final = "pipelines"
|
|
20
|
+
DOCUMENT_SUFFIXES: Final = (".yaml", ".yml", ".json")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ProjectError(Exception):
|
|
24
|
+
"""A project could not be read or scaffolded."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Project(BaseModel):
|
|
28
|
+
"""One pipeline project: where its documents are, and which profile they go to."""
|
|
29
|
+
|
|
30
|
+
model_config = ConfigDict(frozen=True)
|
|
31
|
+
|
|
32
|
+
root: Path
|
|
33
|
+
pipelines: str = DEFAULT_PIPELINES_DIR
|
|
34
|
+
profile: str | None = None
|
|
35
|
+
requires: list[str] = Field(default_factory=list[str])
|
|
36
|
+
"""Block ids every document in this project assumes."""
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def pipelines_dir(self) -> Path:
|
|
40
|
+
"""The directory the project's documents live in."""
|
|
41
|
+
return self.root / self.pipelines
|
|
42
|
+
|
|
43
|
+
def documents(self) -> list[Path]:
|
|
44
|
+
"""List every document in the project, in a stable order."""
|
|
45
|
+
directory = self.pipelines_dir
|
|
46
|
+
if not directory.is_dir():
|
|
47
|
+
raise ProjectError(f"{directory} does not exist; {PROJECT_FILE} points the pipelines directory at it")
|
|
48
|
+
return sorted(path for path in directory.iterdir() if path.suffix in DOCUMENT_SUFFIXES and path.is_file())
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def find_project(start: Path | None = None) -> Project | None:
|
|
52
|
+
"""Find the project the working directory is inside, by walking up to the filesystem root."""
|
|
53
|
+
here = (start or Path.cwd()).resolve()
|
|
54
|
+
for directory in [here, *here.parents]:
|
|
55
|
+
candidate = directory / PROJECT_FILE
|
|
56
|
+
if candidate.is_file():
|
|
57
|
+
return load_project(candidate)
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def load_project(path: Path) -> Project:
|
|
62
|
+
"""Read one ``dirigent.yaml``."""
|
|
63
|
+
try:
|
|
64
|
+
loaded: object = yaml.safe_load(path.read_text())
|
|
65
|
+
except (OSError, yaml.YAMLError) as error:
|
|
66
|
+
raise ProjectError(f"{path} could not be read: {error}") from error
|
|
67
|
+
if loaded is None:
|
|
68
|
+
loaded = {}
|
|
69
|
+
if not isinstance(loaded, dict):
|
|
70
|
+
raise ProjectError(f"{path} should hold a mapping of project settings")
|
|
71
|
+
raw = cast("dict[str, Any]", loaded)
|
|
72
|
+
section = raw.get("project") if isinstance(raw.get("project"), dict) else raw
|
|
73
|
+
body = cast("dict[str, Any]", section)
|
|
74
|
+
pipelines = body.get("pipelines")
|
|
75
|
+
profile = body.get("profile")
|
|
76
|
+
requires = body.get("requires")
|
|
77
|
+
return Project(
|
|
78
|
+
root=path.parent,
|
|
79
|
+
pipelines=pipelines if isinstance(pipelines, str) else DEFAULT_PIPELINES_DIR,
|
|
80
|
+
profile=profile if isinstance(profile, str) else None,
|
|
81
|
+
requires=[str(item) for item in cast("list[object]", requires)] if isinstance(requires, list) else [],
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
PROJECT_TEMPLATE = """\
|
|
86
|
+
# A dirigent pipeline project: a working set of documents, not a source of truth.
|
|
87
|
+
# The server is where definitions actually live; `dg apply` puts these there.
|
|
88
|
+
|
|
89
|
+
project:
|
|
90
|
+
pipelines: pipelines
|
|
91
|
+
profile: local
|
|
92
|
+
|
|
93
|
+
# Block ids every document here assumes. `dg apply` checks these against the target
|
|
94
|
+
# instance's catalog first, and fails with a complete list of what is missing.
|
|
95
|
+
requires: []
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
PROFILES_TEMPLATE = """\
|
|
99
|
+
# Client-side addressing only: which server, and how to get a token for it.
|
|
100
|
+
# Never a database URL -- a CLI that could reach the database would bypass
|
|
101
|
+
# authentication, attribution, and validation entirely.
|
|
102
|
+
|
|
103
|
+
default: local
|
|
104
|
+
|
|
105
|
+
profiles:
|
|
106
|
+
local:
|
|
107
|
+
url: http://127.0.0.1:3333
|
|
108
|
+
token_env: DG_TOKEN
|
|
109
|
+
|
|
110
|
+
# staging:
|
|
111
|
+
# url: https://dirigent-staging.example.org
|
|
112
|
+
# token_env: DG_STAGING_TOKEN
|
|
113
|
+
#
|
|
114
|
+
# prod:
|
|
115
|
+
# url: https://dirigent.example.org
|
|
116
|
+
# token_cmd: pass show dirigent/prod
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
STATE_IGNORE_TEMPLATE = """\
|
|
120
|
+
# A local instance's database and artifacts. Disposable, and never shared.
|
|
121
|
+
# profiles.yaml beside this stays committable: it is addressing, not state.
|
|
122
|
+
state/
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
WORKFLOW_TEMPLATE = """\
|
|
126
|
+
name: dirigent
|
|
127
|
+
|
|
128
|
+
on:
|
|
129
|
+
push:
|
|
130
|
+
branches: [main]
|
|
131
|
+
pull_request:
|
|
132
|
+
|
|
133
|
+
jobs:
|
|
134
|
+
apply:
|
|
135
|
+
runs-on: ubuntu-latest
|
|
136
|
+
steps:
|
|
137
|
+
- uses: actions/checkout@v4
|
|
138
|
+
|
|
139
|
+
- name: Install dirigent
|
|
140
|
+
run: pipx install dirigent-cli
|
|
141
|
+
|
|
142
|
+
# On a pull request this is the whole-project diff and nothing is written.
|
|
143
|
+
- name: Plan
|
|
144
|
+
if: github.event_name == 'pull_request'
|
|
145
|
+
run: dg apply --dry-run
|
|
146
|
+
env:
|
|
147
|
+
DG_URL: ${{ secrets.DG_URL }}
|
|
148
|
+
DG_TOKEN: ${{ secrets.DG_TOKEN }}
|
|
149
|
+
|
|
150
|
+
- name: Apply
|
|
151
|
+
if: github.event_name == 'push'
|
|
152
|
+
run: dg apply
|
|
153
|
+
env:
|
|
154
|
+
DG_URL: ${{ secrets.DG_URL }}
|
|
155
|
+
DG_TOKEN: ${{ secrets.DG_TOKEN }}
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
EXAMPLE_TEMPLATE = """\
|
|
159
|
+
# The smallest thing dirigent can run: one step, one block, no parameters.
|
|
160
|
+
#
|
|
161
|
+
# value.const emits its configured value and touches nothing, so this runs with nothing
|
|
162
|
+
# on the unsafe allowlist:
|
|
163
|
+
#
|
|
164
|
+
# dg apply
|
|
165
|
+
# dg run hello-world --watch
|
|
166
|
+
|
|
167
|
+
format: dirigent/v1
|
|
168
|
+
kind: pipeline
|
|
169
|
+
code: hello-world
|
|
170
|
+
name: Hello world
|
|
171
|
+
description: Emit a greeting, and nothing else.
|
|
172
|
+
|
|
173
|
+
steps:
|
|
174
|
+
greet:
|
|
175
|
+
block: value.const
|
|
176
|
+
config:
|
|
177
|
+
value: "hello from dirigent"
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
COMPOSE_TEMPLATE = """\
|
|
182
|
+
name: dirigent
|
|
183
|
+
|
|
184
|
+
x-dirigent: &dirigent
|
|
185
|
+
build:
|
|
186
|
+
context: .
|
|
187
|
+
dockerfile: Dockerfile
|
|
188
|
+
args:
|
|
189
|
+
DIRIGENT_IMAGE: ${DIRIGENT_IMAGE:-ghcr.io/winterop-com/dirigent:__VERSION__}
|
|
190
|
+
image: dirigent-instance:local
|
|
191
|
+
restart: unless-stopped
|
|
192
|
+
environment: &dirigent-env
|
|
193
|
+
DIRIGENT_DATABASE_URL: ${DIRIGENT_DATABASE_URL:-postgresql+asyncpg://${POSTGRES_USER:-dirigent}:${POSTGRES_PASSWORD:-dirigent}@postgres:5432/${POSTGRES_DB:-dirigent}}
|
|
194
|
+
DIRIGENT_CONFIG_FILE: /etc/dirigent/dirigent.yaml
|
|
195
|
+
DIRIGENT_SECRET_KEY: ${DIRIGENT_SECRET_KEY:?set DIRIGENT_SECRET_KEY in .env}
|
|
196
|
+
DIRIGENT_ENVIRONMENT: ${DIRIGENT_ENVIRONMENT:-prod}
|
|
197
|
+
DIRIGENT_LOG_LEVEL: ${DIRIGENT_LOG_LEVEL:-INFO}
|
|
198
|
+
DIRIGENT_LOG_FORMAT: json
|
|
199
|
+
# `migrate` puts the connection this resolves through in place before anything runs.
|
|
200
|
+
DIRIGENT_ARTIFACT_ROOT: s3://${S3_BUCKET:-dirigent}/artifacts
|
|
201
|
+
DIRIGENT_STORAGE_CONNECTIONS: s3=${S3_CONNECTION:-artifacts}
|
|
202
|
+
# What a tool opens through the filesystem -- a checkout, a build context, a compose
|
|
203
|
+
# file, a bind mount -- rather than through storage. The daemon mounts it at the same
|
|
204
|
+
# path, so a bind docker.run hands over means the same directory on both sides.
|
|
205
|
+
DIRIGENT_WORK_ROOT: /var/lib/dirigent/work
|
|
206
|
+
DIRIGENT_ENABLED_UNSAFE_BLOCKS: ${DIRIGENT_ENABLED_UNSAFE_BLOCKS:-}
|
|
207
|
+
# Pool plus overflow must stay above DIRIGENT_WORKER_CONCURRENCY, which the instance
|
|
208
|
+
# enforces at start-up.
|
|
209
|
+
DIRIGENT_DATABASE_POOL_SIZE: ${DIRIGENT_DATABASE_POOL_SIZE:-12}
|
|
210
|
+
DIRIGENT_DATABASE_MAX_OVERFLOW: ${DIRIGENT_DATABASE_MAX_OVERFLOW:-4}
|
|
211
|
+
# The alert context is built by whichever worker settles the run, so this belongs on
|
|
212
|
+
# every service.
|
|
213
|
+
DIRIGENT_ALERT_BASE_URL: ${DIRIGENT_ALERT_BASE_URL:-http://localhost:3333}
|
|
214
|
+
# Engine spans and metrics are recorded on the worker, so these belong on every service.
|
|
215
|
+
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
|
|
216
|
+
OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-dirigent}
|
|
217
|
+
volumes:
|
|
218
|
+
- ./dirigent.yaml:/etc/dirigent/dirigent.yaml:ro
|
|
219
|
+
depends_on:
|
|
220
|
+
postgres:
|
|
221
|
+
condition: service_healthy
|
|
222
|
+
migrate:
|
|
223
|
+
condition: service_completed_successfully
|
|
224
|
+
s3-bucket:
|
|
225
|
+
condition: service_completed_successfully
|
|
226
|
+
|
|
227
|
+
services:
|
|
228
|
+
postgres:
|
|
229
|
+
image: postgres:17-alpine
|
|
230
|
+
restart: unless-stopped
|
|
231
|
+
environment:
|
|
232
|
+
POSTGRES_USER: ${POSTGRES_USER:-dirigent}
|
|
233
|
+
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dirigent}
|
|
234
|
+
POSTGRES_DB: ${POSTGRES_DB:-dirigent}
|
|
235
|
+
volumes:
|
|
236
|
+
- postgres:/var/lib/postgresql/data
|
|
237
|
+
healthcheck:
|
|
238
|
+
# -d and -U are both given: without them pg_isready answers about the wrong database
|
|
239
|
+
# and reports ready while the one dirigent uses is still being created on first start.
|
|
240
|
+
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-dirigent} -d ${POSTGRES_DB:-dirigent}"]
|
|
241
|
+
interval: 5s
|
|
242
|
+
timeout: 5s
|
|
243
|
+
retries: 20
|
|
244
|
+
start_period: 10s
|
|
245
|
+
ports:
|
|
246
|
+
# Bound to loopback: the stack reaches postgres over the compose network, and the
|
|
247
|
+
# mapping exists only so a person on the host can inspect the database.
|
|
248
|
+
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
|
|
249
|
+
|
|
250
|
+
# Brings the schema forward, and ensures the connection the artifact root resolves
|
|
251
|
+
# through. That is a row, and a container has no API token, so `dg connection ensure`
|
|
252
|
+
# writes it process-side and seals its secret half with the instance key, exactly as the
|
|
253
|
+
# API would. Re-running the stack brings the row to whatever the environment now says.
|
|
254
|
+
migrate:
|
|
255
|
+
<<: *dirigent
|
|
256
|
+
restart: "no"
|
|
257
|
+
entrypoint: ["/bin/sh", "-ec"]
|
|
258
|
+
command:
|
|
259
|
+
- |
|
|
260
|
+
dg db upgrade
|
|
261
|
+
dg connection ensure s3 "$$S3_CONNECTION" \\
|
|
262
|
+
--name "Artifact storage" \\
|
|
263
|
+
--description "The bucket every worker on this stack reads and writes artifacts in." \\
|
|
264
|
+
--set endpoint_url="http://s3:9000" \\
|
|
265
|
+
--set bucket="$$S3_BUCKET" \\
|
|
266
|
+
--set access_key_id="$$S3_ACCESS_KEY" \\
|
|
267
|
+
--set secret_access_key="$$S3_SECRET_KEY" \\
|
|
268
|
+
--set path_style=true
|
|
269
|
+
environment:
|
|
270
|
+
<<: *dirigent-env
|
|
271
|
+
# The credentials reach the command as environment, never as a file or an image layer.
|
|
272
|
+
S3_CONNECTION: ${S3_CONNECTION:-artifacts}
|
|
273
|
+
S3_BUCKET: ${S3_BUCKET:-dirigent}
|
|
274
|
+
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-dirigent}
|
|
275
|
+
S3_SECRET_KEY: ${S3_SECRET_KEY:-dirigent}
|
|
276
|
+
depends_on:
|
|
277
|
+
postgres:
|
|
278
|
+
condition: service_healthy
|
|
279
|
+
|
|
280
|
+
# Artifact storage. The endpoint the connection names is the one containers resolve, not
|
|
281
|
+
# the one a shell on the host does: `localhost` inside a container is the container.
|
|
282
|
+
s3:
|
|
283
|
+
image: ${DIRIGENT_S3_IMAGE:-rustfs/rustfs:1.0.0-rc.4}
|
|
284
|
+
restart: unless-stopped
|
|
285
|
+
environment:
|
|
286
|
+
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY:-dirigent}
|
|
287
|
+
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY:-dirigent}
|
|
288
|
+
volumes:
|
|
289
|
+
- s3:/data
|
|
290
|
+
ports:
|
|
291
|
+
- "${S3_PORT:-9010}:9000"
|
|
292
|
+
healthcheck:
|
|
293
|
+
# An unauthenticated request answers 403 rather than 200, and that is a serving
|
|
294
|
+
# endpoint answering: -f would call it a failure, so only the connection is asserted.
|
|
295
|
+
test: ["CMD-SHELL", "curl -s -o /dev/null http://127.0.0.1:9000/"]
|
|
296
|
+
interval: 5s
|
|
297
|
+
timeout: 5s
|
|
298
|
+
retries: 30
|
|
299
|
+
start_period: 5s
|
|
300
|
+
|
|
301
|
+
# The bucket has to exist before a run writes to it, and nothing else creates it.
|
|
302
|
+
s3-bucket:
|
|
303
|
+
image: minio/mc:RELEASE.2025-04-16T18-13-26Z
|
|
304
|
+
depends_on:
|
|
305
|
+
s3:
|
|
306
|
+
condition: service_healthy
|
|
307
|
+
entrypoint: ["/bin/sh", "-ec"]
|
|
308
|
+
command:
|
|
309
|
+
- |
|
|
310
|
+
mc alias set fs http://s3:9000 "$$S3_ACCESS_KEY" "$$S3_SECRET_KEY"
|
|
311
|
+
mc mb --ignore-existing "fs/$$S3_BUCKET"
|
|
312
|
+
environment:
|
|
313
|
+
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-dirigent}
|
|
314
|
+
S3_SECRET_KEY: ${S3_SECRET_KEY:-dirigent}
|
|
315
|
+
S3_BUCKET: ${S3_BUCKET:-dirigent}
|
|
316
|
+
|
|
317
|
+
server:
|
|
318
|
+
<<: *dirigent
|
|
319
|
+
command: ["server"]
|
|
320
|
+
environment:
|
|
321
|
+
<<: *dirigent-env
|
|
322
|
+
DIRIGENT_BOOTSTRAP_ADMIN_PASSWORD: ${DIRIGENT_BOOTSTRAP_ADMIN_PASSWORD:-}
|
|
323
|
+
DIRIGENT_SCHEDULER_ENABLED: ${DIRIGENT_SCHEDULER_ENABLED:-true}
|
|
324
|
+
# A document in `pipelines/` lands at boot, and `dg apply` sends one now.
|
|
325
|
+
DIRIGENT_APPLY_DIR: ${DIRIGENT_APPLY_DIR:-/etc/dirigent/pipelines}
|
|
326
|
+
DIRIGENT_APPLY_PRUNE: ${DIRIGENT_APPLY_PRUNE:-false}
|
|
327
|
+
# A list under the anchor is replaced, not merged, so the shared mount is respelled here.
|
|
328
|
+
volumes:
|
|
329
|
+
- ./dirigent.yaml:/etc/dirigent/dirigent.yaml:ro
|
|
330
|
+
- ./pipelines:/etc/dirigent/pipelines:ro
|
|
331
|
+
ports:
|
|
332
|
+
- "${DIRIGENT_PORT:-3333}:3333"
|
|
333
|
+
healthcheck:
|
|
334
|
+
test: ["CMD", "dg", "health", "server"]
|
|
335
|
+
interval: 10s
|
|
336
|
+
timeout: 5s
|
|
337
|
+
retries: 10
|
|
338
|
+
start_period: 15s
|
|
339
|
+
|
|
340
|
+
# The daemon the `docker.*` blocks drive. It is the worker's own, so a pipeline's
|
|
341
|
+
# containers are never the host's, and no host socket is mounted anywhere in this stack.
|
|
342
|
+
docker:
|
|
343
|
+
image: docker:28-dind@sha256:2a232a42256f70d78e3cc5d2b5d6b3276710a0de0596c145f627ecfae90282ac
|
|
344
|
+
restart: unless-stopped
|
|
345
|
+
# docker-in-docker needs the full capability set to run its own daemon; there is no
|
|
346
|
+
# unprivileged mode that starts containers.
|
|
347
|
+
privileged: true
|
|
348
|
+
environment:
|
|
349
|
+
# The entrypoint generates the CA, the server certificate and the client certificate
|
|
350
|
+
# under this directory and serves tcp://0.0.0.0:2376 with TLS required.
|
|
351
|
+
DOCKER_TLS_CERTDIR: /certs
|
|
352
|
+
volumes:
|
|
353
|
+
- docker-certs:/certs/client
|
|
354
|
+
- docker-data:/var/lib/docker
|
|
355
|
+
# A bind mount is resolved on the daemon's filesystem, not the worker's, so a path
|
|
356
|
+
# docker.run hands over must mean the same directory here as it does in the worker.
|
|
357
|
+
- work:/var/lib/dirigent/work
|
|
358
|
+
healthcheck:
|
|
359
|
+
test: ["CMD-SHELL", "docker -H unix:///var/run/docker.sock version >/dev/null 2>&1"]
|
|
360
|
+
interval: 5s
|
|
361
|
+
timeout: 5s
|
|
362
|
+
retries: 30
|
|
363
|
+
start_period: 10s
|
|
364
|
+
|
|
365
|
+
worker:
|
|
366
|
+
<<: *dirigent
|
|
367
|
+
command: ["worker"]
|
|
368
|
+
environment:
|
|
369
|
+
<<: *dirigent-env
|
|
370
|
+
DIRIGENT_WORKER_CONCURRENCY: ${DIRIGENT_WORKER_CONCURRENCY:-8}
|
|
371
|
+
# This worker reaches a daemon, so it claims work from a document declaring
|
|
372
|
+
# `requires.workers: [docker]`.
|
|
373
|
+
DIRIGENT_WORKER_TAGS: docker
|
|
374
|
+
# The sidecar daemon, read the same way by the API blocks and by the docker CLI the
|
|
375
|
+
# compose and build blocks shell out to. Only the worker gets these.
|
|
376
|
+
DOCKER_HOST: tcp://docker:2376
|
|
377
|
+
DOCKER_TLS_VERIFY: "1"
|
|
378
|
+
DOCKER_CERT_PATH: /certs/client
|
|
379
|
+
volumes:
|
|
380
|
+
- ./dirigent.yaml:/etc/dirigent/dirigent.yaml:ro
|
|
381
|
+
- ./pipelines:/etc/dirigent/pipelines:ro
|
|
382
|
+
- docker-certs:/certs/client:ro
|
|
383
|
+
# The same volume the daemon mounts, at the same path.
|
|
384
|
+
- work:/var/lib/dirigent/work
|
|
385
|
+
depends_on:
|
|
386
|
+
postgres:
|
|
387
|
+
condition: service_healthy
|
|
388
|
+
migrate:
|
|
389
|
+
condition: service_completed_successfully
|
|
390
|
+
s3-bucket:
|
|
391
|
+
condition: service_completed_successfully
|
|
392
|
+
docker:
|
|
393
|
+
condition: service_healthy
|
|
394
|
+
stop_grace_period: 60s
|
|
395
|
+
healthcheck:
|
|
396
|
+
test: ["CMD", "dg", "health", "worker"]
|
|
397
|
+
interval: 15s
|
|
398
|
+
timeout: 10s
|
|
399
|
+
retries: 3
|
|
400
|
+
start_period: 30s
|
|
401
|
+
|
|
402
|
+
volumes:
|
|
403
|
+
postgres:
|
|
404
|
+
s3:
|
|
405
|
+
# The worker's own working files. Not shared with the server: nothing it does opens one.
|
|
406
|
+
work:
|
|
407
|
+
docker-certs:
|
|
408
|
+
docker-data:
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
ENV_TEMPLATE = """\
|
|
412
|
+
# Fills in ${...} in compose.yaml. Dirigent's own settings are in dirigent.yaml.
|
|
413
|
+
|
|
414
|
+
# Losing this means losing every stored connection secret. Changing it does not re-encrypt
|
|
415
|
+
# what is already stored.
|
|
416
|
+
DIRIGENT_SECRET_KEY=__SECRET_KEY__
|
|
417
|
+
|
|
418
|
+
# One-way: it does nothing the moment any account exists, so leaving it set on every deploy
|
|
419
|
+
# cannot reset a live instance's password.
|
|
420
|
+
DIRIGENT_BOOTSTRAP_ADMIN_PASSWORD=__PASSWORD__
|
|
421
|
+
|
|
422
|
+
POSTGRES_USER=dirigent
|
|
423
|
+
POSTGRES_PASSWORD=dirigent
|
|
424
|
+
POSTGRES_DB=dirigent
|
|
425
|
+
POSTGRES_PORT=5432
|
|
426
|
+
|
|
427
|
+
DIRIGENT_ENVIRONMENT=prod
|
|
428
|
+
DIRIGENT_PORT=3333
|
|
429
|
+
DIRIGENT_LOG_LEVEL=INFO
|
|
430
|
+
|
|
431
|
+
DIRIGENT_ALERT_BASE_URL=http://localhost:3333
|
|
432
|
+
|
|
433
|
+
DIRIGENT_WORKER_CONCURRENCY=8
|
|
434
|
+
|
|
435
|
+
# Pool size plus max overflow must be at least DIRIGENT_WORKER_CONCURRENCY plus 4: one
|
|
436
|
+
# connection per in-flight block call, plus the lease heartbeat, the sweeper, the alert loop
|
|
437
|
+
# and a spare. The instance refuses to start on a configuration where that cannot hold, so
|
|
438
|
+
# raising the concurrency means raising these too.
|
|
439
|
+
DIRIGENT_DATABASE_POOL_SIZE=12
|
|
440
|
+
DIRIGENT_DATABASE_MAX_OVERFLOW=4
|
|
441
|
+
|
|
442
|
+
DIRIGENT_SCHEDULER_ENABLED=true
|
|
443
|
+
|
|
444
|
+
# The `migrate` service writes the connection S3_CONNECTION names from S3_ACCESS_KEY and
|
|
445
|
+
# S3_SECRET_KEY, so the credentials live here and nowhere else; change one and the next `up`
|
|
446
|
+
# brings the row to it.
|
|
447
|
+
S3_ACCESS_KEY=dirigent
|
|
448
|
+
S3_SECRET_KEY=dirigent
|
|
449
|
+
S3_BUCKET=dirigent
|
|
450
|
+
S3_CONNECTION=artifacts
|
|
451
|
+
|
|
452
|
+
# Where the bundled S3 server is published for a person on the host; the stack itself reaches
|
|
453
|
+
# it as http://s3:9000.
|
|
454
|
+
S3_PORT=9010
|
|
455
|
+
DIRIGENT_S3_IMAGE=rustfs/rustfs:1.0.0-rc.4
|
|
456
|
+
|
|
457
|
+
# Comma-separated block ids, empty for none. Listing one means whoever can edit a pipeline
|
|
458
|
+
# can run code on a worker. The docker.* blocks reach the dind sidecar this stack starts.
|
|
459
|
+
#
|
|
460
|
+
# DIRIGENT_ENABLED_UNSAFE_BLOCKS=shell.run,docker.run,docker.compose.up,docker.compose.down
|
|
461
|
+
DIRIGENT_ENABLED_UNSAFE_BLOCKS=
|
|
462
|
+
|
|
463
|
+
# The published image the Dockerfile beside this builds on.
|
|
464
|
+
DIRIGENT_IMAGE=ghcr.io/winterop-com/dirigent:__VERSION__
|
|
465
|
+
"""
|
|
466
|
+
|
|
467
|
+
DOCKERFILE_TEMPLATE = """\
|
|
468
|
+
# This instance's image: the published dirigent, plus whatever is added below.
|
|
469
|
+
ARG DIRIGENT_IMAGE=ghcr.io/winterop-com/dirigent:__VERSION__
|
|
470
|
+
FROM ${DIRIGENT_IMAGE}
|
|
471
|
+
|
|
472
|
+
# A pack is a Python package; install it here and run `docker compose up --build`.
|
|
473
|
+
# RUN uv pip install dirigent-dhis2==__VERSION__
|
|
474
|
+
"""
|
|
475
|
+
|
|
476
|
+
ROOT_IGNORE_TEMPLATE = """\
|
|
477
|
+
# The environment uv sync builds from pyproject.toml.
|
|
478
|
+
.venv/
|
|
479
|
+
__pycache__/
|
|
480
|
+
"""
|
|
481
|
+
|
|
482
|
+
#: Appended to the root ignore by the template whose .env holds the instance key.
|
|
483
|
+
COMPOSE_IGNORE_TEMPLATE = """\
|
|
484
|
+
# Holds this instance's key and its first admin's password.
|
|
485
|
+
.env
|
|
486
|
+
"""
|
|
487
|
+
|
|
488
|
+
PYPROJECT_TEMPLATE = """\
|
|
489
|
+
[project]
|
|
490
|
+
name = "__NAME__"
|
|
491
|
+
version = "0.0.0"
|
|
492
|
+
description = "A dirigent project: pipeline documents, and the runtime that runs them."
|
|
493
|
+
requires-python = ">=3.13"
|
|
494
|
+
dependencies = [
|
|
495
|
+
"dirigent-cli==__VERSION__",
|
|
496
|
+
]
|
|
497
|
+
"""
|
|
498
|
+
|
|
499
|
+
README_TEMPLATE = """\
|
|
500
|
+
# __NAME__
|
|
501
|
+
|
|
502
|
+
A dirigent project. `pipelines/` holds the pipeline documents and `dirigent.yaml` the
|
|
503
|
+
project's settings.
|
|
504
|
+
|
|
505
|
+
`pyproject.toml` pins the dirigent runtime, so the `dg` this project runs on is that pinned
|
|
506
|
+
one and `uv run` is how it is reached.
|
|
507
|
+
|
|
508
|
+
## Run
|
|
509
|
+
|
|
510
|
+
```bash
|
|
511
|
+
__RUN__
|
|
512
|
+
```
|
|
513
|
+
"""
|
|
514
|
+
|
|
515
|
+
#: The commands a scaffolded project's README opens with, per template.
|
|
516
|
+
README_RUN: Final = {
|
|
517
|
+
"basic": (
|
|
518
|
+
"uv sync",
|
|
519
|
+
"uv run dg dev --keep-state",
|
|
520
|
+
"uv run dg apply",
|
|
521
|
+
"uv run dg run hello-world --watch",
|
|
522
|
+
),
|
|
523
|
+
"ci": (
|
|
524
|
+
"uv sync",
|
|
525
|
+
"uv run dg dev --keep-state",
|
|
526
|
+
"uv run dg apply",
|
|
527
|
+
"uv run dg run hello-world --watch",
|
|
528
|
+
),
|
|
529
|
+
"compose": (
|
|
530
|
+
"uv sync",
|
|
531
|
+
"docker compose up -d",
|
|
532
|
+
"uv run dg auth login --username admin",
|
|
533
|
+
"uv run dg run hello-world --watch",
|
|
534
|
+
),
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
#: The templates dg init can scaffold from.
|
|
539
|
+
TEMPLATES: Final = ("basic", "ci", "compose")
|
|
540
|
+
|
|
541
|
+
#: The template whose instance is a container stack rather than a state directory.
|
|
542
|
+
COMPOSE_TEMPLATE_NAME: Final = "compose"
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def check_template(template: str) -> None:
|
|
546
|
+
"""Refuse a template that does not exist, before anything is written or asked for."""
|
|
547
|
+
if template not in TEMPLATES:
|
|
548
|
+
raise ProjectError(f"no template named {template!r}; the built-in templates are basic, ci and compose")
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
class Scaffolded(BaseModel):
|
|
552
|
+
"""What a scaffold wrote, and what it found already there and left alone."""
|
|
553
|
+
|
|
554
|
+
model_config = ConfigDict(frozen=True)
|
|
555
|
+
|
|
556
|
+
files: list[Path] = Field(default_factory=list[Path])
|
|
557
|
+
skipped: list[Path] = Field(default_factory=list[Path])
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def project_name(directory: Path) -> str:
|
|
561
|
+
"""Turn a target directory's name into a project name uv and PEP 621 accept."""
|
|
562
|
+
slug = re.sub(r"[^a-z0-9]+", "-", directory.resolve().name.lower()).strip("-")
|
|
563
|
+
return slug or "dirigent-project"
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def scaffold(
|
|
567
|
+
directory: Path,
|
|
568
|
+
*,
|
|
569
|
+
template: str = "basic",
|
|
570
|
+
version: str = "0.0.0",
|
|
571
|
+
password: str = "",
|
|
572
|
+
) -> Scaffolded:
|
|
573
|
+
"""Create a uv project: pyproject, the project file, a pipelines directory, and an example."""
|
|
574
|
+
check_template(template)
|
|
575
|
+
written: list[Path] = []
|
|
576
|
+
skipped: list[Path] = []
|
|
577
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
578
|
+
written.append(_write(directory / PROJECT_FILE, PROJECT_TEMPLATE + "\n" + project_document()))
|
|
579
|
+
written.append(_write(directory / DEFAULT_PIPELINES_DIR / "hello-world.yaml", EXAMPLE_TEMPLATE))
|
|
580
|
+
written.append(_write(directory / ".dirigent" / "profiles.yaml", PROFILES_TEMPLATE))
|
|
581
|
+
written.append(_write(directory / ".dirigent" / ".gitignore", STATE_IGNORE_TEMPLATE))
|
|
582
|
+
written.append(_write(directory / EXAMPLE_CONFIG_FILE, example_document()))
|
|
583
|
+
name = project_name(directory)
|
|
584
|
+
pyproject = PYPROJECT_TEMPLATE.replace("__NAME__", name).replace("__VERSION__", version)
|
|
585
|
+
_record(directory / "pyproject.toml", pyproject, written, skipped)
|
|
586
|
+
readme = README_TEMPLATE.replace("__NAME__", name).replace("__RUN__", "\n".join(README_RUN[template]))
|
|
587
|
+
_record(directory / "README.md", readme, written, skipped)
|
|
588
|
+
ignore = ROOT_IGNORE_TEMPLATE
|
|
589
|
+
if template == COMPOSE_TEMPLATE_NAME:
|
|
590
|
+
ignore += "\n" + COMPOSE_IGNORE_TEMPLATE
|
|
591
|
+
_merge_ignore(directory / ".gitignore", ignore, written, skipped)
|
|
592
|
+
if template == "ci":
|
|
593
|
+
written.append(_write(directory / ".github" / "workflows" / "dirigent.yml", WORKFLOW_TEMPLATE))
|
|
594
|
+
if template == COMPOSE_TEMPLATE_NAME:
|
|
595
|
+
written.append(_write(directory / "compose.yaml", COMPOSE_TEMPLATE.replace("__VERSION__", version)))
|
|
596
|
+
written.append(_write(directory / "Dockerfile", DOCKERFILE_TEMPLATE.replace("__VERSION__", version)))
|
|
597
|
+
environment = (
|
|
598
|
+
ENV_TEMPLATE.replace("__SECRET_KEY__", _an_instance_key())
|
|
599
|
+
.replace("__PASSWORD__", password)
|
|
600
|
+
.replace("__VERSION__", version)
|
|
601
|
+
)
|
|
602
|
+
written.append(_write(directory / ".env", environment, mode=0o600))
|
|
603
|
+
return Scaffolded(files=written, skipped=skipped)
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _record(path: Path, content: str, written: list[Path], skipped: list[Path]) -> None:
|
|
607
|
+
"""Write a file a project may already have of its own, leaving any existing one alone."""
|
|
608
|
+
if path.exists():
|
|
609
|
+
skipped.append(path)
|
|
610
|
+
return
|
|
611
|
+
written.append(_write(path, content))
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def _merge_ignore(path: Path, content: str, written: list[Path], skipped: list[Path]) -> None:
|
|
615
|
+
"""Write the root ignore, or add to an existing one only the lines it does not have."""
|
|
616
|
+
if not path.exists():
|
|
617
|
+
written.append(_write(path, content))
|
|
618
|
+
return
|
|
619
|
+
current = path.read_text()
|
|
620
|
+
present = set(current.splitlines())
|
|
621
|
+
missing = [line for line in content.splitlines() if line and not line.startswith("#") and line not in present]
|
|
622
|
+
if not missing:
|
|
623
|
+
skipped.append(path)
|
|
624
|
+
return
|
|
625
|
+
separator = "" if current.endswith("\n") or not current else "\n"
|
|
626
|
+
path.write_text(current + separator + "\n".join(missing) + "\n")
|
|
627
|
+
written.append(path)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _an_instance_key() -> str:
|
|
631
|
+
"""Generate the key the stack seals connection secrets with."""
|
|
632
|
+
return base64.urlsafe_b64encode(os.urandom(32)).decode()
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _write(path: Path, content: str, *, mode: int | None = None) -> Path:
|
|
636
|
+
"""Write one scaffolded file, refusing to overwrite one that is already there."""
|
|
637
|
+
if path.exists():
|
|
638
|
+
raise ProjectError(f"{path} already exists; dg init never overwrites")
|
|
639
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
640
|
+
path.write_text(content)
|
|
641
|
+
if mode is not None:
|
|
642
|
+
path.chmod(mode)
|
|
643
|
+
return path
|
dirigent_cli/py.typed
ADDED
|
File without changes
|