datadog-structurizr 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datadog_structurizr/__init__.py +3 -0
- datadog_structurizr/client.py +263 -0
- datadog_structurizr/config.py +183 -0
- datadog_structurizr/emitter.py +212 -0
- datadog_structurizr/main.py +119 -0
- datadog_structurizr/mapper.py +260 -0
- datadog_structurizr/model.py +58 -0
- datadog_structurizr/render.py +39 -0
- datadog_structurizr-0.1.0.dist-info/METADATA +521 -0
- datadog_structurizr-0.1.0.dist-info/RECORD +14 -0
- datadog_structurizr-0.1.0.dist-info/WHEEL +5 -0
- datadog_structurizr-0.1.0.dist-info/entry_points.txt +2 -0
- datadog_structurizr-0.1.0.dist-info/licenses/LICENSE +21 -0
- datadog_structurizr-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Datadog API calls.
|
|
2
|
+
|
|
3
|
+
Plain `requests` so the endpoint being hit is explicit and easy to debug.
|
|
4
|
+
Once every call has succeeded, the responses replace output/raw/ as a whole,
|
|
5
|
+
so a run can be replayed with --offline (no credentials, no API calls) while
|
|
6
|
+
tuning the diagrams, and raw/ never mixes responses from different runs.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import shutil
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Callable
|
|
19
|
+
|
|
20
|
+
import requests
|
|
21
|
+
|
|
22
|
+
from .config import Config
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DatadogAPIError(RuntimeError):
|
|
26
|
+
"""A Datadog call failed; the run stops rather than draw from partial data."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# A 429 is retried after the wait Datadog asks for in x-ratelimit-reset
|
|
30
|
+
# (seconds, checked on a live account), at most this often and this long.
|
|
31
|
+
MAX_RETRIES = 2
|
|
32
|
+
MAX_WAIT_SECONDS = 60
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _retry_wait(resp: requests.Response) -> int | None:
|
|
36
|
+
"""Seconds to wait before retrying a 429, or None to give up."""
|
|
37
|
+
try:
|
|
38
|
+
wait = int(resp.headers.get("x-ratelimit-reset", ""))
|
|
39
|
+
except (TypeError, ValueError):
|
|
40
|
+
return None
|
|
41
|
+
return max(wait, 1) if wait <= MAX_WAIT_SECONDS else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _send(call: Callable[..., requests.Response], url: str, **kwargs: Any) -> requests.Response:
|
|
45
|
+
"""requests.get or requests.post, retrying when Datadog answers 429."""
|
|
46
|
+
for attempt in range(MAX_RETRIES + 1):
|
|
47
|
+
resp = call(url, **kwargs)
|
|
48
|
+
if resp.status_code != 429 or attempt == MAX_RETRIES:
|
|
49
|
+
return resp
|
|
50
|
+
wait = _retry_wait(resp)
|
|
51
|
+
if wait is None:
|
|
52
|
+
return resp
|
|
53
|
+
limit = resp.headers.get("x-ratelimit-name", "Datadog")
|
|
54
|
+
print(f" rate limited ({limit}); retrying in {wait}s", file=sys.stderr)
|
|
55
|
+
time.sleep(wait)
|
|
56
|
+
return resp
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _check(resp: requests.Response, what: str) -> None:
|
|
60
|
+
if resp.status_code != 200:
|
|
61
|
+
raise DatadogAPIError(f"Datadog API error {resp.status_code} for {what}: {resp.text[:500]}")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _headers(cfg: Config) -> dict[str, str]:
|
|
65
|
+
return {
|
|
66
|
+
"DD-API-KEY": cfg.api_key,
|
|
67
|
+
"DD-APPLICATION-KEY": cfg.app_key,
|
|
68
|
+
"Accept": "application/json",
|
|
69
|
+
"Content-Type": "application/json",
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _base(cfg: Config) -> str:
|
|
74
|
+
return f"https://api.{cfg.site}"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _replace_raw(cfg: Config, files: dict[str, Any]) -> None:
|
|
78
|
+
"""Write the responses to a new directory, then swap it in for raw/."""
|
|
79
|
+
staging = Path(tempfile.mkdtemp(prefix=".raw-new-", dir=cfg.output_dir))
|
|
80
|
+
try:
|
|
81
|
+
for name, data in files.items():
|
|
82
|
+
(staging / f"{name}.json").write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
|
83
|
+
except BaseException:
|
|
84
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
85
|
+
raise
|
|
86
|
+
old = staging.with_name(staging.name.replace(".raw-new-", ".raw-old-"))
|
|
87
|
+
if cfg.raw_dir.exists():
|
|
88
|
+
cfg.raw_dir.rename(old)
|
|
89
|
+
staging.rename(cfg.raw_dir)
|
|
90
|
+
shutil.rmtree(old, ignore_errors=True)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _load(cfg: Config, name: str) -> Any:
|
|
94
|
+
path: Path = cfg.raw_dir / f"{name}.json"
|
|
95
|
+
if not path.exists():
|
|
96
|
+
raise SystemExit(f"--offline: {path} not found. Run once online first.")
|
|
97
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _dependencies_name(cfg: Config, service: str) -> str:
|
|
101
|
+
return ("dependencies" if service == cfg.service
|
|
102
|
+
else "dependencies-" + re.sub(r"[^A-Za-z0-9._-]", "_", service))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def fetch_service_dependencies(cfg: Config, service: str | None = None) -> dict[str, Any]:
|
|
106
|
+
"""Upstream (`called_by`) and downstream (`calls`) services of one service.
|
|
107
|
+
|
|
108
|
+
`service` defaults to the target; other services (system.include) are
|
|
109
|
+
saved as raw/dependencies-<service>.json.
|
|
110
|
+
|
|
111
|
+
GET /api/v1/service_dependencies/{service}?env=&start=&end=
|
|
112
|
+
Response: {"name": "...", "calls": [...], "called_by": [...]}
|
|
113
|
+
Docs: https://docs.datadoghq.com/api/latest/service-dependencies/
|
|
114
|
+
"""
|
|
115
|
+
service = service or cfg.service
|
|
116
|
+
if cfg.offline:
|
|
117
|
+
return _load(cfg, _dependencies_name(cfg, service))
|
|
118
|
+
|
|
119
|
+
end = int(time.time())
|
|
120
|
+
params = {"env": cfg.env, "start": end - cfg.lookback_hours * 3600, "end": end}
|
|
121
|
+
resp = _send(
|
|
122
|
+
requests.get, f"{_base(cfg)}/api/v1/service_dependencies/{service}",
|
|
123
|
+
headers=_headers(cfg), params=params, timeout=30,
|
|
124
|
+
)
|
|
125
|
+
_check(resp, f"service dependencies of {service}")
|
|
126
|
+
return resp.json()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _aggregate(cfg: Config, query: str, facets: list[tuple[str, int]]) -> list[dict[str, Any]]:
|
|
130
|
+
"""Span counts grouped by facets, as [{"by": {facet: value}, "compute": {"c0": n}}]."""
|
|
131
|
+
order = {"aggregation": "count", "order": "desc", "type": "measure"}
|
|
132
|
+
body = {
|
|
133
|
+
"data": {
|
|
134
|
+
"type": "aggregate_request",
|
|
135
|
+
"attributes": {
|
|
136
|
+
"filter": {"from": f"now-{cfg.lookback_hours}h", "to": "now", "query": query},
|
|
137
|
+
"compute": [{"aggregation": "count", "type": "total"}],
|
|
138
|
+
"group_by": [{"facet": f, "limit": limit, "sort": order} for f, limit in facets],
|
|
139
|
+
},
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
resp = _send(
|
|
143
|
+
requests.post, f"{_base(cfg)}/api/v2/spans/analytics/aggregate",
|
|
144
|
+
headers=_headers(cfg), json=body, timeout=60,
|
|
145
|
+
)
|
|
146
|
+
_check(resp, f"spans aggregate ({query})")
|
|
147
|
+
# Response: {"data": [{"type": "bucket", "attributes": {"by": {...},
|
|
148
|
+
# "compute": {"c0": n}}}], "meta": {...}}, checked against the live API.
|
|
149
|
+
data = resp.json().get("data")
|
|
150
|
+
if not isinstance(data, list):
|
|
151
|
+
raise DatadogAPIError(f"spans aggregate ({query}): expected a list in 'data', "
|
|
152
|
+
f"got {type(data).__name__}")
|
|
153
|
+
return [b.get("attributes") or {} for b in data]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def fetch_resources(cfg: Config) -> list[dict[str, Any]]:
|
|
157
|
+
"""Entry-point resources (endpoints, handlers) of the target, with hit counts.
|
|
158
|
+
|
|
159
|
+
POST /api/v2/spans/analytics/aggregate grouped by resource_name.
|
|
160
|
+
Only covers indexed spans (retention filters), so counts are relative,
|
|
161
|
+
not absolute traffic. Returns [{"resource": str, "hits": int}, ...].
|
|
162
|
+
"""
|
|
163
|
+
if cfg.offline:
|
|
164
|
+
return _load(cfg, "resources")
|
|
165
|
+
|
|
166
|
+
base = f"service:{cfg.service} env:{cfg.env}"
|
|
167
|
+
# Prefer server/consumer spans so client calls (SQL, outbound HTTP) don't
|
|
168
|
+
# show up as components; fall back only when that query succeeds with no
|
|
169
|
+
# buckets (the tracer doesn't set span.kind), never after a failed call.
|
|
170
|
+
buckets = _aggregate(cfg, f"{base} @span.kind:(server OR consumer)", [("resource_name", 100)])
|
|
171
|
+
if not buckets:
|
|
172
|
+
buckets = _aggregate(cfg, base, [("resource_name", 100)])
|
|
173
|
+
|
|
174
|
+
return [
|
|
175
|
+
{
|
|
176
|
+
"resource": str(b.get("by", {}).get("resource_name", "")),
|
|
177
|
+
"hits": int((b.get("compute") or {}).get("c0") or 0),
|
|
178
|
+
}
|
|
179
|
+
for b in buckets
|
|
180
|
+
if b.get("by", {}).get("resource_name")
|
|
181
|
+
]
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def fetch_service_definition(cfg: Config) -> dict[str, Any]:
|
|
185
|
+
"""Service Catalog metadata (description, team, languages).
|
|
186
|
+
|
|
187
|
+
Optional: 404 (no definition) gives {}; any other error stops the run.
|
|
188
|
+
|
|
189
|
+
GET /api/v2/services/definitions/{service_name}
|
|
190
|
+
"""
|
|
191
|
+
if cfg.offline:
|
|
192
|
+
try:
|
|
193
|
+
return _load(cfg, "definition")
|
|
194
|
+
except SystemExit:
|
|
195
|
+
return {}
|
|
196
|
+
|
|
197
|
+
resp = _send(
|
|
198
|
+
requests.get, f"{_base(cfg)}/api/v2/services/definitions/{cfg.service}",
|
|
199
|
+
headers=_headers(cfg), timeout=30,
|
|
200
|
+
)
|
|
201
|
+
if resp.status_code == 404:
|
|
202
|
+
# No definition in the Service Catalog: metadata is optional.
|
|
203
|
+
return {}
|
|
204
|
+
_check(resp, f"service definition of {cfg.service}")
|
|
205
|
+
return resp.json()
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def fetch_span_types(cfg: Config, services: list[str]) -> dict[str, dict[str, int]]:
|
|
209
|
+
"""Span counts per span type for each service, from the services' own spans.
|
|
210
|
+
|
|
211
|
+
One spans aggregate grouped by service, then type. A datastore reports one
|
|
212
|
+
type (sql, redis, ...); an application service mostly web or http.
|
|
213
|
+
Returns {service: {type: span count}}; "" is spans with no type.
|
|
214
|
+
"""
|
|
215
|
+
if cfg.offline:
|
|
216
|
+
try:
|
|
217
|
+
return _load(cfg, "types")
|
|
218
|
+
except SystemExit:
|
|
219
|
+
return {} # raw/ from before types were fetched: classify by name
|
|
220
|
+
if not services:
|
|
221
|
+
return {}
|
|
222
|
+
names = " OR ".join('"' + s.replace('"', '\\"') + '"' for s in services)
|
|
223
|
+
buckets = _aggregate(cfg, f"env:{cfg.env} service:({names})",
|
|
224
|
+
[("service", len(services)), ("type", 10)])
|
|
225
|
+
types: dict[str, dict[str, int]] = {}
|
|
226
|
+
for b in buckets:
|
|
227
|
+
by = b.get("by", {})
|
|
228
|
+
if by.get("service"):
|
|
229
|
+
types.setdefault(str(by["service"]), {})[str(by.get("type") or "")] = \
|
|
230
|
+
int((b.get("compute") or {}).get("c0") or 0)
|
|
231
|
+
return types
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@dataclass
|
|
235
|
+
class Fetched:
|
|
236
|
+
deps: dict[str, Any]
|
|
237
|
+
member_deps: dict[str, dict[str, Any]]
|
|
238
|
+
resources: list[dict[str, Any]]
|
|
239
|
+
definition: dict[str, Any]
|
|
240
|
+
types: dict[str, dict[str, int]]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def fetch_all(cfg: Config) -> Fetched:
|
|
244
|
+
"""Every input of one run. Online, raw/ is replaced only after all calls succeed."""
|
|
245
|
+
deps = fetch_service_dependencies(cfg)
|
|
246
|
+
member_deps = {name: fetch_service_dependencies(cfg, name) for name in cfg.include}
|
|
247
|
+
neighbours = sorted({n for d in (deps, *member_deps.values())
|
|
248
|
+
for n in (d.get("calls") or []) + (d.get("called_by") or [])}
|
|
249
|
+
- {cfg.service, *cfg.include})
|
|
250
|
+
fetched = Fetched(
|
|
251
|
+
deps=deps, member_deps=member_deps,
|
|
252
|
+
resources=fetch_resources(cfg),
|
|
253
|
+
definition=fetch_service_definition(cfg),
|
|
254
|
+
types=fetch_span_types(cfg, neighbours),
|
|
255
|
+
)
|
|
256
|
+
if not cfg.offline:
|
|
257
|
+
files = {"dependencies": fetched.deps, "resources": fetched.resources,
|
|
258
|
+
"types": fetched.types}
|
|
259
|
+
files.update((_dependencies_name(cfg, name), deps) for name, deps in fetched.member_deps.items())
|
|
260
|
+
if fetched.definition:
|
|
261
|
+
files["definition"] = fetched.definition
|
|
262
|
+
_replace_raw(cfg, files)
|
|
263
|
+
return fetched
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Configuration loading and validation.
|
|
2
|
+
|
|
3
|
+
Precedence, highest first: command-line flags, the --config TOML file,
|
|
4
|
+
environment variables (or a .env file). Datadog keys are only read from the
|
|
5
|
+
environment, never from the config file or the command line.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Callable, Optional
|
|
14
|
+
|
|
15
|
+
from dotenv import load_dotenv
|
|
16
|
+
|
|
17
|
+
if sys.version_info >= (3, 11):
|
|
18
|
+
import tomllib
|
|
19
|
+
else: # pragma: no cover - exercised on Python 3.9/3.10 in CI
|
|
20
|
+
import tomli as tomllib
|
|
21
|
+
|
|
22
|
+
# Top-level and per-table keys accepted in the config file; anything else is a typo.
|
|
23
|
+
_FILE_SCHEMA: dict[str, Any] = {
|
|
24
|
+
"service": str, "env": str, "hours": int, "site": str, "output": str,
|
|
25
|
+
"ignore": list,
|
|
26
|
+
"system": {"name": str, "description": str, "include": list},
|
|
27
|
+
"person": {"enabled": bool, "name": str, "description": str},
|
|
28
|
+
"classify": {"datastores": list, "external": list, "internal": list},
|
|
29
|
+
"components": {"max": int, "strip_prefixes": list},
|
|
30
|
+
}
|
|
31
|
+
_SECRET_KEYS = {"api_key", "app_key", "dd_api_key", "dd_app_key", "api-key", "app-key"}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Config:
|
|
36
|
+
api_key: str
|
|
37
|
+
app_key: str
|
|
38
|
+
site: str
|
|
39
|
+
service: str
|
|
40
|
+
env: str
|
|
41
|
+
lookback_hours: int
|
|
42
|
+
output_dir: Path
|
|
43
|
+
offline: bool # rebuild from output/raw/*.json instead of calling Datadog
|
|
44
|
+
# Model inputs. Names in include are exact; the other lists accept fnmatch globs.
|
|
45
|
+
system_name: str = ""
|
|
46
|
+
system_description: str = ""
|
|
47
|
+
include: tuple[str, ...] = ()
|
|
48
|
+
ignore: tuple[str, ...] = ()
|
|
49
|
+
datastores: tuple[str, ...] = ()
|
|
50
|
+
external: tuple[str, ...] = ()
|
|
51
|
+
internal: tuple[str, ...] = ()
|
|
52
|
+
person_enabled: bool = True
|
|
53
|
+
person_name: str = "User"
|
|
54
|
+
person_description: str = "End user of the web component."
|
|
55
|
+
max_components: int = 12 # L2 keeps this many, the last one being "Other"
|
|
56
|
+
strip_prefixes: tuple[str, ...] = () # path prefixes removed before grouping routes
|
|
57
|
+
source_file: Optional[Path] = field(default=None, compare=False)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def raw_dir(self) -> Path:
|
|
61
|
+
return self.output_dir / "raw"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _check_schema(data: dict[str, Any], schema: dict[str, Any], where: str) -> None:
|
|
65
|
+
for key, value in data.items():
|
|
66
|
+
if key.lower() in _SECRET_KEYS:
|
|
67
|
+
raise SystemExit(f"{where}: '{key}' is not allowed here. "
|
|
68
|
+
"Put Datadog keys in DD_API_KEY / DD_APP_KEY environment variables.")
|
|
69
|
+
if key not in schema:
|
|
70
|
+
raise SystemExit(f"{where}: unknown key '{key}'. Allowed: {', '.join(schema)}.")
|
|
71
|
+
expected = schema[key]
|
|
72
|
+
if isinstance(expected, dict):
|
|
73
|
+
if not isinstance(value, dict):
|
|
74
|
+
raise SystemExit(f"{where}: '{key}' must be a table ([{key}]).")
|
|
75
|
+
_check_schema(value, expected, f"{where} [{key}]")
|
|
76
|
+
elif expected is list:
|
|
77
|
+
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
|
78
|
+
raise SystemExit(f"{where}: '{key}' must be a list of strings.")
|
|
79
|
+
elif not isinstance(value, expected) or (expected is int and isinstance(value, bool)):
|
|
80
|
+
raise SystemExit(f"{where}: '{key}' must be {expected.__name__}.")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def read_config_file(path: Path) -> dict[str, Any]:
|
|
84
|
+
if not path.is_file():
|
|
85
|
+
raise SystemExit(f"Config file not found: {path}")
|
|
86
|
+
try:
|
|
87
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
88
|
+
except tomllib.TOMLDecodeError as exc:
|
|
89
|
+
raise SystemExit(f"{path}: invalid TOML: {exc}") from None
|
|
90
|
+
_check_schema(data, _FILE_SCHEMA, str(path))
|
|
91
|
+
return data
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def normalize_site(site: str) -> str:
|
|
95
|
+
"""The Datadog site as the tool expects it ("datadoghq.eu"), from forms people
|
|
96
|
+
copy from the browser or the API docs: "https://api.datadoghq.eu/" or
|
|
97
|
+
"api.datadoghq.eu". The tool adds "https://api." itself."""
|
|
98
|
+
site = site.strip().lower()
|
|
99
|
+
for prefix in ("https://", "http://", "api.", "app."):
|
|
100
|
+
site = site.removeprefix(prefix)
|
|
101
|
+
return site.rstrip("/")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def load_config(overrides: dict[str, Any] | None = None,
|
|
105
|
+
config_file: Path | None = None,
|
|
106
|
+
prompt: Callable[[str], str] | None = None) -> Config:
|
|
107
|
+
"""Build the Config.
|
|
108
|
+
|
|
109
|
+
overrides: command-line values (None means not given).
|
|
110
|
+
prompt: asks for a missing service or env; pass None to never prompt.
|
|
111
|
+
"""
|
|
112
|
+
# Only ./.env: without a path, load_dotenv searches upwards from this
|
|
113
|
+
# module's folder, so an editable install would read the repository's .env.
|
|
114
|
+
load_dotenv(Path.cwd() / ".env")
|
|
115
|
+
cli = {k: v for k, v in (overrides or {}).items() if v not in (None, "", ())}
|
|
116
|
+
file = read_config_file(config_file) if config_file else {}
|
|
117
|
+
system, person, classify = file.get("system", {}), file.get("person", {}), file.get("classify", {})
|
|
118
|
+
components = file.get("components", {})
|
|
119
|
+
max_components = components.get("max", 12)
|
|
120
|
+
if max_components < 2:
|
|
121
|
+
raise SystemExit(f"{config_file}: [components] 'max' must be at least 2.")
|
|
122
|
+
strip_prefixes = tuple(components.get("strip_prefixes", []))
|
|
123
|
+
for prefix in strip_prefixes:
|
|
124
|
+
if not prefix.startswith("/") or prefix == "/":
|
|
125
|
+
raise SystemExit(f"{config_file}: [components] 'strip_prefixes' entries are paths "
|
|
126
|
+
f"such as \"/shop\", got {prefix!r}.")
|
|
127
|
+
|
|
128
|
+
def get(cli_key: str, file_value: Any, env_key: str | None, default: Any = None) -> Any:
|
|
129
|
+
if cli_key in cli:
|
|
130
|
+
return cli[cli_key]
|
|
131
|
+
if file_value is not None:
|
|
132
|
+
return file_value
|
|
133
|
+
if env_key and os.getenv(env_key):
|
|
134
|
+
return os.getenv(env_key)
|
|
135
|
+
return default
|
|
136
|
+
|
|
137
|
+
offline = bool(cli.get("OFFLINE"))
|
|
138
|
+
service = get("DD_SERVICE", file.get("service"), "DD_SERVICE")
|
|
139
|
+
env = get("DD_ENV", file.get("env"), "DD_ENV")
|
|
140
|
+
if prompt:
|
|
141
|
+
if not service:
|
|
142
|
+
service = prompt("APM service name: ").strip()
|
|
143
|
+
if not env:
|
|
144
|
+
env = prompt("APM environment (for example prod): ").strip()
|
|
145
|
+
|
|
146
|
+
api_key, app_key = os.getenv("DD_API_KEY", ""), os.getenv("DD_APP_KEY", "")
|
|
147
|
+
missing = [name for name, value in (("DD_SERVICE", service), ("DD_ENV", env)) if not value]
|
|
148
|
+
if not offline:
|
|
149
|
+
missing += [name for name, value in (("DD_API_KEY", api_key), ("DD_APP_KEY", app_key)) if not value]
|
|
150
|
+
if missing:
|
|
151
|
+
raise SystemExit(
|
|
152
|
+
f"Missing required settings: {', '.join(missing)}. "
|
|
153
|
+
"Set them in the environment or .env (see .env.example); "
|
|
154
|
+
"service and env can also come from --service/--env or the config file."
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
output_dir = Path(get("OUTPUT_DIR", file.get("output"), "OUTPUT_DIR", "output"))
|
|
158
|
+
(output_dir / "raw").mkdir(parents=True, exist_ok=True)
|
|
159
|
+
|
|
160
|
+
include = [*system.get("include", []), *cli.get("INCLUDE", ())]
|
|
161
|
+
return Config(
|
|
162
|
+
api_key=api_key,
|
|
163
|
+
app_key=app_key,
|
|
164
|
+
site=normalize_site(get("DD_SITE", file.get("site"), "DD_SITE", "datadoghq.com")),
|
|
165
|
+
service=service,
|
|
166
|
+
env=env,
|
|
167
|
+
lookback_hours=int(get("DD_LOOKBACK_HOURS", file.get("hours"), "DD_LOOKBACK_HOURS", 24)),
|
|
168
|
+
output_dir=output_dir,
|
|
169
|
+
offline=offline,
|
|
170
|
+
system_name=get("SYSTEM_NAME", system.get("name"), None, ""),
|
|
171
|
+
system_description=system.get("description", ""),
|
|
172
|
+
include=tuple(dict.fromkeys(n for n in include if n != service)),
|
|
173
|
+
ignore=tuple(file.get("ignore", [])),
|
|
174
|
+
datastores=tuple(classify.get("datastores", [])),
|
|
175
|
+
external=tuple(classify.get("external", [])),
|
|
176
|
+
internal=tuple(classify.get("internal", [])),
|
|
177
|
+
person_enabled="NO_PERSON" not in cli and person.get("enabled", True),
|
|
178
|
+
person_name=person.get("name", "User"),
|
|
179
|
+
person_description=person.get("description", "End user of the web component."),
|
|
180
|
+
max_components=max_components,
|
|
181
|
+
strip_prefixes=strip_prefixes,
|
|
182
|
+
source_file=config_file,
|
|
183
|
+
)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Write the C4 model as a Structurizr DSL workspace and as Mermaid C4 diagrams."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .model import C4Model, Element
|
|
7
|
+
|
|
8
|
+
# --------------------------------------------------------------------------
|
|
9
|
+
# Structurizr DSL
|
|
10
|
+
# --------------------------------------------------------------------------
|
|
11
|
+
|
|
12
|
+
def _q(text: str) -> str:
|
|
13
|
+
return '"' + text.replace("\\", "\\\\").replace('"', "'") + '"'
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
MODEL_FILE = "datadog-model.dsl"
|
|
17
|
+
VIEWS_FILE = "datadog-views.dsl"
|
|
18
|
+
_GENERATED = ("# Generated by datadog-structurizr and rewritten on every run: do not edit.\n"
|
|
19
|
+
"# Add what Datadog cannot see to workspace.dsl, which includes this file.\n")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def emit_dsl_model(model: C4Model) -> str:
|
|
23
|
+
"""Elements and relationships, for !include inside the model block of workspace.dsl."""
|
|
24
|
+
lines: list[str] = []
|
|
25
|
+
w = lines.append
|
|
26
|
+
w(f"# {model.description}")
|
|
27
|
+
p = model.person
|
|
28
|
+
if p:
|
|
29
|
+
w(f"{p.key} = person {_q(p.name)} {_q(p.description)}")
|
|
30
|
+
w("")
|
|
31
|
+
s = model.target_system
|
|
32
|
+
w(f"{s.key} = softwareSystem {_q(s.name)} {_q(s.description)} {{")
|
|
33
|
+
for c in s.children:
|
|
34
|
+
w(f" {c.key} = container {_q(c.name)} {_q(c.description)} {_q(c.technology)} {{")
|
|
35
|
+
if c.database:
|
|
36
|
+
w(' tags "Database"')
|
|
37
|
+
for comp in c.children:
|
|
38
|
+
w(f" {comp.key} = component {_q(comp.name)} {_q(comp.description)} {_q(comp.technology)}")
|
|
39
|
+
w(" }")
|
|
40
|
+
w("}")
|
|
41
|
+
w("")
|
|
42
|
+
for e in model.systems:
|
|
43
|
+
tag = "External" if e.external else "Internal Service"
|
|
44
|
+
w(f"{e.key} = softwareSystem {_q(e.name)} {_q(e.description)} {_q(tag)}")
|
|
45
|
+
w("")
|
|
46
|
+
for r in model.relationships:
|
|
47
|
+
tech = f" {_q(r.technology)}" if r.technology else ""
|
|
48
|
+
w(f"{r.source_key} -> {r.target_key} {_q(r.description)}{tech}")
|
|
49
|
+
return _GENERATED + "\n".join(lines) + "\n"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def emit_dsl_views(model: C4Model) -> str:
|
|
53
|
+
"""The three views and styles, for !include inside the views block of workspace.dsl.
|
|
54
|
+
|
|
55
|
+
`include *` is resolved after the whole model is parsed, so the views also
|
|
56
|
+
show what workspace.dsl adds by hand.
|
|
57
|
+
"""
|
|
58
|
+
lines: list[str] = []
|
|
59
|
+
w = lines.append
|
|
60
|
+
s = model.target_system
|
|
61
|
+
w(f'systemContext {s.key} "L0-SystemContext" {{')
|
|
62
|
+
w(" include *")
|
|
63
|
+
w(" autolayout lr")
|
|
64
|
+
w("}")
|
|
65
|
+
w(f'container {s.key} "L1-Containers" {{')
|
|
66
|
+
w(" include *")
|
|
67
|
+
w(" autolayout lr")
|
|
68
|
+
w("}")
|
|
69
|
+
w(f'component {model.target_container.key} "L2-Components" '
|
|
70
|
+
+ _q(f"Entry points of {model.target_container.name} seen in Datadog APM "
|
|
71
|
+
"(HTTP endpoint groups and handlers), not its code structure.") + " {")
|
|
72
|
+
w(" include *")
|
|
73
|
+
w(" autolayout lr")
|
|
74
|
+
w("}")
|
|
75
|
+
w("styles {")
|
|
76
|
+
for tag, props in (("Person", ["shape Person"]),
|
|
77
|
+
("Database", ["shape Cylinder"]),
|
|
78
|
+
("External", ["background #999999", "color #ffffff"])):
|
|
79
|
+
w(f' element "{tag}" {{')
|
|
80
|
+
for prop in props:
|
|
81
|
+
w(f" {prop}")
|
|
82
|
+
w(" }")
|
|
83
|
+
w("}")
|
|
84
|
+
return _GENERATED + "\n".join(lines) + "\n"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def emit_workspace(model: C4Model) -> str:
|
|
88
|
+
"""The workspace.dsl written once, when none exists; never overwritten."""
|
|
89
|
+
comp = next(iter(model.target_container.children), None)
|
|
90
|
+
target = next(iter(model.systems), None)
|
|
91
|
+
example = (f" # {comp.key} -> {target.key} \"Calls\"\n"
|
|
92
|
+
if comp and target else "")
|
|
93
|
+
return (
|
|
94
|
+
f"workspace {_q(model.name)} {{\n"
|
|
95
|
+
" # Created once by datadog-structurizr and never overwritten: keep and edit it.\n"
|
|
96
|
+
f" # {MODEL_FILE} and {VIEWS_FILE} are rewritten on every run.\n"
|
|
97
|
+
"\n"
|
|
98
|
+
" model {\n"
|
|
99
|
+
f" !include {MODEL_FILE}\n"
|
|
100
|
+
"\n"
|
|
101
|
+
" # Elements and relationships Datadog cannot see, for example:\n"
|
|
102
|
+
f"{example}"
|
|
103
|
+
" }\n"
|
|
104
|
+
"\n"
|
|
105
|
+
" views {\n"
|
|
106
|
+
f" !include {VIEWS_FILE}\n"
|
|
107
|
+
" }\n"
|
|
108
|
+
"}\n"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def inline_includes(workspace: str, directory: Path) -> str:
|
|
113
|
+
"""workspace.dsl with the generated fragments pasted in, for the Structurizr playground."""
|
|
114
|
+
out = []
|
|
115
|
+
for line in workspace.splitlines():
|
|
116
|
+
name = line.strip().removeprefix("!include").strip()
|
|
117
|
+
if line.strip().startswith("!include") and name in (MODEL_FILE, VIEWS_FILE):
|
|
118
|
+
indent = line[:len(line) - len(line.lstrip())]
|
|
119
|
+
text = (directory / name).read_text(encoding="utf-8")
|
|
120
|
+
out += [indent + x if x else x for x in text.splitlines()]
|
|
121
|
+
else:
|
|
122
|
+
out.append(line)
|
|
123
|
+
return "\n".join(out) + "\n"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# --------------------------------------------------------------------------
|
|
127
|
+
# Mermaid C4 (renders with mmdc, GitHub, VS Code - no Java)
|
|
128
|
+
# --------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
def _m(text: str) -> str:
|
|
131
|
+
return '"' + text.replace('"', "'") + '"'
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _owner(key: str, model: C4Model, level: str) -> str | None:
|
|
135
|
+
"""Map an element key onto the element shown at this diagram level."""
|
|
136
|
+
s, c = model.target_system, model.target_container
|
|
137
|
+
if (model.person and key == model.person.key) or any(e.key == key for e in model.systems):
|
|
138
|
+
return key
|
|
139
|
+
if level == "L0":
|
|
140
|
+
return s.key
|
|
141
|
+
if any(x.key == key for x in s.children):
|
|
142
|
+
return key
|
|
143
|
+
if any(x.key == key for x in c.children):
|
|
144
|
+
return key if level == "L2" else c.key
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _system_line(e: Element) -> str:
|
|
149
|
+
fn = "System_Ext" if e.external else "System"
|
|
150
|
+
return f" {fn}({e.key}, {_m(e.name)}, {_m(e.description)})"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _container_line(x: Element) -> str:
|
|
154
|
+
fn = "ContainerDb" if x.database else "Container"
|
|
155
|
+
return f" {fn}({x.key}, {_m(x.name)}, {_m(x.technology)}, {_m(x.description)})"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _rels(model: C4Model, level: str, visible: set[str]) -> list[str]:
|
|
159
|
+
seen: set[tuple[str, str]] = set()
|
|
160
|
+
out = []
|
|
161
|
+
for r in model.relationships:
|
|
162
|
+
a, b = _owner(r.source_key, model, level), _owner(r.target_key, model, level)
|
|
163
|
+
if not a or not b or a == b or (a, b) in seen or a not in visible or b not in visible:
|
|
164
|
+
continue
|
|
165
|
+
seen.add((a, b))
|
|
166
|
+
out.append(f" Rel({a}, {b}, {_m(r.description)}"
|
|
167
|
+
+ (f", {_m(r.technology)})" if r.technology else ")"))
|
|
168
|
+
return out
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def emit_mermaid(model: C4Model) -> dict[str, str]:
|
|
172
|
+
p, s, c = model.person, model.target_system, model.target_container
|
|
173
|
+
person_line = [f" Person({p.key}, {_m(p.name)}, {_m(p.description)})"] if p else []
|
|
174
|
+
person_keys = {p.key} if p else set()
|
|
175
|
+
|
|
176
|
+
# L0 - System Context
|
|
177
|
+
l0 = ["C4Context", f" title L0 System Context - {s.name}", *person_line,
|
|
178
|
+
f" System({s.key}, {_m(s.name)}, {_m(s.description)})"]
|
|
179
|
+
l0 += [_system_line(e) for e in model.systems]
|
|
180
|
+
l0 += _rels(model, "L0", {*person_keys, s.key, *(e.key for e in model.systems)})
|
|
181
|
+
|
|
182
|
+
# L1 - Containers
|
|
183
|
+
l1 = ["C4Container", f" title L1 Containers - {s.name}", *person_line,
|
|
184
|
+
f" System_Boundary({s.key}, {_m(s.name)}) {{"]
|
|
185
|
+
l1 += [_container_line(x) for x in s.children]
|
|
186
|
+
l1.append(" }")
|
|
187
|
+
l1 += [_system_line(e) for e in model.systems]
|
|
188
|
+
l1 += _rels(model, "L1", {*person_keys, *(x.key for x in s.children), *(e.key for e in model.systems)})
|
|
189
|
+
|
|
190
|
+
# L2 - Components (only elements with a relationship to a component)
|
|
191
|
+
comp_keys = {x.key for x in c.children}
|
|
192
|
+
related = {k for r in model.relationships for k in (r.source_key, r.target_key)
|
|
193
|
+
if r.source_key in comp_keys or r.target_key in comp_keys} - comp_keys
|
|
194
|
+
l2 = ["C4Component", f" title L2 Components - {c.name} (entry points seen in APM)"]
|
|
195
|
+
if person_keys & related:
|
|
196
|
+
l2 += person_line
|
|
197
|
+
l2.append(f" Container_Boundary({c.key}, {_m(c.name)}) {{")
|
|
198
|
+
for x in c.children:
|
|
199
|
+
l2.append(f" Component({x.key}, {_m(x.name)}, {_m(x.technology)}, {_m(x.description)})")
|
|
200
|
+
l2.append(" }")
|
|
201
|
+
l2 += [_system_line(e) for e in model.systems if e.key in related]
|
|
202
|
+
l2 += [_container_line(x) for x in s.children if x.key in related]
|
|
203
|
+
l2 += _rels(model, "L2", comp_keys | related)
|
|
204
|
+
|
|
205
|
+
layout = ' UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1")'
|
|
206
|
+
for diagram in (l0, l1, l2):
|
|
207
|
+
diagram.append(layout)
|
|
208
|
+
return {
|
|
209
|
+
"L0-SystemContext": "\n".join(l0) + "\n",
|
|
210
|
+
"L1-Containers": "\n".join(l1) + "\n",
|
|
211
|
+
"L2-Components": "\n".join(l2) + "\n",
|
|
212
|
+
}
|