cloudmap 1.3.3__tar.gz → 1.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudmap-1.3.4/.github/workflows/ci.yml +51 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/ARCHITECTURE.md +19 -8
- {cloudmap-1.3.3 → cloudmap-1.3.4}/PKG-INFO +36 -3
- {cloudmap-1.3.3 → cloudmap-1.3.4}/README.md +35 -2
- {cloudmap-1.3.3 → cloudmap-1.3.4}/SECURITY.md +10 -3
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/__init__.py +1 -1
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/cli.py +146 -22
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/extract/extractors.py +58 -2
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ingest/azure.py +70 -2
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/csv_export.py +7 -1
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/html.py +12 -2
- cloudmap-1.3.4/tests/test_artifact_bytes.py +84 -0
- cloudmap-1.3.4/tests/test_az_invocation.py +92 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_deterministic_output.py +8 -3
- cloudmap-1.3.4/tests/test_hidden_config_types.py +96 -0
- cloudmap-1.3.4/tests/test_hostile_names.py +189 -0
- cloudmap-1.3.4/tests/test_large_graphs.py +125 -0
- cloudmap-1.3.4/tests/test_output_encoding.py +67 -0
- cloudmap-1.3.4/tests/test_portal_links.py +74 -0
- cloudmap-1.3.4/tests/test_sovereign_clouds.py +112 -0
- cloudmap-1.3.3/.github/workflows/ci.yml +0 -26
- {cloudmap-1.3.3 → cloudmap-1.3.4}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/.github/workflows/publish.yml +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/.gitignore +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/CONTRIBUTING.md +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/FORMAT.md +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/LICENSE +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/PLAN.md +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/__main__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/adapters/__init__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ask/__init__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ask/intent.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ask/narration.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ask/queries.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/data/contoso.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/extract/__init__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/extract/llm.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/graph.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ingest/__init__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/ingest/fixture.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/interactive.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/local_model.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/model.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/__init__.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/azure_icons.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/drawio.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/json_out.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/render/mermaid.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/cloudmap/scrub.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/docs/social-preview.html +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/docs/social-preview.png +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/estate-viewer.png +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/fixtures/acme_orders.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/fixtures/contoso.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/fixtures/estate.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/pyproject.toml +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/01_input_complex_random.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/04_scrubbed_output.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/06_trace_output.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/07_trace_output.html +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/08_trace_output.csv +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/09_trace_output.drawio +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/10_input_enterprise_architecture.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/12_enterprise_trace.html +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/13_enterprise_trace.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/14_enterprise_scrubbed.json +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/complex_mock_demo/README.md +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/data/azure_resource_types.txt +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_adapters.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_arg_rows.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_ask.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_azure.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_cli_exports.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_common_types.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_containerapps.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_containment_vs_association.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_drawio_xml.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_enrich.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_estate.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_fixtures_safe.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_golden_orders.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_graph.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_html.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_ingest_paging.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_interactive_wizard.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_llm.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_local_model.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_render_text.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_scrub.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_trust.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/tests/test_type_agnostic.py +0 -0
- {cloudmap-1.3.3 → cloudmap-1.3.4}/uv.lock +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ${{ matrix.os }}
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
# 3.9 is the floor the README promises; 3.13 catches what is about to break.
|
|
15
|
+
# Windows and macOS are here because the package claims OS Independent and
|
|
16
|
+
# an ubuntu-only matrix let a Windows-breaking bug ship: `az` installs as
|
|
17
|
+
# az.cmd there, and subprocess without shell=True never finds it. macOS is
|
|
18
|
+
# the cheap third data point for path and encoding assumptions.
|
|
19
|
+
include:
|
|
20
|
+
- {os: ubuntu-latest, python-version: "3.9"}
|
|
21
|
+
- {os: ubuntu-latest, python-version: "3.13"}
|
|
22
|
+
- {os: windows-latest, python-version: "3.13"}
|
|
23
|
+
- {os: macos-latest, python-version: "3.13"}
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
- uses: actions/setup-python@v5
|
|
27
|
+
with:
|
|
28
|
+
python-version: ${{ matrix.python-version }}
|
|
29
|
+
- run: pip install -e ".[dev]"
|
|
30
|
+
- run: ruff check .
|
|
31
|
+
- run: pytest
|
|
32
|
+
# The suite includes tests/test_fixtures_safe.py, which fails if a fixture
|
|
33
|
+
# ever carries a credential or an unscrubbed GUID. That guard is the reason
|
|
34
|
+
# CI runs on pull requests too.
|
|
35
|
+
|
|
36
|
+
# A smoke test of the path a stranger actually takes first, on every OS:
|
|
37
|
+
# writing all five artifacts to disk exercises the file encodings and the
|
|
38
|
+
# path joining that a pure-unit suite never touches.
|
|
39
|
+
- name: demo trace writes every artifact
|
|
40
|
+
run: >
|
|
41
|
+
cloudmap trace contoso-web --from demo -d smoke
|
|
42
|
+
--json smoke/x.json --html smoke/x.html --csv smoke/x.csv --mermaid smoke/x.mmd
|
|
43
|
+
- name: the artifacts exist and are not empty
|
|
44
|
+
shell: python
|
|
45
|
+
run: |
|
|
46
|
+
import pathlib, sys
|
|
47
|
+
missing = [str(p) for p in [pathlib.Path("smoke", f"x.{e}")
|
|
48
|
+
for e in ("json", "html", "csv", "mmd")]
|
|
49
|
+
if not pathlib.Path(p).exists() or pathlib.Path(p).stat().st_size == 0]
|
|
50
|
+
if missing:
|
|
51
|
+
sys.exit("empty or missing: " + ", ".join(missing))
|
|
@@ -78,7 +78,7 @@ flowchart TD
|
|
|
78
78
|
| `cloudmap/cli.py` | Orchestration + argparse | `_cmd_trace` wires the stages; `_cmd_capture`/`_cmd_scrub` produce committable fixtures; `_cmd_ask` loads a saved map and prints the computed answer, proof lines included. |
|
|
79
79
|
| `cloudmap/data/contoso.json` | The packaged demo estate | `--from demo` must work for a pip-only install - the first command a stranger runs cannot depend on cloning the repo. |
|
|
80
80
|
| `fixtures/` | Synthetic + scrubbed-golden estates | Fixture-first development → zero cloud contact in tests. |
|
|
81
|
-
| `tests/` |
|
|
81
|
+
| `tests/` | 536 tests | Behaviour-named; includes a conformance sweep of all 4,687 published Azure resource types (`tests/data/azure_resource_types.txt`) through every dependency channel, a realistic-estate suite with the property names Azure genuinely returns, `test_fixtures_safe.py` (fails CI if a fixture ever carries a credential), `test_deterministic_output.py` (subprocesses with different hash seeds - the in-process suite cannot see that class of bug) and `test_sovereign_clouds.py`. |
|
|
82
82
|
| `.gitignore` | Safety | `live/`, `*.live.json`, `*.blast.drawio` and every artifact extension at the repo root are ignored → output from a real tenant cannot be committed by accident. |
|
|
83
83
|
|
|
84
84
|
## Execution Flow
|
|
@@ -149,13 +149,24 @@ flowchart TD
|
|
|
149
149
|
## Open Questions / Risks
|
|
150
150
|
|
|
151
151
|
- **Live validation breadth**: the engine is conformance-tested against every
|
|
152
|
-
published type, but has been exercised against one real tenant
|
|
153
|
-
|
|
154
|
-
the shared `ftpsHostName` one (found live, fixed,
|
|
155
|
-
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
152
|
+
published type, but has been exercised against one real tenant, in the public
|
|
153
|
+
cloud, on Linux. Property shapes elsewhere may surface new false-positive
|
|
154
|
+
classes like the shared `ftpsHostName` one (found live, fixed,
|
|
155
|
+
regression-pinned). The sovereign clouds are covered by construction rather
|
|
156
|
+
than by a live run: an edge's kind now comes from the resolved target's ARM
|
|
157
|
+
type, not from its DNS suffix (`extractors.kind_for_target`), so
|
|
158
|
+
`vault.usgovcloudapi.net` and `vault.azure.cn` need no table entry. Windows
|
|
159
|
+
and macOS are covered by CI rather than by a table of assumptions.
|
|
160
|
+
- **Deep config readers exist for App Service and AKS only** - the two common
|
|
161
|
+
types whose config Resource Graph withholds. Checked, not assumed: Logic Apps,
|
|
162
|
+
Container Apps, ML workspaces, Cognitive Services and API Management all return
|
|
163
|
+
theirs in `properties`. What remains hidden is a different shape - child
|
|
164
|
+
resources a provider never registered with Resource Graph (Data Factory and
|
|
165
|
+
Synapse linked services, Automation runbooks, Stream Analytics inputs) and
|
|
166
|
+
data planes with their own API (Databricks secret scopes). Those types are
|
|
167
|
+
listed in `cli._CONFIG_HIDDEN_TYPES` and declared as a blind spot on every
|
|
168
|
+
map that contains one, live or fixture, so an empty result cannot pass for an
|
|
169
|
+
answer. A reader for them is roadmap, not present.
|
|
159
170
|
- **Cost of `--enrich all`** on large tenants: one `az` round-trip per app,
|
|
160
171
|
`CLOUDMAP_ENRICH_WORKERS` (default 12) at a time. No cache between runs yet -
|
|
161
172
|
every trace pays the scan again.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloudmap
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.4
|
|
4
4
|
Summary: Trace the blast radius of an Azure resource: one name in, a verified dependency graph out.
|
|
5
5
|
Project-URL: Homepage, https://github.com/KatsaounisThanasis/cloudmap
|
|
6
6
|
Project-URL: Repository, https://github.com/KatsaounisThanasis/cloudmap
|
|
@@ -70,8 +70,9 @@ graph, and draws it.
|
|
|
70
70
|
pip install cloudmap
|
|
71
71
|
```
|
|
72
72
|
|
|
73
|
-
Python 3.9
|
|
74
|
-
|
|
73
|
+
Python 3.9+, on Linux, macOS or Windows (CI runs all three). Two runtime
|
|
74
|
+
dependencies (`rich` and `questionary`, both for the terminal UI). Live mode
|
|
75
|
+
additionally needs the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli)
|
|
75
76
|
on your PATH. The optional AI passes need a local model server - [ollama](https://ollama.com)
|
|
76
77
|
works out of the box, and any OpenAI-compatible server (LM Studio, llama.cpp,
|
|
77
78
|
vLLM, LocalAI) works via two env vars:
|
|
@@ -318,6 +319,38 @@ from that map):
|
|
|
318
319
|
answer from an incomplete map is never allowed to read as "nothing depends on
|
|
319
320
|
this".
|
|
320
321
|
|
|
322
|
+
### Types cloudmap cannot map honestly, and says so
|
|
323
|
+
|
|
324
|
+
Five types keep their dependencies somewhere Resource Graph does not return, and
|
|
325
|
+
have no reader: **Data Factory** and **Synapse** (linked services are child
|
|
326
|
+
resources Resource Graph does not index), **Automation accounts** (runbooks,
|
|
327
|
+
variables, connections), **Stream Analytics** (inputs and outputs) and
|
|
328
|
+
**Databricks** (secret scopes live in the workspace's own data plane).
|
|
329
|
+
|
|
330
|
+
Tracing one of these still works - it appears on maps, RBAC and ARM references
|
|
331
|
+
resolve normally - but a **blind spot is written on the map**, because an empty
|
|
332
|
+
result for a Data Factory is not evidence that the factory depends on nothing.
|
|
333
|
+
|
|
334
|
+
This list is short because it was checked rather than assumed. Logic Apps
|
|
335
|
+
(`definition`), Container Apps (`configuration`), ML workspaces, Cognitive
|
|
336
|
+
Services and API Management all return their config in `properties`, so the four
|
|
337
|
+
generic channels find their dependencies with no special handling. App Service
|
|
338
|
+
and AKS hide config too - they are the two that have readers.
|
|
339
|
+
|
|
340
|
+
### Government, China and the other sovereign clouds
|
|
341
|
+
|
|
342
|
+
Supported, with nothing to configure. The DNS suffixes differ per cloud
|
|
343
|
+
(`vault.usgovcloudapi.net`, `vault.azure.cn`, `database.chinacloudapi.cn`), but
|
|
344
|
+
cloudmap does not read the service off the suffix: once a hostname resolves to a
|
|
345
|
+
scanned resource, that resource's ARM type is what decides the edge kind. The
|
|
346
|
+
HTML viewer's portal deep-links follow the cloud your account is in
|
|
347
|
+
(`portal.azure.us`, `portal.azure.cn`, ...), taken from `az account show`.
|
|
348
|
+
|
|
349
|
+
The one place a suffix still matters is a host that resolves to **nothing
|
|
350
|
+
scanned**: with no resource to consult, only the public-cloud suffixes are
|
|
351
|
+
recognised by name, so an unresolved sovereign host is surfaced as a generic
|
|
352
|
+
external reference rather than a categorised one. It is never dropped.
|
|
353
|
+
|
|
321
354
|
### My map came back empty or thinner than expected
|
|
322
355
|
|
|
323
356
|
In order:
|
|
@@ -36,8 +36,9 @@ graph, and draws it.
|
|
|
36
36
|
pip install cloudmap
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
Python 3.9
|
|
40
|
-
|
|
39
|
+
Python 3.9+, on Linux, macOS or Windows (CI runs all three). Two runtime
|
|
40
|
+
dependencies (`rich` and `questionary`, both for the terminal UI). Live mode
|
|
41
|
+
additionally needs the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli)
|
|
41
42
|
on your PATH. The optional AI passes need a local model server - [ollama](https://ollama.com)
|
|
42
43
|
works out of the box, and any OpenAI-compatible server (LM Studio, llama.cpp,
|
|
43
44
|
vLLM, LocalAI) works via two env vars:
|
|
@@ -284,6 +285,38 @@ from that map):
|
|
|
284
285
|
answer from an incomplete map is never allowed to read as "nothing depends on
|
|
285
286
|
this".
|
|
286
287
|
|
|
288
|
+
### Types cloudmap cannot map honestly, and says so
|
|
289
|
+
|
|
290
|
+
Five types keep their dependencies somewhere Resource Graph does not return, and
|
|
291
|
+
have no reader: **Data Factory** and **Synapse** (linked services are child
|
|
292
|
+
resources Resource Graph does not index), **Automation accounts** (runbooks,
|
|
293
|
+
variables, connections), **Stream Analytics** (inputs and outputs) and
|
|
294
|
+
**Databricks** (secret scopes live in the workspace's own data plane).
|
|
295
|
+
|
|
296
|
+
Tracing one of these still works - it appears on maps, RBAC and ARM references
|
|
297
|
+
resolve normally - but a **blind spot is written on the map**, because an empty
|
|
298
|
+
result for a Data Factory is not evidence that the factory depends on nothing.
|
|
299
|
+
|
|
300
|
+
This list is short because it was checked rather than assumed. Logic Apps
|
|
301
|
+
(`definition`), Container Apps (`configuration`), ML workspaces, Cognitive
|
|
302
|
+
Services and API Management all return their config in `properties`, so the four
|
|
303
|
+
generic channels find their dependencies with no special handling. App Service
|
|
304
|
+
and AKS hide config too - they are the two that have readers.
|
|
305
|
+
|
|
306
|
+
### Government, China and the other sovereign clouds
|
|
307
|
+
|
|
308
|
+
Supported, with nothing to configure. The DNS suffixes differ per cloud
|
|
309
|
+
(`vault.usgovcloudapi.net`, `vault.azure.cn`, `database.chinacloudapi.cn`), but
|
|
310
|
+
cloudmap does not read the service off the suffix: once a hostname resolves to a
|
|
311
|
+
scanned resource, that resource's ARM type is what decides the edge kind. The
|
|
312
|
+
HTML viewer's portal deep-links follow the cloud your account is in
|
|
313
|
+
(`portal.azure.us`, `portal.azure.cn`, ...), taken from `az account show`.
|
|
314
|
+
|
|
315
|
+
The one place a suffix still matters is a host that resolves to **nothing
|
|
316
|
+
scanned**: with no resource to consult, only the public-cloud suffixes are
|
|
317
|
+
recognised by name, so an unresolved sovereign host is surfaced as a generic
|
|
318
|
+
external reference rather than a categorised one. It is never dropped.
|
|
319
|
+
|
|
287
320
|
### My map came back empty or thinner than expected
|
|
288
321
|
|
|
289
322
|
In order:
|
|
@@ -29,10 +29,17 @@ The interesting failure modes for a tool like this:
|
|
|
29
29
|
- **Scope escalation**: cloudmap must only ever read what the caller's own
|
|
30
30
|
`az login` token can read, and must never perform a write operation against
|
|
31
31
|
the tenant.
|
|
32
|
-
- **Injection through cloud-controlled data**: resource names, tags and
|
|
33
|
-
properties are attacker-influenceable in shared tenants
|
|
32
|
+
- **Injection through cloud-controlled data**: resource names, ids, tags and
|
|
33
|
+
properties are attacker-influenceable in shared tenants, and a fixture or a
|
|
34
|
+
capture from someone else is untrusted input by definition; anything that lets
|
|
34
35
|
them break out of an `az` argument list, the HTML viewer, or the draw.io XML
|
|
35
|
-
is in scope.
|
|
36
|
+
is in scope. One shipped: the viewer's portal button interpolated a resource
|
|
37
|
+
id straight into an `href="..."`, so an id carrying a double quote closed the
|
|
38
|
+
attribute and the rest parsed as attributes - `kv" onmouseover="..."` became a
|
|
39
|
+
live event handler in a file meant to be handed to a colleague. Fixed with
|
|
40
|
+
encodeURI + escaping at that interpolation, and `tests/test_hostile_names.py`
|
|
41
|
+
now runs every renderer against names built to break each escape context,
|
|
42
|
+
asserting on the parsed DOM and XML tree rather than on substrings.
|
|
36
43
|
|
|
37
44
|
## If you are about to point this at a real tenant
|
|
38
45
|
|
|
@@ -12,7 +12,30 @@ from .render.json_out import to_json
|
|
|
12
12
|
from .render.mermaid import to_mermaid
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def _force_utf8_output():
|
|
16
|
+
"""Make our own stdout/stderr able to carry the map.
|
|
17
|
+
|
|
18
|
+
The summary is drawn with box characters and one emoji per resource type. A
|
|
19
|
+
Python stream not attached to a terminal encodes with
|
|
20
|
+
locale.getpreferredencoding() - cp1252 on a stock Windows install - and the
|
|
21
|
+
first 🌐 raises UnicodeEncodeError mid-render, exit code 1. So on Windows
|
|
22
|
+
`cloudmap trace ... > map.txt`, or any CI step that captures the output,
|
|
23
|
+
crashed while the same command in the console worked.
|
|
24
|
+
|
|
25
|
+
errors="replace" is the belt: a terminal that genuinely cannot do UTF-8
|
|
26
|
+
should print a '?' and carry on, never lose the run. Streams that are not
|
|
27
|
+
TextIOWrappers (a test harness, a pytest capture) have no reconfigure() and
|
|
28
|
+
are left exactly as they are.
|
|
29
|
+
"""
|
|
30
|
+
for stream in (sys.stdout, sys.stderr):
|
|
31
|
+
try:
|
|
32
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
33
|
+
except Exception:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
15
37
|
def main(argv=None):
|
|
38
|
+
_force_utf8_output()
|
|
16
39
|
# Interactive mode if no arguments are provided
|
|
17
40
|
if (argv is None and len(sys.argv) == 1) or (argv is not None and len(argv) == 0):
|
|
18
41
|
# The wizard hands the terminal to questionary; in a pipe / cron / CI
|
|
@@ -116,6 +139,18 @@ def main(argv=None):
|
|
|
116
139
|
return 1
|
|
117
140
|
|
|
118
141
|
|
|
142
|
+
def _active_portal():
|
|
143
|
+
"""The portal base URL of the cloud the scan ran against.
|
|
144
|
+
|
|
145
|
+
Imported lazily and defensively: the fixture path must keep working with no
|
|
146
|
+
Azure CLI installed at all, which is the whole point of `--from demo`."""
|
|
147
|
+
try:
|
|
148
|
+
from .ingest.azure import active_portal
|
|
149
|
+
return active_portal()
|
|
150
|
+
except Exception:
|
|
151
|
+
return "https://portal.azure.com"
|
|
152
|
+
|
|
153
|
+
|
|
119
154
|
def _cmd_trace(args):
|
|
120
155
|
read_gaps, blind_spots = [], []
|
|
121
156
|
if args.live:
|
|
@@ -217,7 +252,23 @@ def _cmd_trace(args):
|
|
|
217
252
|
if args.level == "high":
|
|
218
253
|
sub = collapse_high_level(sub, seed)
|
|
219
254
|
|
|
220
|
-
|
|
255
|
+
# A type whose dependencies Resource Graph never returns is a blind spot on
|
|
256
|
+
# every path, live or fixture, enriched or not - so it is declared here
|
|
257
|
+
# rather than in the live-only enrichment step. Without this, tracing a Data
|
|
258
|
+
# Factory returns a confident empty map.
|
|
259
|
+
for hidden in sorted({n.type for n in sub.nodes.values() if n.type in _CONFIG_HIDDEN_TYPES}):
|
|
260
|
+
names = sorted(n.name for n in sub.nodes.values() if n.type == hidden)
|
|
261
|
+
blind_spots.append(
|
|
262
|
+
f"{hidden} ({', '.join(names[:4])}{', ...' if len(names) > 4 else ''}): "
|
|
263
|
+
f"{_CONFIG_HIDDEN_TYPES[hidden]}. Those dependencies cannot appear on this map, "
|
|
264
|
+
f"and cloudmap has no reader for them - an empty result here is not evidence."
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
meta = {"truncated": truncated, "read_gaps": read_gaps, "blind_spots": blind_spots,
|
|
268
|
+
# Which cloud's portal the viewer should deep-link into. Live runs
|
|
269
|
+
# learn it from the account; a fixture has no cloud to ask, so the
|
|
270
|
+
# public portal stands in.
|
|
271
|
+
"portal": _active_portal()}
|
|
221
272
|
|
|
222
273
|
_export_outputs(sub, seed, args, meta)
|
|
223
274
|
return 0
|
|
@@ -241,29 +292,39 @@ def _export_outputs(sub, seed, args, meta):
|
|
|
241
292
|
out = args.out or (None if other_asked else f"{name}.blast.drawio")
|
|
242
293
|
if out:
|
|
243
294
|
_ensure_parent(out)
|
|
244
|
-
with open(out, "w", encoding="utf-8") as f:
|
|
295
|
+
with open(out, "w", encoding="utf-8", newline="\n") as f:
|
|
245
296
|
f.write(to_drawio(sub, seed))
|
|
246
297
|
if args.mermaid:
|
|
247
298
|
_ensure_parent(args.mermaid)
|
|
248
|
-
with open(args.mermaid, "w", encoding="utf-8") as f:
|
|
299
|
+
with open(args.mermaid, "w", encoding="utf-8", newline="\n") as f:
|
|
249
300
|
f.write(to_mermaid(sub, seed))
|
|
250
301
|
if args.json_out:
|
|
251
302
|
_ensure_parent(args.json_out)
|
|
252
|
-
with open(args.json_out, "w", encoding="utf-8") as f:
|
|
303
|
+
with open(args.json_out, "w", encoding="utf-8", newline="\n") as f:
|
|
253
304
|
f.write(to_json(sub, seed, meta=meta))
|
|
254
305
|
if args.html_out:
|
|
255
306
|
_ensure_parent(args.html_out)
|
|
256
|
-
with open(args.html_out, "w", encoding="utf-8") as f:
|
|
307
|
+
with open(args.html_out, "w", encoding="utf-8", newline="\n") as f:
|
|
257
308
|
f.write(to_html(sub, seed, meta=meta))
|
|
258
309
|
if args.csv_out:
|
|
259
310
|
_ensure_parent(args.csv_out)
|
|
260
311
|
from .render.csv_export import to_csv
|
|
261
|
-
|
|
312
|
+
# newline="" is mandatory for csv output: without it Windows translates
|
|
313
|
+
# the writer's line terminator a second time and every record gains a
|
|
314
|
+
# blank line after it.
|
|
315
|
+
with open(args.csv_out, "w", encoding="utf-8", newline="") as f:
|
|
262
316
|
f.write(to_csv(sub, seed, meta=meta))
|
|
263
317
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
318
|
+
# Every artifact is on disk by now, so a failure to DRAW the summary must not
|
|
319
|
+
# throw away a scan that may have taken minutes against a live tenant. The
|
|
320
|
+
# terminal view is a convenience; the files are the result.
|
|
321
|
+
try:
|
|
322
|
+
_print_summary(sub, seed, out, truncated=meta.get("truncated", False),
|
|
323
|
+
blind_spots=meta.get("blind_spots", []),
|
|
324
|
+
single_sub=bool(getattr(args, "single_sub", False)))
|
|
325
|
+
except Exception as e:
|
|
326
|
+
print(f"cloudmap: could not draw the terminal summary ({type(e).__name__}: {e}). "
|
|
327
|
+
f"The exported files are complete and unaffected.", file=sys.stderr)
|
|
267
328
|
|
|
268
329
|
|
|
269
330
|
_WEBAPP = "microsoft.web/sites"
|
|
@@ -303,6 +364,34 @@ _CONFIG_REFERENCED_TYPES = frozenset({
|
|
|
303
364
|
"microsoft.app/containerapps",
|
|
304
365
|
})
|
|
305
366
|
|
|
367
|
+
# Types that keep their dependencies somewhere Resource Graph does not return,
|
|
368
|
+
# and for which cloudmap has no reader. Two shapes of hiding place:
|
|
369
|
+
#
|
|
370
|
+
# - CHILD resources a provider never registered with Resource Graph. A Data
|
|
371
|
+
# Factory's linked services are `factories/linkedservices` objects; the
|
|
372
|
+
# factory's own `properties` names none of them.
|
|
373
|
+
# - A DATA PLANE with its own API, like Databricks secret scopes.
|
|
374
|
+
#
|
|
375
|
+
# App Service and AKS belong to this list by nature and are absent from it
|
|
376
|
+
# because they are the two that HAVE readers (`enrich_webapps`,
|
|
377
|
+
# `enrich_aks_clusters`). Everything here is a declared gap instead: an empty map
|
|
378
|
+
# for one of these types is not evidence of anything, and must not be presented
|
|
379
|
+
# as if it were. Verified against Resource Graph rather than assumed - Logic
|
|
380
|
+
# Apps, Container Apps, ML workspaces, Cognitive Services and API Management
|
|
381
|
+
# service objects all return their config in `properties`, so they are NOT here.
|
|
382
|
+
_CONFIG_HIDDEN_TYPES = {
|
|
383
|
+
"microsoft.datafactory/factories":
|
|
384
|
+
"linked services and pipelines are child resources Resource Graph does not index",
|
|
385
|
+
"microsoft.synapse/workspaces":
|
|
386
|
+
"linked services are child resources Resource Graph does not index",
|
|
387
|
+
"microsoft.databricks/workspaces":
|
|
388
|
+
"secret scopes and notebooks live in the workspace's own data plane",
|
|
389
|
+
"microsoft.automation/automationaccounts":
|
|
390
|
+
"runbooks, variables and connections are child resources Resource Graph does not index",
|
|
391
|
+
"microsoft.streamanalytics/streamingjobs":
|
|
392
|
+
"inputs and outputs are child resources Resource Graph does not index",
|
|
393
|
+
}
|
|
394
|
+
|
|
306
395
|
|
|
307
396
|
def _enrichment_targets(graph, seed, mode, direction):
|
|
308
397
|
"""Which workloads to deep-enrich, and which stay a blind spot.
|
|
@@ -498,7 +587,7 @@ def _write_export(path, resources, scrubbed, meta=None):
|
|
|
498
587
|
|
|
499
588
|
_ensure_parent(path)
|
|
500
589
|
doc = {"meta": dict(meta or {}, scrubbed=scrubbed), "data": resources}
|
|
501
|
-
with open(path, "w", encoding="utf-8") as f:
|
|
590
|
+
with open(path, "w", encoding="utf-8", newline="\n") as f:
|
|
502
591
|
_json.dump(doc, f, indent=2)
|
|
503
592
|
|
|
504
593
|
|
|
@@ -577,6 +666,13 @@ def _ensure_parent(path):
|
|
|
577
666
|
os.makedirs(parent, exist_ok=True)
|
|
578
667
|
|
|
579
668
|
|
|
669
|
+
# How many levels of the dependency tree the terminal prints. Past roughly this
|
|
670
|
+
# the tree stops being readable, and past a few hundred the recursion that builds
|
|
671
|
+
# it hits Python's frame limit - so the cut is both a UX and a safety one. Every
|
|
672
|
+
# exported format carries the full graph regardless.
|
|
673
|
+
_MAX_TREE_DEPTH = 40
|
|
674
|
+
|
|
675
|
+
|
|
580
676
|
def _print_summary(graph, seed, out, truncated=False, blind_spots=(), single_sub=False):
|
|
581
677
|
try:
|
|
582
678
|
from rich.console import Console
|
|
@@ -616,10 +712,18 @@ def _print_summary(graph, seed, out, truncated=False, blind_spots=(), single_sub
|
|
|
616
712
|
if "observes" in kind: return "dim white"
|
|
617
713
|
return "blue"
|
|
618
714
|
|
|
619
|
-
|
|
715
|
+
cut = [] # branches the depth cap stopped, explained under the panel
|
|
716
|
+
|
|
717
|
+
def _build_tree(node_id, seen, upward, depth=0):
|
|
620
718
|
"""Walk one direction only. Upward reads "what depends on me", so its
|
|
621
719
|
arrows are drawn pointing back at the parent - printing them like
|
|
622
|
-
downstream edges would state the dependency backwards.
|
|
720
|
+
downstream edges would state the dependency backwards.
|
|
721
|
+
|
|
722
|
+
The depth cap is not cosmetic. This recurses once per level, so a long
|
|
723
|
+
enough chain raised RecursionError and killed the summary outright -
|
|
724
|
+
with a traceback, which reads as a broken tool. A terminal tree that
|
|
725
|
+
deep is unreadable long before it is dangerous, so it is cut with a
|
|
726
|
+
note pointing at the exports, which carry the whole chain."""
|
|
623
727
|
tree = Tree(_label(graph.nodes[node_id]))
|
|
624
728
|
seen.add(node_id)
|
|
625
729
|
for e in graph.edges:
|
|
@@ -630,8 +734,15 @@ def _print_summary(graph, seed, out, truncated=False, blind_spots=(), single_sub
|
|
|
630
734
|
else f"[{_kind_colour(e.kind)}]--{e.kind}-->[/{_kind_colour(e.kind)}]")
|
|
631
735
|
if nxt in seen:
|
|
632
736
|
tree.add(f"{lbl} [dim]{graph.nodes[nxt].name} (cycle)[/dim]")
|
|
737
|
+
elif depth >= _MAX_TREE_DEPTH:
|
|
738
|
+
# Recorded, not explained here: at this depth the indentation
|
|
739
|
+
# alone is wider than a normal terminal, so a note attached to
|
|
740
|
+
# the node is wrapped away and the tree just appears to stop.
|
|
741
|
+
# The explanation is printed under the panel instead.
|
|
742
|
+
cut.append(graph.nodes[nxt].name)
|
|
743
|
+
tree.add(f"{lbl} [dim]{graph.nodes[nxt].name} …[/dim]")
|
|
633
744
|
else:
|
|
634
|
-
branch = _build_tree(nxt, set(seen), upward)
|
|
745
|
+
branch = _build_tree(nxt, set(seen), upward, depth + 1)
|
|
635
746
|
branch.label = f"{lbl} " + str(branch.label)
|
|
636
747
|
tree.add(branch)
|
|
637
748
|
return tree
|
|
@@ -645,21 +756,34 @@ def _print_summary(graph, seed, out, truncated=False, blind_spots=(), single_sub
|
|
|
645
756
|
if not graph.edges:
|
|
646
757
|
# A lonely one-node map is a real answer, but an unexplained one is
|
|
647
758
|
# indistinguishable from a broken tool. Say what could hide it.
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
"
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
"
|
|
656
|
-
|
|
759
|
+
reasons = ["the resource genuinely has no dependencies in the scanned scope"]
|
|
760
|
+
if single_sub:
|
|
761
|
+
reasons.append("its dependents live in another subscription "
|
|
762
|
+
"(this scan was scoped with --single-sub)")
|
|
763
|
+
reasons.append("the reference exists only in config that was not deep-read "
|
|
764
|
+
"(try --enrich all)")
|
|
765
|
+
if graph.nodes[seed].type in _CONFIG_HIDDEN_TYPES:
|
|
766
|
+
reasons.append("[bold]this type keeps its dependencies where Resource Graph "
|
|
767
|
+
"cannot see them[/bold] - see the blind spot above, which is "
|
|
768
|
+
"the likeliest explanation of the three")
|
|
769
|
+
console.print("[bold yellow]No dependencies found.[/bold yellow] "
|
|
770
|
+
"Any of these could be why:")
|
|
771
|
+
for reason in reasons:
|
|
772
|
+
console.print(f" - {reason}")
|
|
657
773
|
if down or not up:
|
|
658
774
|
console.print(Panel(_build_tree(seed, set(), upward=False),
|
|
659
775
|
title=f"Depends on ({len(down)})", border_style="blue"))
|
|
660
776
|
if up:
|
|
661
777
|
console.print(Panel(_build_tree(seed, set(), upward=True),
|
|
662
778
|
title=f"What depends on it ({len(up)})", border_style="magenta"))
|
|
779
|
+
if cut:
|
|
780
|
+
# Flush left, outside the panel: at 40 levels of indentation any note
|
|
781
|
+
# attached to the node itself is wrapped off the edge of the screen,
|
|
782
|
+
# and a tree that silently stops looks like a bug rather than a cut.
|
|
783
|
+
console.print(
|
|
784
|
+
f"[yellow]Tree cut at {_MAX_TREE_DEPTH} levels[/yellow] "
|
|
785
|
+
f"({len(cut)} branch(es), e.g. {cut[0]}). The printed tree is a "
|
|
786
|
+
f"preview - every exported format has the whole graph.")
|
|
663
787
|
if out:
|
|
664
788
|
console.print(f"🔗 [bold]draw.io:[/bold] {out}\n")
|
|
665
789
|
else:
|
|
@@ -129,6 +129,58 @@ def domain_kind(host):
|
|
|
129
129
|
return "references", "external service"
|
|
130
130
|
|
|
131
131
|
|
|
132
|
+
# The same relationships as _DOMAIN_KIND, keyed on the ARM type instead of the
|
|
133
|
+
# DNS suffix. This is the ground truth: `_DOMAIN_KIND` guesses a service from
|
|
134
|
+
# the shape of a hostname, but once a host has resolved to a SCANNED resource we
|
|
135
|
+
# know exactly what it is.
|
|
136
|
+
#
|
|
137
|
+
# It also makes the tool cloud-agnostic for free. The suffixes above are the
|
|
138
|
+
# public cloud's; Azure Government, China and the other sovereign clouds use
|
|
139
|
+
# different ones (`vault.usgovcloudapi.net`, `vault.azure.cn`,
|
|
140
|
+
# `database.chinacloudapi.cn`, ...). Before this, a Gov tenant still got the
|
|
141
|
+
# right EDGES - the generic host index resolves on the first DNS label, which is
|
|
142
|
+
# cloud-independent - but every one of them came out as a bare `references`,
|
|
143
|
+
# losing exactly the distinction ("who reads this vault") the map is for.
|
|
144
|
+
_TYPE_KIND = {
|
|
145
|
+
"microsoft.keyvault/vaults": "reads-secret",
|
|
146
|
+
"microsoft.keyvault/managedhsms": "reads-secret",
|
|
147
|
+
"microsoft.sql/servers": "connects-to",
|
|
148
|
+
"microsoft.sql/servers/databases": "connects-to",
|
|
149
|
+
"microsoft.documentdb/databaseaccounts": "connects-to",
|
|
150
|
+
"microsoft.cache/redis": "connects-to",
|
|
151
|
+
"microsoft.cache/redisenterprise": "connects-to",
|
|
152
|
+
"microsoft.servicebus/namespaces": "connects-to",
|
|
153
|
+
"microsoft.eventhub/namespaces": "connects-to",
|
|
154
|
+
"microsoft.search/searchservices": "connects-to",
|
|
155
|
+
"microsoft.cognitiveservices/accounts": "connects-to",
|
|
156
|
+
"microsoft.storage/storageaccounts": "connects-to",
|
|
157
|
+
"microsoft.containerregistry/registries": "pulls-image",
|
|
158
|
+
"microsoft.web/sites": "calls",
|
|
159
|
+
"microsoft.app/containerapps": "calls",
|
|
160
|
+
"microsoft.insights/components": "sends-telemetry",
|
|
161
|
+
"microsoft.operationalinsights/workspaces": "uses-workspace",
|
|
162
|
+
}
|
|
163
|
+
for _prefix, _kind in (("microsoft.dbforpostgresql/", "connects-to"),
|
|
164
|
+
("microsoft.dbformysql/", "connects-to"),
|
|
165
|
+
("microsoft.dbformariadb/", "connects-to")):
|
|
166
|
+
_TYPE_KIND[_prefix] = _kind
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def kind_for_target(node_type, host):
|
|
170
|
+
"""The edge kind for a host reference that RESOLVED to a scanned resource.
|
|
171
|
+
|
|
172
|
+
Prefers what the target actually is over what its hostname looks like, and
|
|
173
|
+
falls back to the suffix table for a type with no entry (which still catches
|
|
174
|
+
the public-cloud hosts of services this map does not name)."""
|
|
175
|
+
t = (node_type or "").lower()
|
|
176
|
+
if t in _TYPE_KIND:
|
|
177
|
+
return _TYPE_KIND[t]
|
|
178
|
+
for prefix, kind in _TYPE_KIND.items():
|
|
179
|
+
if prefix.endswith("/") and t.startswith(prefix):
|
|
180
|
+
return kind
|
|
181
|
+
return domain_kind(host)[0]
|
|
182
|
+
|
|
183
|
+
|
|
132
184
|
def _vault_refs(text):
|
|
133
185
|
names = set()
|
|
134
186
|
for inner in _KV_REF.findall(text):
|
|
@@ -233,6 +285,8 @@ class Resolver:
|
|
|
233
285
|
can look up whether an endpoint host corresponds to a scanned resource."""
|
|
234
286
|
|
|
235
287
|
def __init__(self, nodes):
|
|
288
|
+
self.nodes = nodes # id -> Node, so a resolved target's ARM type
|
|
289
|
+
# can decide an edge kind (see kind_for_target)
|
|
236
290
|
self.by_id = {}
|
|
237
291
|
self.by_host = {} # endpoint hostname -> node_id
|
|
238
292
|
self.by_principal = {}
|
|
@@ -656,7 +710,8 @@ def extract_edges(nodes):
|
|
|
656
710
|
continue
|
|
657
711
|
if _is_containment(path):
|
|
658
712
|
continue
|
|
659
|
-
kind = "observes" if n.type in _OBSERVER_TYPES
|
|
713
|
+
kind = ("observes" if n.type in _OBSERVER_TYPES
|
|
714
|
+
else kind_for_target(nodes[tgt].type, host))
|
|
660
715
|
add(n.id, tgt, kind, f"references host {host} at {path}")
|
|
661
716
|
known.add((n.id, tgt))
|
|
662
717
|
|
|
@@ -734,7 +789,8 @@ def _config_edges(n, values, r, add, label="app config"):
|
|
|
734
789
|
for word in words:
|
|
735
790
|
if word in r.by_host:
|
|
736
791
|
hit_hosts[r.by_host[word].lower()] = word
|
|
737
|
-
add(n.id, r.by_host[word],
|
|
792
|
+
add(n.id, r.by_host[word], kind_for_target(r.nodes[r.by_host[word]].type, word),
|
|
793
|
+
f"{label} references host {word}")
|
|
738
794
|
if word in r.by_ik:
|
|
739
795
|
add(n.id, r.by_ik[word], "sends-telemetry", f"{label} contains instrumentation key")
|
|
740
796
|
|