cloudmap 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloudmap-1.0.0 → cloudmap-1.1.0}/ARCHITECTURE.md +1 -1
- {cloudmap-1.0.0 → cloudmap-1.1.0}/PKG-INFO +106 -54
- {cloudmap-1.0.0 → cloudmap-1.1.0}/README.md +105 -53
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/__init__.py +1 -1
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ask/narration.py +1 -1
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/cli.py +3 -3
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/extract/llm.py +5 -3
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/interactive.py +1 -1
- cloudmap-1.1.0/cloudmap/local_model.py +92 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_ask.py +1 -1
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_llm.py +3 -3
- cloudmap-1.1.0/tests/test_local_model.py +92 -0
- cloudmap-1.0.0/cloudmap/local_model.py +0 -49
- {cloudmap-1.0.0 → cloudmap-1.1.0}/.github/workflows/ci.yml +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/.github/workflows/publish.yml +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/.gitignore +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/FORMAT.md +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/LICENSE +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/PLAN.md +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/__main__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/adapters/__init__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ask/__init__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ask/intent.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ask/queries.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/extract/__init__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/extract/extractors.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/graph.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ingest/__init__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ingest/azure.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/ingest/fixture.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/model.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/__init__.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/azure_icons.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/csv_export.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/drawio.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/html.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/json_out.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/render/mermaid.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/cloudmap/scrub.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/docs/social-preview.html +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/docs/social-preview.png +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/estate-viewer.png +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/fixtures/acme_orders.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/fixtures/contoso.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/fixtures/estate.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/pyproject.toml +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/01_input_complex_random.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/04_scrubbed_output.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/06_trace_output.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/07_trace_output.html +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/08_trace_output.csv +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/09_trace_output.drawio +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/10_input_enterprise_architecture.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/12_enterprise_trace.html +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/13_enterprise_trace.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/14_enterprise_scrubbed.json +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/README.md +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_adapters.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_arg_rows.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_azure.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_cli_exports.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_containerapps.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_drawio_xml.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_enrich.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_estate.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_fixtures_safe.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_golden_orders.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_graph.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_html.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_ingest_paging.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_interactive_wizard.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_scrub.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/tests/test_trust.py +0 -0
- {cloudmap-1.0.0 → cloudmap-1.1.0}/uv.lock +0 -0
|
@@ -67,7 +67,7 @@ flowchart TD
|
|
|
67
67
|
| `cloudmap/ask/queries.py` | **The Ask layer's heart**: the six queries, each computed by traversal | An answer must be auditable, so it is derived from edges, never generated. `_trust()` grades a whole path by its weakest hop, and distinguishes *passing through* an unverified node (whole finding becomes a guess) from *ending* at one (the reference is proven; the target is flagged). |
|
|
68
68
|
| `cloudmap/ask/intent.py` | Question → one query | Rules first so the common phrasings need no model at all (including the "rotate / restart / decommission" verbs). The model is a fallback that may only name a query from a fixed list and a resource, both validated against the graph — it routes, it never answers. |
|
|
69
69
|
| `cloudmap/ask/narration.py` | Optional prose (`--explain`) | Handed the computed facts only, and printed *below* them, so drifting prose is visibly a narration disagreeing with the facts, not a wrong answer. |
|
|
70
|
-
| `cloudmap/local_model.py` | The single outbound model call | One module = one auditable promise: the call goes to localhost (ollama) and failure returns an empty value, because cloudmap must be fully useful with no model installed. |
|
|
70
|
+
| `cloudmap/local_model.py` | The single outbound model call | One module = one auditable promise: the call goes to localhost (ollama's API by default, any OpenAI-compatible server via `CLOUDMAP_LLM_URL`) and failure returns an empty value, because cloudmap must be fully useful with no model installed. |
|
|
71
71
|
| `cloudmap/cli.py` | Orchestration + argparse | `_cmd_trace` wires the trace stages together (live enrichment + external merge live here); `_cmd_ask` loads a saved map and prints the computed answer, proof lines included. |
|
|
72
72
|
| `fixtures/contoso.json` | 100% synthetic estate | Fixture-first development → zero cloud contact in tests. |
|
|
73
73
|
| `tests/test_graph.py` | 5 unit tests | Lock in the hub-boundary behaviour + edge-kinds against the fixture. |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloudmap
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Trace the blast radius of an Azure resource: one name in, a verified dependency graph out.
|
|
5
5
|
Project-URL: Homepage, https://github.com/KatsaounisThanasis/cloudmap
|
|
6
6
|
Project-URL: Repository, https://github.com/KatsaounisThanasis/cloudmap
|
|
@@ -32,16 +32,29 @@ Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
|
32
32
|
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
33
33
|
Description-Content-Type: text/markdown
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
<div align="center">
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
[](https://pypi.org/project/cloudmap/)
|
|
39
|
-
[](https://pypi.org/project/cloudmap/)
|
|
40
|
-
[](LICENSE)
|
|
37
|
+
<img src="docs/social-preview.png" alt="cloudmap — one Azure resource name in, its whole blast radius out" width="820">
|
|
41
38
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
<p>
|
|
40
|
+
<a href="https://github.com/KatsaounisThanasis/cloudmap/actions/workflows/ci.yml"><img src="https://github.com/KatsaounisThanasis/cloudmap/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
41
|
+
<a href="https://pypi.org/project/cloudmap/"><img src="https://img.shields.io/pypi/v/cloudmap?color=1f7a8c" alt="PyPI"></a>
|
|
42
|
+
<a href="https://pypi.org/project/cloudmap/"><img src="https://img.shields.io/pypi/pyversions/cloudmap?color=1f7a8c" alt="Python versions"></a>
|
|
43
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT"></a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
<p>
|
|
47
|
+
<a href="#install"><b>Install</b></a> ·
|
|
48
|
+
<a href="#60-second-demo"><b>Demo</b></a> ·
|
|
49
|
+
<a href="#interactive-wizard"><b>Wizard</b></a> ·
|
|
50
|
+
<a href="#why"><b>Why</b></a> ·
|
|
51
|
+
<a href="#how-it-works"><b>How it works</b></a> ·
|
|
52
|
+
<a href="#ask-a-map-questions"><b>Ask</b></a>
|
|
53
|
+
</p>
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
---
|
|
45
58
|
|
|
46
59
|
Azure Resource Graph has no "dependencies" table. The relationships that matter
|
|
47
60
|
- what an App Service is hosted on, which Key Vault it reads, which subnet it
|
|
@@ -50,15 +63,6 @@ Endpoint fronts, which App Gateway routes to it - are buried inside each
|
|
|
50
63
|
resource's `properties`. cloudmap reads them out, correlates them into one
|
|
51
64
|
graph, and draws it.
|
|
52
65
|
|
|
53
|
-

|
|
54
|
-
|
|
55
|
-
**Contents** · [Install](#install) · [60-second demo](#60-second-demo) ·
|
|
56
|
-
[Interactive wizard](#interactive-wizard) · [Why](#why) ·
|
|
57
|
-
[What it maps](#what-it-maps) · [How it works](#how-it-works) ·
|
|
58
|
-
[Ask a map questions](#ask-a-map-questions) · [Live Azure](#live-azure-opt-in) ·
|
|
59
|
-
[Capture and scrub](#capture-a-real-export-so-the-tests-can-be-wrong) ·
|
|
60
|
-
[Usage reference](#usage-reference)
|
|
61
|
-
|
|
62
66
|
## Install
|
|
63
67
|
|
|
64
68
|
```
|
|
@@ -67,15 +71,25 @@ pip install cloudmap
|
|
|
67
71
|
|
|
68
72
|
Python 3.9+. Two runtime dependencies (`rich` and `questionary`, both for the
|
|
69
73
|
terminal UI). Live mode additionally needs the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli)
|
|
70
|
-
on your PATH
|
|
74
|
+
on your PATH. The optional AI passes need a local model server - [ollama](https://ollama.com)
|
|
75
|
+
works out of the box, and any OpenAI-compatible server (LM Studio, llama.cpp,
|
|
76
|
+
vLLM, LocalAI) works via two env vars:
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
export CLOUDMAP_LLM_URL=http://localhost:1234/v1/chat/completions # your server
|
|
80
|
+
export CLOUDMAP_LLM_MODEL=<model name> # default: qwen2.5-coder:3b
|
|
81
|
+
```
|
|
71
82
|
|
|
72
|
-
|
|
83
|
+
<details>
|
|
84
|
+
<summary>Working on it instead</summary>
|
|
73
85
|
|
|
74
86
|
```
|
|
75
87
|
git clone https://github.com/KatsaounisThanasis/cloudmap && cd cloudmap
|
|
76
88
|
pip install -e ".[dev]" && pytest
|
|
77
89
|
```
|
|
78
90
|
|
|
91
|
+
</details>
|
|
92
|
+
|
|
79
93
|
## 60-second demo
|
|
80
94
|
|
|
81
95
|
No Azure account needed - the repo ships a synthetic estate:
|
|
@@ -99,7 +113,12 @@ Blast radius: 9 resources (0 external), 9 dependencies
|
|
|
99
113
|
🔗 draw.io: contoso-web.drawio
|
|
100
114
|
```
|
|
101
115
|
|
|
102
|
-
|
|
116
|
+
That is the default **high-level** view - resources grouped by type. Add
|
|
117
|
+
`--level detail` to see every instance with its real name, and on a real estate
|
|
118
|
+
it goes several layers deep:
|
|
119
|
+
|
|
120
|
+
<details>
|
|
121
|
+
<summary>A deeper map</summary>
|
|
103
122
|
|
|
104
123
|
```text
|
|
105
124
|
Blast radius: 15 resources (0 external), 14 dependencies
|
|
@@ -122,19 +141,26 @@ Blast radius: 15 resources (0 external), 14 dependencies
|
|
|
122
141
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
123
142
|
```
|
|
124
143
|
|
|
125
|
-
|
|
126
|
-
`--level detail` to see every instance with its real name.)
|
|
144
|
+
</details>
|
|
127
145
|
|
|
128
|
-
###
|
|
146
|
+
### What you get out
|
|
129
147
|
|
|
130
|
-
| Flag |
|
|
148
|
+
| Flag | Output |
|
|
131
149
|
|---|---|
|
|
132
150
|
| `-o FILE` | **draw.io** diagram with native Azure icons - open it in [draw.io](https://app.diagrams.net), the desktop app or the VS Code extension and edit it like any hand-drawn diagram |
|
|
133
|
-
| `--html FILE` | **Interactive viewer
|
|
151
|
+
| `--html FILE` | **Interactive viewer** - one self-contained file, no server and no CDN |
|
|
134
152
|
| `--mermaid FILE` | Mermaid source, for embedding in Markdown docs |
|
|
135
153
|
| `--json FILE` | The graph itself - this is what `cloudmap ask` reads |
|
|
136
154
|
| `--csv FILE` | Flat edge list with evidence, for a spreadsheet or an auditor |
|
|
137
155
|
|
|
156
|
+
<div align="center">
|
|
157
|
+
<img src="estate-viewer.png" alt="The interactive HTML viewer: the seed at the centre, dependencies fanned out with native Azure icons, each edge carrying its relationship and proof">
|
|
158
|
+
</div>
|
|
159
|
+
|
|
160
|
+
The `--html` viewer has a dark mode, edges colour-coded by relationship type
|
|
161
|
+
(security / data / network), resource-group filters and search, SVG and PNG
|
|
162
|
+
export, and direct links into the Azure portal.
|
|
163
|
+
|
|
138
164
|
## Interactive wizard
|
|
139
165
|
|
|
140
166
|
Run it with no arguments and it walks you through the whole thing:
|
|
@@ -198,20 +224,6 @@ App Insights, VNets, Private Endpoints and managed identities.
|
|
|
198
224
|
|
|
199
225
|
## Ask a map questions
|
|
200
226
|
|
|
201
|
-
```
|
|
202
|
-
cloudmap ask <map.json> "<question>"
|
|
203
|
-
|
|
204
|
-
--explain also narrate the answer with a LOCAL model (ollama)
|
|
205
|
-
--llm if no built-in rule understands the phrasing, let a LOCAL model
|
|
206
|
-
pick the query (its choice is validated against the map)
|
|
207
|
-
--max-hops N limit traversal depth
|
|
208
|
-
--json print the answer as JSON (for scripting)
|
|
209
|
-
```
|
|
210
|
-
|
|
211
|
-
Instance names only exist in a `--level detail` map; the default high-level map
|
|
212
|
-
groups by type, so ask it about a group (`"what breaks if I touch Key Vault"`) or
|
|
213
|
-
trace with `--level detail` first:
|
|
214
|
-
|
|
215
227
|
```
|
|
216
228
|
$ cloudmap trace contoso-web --from fixtures/contoso.json --level detail --json out.json
|
|
217
229
|
$ cloudmap ask out.json "what breaks if I touch contoso-kv"
|
|
@@ -238,11 +250,50 @@ cannot promote a guess to one. Every finding shows the hops behind it and the pr
|
|
|
238
250
|
of each hop, and a finding that leans on a model-proposed edge is marked `[GUESS]`.
|
|
239
251
|
If the map itself says it is incomplete, every answer from it repeats that warning.
|
|
240
252
|
|
|
241
|
-
|
|
253
|
+
<details>
|
|
254
|
+
<summary><b>Flags</b></summary>
|
|
255
|
+
|
|
256
|
+
```
|
|
257
|
+
cloudmap ask <map.json> "<question>"
|
|
258
|
+
|
|
259
|
+
--explain also narrate the answer with a LOCAL model
|
|
260
|
+
--llm if no built-in rule understands the phrasing, let a LOCAL model
|
|
261
|
+
pick the query (its choice is validated against the map)
|
|
262
|
+
--max-hops N limit traversal depth
|
|
263
|
+
--json print the answer as JSON (for scripting)
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Instance names only exist in a `--level detail` map; the default high-level map
|
|
267
|
+
groups by type, so ask it about a group (`"what breaks if I touch Key Vault"`) or
|
|
268
|
+
trace with `--level detail` first.
|
|
269
|
+
|
|
270
|
+
</details>
|
|
271
|
+
|
|
272
|
+
## Live Azure
|
|
273
|
+
|
|
274
|
+
Fixtures are the default. Live mode is opt-in and **not sandboxed**: `--allow-live`
|
|
275
|
+
is the deliberate switch, and cloudmap reads whatever subscription `az` is pointed
|
|
276
|
+
at. The read is read-only, but it is a read of live infrastructure, so point it on
|
|
277
|
+
purpose. **Do not point this at data you are not authorized to read.**
|
|
242
278
|
|
|
243
279
|
```
|
|
244
280
|
cloudmap trace my-app --live --allow-live
|
|
281
|
+
```
|
|
245
282
|
|
|
283
|
+
**cloudmap runs as you.** It has no credentials of its own - every read goes through
|
|
284
|
+
the Azure CLI with your `az login` token, so it sees exactly what your account can
|
|
285
|
+
see and nothing more. Resource Graph filters by RBAC, so resources you cannot read
|
|
286
|
+
simply do not appear (Azure raises no error for them - they are invisible, not
|
|
287
|
+
denied). Deep reads your role does not allow (app settings, Key Vault values, AKS
|
|
288
|
+
manifests) fail visibly instead: cloudmap **reports the gap** on the map rather than
|
|
289
|
+
silently dropping edges, and warns when a scan is truncated - so a small graph on a
|
|
290
|
+
low-privilege account reads as "this is what I was allowed to see", not "this is
|
|
291
|
+
everything".
|
|
292
|
+
|
|
293
|
+
<details>
|
|
294
|
+
<summary><b>Flags, enrichment and the subscription pin</b></summary>
|
|
295
|
+
|
|
296
|
+
```
|
|
246
297
|
--single-sub query only the active subscription (default: every enabled
|
|
247
298
|
subscription in the tenant)
|
|
248
299
|
--enrich MODE which web apps to deep-enrich for the dependencies that only
|
|
@@ -251,7 +302,7 @@ cloudmap trace my-app --live --allow-live
|
|
|
251
302
|
all | seed | none
|
|
252
303
|
--resolve-secrets read KV secret values in-memory to see through KV-backed
|
|
253
304
|
connection strings (never printed or written)
|
|
254
|
-
--llm let a LOCAL model
|
|
305
|
+
--llm let a LOCAL model propose extra edges, each
|
|
255
306
|
verified against scanned resources before it is trusted
|
|
256
307
|
```
|
|
257
308
|
|
|
@@ -265,22 +316,18 @@ Anything left un-enriched is reported as a **blind spot** on the map and repeate
|
|
|
265
316
|
every `ask` answer drawn from it, so an empty result never passes for "nothing depends
|
|
266
317
|
on this".
|
|
267
318
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
against a stale `az` context silently redirecting a scan, pin the subscription you
|
|
272
|
-
mean - if set, cloudmap refuses to run against any other active subscription:
|
|
319
|
+
As an optional guard against a stale `az` context silently redirecting a scan, pin the
|
|
320
|
+
subscription you mean - if set, cloudmap refuses to run against any other active
|
|
321
|
+
subscription:
|
|
273
322
|
|
|
274
323
|
```
|
|
275
324
|
export CLOUDMAP_ALLOW_SUBSCRIPTION=<subscription-id> # optional
|
|
276
325
|
```
|
|
277
326
|
|
|
278
|
-
|
|
279
|
-
silently dropping edges, and warns when a scan is truncated - so you know when the
|
|
280
|
-
picture is incomplete. Fixtures are always the default.
|
|
281
|
-
**Do not point this at data you are not authorized to read.**
|
|
327
|
+
</details>
|
|
282
328
|
|
|
283
|
-
|
|
329
|
+
<details>
|
|
330
|
+
<summary><b>Capture a real export (so the tests can be wrong)</b></summary>
|
|
284
331
|
|
|
285
332
|
A fixture you wrote yourself can only confirm what you already believe. `capture`
|
|
286
333
|
saves what Azure actually returned, scrubs it, and gives you something that can
|
|
@@ -305,7 +352,10 @@ printed, the mapping is not. **A scrubber is not a proof: read the file before y
|
|
|
305
352
|
commit it.** `--no-scrub` exists for local debugging and writes credentials to
|
|
306
353
|
disk; keep those files named `*.live.json` so `.gitignore` catches them.
|
|
307
354
|
|
|
308
|
-
|
|
355
|
+
</details>
|
|
356
|
+
|
|
357
|
+
<details>
|
|
358
|
+
<summary><b>Usage reference</b></summary>
|
|
309
359
|
|
|
310
360
|
```
|
|
311
361
|
cloudmap trace <name> (--from <fixture.json> | --live) [options]
|
|
@@ -325,7 +375,9 @@ cloudmap trace <name> (--from <fixture.json> | --live) [options]
|
|
|
325
375
|
-d, --out-dir DIR write every artifact into DIR, named after the seed
|
|
326
376
|
```
|
|
327
377
|
|
|
328
|
-
Other subcommands: `cloudmap capture`, `cloudmap scrub`, `cloudmap ask
|
|
378
|
+
Other subcommands: `cloudmap capture`, `cloudmap scrub`, `cloudmap ask`.
|
|
379
|
+
|
|
380
|
+
</details>
|
|
329
381
|
|
|
330
382
|
## Roadmap
|
|
331
383
|
|
|
@@ -1,13 +1,26 @@
|
|
|
1
|
-
|
|
1
|
+
<div align="center">
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
[](https://pypi.org/project/cloudmap/)
|
|
5
|
-
[](https://pypi.org/project/cloudmap/)
|
|
6
|
-
[](LICENSE)
|
|
3
|
+
<img src="docs/social-preview.png" alt="cloudmap — one Azure resource name in, its whole blast radius out" width="820">
|
|
7
4
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
<p>
|
|
6
|
+
<a href="https://github.com/KatsaounisThanasis/cloudmap/actions/workflows/ci.yml"><img src="https://github.com/KatsaounisThanasis/cloudmap/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
7
|
+
<a href="https://pypi.org/project/cloudmap/"><img src="https://img.shields.io/pypi/v/cloudmap?color=1f7a8c" alt="PyPI"></a>
|
|
8
|
+
<a href="https://pypi.org/project/cloudmap/"><img src="https://img.shields.io/pypi/pyversions/cloudmap?color=1f7a8c" alt="Python versions"></a>
|
|
9
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT"></a>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
<p>
|
|
13
|
+
<a href="#install"><b>Install</b></a> ·
|
|
14
|
+
<a href="#60-second-demo"><b>Demo</b></a> ·
|
|
15
|
+
<a href="#interactive-wizard"><b>Wizard</b></a> ·
|
|
16
|
+
<a href="#why"><b>Why</b></a> ·
|
|
17
|
+
<a href="#how-it-works"><b>How it works</b></a> ·
|
|
18
|
+
<a href="#ask-a-map-questions"><b>Ask</b></a>
|
|
19
|
+
</p>
|
|
20
|
+
|
|
21
|
+
</div>
|
|
22
|
+
|
|
23
|
+
---
|
|
11
24
|
|
|
12
25
|
Azure Resource Graph has no "dependencies" table. The relationships that matter
|
|
13
26
|
- what an App Service is hosted on, which Key Vault it reads, which subnet it
|
|
@@ -16,15 +29,6 @@ Endpoint fronts, which App Gateway routes to it - are buried inside each
|
|
|
16
29
|
resource's `properties`. cloudmap reads them out, correlates them into one
|
|
17
30
|
graph, and draws it.
|
|
18
31
|
|
|
19
|
-

|
|
20
|
-
|
|
21
|
-
**Contents** · [Install](#install) · [60-second demo](#60-second-demo) ·
|
|
22
|
-
[Interactive wizard](#interactive-wizard) · [Why](#why) ·
|
|
23
|
-
[What it maps](#what-it-maps) · [How it works](#how-it-works) ·
|
|
24
|
-
[Ask a map questions](#ask-a-map-questions) · [Live Azure](#live-azure-opt-in) ·
|
|
25
|
-
[Capture and scrub](#capture-a-real-export-so-the-tests-can-be-wrong) ·
|
|
26
|
-
[Usage reference](#usage-reference)
|
|
27
|
-
|
|
28
32
|
## Install
|
|
29
33
|
|
|
30
34
|
```
|
|
@@ -33,15 +37,25 @@ pip install cloudmap
|
|
|
33
37
|
|
|
34
38
|
Python 3.9+. Two runtime dependencies (`rich` and `questionary`, both for the
|
|
35
39
|
terminal UI). Live mode additionally needs the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli)
|
|
36
|
-
on your PATH
|
|
40
|
+
on your PATH. The optional AI passes need a local model server - [ollama](https://ollama.com)
|
|
41
|
+
works out of the box, and any OpenAI-compatible server (LM Studio, llama.cpp,
|
|
42
|
+
vLLM, LocalAI) works via two env vars:
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
export CLOUDMAP_LLM_URL=http://localhost:1234/v1/chat/completions # your server
|
|
46
|
+
export CLOUDMAP_LLM_MODEL=<model name> # default: qwen2.5-coder:3b
|
|
47
|
+
```
|
|
37
48
|
|
|
38
|
-
|
|
49
|
+
<details>
|
|
50
|
+
<summary>Working on it instead</summary>
|
|
39
51
|
|
|
40
52
|
```
|
|
41
53
|
git clone https://github.com/KatsaounisThanasis/cloudmap && cd cloudmap
|
|
42
54
|
pip install -e ".[dev]" && pytest
|
|
43
55
|
```
|
|
44
56
|
|
|
57
|
+
</details>
|
|
58
|
+
|
|
45
59
|
## 60-second demo
|
|
46
60
|
|
|
47
61
|
No Azure account needed - the repo ships a synthetic estate:
|
|
@@ -65,7 +79,12 @@ Blast radius: 9 resources (0 external), 9 dependencies
|
|
|
65
79
|
🔗 draw.io: contoso-web.drawio
|
|
66
80
|
```
|
|
67
81
|
|
|
68
|
-
|
|
82
|
+
That is the default **high-level** view - resources grouped by type. Add
|
|
83
|
+
`--level detail` to see every instance with its real name, and on a real estate
|
|
84
|
+
it goes several layers deep:
|
|
85
|
+
|
|
86
|
+
<details>
|
|
87
|
+
<summary>A deeper map</summary>
|
|
69
88
|
|
|
70
89
|
```text
|
|
71
90
|
Blast radius: 15 resources (0 external), 14 dependencies
|
|
@@ -88,19 +107,26 @@ Blast radius: 15 resources (0 external), 14 dependencies
|
|
|
88
107
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
89
108
|
```
|
|
90
109
|
|
|
91
|
-
|
|
92
|
-
`--level detail` to see every instance with its real name.)
|
|
110
|
+
</details>
|
|
93
111
|
|
|
94
|
-
###
|
|
112
|
+
### What you get out
|
|
95
113
|
|
|
96
|
-
| Flag |
|
|
114
|
+
| Flag | Output |
|
|
97
115
|
|---|---|
|
|
98
116
|
| `-o FILE` | **draw.io** diagram with native Azure icons - open it in [draw.io](https://app.diagrams.net), the desktop app or the VS Code extension and edit it like any hand-drawn diagram |
|
|
99
|
-
| `--html FILE` | **Interactive viewer
|
|
117
|
+
| `--html FILE` | **Interactive viewer** - one self-contained file, no server and no CDN |
|
|
100
118
|
| `--mermaid FILE` | Mermaid source, for embedding in Markdown docs |
|
|
101
119
|
| `--json FILE` | The graph itself - this is what `cloudmap ask` reads |
|
|
102
120
|
| `--csv FILE` | Flat edge list with evidence, for a spreadsheet or an auditor |
|
|
103
121
|
|
|
122
|
+
<div align="center">
|
|
123
|
+
<img src="estate-viewer.png" alt="The interactive HTML viewer: the seed at the centre, dependencies fanned out with native Azure icons, each edge carrying its relationship and proof">
|
|
124
|
+
</div>
|
|
125
|
+
|
|
126
|
+
The `--html` viewer has a dark mode, edges colour-coded by relationship type
|
|
127
|
+
(security / data / network), resource-group filters and search, SVG and PNG
|
|
128
|
+
export, and direct links into the Azure portal.
|
|
129
|
+
|
|
104
130
|
## Interactive wizard
|
|
105
131
|
|
|
106
132
|
Run it with no arguments and it walks you through the whole thing:
|
|
@@ -164,20 +190,6 @@ App Insights, VNets, Private Endpoints and managed identities.
|
|
|
164
190
|
|
|
165
191
|
## Ask a map questions
|
|
166
192
|
|
|
167
|
-
```
|
|
168
|
-
cloudmap ask <map.json> "<question>"
|
|
169
|
-
|
|
170
|
-
--explain also narrate the answer with a LOCAL model (ollama)
|
|
171
|
-
--llm if no built-in rule understands the phrasing, let a LOCAL model
|
|
172
|
-
pick the query (its choice is validated against the map)
|
|
173
|
-
--max-hops N limit traversal depth
|
|
174
|
-
--json print the answer as JSON (for scripting)
|
|
175
|
-
```
|
|
176
|
-
|
|
177
|
-
Instance names only exist in a `--level detail` map; the default high-level map
|
|
178
|
-
groups by type, so ask it about a group (`"what breaks if I touch Key Vault"`) or
|
|
179
|
-
trace with `--level detail` first:
|
|
180
|
-
|
|
181
193
|
```
|
|
182
194
|
$ cloudmap trace contoso-web --from fixtures/contoso.json --level detail --json out.json
|
|
183
195
|
$ cloudmap ask out.json "what breaks if I touch contoso-kv"
|
|
@@ -204,11 +216,50 @@ cannot promote a guess to one. Every finding shows the hops behind it and the pr
|
|
|
204
216
|
of each hop, and a finding that leans on a model-proposed edge is marked `[GUESS]`.
|
|
205
217
|
If the map itself says it is incomplete, every answer from it repeats that warning.
|
|
206
218
|
|
|
207
|
-
|
|
219
|
+
<details>
|
|
220
|
+
<summary><b>Flags</b></summary>
|
|
221
|
+
|
|
222
|
+
```
|
|
223
|
+
cloudmap ask <map.json> "<question>"
|
|
224
|
+
|
|
225
|
+
--explain also narrate the answer with a LOCAL model
|
|
226
|
+
--llm if no built-in rule understands the phrasing, let a LOCAL model
|
|
227
|
+
pick the query (its choice is validated against the map)
|
|
228
|
+
--max-hops N limit traversal depth
|
|
229
|
+
--json print the answer as JSON (for scripting)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Instance names only exist in a `--level detail` map; the default high-level map
|
|
233
|
+
groups by type, so ask it about a group (`"what breaks if I touch Key Vault"`) or
|
|
234
|
+
trace with `--level detail` first.
|
|
235
|
+
|
|
236
|
+
</details>
|
|
237
|
+
|
|
238
|
+
## Live Azure
|
|
239
|
+
|
|
240
|
+
Fixtures are the default. Live mode is opt-in and **not sandboxed**: `--allow-live`
|
|
241
|
+
is the deliberate switch, and cloudmap reads whatever subscription `az` is pointed
|
|
242
|
+
at. The read is read-only, but it is a read of live infrastructure, so point it on
|
|
243
|
+
purpose. **Do not point this at data you are not authorized to read.**
|
|
208
244
|
|
|
209
245
|
```
|
|
210
246
|
cloudmap trace my-app --live --allow-live
|
|
247
|
+
```
|
|
211
248
|
|
|
249
|
+
**cloudmap runs as you.** It has no credentials of its own - every read goes through
|
|
250
|
+
the Azure CLI with your `az login` token, so it sees exactly what your account can
|
|
251
|
+
see and nothing more. Resource Graph filters by RBAC, so resources you cannot read
|
|
252
|
+
simply do not appear (Azure raises no error for them - they are invisible, not
|
|
253
|
+
denied). Deep reads your role does not allow (app settings, Key Vault values, AKS
|
|
254
|
+
manifests) fail visibly instead: cloudmap **reports the gap** on the map rather than
|
|
255
|
+
silently dropping edges, and warns when a scan is truncated - so a small graph on a
|
|
256
|
+
low-privilege account reads as "this is what I was allowed to see", not "this is
|
|
257
|
+
everything".
|
|
258
|
+
|
|
259
|
+
<details>
|
|
260
|
+
<summary><b>Flags, enrichment and the subscription pin</b></summary>
|
|
261
|
+
|
|
262
|
+
```
|
|
212
263
|
--single-sub query only the active subscription (default: every enabled
|
|
213
264
|
subscription in the tenant)
|
|
214
265
|
--enrich MODE which web apps to deep-enrich for the dependencies that only
|
|
@@ -217,7 +268,7 @@ cloudmap trace my-app --live --allow-live
|
|
|
217
268
|
all | seed | none
|
|
218
269
|
--resolve-secrets read KV secret values in-memory to see through KV-backed
|
|
219
270
|
connection strings (never printed or written)
|
|
220
|
-
--llm let a LOCAL model
|
|
271
|
+
--llm let a LOCAL model propose extra edges, each
|
|
221
272
|
verified against scanned resources before it is trusted
|
|
222
273
|
```
|
|
223
274
|
|
|
@@ -231,22 +282,18 @@ Anything left un-enriched is reported as a **blind spot** on the map and repeate
|
|
|
231
282
|
every `ask` answer drawn from it, so an empty result never passes for "nothing depends
|
|
232
283
|
on this".
|
|
233
284
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
against a stale `az` context silently redirecting a scan, pin the subscription you
|
|
238
|
-
mean - if set, cloudmap refuses to run against any other active subscription:
|
|
285
|
+
As an optional guard against a stale `az` context silently redirecting a scan, pin the
|
|
286
|
+
subscription you mean - if set, cloudmap refuses to run against any other active
|
|
287
|
+
subscription:
|
|
239
288
|
|
|
240
289
|
```
|
|
241
290
|
export CLOUDMAP_ALLOW_SUBSCRIPTION=<subscription-id> # optional
|
|
242
291
|
```
|
|
243
292
|
|
|
244
|
-
|
|
245
|
-
silently dropping edges, and warns when a scan is truncated - so you know when the
|
|
246
|
-
picture is incomplete. Fixtures are always the default.
|
|
247
|
-
**Do not point this at data you are not authorized to read.**
|
|
293
|
+
</details>
|
|
248
294
|
|
|
249
|
-
|
|
295
|
+
<details>
|
|
296
|
+
<summary><b>Capture a real export (so the tests can be wrong)</b></summary>
|
|
250
297
|
|
|
251
298
|
A fixture you wrote yourself can only confirm what you already believe. `capture`
|
|
252
299
|
saves what Azure actually returned, scrubs it, and gives you something that can
|
|
@@ -271,7 +318,10 @@ printed, the mapping is not. **A scrubber is not a proof: read the file before y
|
|
|
271
318
|
commit it.** `--no-scrub` exists for local debugging and writes credentials to
|
|
272
319
|
disk; keep those files named `*.live.json` so `.gitignore` catches them.
|
|
273
320
|
|
|
274
|
-
|
|
321
|
+
</details>
|
|
322
|
+
|
|
323
|
+
<details>
|
|
324
|
+
<summary><b>Usage reference</b></summary>
|
|
275
325
|
|
|
276
326
|
```
|
|
277
327
|
cloudmap trace <name> (--from <fixture.json> | --live) [options]
|
|
@@ -291,7 +341,9 @@ cloudmap trace <name> (--from <fixture.json> | --live) [options]
|
|
|
291
341
|
-d, --out-dir DIR write every artifact into DIR, named after the seed
|
|
292
342
|
```
|
|
293
343
|
|
|
294
|
-
Other subcommands: `cloudmap capture`, `cloudmap scrub`, `cloudmap ask
|
|
344
|
+
Other subcommands: `cloudmap capture`, `cloudmap scrub`, `cloudmap ask`.
|
|
345
|
+
|
|
346
|
+
</details>
|
|
295
347
|
|
|
296
348
|
## Roadmap
|
|
297
349
|
|
|
@@ -5,7 +5,7 @@ trust labels - and told to add nothing. The deterministic answer is printed abov
|
|
|
5
5
|
the narration either way, so if the prose drifts, what the reader sees is a
|
|
6
6
|
narration disagreeing with the facts printed right above it, not a wrong answer.
|
|
7
7
|
|
|
8
|
-
Empty string on any failure: no
|
|
8
|
+
Empty string on any failure: no local model means no prose, never a missing answer.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import json
|
|
@@ -38,7 +38,7 @@ def main(argv=None):
|
|
|
38
38
|
help="live: read KV secret values in-memory to see through KV-backed "
|
|
39
39
|
"connection strings (never printed or written)")
|
|
40
40
|
t.add_argument("--llm", action="store_true",
|
|
41
|
-
help="also let a LOCAL model
|
|
41
|
+
help="also let a LOCAL model propose edges from the seed's JSON; "
|
|
42
42
|
"each proposal is verified against scanned resources (nothing leaves the machine)")
|
|
43
43
|
t.add_argument("--enrich", choices=["auto", "seed", "all", "none"], default="auto",
|
|
44
44
|
help="live: which web apps to deep-enrich for the dependencies that live "
|
|
@@ -85,7 +85,7 @@ def main(argv=None):
|
|
|
85
85
|
a.add_argument("map", help="a cloudmap graph JSON (written by `trace --json`) or a raw export")
|
|
86
86
|
a.add_argument("question", help='e.g. "what breaks if I touch kv-orders-dev"')
|
|
87
87
|
a.add_argument("--explain", action="store_true",
|
|
88
|
-
help="also narrate the answer with a LOCAL model
|
|
88
|
+
help="also narrate the answer with a LOCAL model; the computed "
|
|
89
89
|
"answer is printed either way")
|
|
90
90
|
a.add_argument("--llm", action="store_true",
|
|
91
91
|
help="if no built-in rule understands the question, let a LOCAL model pick "
|
|
@@ -142,7 +142,7 @@ def _cmd_trace(args):
|
|
|
142
142
|
from rich.console import Console
|
|
143
143
|
from rich.progress import Progress, SpinnerColumn, TextColumn
|
|
144
144
|
console = Console(stderr=True)
|
|
145
|
-
console.print("[bold yellow]Running local model
|
|
145
|
+
console.print("[bold yellow]Running local model on blast radius nodes to propose hidden edges...[/bold yellow]")
|
|
146
146
|
from .extract.extractors import Resolver, merge_model_edges
|
|
147
147
|
from .extract.llm import llm_edges_for_seed
|
|
148
148
|
|
|
@@ -12,13 +12,15 @@ Trust is preserved by two rails, not by trusting the model:
|
|
|
12
12
|
is exactly what must not reach the map. Verified edges are still marked
|
|
13
13
|
origin="model" (drawn dashed), because the model supplied the relationship.
|
|
14
14
|
|
|
15
|
-
Local by design
|
|
15
|
+
Local by design - the model runs on your machine (ollama by default, any
|
|
16
|
+
OpenAI-compatible server via CLOUDMAP_LLM_URL) and the resource JSON never
|
|
17
|
+
leaves it.
|
|
16
18
|
"""
|
|
17
19
|
|
|
18
20
|
import json
|
|
19
21
|
import re
|
|
20
22
|
|
|
21
|
-
from ..local_model import DEFAULT_MODEL, OLLAMA_URL, generate_json # noqa: F401 (re-exported)
|
|
23
|
+
from ..local_model import DEFAULT_MODEL, LLM_URL, OLLAMA_URL, generate_json # noqa: F401 (re-exported)
|
|
22
24
|
from ..model import Edge
|
|
23
25
|
|
|
24
26
|
# A plausible dependency target is a hostname / resource name / ARM id - never a
|
|
@@ -47,7 +49,7 @@ Resource JSON:
|
|
|
47
49
|
|
|
48
50
|
def propose_edges(resource_raw, model=None, timeout=600):
|
|
49
51
|
"""Ask the local model for candidate edges. Returns [(target, relationship)].
|
|
50
|
-
Empty list on any failure (
|
|
52
|
+
Empty list on any failure (model server down, bad JSON, timeout)."""
|
|
51
53
|
prompt = _PROMPT.format(rtype=resource_raw.get("type", "unknown"),
|
|
52
54
|
resource=json.dumps(resource_raw, indent=2))
|
|
53
55
|
parsed = generate_json(prompt, model=model, timeout=timeout)
|
|
@@ -175,7 +175,7 @@ def interactive_main():
|
|
|
175
175
|
|
|
176
176
|
# 3.2 AI Edge Proposals (Phase 3)
|
|
177
177
|
use_llm = questionary.confirm(
|
|
178
|
-
"Use local AI
|
|
178
|
+
"Use a local AI model to discover hidden dependencies? (Slower, but finds undocumented links)",
|
|
179
179
|
default=False,
|
|
180
180
|
style=custom_style
|
|
181
181
|
).ask()
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The one place cloudmap talks to a model - and it talks to a LOCAL one.
|
|
2
|
+
|
|
3
|
+
Keeping the transport in a single module is a design promise rather than tidiness:
|
|
4
|
+
there is exactly one outbound call in the whole tool, it points at localhost, and
|
|
5
|
+
it serves the only two jobs a model is allowed to do here - proposing candidate
|
|
6
|
+
edges that a deterministic rule then verifies, and narrating an answer the graph
|
|
7
|
+
already produced. No resource JSON ever reaches a third party.
|
|
8
|
+
|
|
9
|
+
Two wire formats are spoken, chosen from the URL:
|
|
10
|
+
- ollama's native /api/generate (the default, http://localhost:11434)
|
|
11
|
+
- the OpenAI-compatible /v1/chat/completions that LM Studio, llama.cpp server,
|
|
12
|
+
vLLM, LocalAI (and ollama itself) expose - any URL whose path contains /v1/
|
|
13
|
+
or ends in /chat/completions is treated as OpenAI-compatible.
|
|
14
|
+
|
|
15
|
+
Configure with CLOUDMAP_LLM_URL and CLOUDMAP_LLM_MODEL. CLOUDMAP_OLLAMA_URL is
|
|
16
|
+
honoured as a legacy alias.
|
|
17
|
+
|
|
18
|
+
Every failure returns an empty value instead of raising: cloudmap must stay fully
|
|
19
|
+
useful with no model installed, so "no model" is a normal state, not an error.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import urllib.parse
|
|
25
|
+
import urllib.request
|
|
26
|
+
|
|
27
|
+
LLM_URL = (os.environ.get("CLOUDMAP_LLM_URL")
|
|
28
|
+
or os.environ.get("CLOUDMAP_OLLAMA_URL")
|
|
29
|
+
or "http://localhost:11434/api/generate")
|
|
30
|
+
DEFAULT_MODEL = os.environ.get("CLOUDMAP_LLM_MODEL", "qwen2.5-coder:3b")
|
|
31
|
+
|
|
32
|
+
# Legacy alias: extract.llm re-exported this name in earlier releases.
|
|
33
|
+
OLLAMA_URL = LLM_URL
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _is_openai_compatible(url):
|
|
37
|
+
path = urllib.parse.urlparse(url).path
|
|
38
|
+
return "/v1/" in path or path.rstrip("/").endswith("chat/completions")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _request_body(prompt, model, json_format):
|
|
42
|
+
if _is_openai_compatible(LLM_URL):
|
|
43
|
+
body = {
|
|
44
|
+
"model": model or DEFAULT_MODEL,
|
|
45
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
46
|
+
"temperature": 0,
|
|
47
|
+
"stream": False,
|
|
48
|
+
}
|
|
49
|
+
if json_format:
|
|
50
|
+
body["response_format"] = {"type": "json_object"}
|
|
51
|
+
return body
|
|
52
|
+
body = {
|
|
53
|
+
"model": model or DEFAULT_MODEL,
|
|
54
|
+
"prompt": prompt,
|
|
55
|
+
"stream": False,
|
|
56
|
+
"options": {"temperature": 0},
|
|
57
|
+
}
|
|
58
|
+
if json_format:
|
|
59
|
+
body["format"] = "json"
|
|
60
|
+
return body
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _response_text(resp):
|
|
64
|
+
if _is_openai_compatible(LLM_URL):
|
|
65
|
+
choices = resp.get("choices") or []
|
|
66
|
+
if not choices:
|
|
67
|
+
return ""
|
|
68
|
+
return (choices[0].get("message") or {}).get("content", "") or ""
|
|
69
|
+
return resp.get("response", "") or ""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def generate(prompt, model=None, timeout=600, json_format=True):
|
|
73
|
+
"""Ask the local model once. Returns the response text, or "" on any failure
|
|
74
|
+
(server absent, not running, timeout, malformed payload)."""
|
|
75
|
+
try:
|
|
76
|
+
req = urllib.request.Request(
|
|
77
|
+
LLM_URL, data=json.dumps(_request_body(prompt, model, json_format)).encode(),
|
|
78
|
+
headers={"Content-Type": "application/json"})
|
|
79
|
+
resp = json.load(urllib.request.urlopen(req, timeout=timeout))
|
|
80
|
+
return _response_text(resp)
|
|
81
|
+
except Exception:
|
|
82
|
+
return ""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def generate_json(prompt, model=None, timeout=600):
|
|
86
|
+
"""generate() plus a strict JSON parse. Returns {} on any failure, so callers
|
|
87
|
+
never have to distinguish "model down" from "model answered nonsense"."""
|
|
88
|
+
try:
|
|
89
|
+
parsed = json.loads(generate(prompt, model=model, timeout=timeout, json_format=True))
|
|
90
|
+
except Exception:
|
|
91
|
+
return {}
|
|
92
|
+
return parsed if isinstance(parsed, dict) else {}
|
|
@@ -300,7 +300,7 @@ def test_llm_routing_result_is_still_computed_by_the_query(monkeypatch):
|
|
|
300
300
|
# --- degradation and honesty ----------------------------------------------------
|
|
301
301
|
|
|
302
302
|
def test_answer_works_with_no_model_available():
|
|
303
|
-
# local_model.generate() returns "" when
|
|
303
|
+
# local_model.generate() returns "" when no local model is running; the deterministic
|
|
304
304
|
# answer must be complete anyway and the narration simply empty.
|
|
305
305
|
graph, seed = _real()
|
|
306
306
|
res = answer(graph, f"what does {graph.nodes[seed].name} depend on")
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""The LLM path must stay a proposer behind a verifier: only proposals whose
|
|
2
2
|
target resolves to a real scanned resource survive, secret-looking targets are
|
|
3
|
-
rejected, and with no model available it contributes nothing. No
|
|
4
|
-
generate_json is monkeypatched."""
|
|
3
|
+
rejected, and with no model available it contributes nothing. No model server
|
|
4
|
+
needed - generate_json is monkeypatched."""
|
|
5
5
|
|
|
6
6
|
from cloudmap.extract import llm
|
|
7
7
|
from cloudmap.extract.extractors import Resolver
|
|
@@ -41,6 +41,6 @@ def test_secretish_targets_are_rejected(monkeypatch):
|
|
|
41
41
|
|
|
42
42
|
def test_no_model_available_contributes_nothing(monkeypatch):
|
|
43
43
|
seed, resolver = _seed_and_resolver()
|
|
44
|
-
monkeypatch.setattr(llm, "generate_json", lambda *a, **k: {}) #
|
|
44
|
+
monkeypatch.setattr(llm, "generate_json", lambda *a, **k: {}) # model down -> {}
|
|
45
45
|
|
|
46
46
|
assert llm_edges_for_seed(seed, resolver) == ([], [])
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The model transport: one module, two wire formats, and a hard promise that
|
|
2
|
+
every failure returns an empty value - cloudmap must stay fully useful with no
|
|
3
|
+
model installed, so "no model" is a normal state, not an error.
|
|
4
|
+
|
|
5
|
+
No real server is contacted: urllib.request.urlopen is faked per test.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import io
|
|
9
|
+
import json
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
from cloudmap import local_model
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class _Served:
|
|
17
|
+
"""Capture the request body and serve a canned JSON response."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, response):
|
|
20
|
+
self.response, self.body = response, None
|
|
21
|
+
|
|
22
|
+
def __call__(self, req, timeout=None):
|
|
23
|
+
self.body = json.loads(req.data.decode())
|
|
24
|
+
return io.BytesIO(json.dumps(self.response).encode())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# --- wire-format selection --------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
@pytest.mark.parametrize("url,openai", [
|
|
30
|
+
("http://localhost:11434/api/generate", False),
|
|
31
|
+
("http://localhost:11434", False),
|
|
32
|
+
("http://localhost:1234/v1/chat/completions", True),
|
|
33
|
+
("http://localhost:8080/v1/completions", True),
|
|
34
|
+
("http://gpu-box:8000/openai/v1/chat/completions", True),
|
|
35
|
+
("http://localhost:9000/chat/completions", True),
|
|
36
|
+
])
|
|
37
|
+
def test_the_url_shape_selects_the_wire_format(url, openai):
|
|
38
|
+
assert local_model._is_openai_compatible(url) is openai
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_ollama_speaks_prompt_and_reads_response(monkeypatch):
|
|
42
|
+
served = _Served({"response": "hello"})
|
|
43
|
+
monkeypatch.setattr(local_model, "LLM_URL", "http://localhost:11434/api/generate")
|
|
44
|
+
monkeypatch.setattr(local_model.urllib.request, "urlopen", served)
|
|
45
|
+
|
|
46
|
+
assert local_model.generate("hi") == "hello"
|
|
47
|
+
assert served.body["prompt"] == "hi"
|
|
48
|
+
assert served.body["format"] == "json"
|
|
49
|
+
assert served.body["options"] == {"temperature": 0}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_openai_compatible_speaks_messages_and_reads_choices(monkeypatch):
|
|
53
|
+
served = _Served({"choices": [{"message": {"content": "hello"}}]})
|
|
54
|
+
monkeypatch.setattr(local_model, "LLM_URL", "http://localhost:1234/v1/chat/completions")
|
|
55
|
+
monkeypatch.setattr(local_model.urllib.request, "urlopen", served)
|
|
56
|
+
|
|
57
|
+
assert local_model.generate("hi") == "hello"
|
|
58
|
+
assert served.body["messages"] == [{"role": "user", "content": "hi"}]
|
|
59
|
+
assert served.body["response_format"] == {"type": "json_object"}
|
|
60
|
+
assert "prompt" not in served.body
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# --- the never-raises contract ----------------------------------------------------
|
|
64
|
+
|
|
65
|
+
def test_a_dead_server_returns_empty_never_raises(monkeypatch):
|
|
66
|
+
def down(req, timeout=None):
|
|
67
|
+
raise OSError("connection refused")
|
|
68
|
+
|
|
69
|
+
monkeypatch.setattr(local_model.urllib.request, "urlopen", down)
|
|
70
|
+
|
|
71
|
+
assert local_model.generate("hi") == ""
|
|
72
|
+
assert local_model.generate_json("hi") == {}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@pytest.mark.parametrize("response", [
|
|
76
|
+
{}, # no recognisable field
|
|
77
|
+
{"choices": []}, # openai shape, empty
|
|
78
|
+
{"response": None}, # ollama shape, null
|
|
79
|
+
])
|
|
80
|
+
def test_a_malformed_payload_returns_empty(monkeypatch, response):
|
|
81
|
+
monkeypatch.setattr(local_model, "LLM_URL", "http://localhost:1234/v1/chat/completions"
|
|
82
|
+
if "choices" in response else "http://localhost:11434/api/generate")
|
|
83
|
+
monkeypatch.setattr(local_model.urllib.request, "urlopen", _Served(response))
|
|
84
|
+
|
|
85
|
+
assert local_model.generate("hi") == ""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.mark.parametrize("text", ["not json", "[]", "null", ""])
|
|
89
|
+
def test_generate_json_returns_a_dict_or_nothing(monkeypatch, text):
|
|
90
|
+
monkeypatch.setattr(local_model, "generate", lambda *a, **k: text)
|
|
91
|
+
|
|
92
|
+
assert local_model.generate_json("hi") == {}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
"""The one place cloudmap talks to a model - and it talks to a LOCAL one.
|
|
2
|
-
|
|
3
|
-
Keeping the transport in a single module is a design promise rather than tidiness:
|
|
4
|
-
there is exactly one outbound call in the whole tool, it points at localhost
|
|
5
|
-
(ollama), and it serves the only two jobs a model is allowed to do here -
|
|
6
|
-
proposing candidate edges that a deterministic rule then verifies, and narrating
|
|
7
|
-
an answer the graph already produced. No resource JSON ever reaches a third party.
|
|
8
|
-
|
|
9
|
-
Every failure returns an empty value instead of raising: cloudmap must stay fully
|
|
10
|
-
useful with no model installed, so "no model" is a normal state, not an error.
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
import json
|
|
14
|
-
import os
|
|
15
|
-
import urllib.request
|
|
16
|
-
|
|
17
|
-
OLLAMA_URL = os.environ.get("CLOUDMAP_OLLAMA_URL", "http://localhost:11434/api/generate")
|
|
18
|
-
DEFAULT_MODEL = os.environ.get("CLOUDMAP_LLM_MODEL", "qwen2.5-coder:3b")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def generate(prompt, model=None, timeout=600, json_format=True):
|
|
22
|
-
"""Ask the local model once. Returns the response text, or "" on any failure
|
|
23
|
-
(ollama absent, not running, timeout, malformed payload)."""
|
|
24
|
-
body = {
|
|
25
|
-
"model": model or DEFAULT_MODEL,
|
|
26
|
-
"prompt": prompt,
|
|
27
|
-
"stream": False,
|
|
28
|
-
"options": {"temperature": 0},
|
|
29
|
-
}
|
|
30
|
-
if json_format:
|
|
31
|
-
body["format"] = "json"
|
|
32
|
-
try:
|
|
33
|
-
req = urllib.request.Request(
|
|
34
|
-
OLLAMA_URL, data=json.dumps(body).encode(),
|
|
35
|
-
headers={"Content-Type": "application/json"})
|
|
36
|
-
resp = json.load(urllib.request.urlopen(req, timeout=timeout))
|
|
37
|
-
except Exception:
|
|
38
|
-
return ""
|
|
39
|
-
return resp.get("response", "") or ""
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def generate_json(prompt, model=None, timeout=600):
|
|
43
|
-
"""generate() plus a strict JSON parse. Returns {} on any failure, so callers
|
|
44
|
-
never have to distinguish "model down" from "model answered nonsense"."""
|
|
45
|
-
try:
|
|
46
|
-
parsed = json.loads(generate(prompt, model=model, timeout=timeout, json_format=True))
|
|
47
|
-
except Exception:
|
|
48
|
-
return {}
|
|
49
|
-
return parsed if isinstance(parsed, dict) else {}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloudmap-1.0.0 → cloudmap-1.1.0}/tests/complex_mock_demo/10_input_enterprise_architecture.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|