proxy2model 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proxy2model/__init__.py +19 -0
- proxy2model/__main__.py +6 -0
- proxy2model/atomic.py +68 -0
- proxy2model/cli/__init__.py +3 -0
- proxy2model/cli/analyze.py +501 -0
- proxy2model/cli/app.py +408 -0
- proxy2model/cli/browse.py +279 -0
- proxy2model/cli/build.py +399 -0
- proxy2model/cli/common.py +190 -0
- proxy2model/cli/history.py +224 -0
- proxy2model/cli/mcp.py +184 -0
- proxy2model/cli/ui.py +62 -0
- proxy2model/core/__init__.py +1 -0
- proxy2model/core/changes.py +124 -0
- proxy2model/core/context.py +571 -0
- proxy2model/core/diff.py +330 -0
- proxy2model/core/domains.py +98 -0
- proxy2model/core/graph.py +562 -0
- proxy2model/core/methodology.py +343 -0
- proxy2model/core/model.py +998 -0
- proxy2model/core/stack.py +218 -0
- proxy2model/data/fonts/LICENSES.md +16 -0
- proxy2model/data/fonts/inter-latin-wght-normal.woff2 +0 -0
- proxy2model/data/fonts/jetbrains-mono-latin-wght-normal.woff2 +0 -0
- proxy2model/data/fonts/nippo-200.woff2 +0 -0
- proxy2model/data/fonts/nippo-300.woff2 +0 -0
- proxy2model/data/fonts/nippo-400.woff2 +0 -0
- proxy2model/data/fonts/nippo-500.woff2 +0 -0
- proxy2model/data/fonts/nippo-700.woff2 +0 -0
- proxy2model/data/public_suffix_list.dat +16497 -0
- proxy2model/db/__init__.py +1 -0
- proxy2model/db/bql.py +710 -0
- proxy2model/db/bql_js.py +278 -0
- proxy2model/db/store.py +547 -0
- proxy2model/export/__init__.py +1 -0
- proxy2model/export/openapi.py +248 -0
- proxy2model/ingest/__init__.py +1 -0
- proxy2model/ingest/browse.py +181 -0
- proxy2model/ingest/browser.py +1058 -0
- proxy2model/ingest/crawl.py +673 -0
- proxy2model/ingest/extract.py +208 -0
- proxy2model/ingest/infer.py +75 -0
- proxy2model/ingest/journey.py +148 -0
- proxy2model/ingest/live.py +96 -0
- proxy2model/ingest/parse.py +1275 -0
- proxy2model/ingest/redact.py +535 -0
- proxy2model/py.typed +0 -0
- proxy2model/recon/__init__.py +1 -0
- proxy2model/recon/osint.py +607 -0
- proxy2model/report/__init__.py +1 -0
- proxy2model/report/animate.py +163 -0
- proxy2model/report/dash.py +469 -0
- proxy2model/report/fonts.py +51 -0
- proxy2model/report/html.py +988 -0
- proxy2model/report/inventory.py +258 -0
- proxy2model/report/map.py +515 -0
- proxy2model/report/pages.py +537 -0
- proxy2model/report/query.py +306 -0
- proxy2model/report/shot.py +55 -0
- proxy2model/report/view.py +213 -0
- proxy2model-2.0.0.dist-info/METADATA +148 -0
- proxy2model-2.0.0.dist-info/RECORD +66 -0
- proxy2model-2.0.0.dist-info/WHEEL +5 -0
- proxy2model-2.0.0.dist-info/entry_points.txt +2 -0
- proxy2model-2.0.0.dist-info/licenses/LICENSE +201 -0
- proxy2model-2.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Shared by the commands: names, the fingerprint key, saved inputs, loading a model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from ..atomic import atomic_write
|
|
12
|
+
from ..ingest.parse import Exchange
|
|
13
|
+
from ..core.model import from_dict
|
|
14
|
+
from ..ingest.redact import set_fingerprint_key
|
|
15
|
+
from ..db import store
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
DEFAULT_INPUT = "_default"
|
|
22
|
+
EXPORTS = "exports" # graph.json, graph.graphml and build.svg, inside the model folder
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _name(kind: str):
|
|
26
|
+
def check(value: str) -> str:
|
|
27
|
+
if not _NAME_RE.match(value):
|
|
28
|
+
raise argparse.ArgumentTypeError(
|
|
29
|
+
f"{kind} must be letters, digits, '.', '_' or '-' (got {value!r})")
|
|
30
|
+
return value
|
|
31
|
+
return check
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _init_key() -> None:
|
|
35
|
+
"""Keep one fingerprint key per user, outside any output directory.
|
|
36
|
+
|
|
37
|
+
Fingerprints from separate runs (e.g. one per role) must be comparable, but
|
|
38
|
+
the key must not travel with the outputs — otherwise anyone holding
|
|
39
|
+
model.json could confirm a guessed secret against its fingerprint.
|
|
40
|
+
"""
|
|
41
|
+
if os.environ.get("PROXY2MODEL_FP_KEY"):
|
|
42
|
+
return
|
|
43
|
+
base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
|
|
44
|
+
path = os.path.join(base, "proxy2model", "fingerprint.key")
|
|
45
|
+
old = os.path.join(base, "burp2model", "fingerprint.key")
|
|
46
|
+
if not os.path.exists(path) and os.path.exists(old) and os.path.getsize(old) >= 16:
|
|
47
|
+
path = old # keep fingerprints comparable with builds made before the rename
|
|
48
|
+
try:
|
|
49
|
+
if not os.path.exists(path) or os.path.getsize(path) < 16:
|
|
50
|
+
# never run with a missing, empty or truncated key
|
|
51
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
52
|
+
with atomic_write(path, "wb", perms=0o600) as f: # whole or absent, never half a key
|
|
53
|
+
f.write(os.urandom(32))
|
|
54
|
+
with open(path, "rb") as f:
|
|
55
|
+
set_fingerprint_key(f.read())
|
|
56
|
+
except OSError as e:
|
|
57
|
+
print(f"warning: no persistent fingerprint key ({e}); fingerprints from "
|
|
58
|
+
"separate runs will not match", file=sys.stderr)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _inputs_dir(outdir: str) -> str:
|
|
62
|
+
return os.path.join(outdir, "inputs")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
INPUTS_VERSION = 1
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _save_input(outdir: str, key: str, exchanges: list[Exchange], stats: dict) -> None:
|
|
69
|
+
d = _inputs_dir(outdir)
|
|
70
|
+
os.makedirs(d, exist_ok=True)
|
|
71
|
+
with atomic_write(os.path.join(d, key + ".jsonl")) as f:
|
|
72
|
+
f.write(json.dumps({"_stats": stats, "_version": INPUTS_VERSION}) + "\n")
|
|
73
|
+
for ex in exchanges:
|
|
74
|
+
f.write(json.dumps(ex.to_record()) + "\n")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _stage_input(outdir: str, exchanges, stats: dict) -> tuple[str, int]:
|
|
78
|
+
"""Write parsed exchanges straight to a staging file as they stream in, so a large export is never
|
|
79
|
+
held in memory. The stats line goes last, when the counts are final. Returns (path, exchanges written)."""
|
|
80
|
+
d = _inputs_dir(outdir)
|
|
81
|
+
os.makedirs(d, exist_ok=True)
|
|
82
|
+
path = os.path.join(d, ".staging")
|
|
83
|
+
n = 0
|
|
84
|
+
try:
|
|
85
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
86
|
+
for ex in exchanges:
|
|
87
|
+
f.write(json.dumps(ex.to_record()) + "\n")
|
|
88
|
+
n += 1
|
|
89
|
+
f.write(json.dumps({"_stats": stats, "_version": INPUTS_VERSION}) + "\n")
|
|
90
|
+
except BaseException:
|
|
91
|
+
if os.path.exists(path):
|
|
92
|
+
os.remove(path)
|
|
93
|
+
raise
|
|
94
|
+
return path, n
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _drop_if_empty(outdir: str) -> None:
|
|
98
|
+
"""A build that read nothing leaves no folders behind."""
|
|
99
|
+
for d in (_inputs_dir(outdir), outdir):
|
|
100
|
+
try:
|
|
101
|
+
os.rmdir(d)
|
|
102
|
+
except OSError:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _clear_inputs(outdir: str, only: str | None = None, keep: str | None = None) -> None:
|
|
107
|
+
d = _inputs_dir(outdir)
|
|
108
|
+
if not os.path.isdir(d):
|
|
109
|
+
return
|
|
110
|
+
for fn in os.listdir(d):
|
|
111
|
+
if fn.endswith(".jsonl") and (only is None or fn == only + ".jsonl") and fn != f"{keep}.jsonl":
|
|
112
|
+
os.remove(os.path.join(d, fn))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _load_inputs(outdir: str) -> tuple[list[Exchange], dict, list[str]]:
|
|
116
|
+
"""All saved inputs, re-indexed so evidence ids are unique across them."""
|
|
117
|
+
d = _inputs_dir(outdir)
|
|
118
|
+
exchanges: list[Exchange] = []
|
|
119
|
+
stats = {"items": 0, "parsed": 0, "skipped": 0}
|
|
120
|
+
names = []
|
|
121
|
+
for fn in sorted(os.listdir(d)) if os.path.isdir(d) else []:
|
|
122
|
+
if not fn.endswith(".jsonl"):
|
|
123
|
+
continue
|
|
124
|
+
try:
|
|
125
|
+
with open(os.path.join(d, fn), encoding="utf-8") as f:
|
|
126
|
+
loaded = []
|
|
127
|
+
for line in f:
|
|
128
|
+
rec = json.loads(line)
|
|
129
|
+
if "_stats" in rec:
|
|
130
|
+
if rec.get("_version", 1) > INPUTS_VERSION:
|
|
131
|
+
raise ValueError(f"saved by a newer proxy2model "
|
|
132
|
+
f"(inputs v{rec['_version']})")
|
|
133
|
+
for k in stats:
|
|
134
|
+
stats[k] += rec["_stats"].get(k, 0)
|
|
135
|
+
continue
|
|
136
|
+
loaded.append(Exchange.from_record(rec))
|
|
137
|
+
except (ValueError, KeyError, TypeError) as e:
|
|
138
|
+
print(f"warning: skipping unreadable saved input {fn}: {e}\n"
|
|
139
|
+
f" re-run the build for that capture (or use --fresh)", file=sys.stderr)
|
|
140
|
+
continue
|
|
141
|
+
names.append(fn[:-6])
|
|
142
|
+
for ex in loaded:
|
|
143
|
+
ex.index = len(exchanges) + 1
|
|
144
|
+
exchanges.append(ex)
|
|
145
|
+
return exchanges, stats, names
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _scope(args) -> list[str] | None:
|
|
149
|
+
"""Normalise --scope values: accept `https://shop.com:443/x` for `shop.com`."""
|
|
150
|
+
if not args.scope:
|
|
151
|
+
return None
|
|
152
|
+
out = []
|
|
153
|
+
for s in args.scope:
|
|
154
|
+
for x in s.split(","):
|
|
155
|
+
x = x.strip().lower()
|
|
156
|
+
if not x:
|
|
157
|
+
continue
|
|
158
|
+
x = re.sub(r"^[a-z][a-z0-9+.-]*://", "", x) # scheme
|
|
159
|
+
x = x.split("/")[0] # path
|
|
160
|
+
if not x.startswith("[") and x.count(":") == 1:
|
|
161
|
+
x = x.split(":")[0] # port
|
|
162
|
+
if x:
|
|
163
|
+
out.append(x)
|
|
164
|
+
return out or None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _db_path(args) -> str | None:
|
|
168
|
+
path = os.path.join(args.out, args.webapp, store.DB_NAME)
|
|
169
|
+
if not os.path.exists(path):
|
|
170
|
+
print(f"no graph database at {path} — run `proxy2model build` first", file=sys.stderr)
|
|
171
|
+
return None
|
|
172
|
+
return path
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _model_path(args) -> str | None:
|
|
176
|
+
path = os.path.join(args.out, args.webapp, "model.json")
|
|
177
|
+
if not os.path.exists(path):
|
|
178
|
+
print(f"no model at {path} — run `proxy2model build` first", file=sys.stderr)
|
|
179
|
+
return None
|
|
180
|
+
return path
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _reload(path: str):
|
|
184
|
+
try:
|
|
185
|
+
with open(path, encoding="utf-8") as f:
|
|
186
|
+
return from_dict(json.load(f))
|
|
187
|
+
except (OSError, ValueError, KeyError, TypeError) as e:
|
|
188
|
+
print(f"could not load {path}: {e}\nre-run `proxy2model build` to regenerate it.",
|
|
189
|
+
file=sys.stderr)
|
|
190
|
+
return None
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""History commands: what changed between builds, the build list, and watching a folder of exports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from ..core import diff as diffmod
|
|
11
|
+
from ..db import store
|
|
12
|
+
from ..atomic import atomic_write, keep_aside
|
|
13
|
+
from ..core.model import from_dict
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _main(argv):
|
|
17
|
+
from .app import main
|
|
18
|
+
return main(argv)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _db_path(args):
|
|
22
|
+
from .common import _db_path as db_path
|
|
23
|
+
return db_path(args)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _changes_since_previous(db_path: str, build_id: int | None) -> dict | None:
|
|
27
|
+
"""The drift from the build before this one, for the report. None on a first build or any trouble."""
|
|
28
|
+
if not build_id or build_id < 2:
|
|
29
|
+
return None
|
|
30
|
+
try:
|
|
31
|
+
conn = store.open_db(db_path)
|
|
32
|
+
try:
|
|
33
|
+
ids = [r[0] for r in conn.execute("SELECT id FROM builds WHERE id<? ORDER BY id DESC LIMIT 1", (build_id,))]
|
|
34
|
+
if not ids:
|
|
35
|
+
return None
|
|
36
|
+
old, ometa = _snapshot_of(conn, ids[0])
|
|
37
|
+
new, nmeta = _snapshot_of(conn, build_id)
|
|
38
|
+
return diffmod.diff_models(old, new, ometa, nmeta, hist=lambda k, key: store.history(conn, k, key))
|
|
39
|
+
finally:
|
|
40
|
+
conn.close()
|
|
41
|
+
except (ValueError, KeyError, TypeError, OSError, sqlite3.Error):
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _snapshot_of(conn, bid: int):
|
|
46
|
+
snap = store.load_build(conn, bid)
|
|
47
|
+
if snap is None:
|
|
48
|
+
raise ValueError(f"no build #{bid} in graph.db")
|
|
49
|
+
row = conn.execute("SELECT built_at, label FROM builds WHERE id=?", (bid,)).fetchone()
|
|
50
|
+
return from_dict(snap), {"id": bid, "built_at": row["built_at"], "label": row["label"] or ""}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def cmd_watch(args) -> int:
|
|
54
|
+
"""Poll a folder: every settled new export is built, labelled with its file name, and diffed."""
|
|
55
|
+
import fnmatch
|
|
56
|
+
import time
|
|
57
|
+
folder = args.folder
|
|
58
|
+
if not os.path.isdir(folder):
|
|
59
|
+
print(f"watch: {folder} is not a folder", file=sys.stderr)
|
|
60
|
+
return 2
|
|
61
|
+
pats = [p.strip() for p in args.pattern.split(",") if p.strip()]
|
|
62
|
+
outdir = os.path.join(args.out, args.webapp)
|
|
63
|
+
state_path = os.path.join(outdir, "watch.json")
|
|
64
|
+
done = _load_watch_state(state_path)
|
|
65
|
+
|
|
66
|
+
def stat(fn):
|
|
67
|
+
try:
|
|
68
|
+
st = os.stat(os.path.join(folder, fn))
|
|
69
|
+
except OSError: # gone between the listing and the look
|
|
70
|
+
return None
|
|
71
|
+
return (st.st_mtime_ns, fn, st.st_size) if os.path.isfile(os.path.join(folder, fn)) else None
|
|
72
|
+
|
|
73
|
+
def listing():
|
|
74
|
+
try:
|
|
75
|
+
names = os.listdir(folder)
|
|
76
|
+
except OSError as e:
|
|
77
|
+
print(f"watch: cannot list {folder} ({e}); will look again", file=sys.stderr)
|
|
78
|
+
return []
|
|
79
|
+
found = [stat(fn) for fn in names if any(fnmatch.fnmatch(fn.lower(), x.lower()) for x in pats)]
|
|
80
|
+
return sorted(t for t in found if t)
|
|
81
|
+
|
|
82
|
+
sig = lambda t: [t[0], t[2]] # noqa: E731
|
|
83
|
+
if args.skip_existing and not done:
|
|
84
|
+
done = {fn: sig((m, fn, sz)) for m, fn, sz in listing()}
|
|
85
|
+
last: dict[str, list] = {}
|
|
86
|
+
built = 0
|
|
87
|
+
print(f"watch: {folder} ({', '.join(pats)}) → {args.webapp}"
|
|
88
|
+
+ ("" if args.once else f" every {args.interval:g}s, Ctrl-C to stop"), file=sys.stderr)
|
|
89
|
+
try:
|
|
90
|
+
while True:
|
|
91
|
+
for mtime, fn, size in listing():
|
|
92
|
+
if done.get(fn) == sig((mtime, fn, size)) or size == 0:
|
|
93
|
+
continue
|
|
94
|
+
# still being written (or copied): the size and the time must hold still for one interval
|
|
95
|
+
if not args.once and last.get(fn) != sig((mtime, fn, size)):
|
|
96
|
+
last[fn] = sig((mtime, fn, size))
|
|
97
|
+
continue
|
|
98
|
+
label = os.path.splitext(fn)[0]
|
|
99
|
+
argv = ["build", os.path.join(folder, fn), "-w", args.webapp, "--out", args.out, "--label", label]
|
|
100
|
+
for flag, val in (("--role", args.role), ("--skip-tools", args.skip_tools)):
|
|
101
|
+
if val:
|
|
102
|
+
argv += [flag, val]
|
|
103
|
+
for sc in args.scope or []:
|
|
104
|
+
argv += ["--scope", sc]
|
|
105
|
+
if args.osint:
|
|
106
|
+
argv.append("--osint")
|
|
107
|
+
print(f"\nwatch: new export {fn}", file=sys.stderr)
|
|
108
|
+
rc = _main(argv)
|
|
109
|
+
after = stat(fn)
|
|
110
|
+
if after is None or sig(after) != sig((mtime, fn, size)):
|
|
111
|
+
# the file moved under the build: what was read may be part of it. Build it again once it settles.
|
|
112
|
+
print(f"watch: {fn} changed while it was being read; will build it again once it settles",
|
|
113
|
+
file=sys.stderr)
|
|
114
|
+
last.pop(fn, None)
|
|
115
|
+
continue
|
|
116
|
+
if rc != 0:
|
|
117
|
+
print(f"watch: could not build from {fn}; will try again if it changes", file=sys.stderr)
|
|
118
|
+
done[fn] = sig((mtime, fn, size))
|
|
119
|
+
continue
|
|
120
|
+
done[fn] = sig((mtime, fn, size))
|
|
121
|
+
built += 1
|
|
122
|
+
os.makedirs(outdir, exist_ok=True)
|
|
123
|
+
with atomic_write(state_path) as f:
|
|
124
|
+
json.dump(done, f, indent=1, sort_keys=True)
|
|
125
|
+
try:
|
|
126
|
+
conn = store.open_db(os.path.join(outdir, store.DB_NAME))
|
|
127
|
+
try:
|
|
128
|
+
n = len(store.list_builds(conn))
|
|
129
|
+
finally:
|
|
130
|
+
conn.close()
|
|
131
|
+
except (OSError, ValueError, sqlite3.Error):
|
|
132
|
+
n = 0
|
|
133
|
+
if n > 1:
|
|
134
|
+
print("", file=sys.stderr)
|
|
135
|
+
_main(["diff", args.webapp, "--out", args.out])
|
|
136
|
+
if args.once:
|
|
137
|
+
break
|
|
138
|
+
time.sleep(max(0.2, args.interval))
|
|
139
|
+
except KeyboardInterrupt:
|
|
140
|
+
print("\nwatch: stopped", file=sys.stderr)
|
|
141
|
+
if args.once and not built:
|
|
142
|
+
print("watch: nothing new to build", file=sys.stderr)
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _load_watch_state(path: str) -> dict:
|
|
147
|
+
"""Which exports were already built. A damaged state file is kept aside rather than read as "nothing
|
|
148
|
+
built yet", which would rebuild every export in the folder as a new build."""
|
|
149
|
+
try:
|
|
150
|
+
with open(path, encoding="utf-8") as f:
|
|
151
|
+
done = json.load(f)
|
|
152
|
+
except FileNotFoundError:
|
|
153
|
+
return {}
|
|
154
|
+
except (OSError, ValueError) as e:
|
|
155
|
+
done = e
|
|
156
|
+
if isinstance(done, dict):
|
|
157
|
+
return done
|
|
158
|
+
try:
|
|
159
|
+
aside = keep_aside(path)
|
|
160
|
+
print(f"watch: {path} was unreadable; kept it aside as {aside}. Exports already built "
|
|
161
|
+
f"will be built again (use --skip-existing to skip them)", file=sys.stderr)
|
|
162
|
+
except OSError:
|
|
163
|
+
pass
|
|
164
|
+
return {}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def cmd_builds(args) -> int:
|
|
168
|
+
dbp = _db_path(args)
|
|
169
|
+
if dbp is None:
|
|
170
|
+
return 2
|
|
171
|
+
try:
|
|
172
|
+
conn = store.open_db(dbp)
|
|
173
|
+
try:
|
|
174
|
+
rows = store.list_builds(conn)
|
|
175
|
+
finally:
|
|
176
|
+
conn.close()
|
|
177
|
+
except (ValueError, sqlite3.Error) as e:
|
|
178
|
+
print(f"could not read {dbp}: {e}", file=sys.stderr)
|
|
179
|
+
return 2
|
|
180
|
+
print(f"{args.webapp}: {len(rows)} build(s) kept in graph.db")
|
|
181
|
+
print(f" {'id':>3} {'built (UTC)':<20} {'nodes':>6} {'edges':>6} {'requests':>8} label")
|
|
182
|
+
for r in rows:
|
|
183
|
+
print(f" {r['id']:>3} {r['built_at']:<20} {r['nodes']:>6} {r['edges']:>6} {r['exchanges']:>8} {r['label']}")
|
|
184
|
+
if len(rows) > 1:
|
|
185
|
+
print(f"\ncompare two: proxy2model diff {args.webapp} {rows[-2]['id']} {rows[-1]['id']}")
|
|
186
|
+
return 0
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def cmd_diff(args) -> int:
|
|
190
|
+
dbp = _db_path(args)
|
|
191
|
+
if dbp is None:
|
|
192
|
+
return 2
|
|
193
|
+
try:
|
|
194
|
+
conn = store.open_db(dbp)
|
|
195
|
+
except (ValueError, sqlite3.Error) as e:
|
|
196
|
+
print(f"could not diff: {e}", file=sys.stderr)
|
|
197
|
+
return 2
|
|
198
|
+
try:
|
|
199
|
+
b_ref = args.b if args.b is not None else "latest"
|
|
200
|
+
a_ref = args.a if args.a is not None else "previous"
|
|
201
|
+
newer = store.resolve_build(conn, b_ref)
|
|
202
|
+
if args.a is None and args.b is None:
|
|
203
|
+
older = store.resolve_build(conn, "previous")
|
|
204
|
+
else:
|
|
205
|
+
older = store.resolve_build(conn, a_ref if args.a is not None else "previous")
|
|
206
|
+
if older == newer:
|
|
207
|
+
print(f"build #{older} is compared with itself; pick two different builds "
|
|
208
|
+
f"(`proxy2model builds {args.webapp}` lists them)", file=sys.stderr)
|
|
209
|
+
return 2
|
|
210
|
+
if older > newer:
|
|
211
|
+
older, newer = newer, older
|
|
212
|
+
old, ometa = _snapshot_of(conn, older)
|
|
213
|
+
new, nmeta = _snapshot_of(conn, newer)
|
|
214
|
+
d = diffmod.diff_models(old, new, ometa, nmeta, hist=lambda k, key: store.history(conn, k, key))
|
|
215
|
+
except (ValueError, KeyError, TypeError, OSError, sqlite3.Error) as e:
|
|
216
|
+
print(f"could not diff: {e}", file=sys.stderr)
|
|
217
|
+
return 2
|
|
218
|
+
finally:
|
|
219
|
+
conn.close()
|
|
220
|
+
if args.json:
|
|
221
|
+
print(json.dumps(d, indent=2))
|
|
222
|
+
else:
|
|
223
|
+
print(diffmod.format_text(d))
|
|
224
|
+
return 0
|
proxy2model/cli/mcp.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""`mcp`: serve built models to an AI agent over the Model Context Protocol (stdio).
|
|
2
|
+
|
|
3
|
+
Each tool runs the same command a researcher would type, so an agent gets exactly the answers
|
|
4
|
+
the CLI gives: cited, deterministic, no model call inside. It reads models under one output
|
|
5
|
+
folder and never builds, crawls or contacts anything.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import io
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
from .. import __version__
|
|
17
|
+
from .common import _NAME_RE
|
|
18
|
+
|
|
19
|
+
PROTOCOLS = ("2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05")
|
|
20
|
+
|
|
21
|
+
_APP = {"type": "string", "description": "model name used at build time (the -w value)"}
|
|
22
|
+
TOOLS = [
|
|
23
|
+
{"name": "list_models", "description": "The models built in this output folder, with their size and roles.",
|
|
24
|
+
"inputSchema": {"type": "object", "properties": {}}},
|
|
25
|
+
{"name": "query", "description": "Ask a model a factual question; every line cites evidence ids. "
|
|
26
|
+
"Ask `help` first for what it answers (list-apis, reconcile, auth-surface, "
|
|
27
|
+
"unknowns, provenance /path, ...). Questions it cannot answer are refused.",
|
|
28
|
+
"inputSchema": {"type": "object", "required": ["app", "question"],
|
|
29
|
+
"properties": {"app": _APP, "question": {"type": "string"}}}},
|
|
30
|
+
{"name": "bql", "description": "Filter and walk graph.db with BQL, e.g. `req.method:POST AND resp.code.gte:400`, "
|
|
31
|
+
"`static-only`, `reach \"POST /api/checkout\"`. `help` prints the language.",
|
|
32
|
+
"inputSchema": {"type": "object", "required": ["app", "expression"],
|
|
33
|
+
"properties": {"app": _APP, "expression": {"type": "string"},
|
|
34
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 1000}}}},
|
|
35
|
+
{"name": "cross_role", "description": "What the lower-privileged role reached that only the higher one should.",
|
|
36
|
+
"inputSchema": {"type": "object", "required": ["app", "low", "high"],
|
|
37
|
+
"properties": {"app": _APP, "low": {"type": "string"}, "high": {"type": "string"}}}},
|
|
38
|
+
{"name": "builds", "description": "The builds of a model kept in graph.db, to pick two for diff.",
|
|
39
|
+
"inputSchema": {"type": "object", "required": ["app"], "properties": {"app": _APP}}},
|
|
40
|
+
{"name": "diff", "description": "What changed between two builds: new workflows, changed fingerprints, gone things.",
|
|
41
|
+
"inputSchema": {"type": "object", "required": ["app"],
|
|
42
|
+
"properties": {"app": _APP,
|
|
43
|
+
"from": {"type": "string", "description": "build id or label (default: previous)"},
|
|
44
|
+
"to": {"type": "string", "description": "build id or label (default: latest)"}}}},
|
|
45
|
+
{"name": "gaps", "description": "The model's open questions, ranked by how much resolving each would tell you.",
|
|
46
|
+
"inputSchema": {"type": "object", "required": ["app"], "properties": {"app": _APP}}},
|
|
47
|
+
{"name": "openapi", "description": "The observed API as an OpenAPI 3.0 document (JSON).",
|
|
48
|
+
"inputSchema": {"type": "object", "required": ["app"], "properties": {"app": _APP}}},
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ToolError(Exception):
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _name(value, what: str) -> str:
|
|
57
|
+
if not isinstance(value, str) or not _NAME_RE.match(value):
|
|
58
|
+
raise ToolError(f"{what} must be letters, digits, '.', '_' or '-'")
|
|
59
|
+
return value
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _text(value, what: str) -> str:
|
|
63
|
+
if not isinstance(value, str) or not value.strip():
|
|
64
|
+
raise ToolError(f"{what} is required")
|
|
65
|
+
return value
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _run(argv: list[str]) -> tuple[int, str]:
|
|
69
|
+
"""Run one CLI command and capture what it prints."""
|
|
70
|
+
from .app import main
|
|
71
|
+
out, err = io.StringIO(), io.StringIO()
|
|
72
|
+
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
73
|
+
try:
|
|
74
|
+
rc = main(argv)
|
|
75
|
+
except SystemExit as e: # argparse refusing an argument
|
|
76
|
+
rc = e.code if isinstance(e.code, int) else 2
|
|
77
|
+
text = out.getvalue()
|
|
78
|
+
if err.getvalue().strip():
|
|
79
|
+
text = (text + "\n" + err.getvalue()).strip()
|
|
80
|
+
return rc or 0, text.strip()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def list_models(root: str) -> str:
|
|
84
|
+
rows = []
|
|
85
|
+
for name in sorted(os.listdir(root)) if os.path.isdir(root) else []:
|
|
86
|
+
path = os.path.join(root, name, "model.json")
|
|
87
|
+
if not os.path.isfile(path):
|
|
88
|
+
continue
|
|
89
|
+
try:
|
|
90
|
+
with open(path, encoding="utf-8") as f:
|
|
91
|
+
m = json.load(f)
|
|
92
|
+
except (OSError, ValueError):
|
|
93
|
+
continue
|
|
94
|
+
c = m.get("counts", {})
|
|
95
|
+
rows.append(f"{name}: {c.get('endpoints', 0)} APIs, {c.get('routes', 0)} routes, "
|
|
96
|
+
f"{c.get('hosts', 0)} hosts, {len(m.get('evidence', []))} requests"
|
|
97
|
+
+ (f", roles {', '.join(m.get('roles', []))}" if m.get("roles") else ""))
|
|
98
|
+
return "\n".join(rows) or f"no models in {root} (build one with `proxy2model FILE -w NAME`)"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def call(tool: str, a: dict, root: str) -> tuple[bool, str]:
|
|
102
|
+
"""(ok, text) for one tool call. Options go before `--`, values after, so no value is read as a flag."""
|
|
103
|
+
out = ["--out", root]
|
|
104
|
+
if tool == "list_models":
|
|
105
|
+
return True, list_models(root)
|
|
106
|
+
app = _name(a.get("app"), "app")
|
|
107
|
+
if tool == "query":
|
|
108
|
+
argv = ["query", *out, "--", app, _text(a.get("question"), "question")]
|
|
109
|
+
elif tool == "bql":
|
|
110
|
+
limit = a.get("limit", 100)
|
|
111
|
+
if not isinstance(limit, int) or not 1 <= limit <= 1000:
|
|
112
|
+
raise ToolError("limit must be 1 to 1000")
|
|
113
|
+
argv = ["q", *out, "--limit", str(limit), "--", app, _text(a.get("expression"), "expression")]
|
|
114
|
+
elif tool == "cross_role":
|
|
115
|
+
argv = ["cross-role", *out, "--low", _name(a.get("low"), "low"), "--high", _name(a.get("high"), "high"),
|
|
116
|
+
"--", app]
|
|
117
|
+
elif tool == "builds":
|
|
118
|
+
argv = ["builds", *out, "--", app]
|
|
119
|
+
elif tool == "diff":
|
|
120
|
+
refs = [_text(a[k], k) for k in ("from", "to") if a.get(k) is not None]
|
|
121
|
+
if a.get("to") is not None and a.get("from") is None:
|
|
122
|
+
refs = ["previous"] + refs
|
|
123
|
+
argv = ["diff", *out, "--", app, *refs]
|
|
124
|
+
elif tool == "gaps":
|
|
125
|
+
argv = ["gaps", *out, "--", app]
|
|
126
|
+
elif tool == "openapi":
|
|
127
|
+
argv = ["openapi", *out, "--", app]
|
|
128
|
+
else:
|
|
129
|
+
raise ToolError(f"unknown tool {tool!r}")
|
|
130
|
+
rc, text = _run(argv)
|
|
131
|
+
return rc == 0, text or "(no output)"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def handle(msg: dict, root: str) -> dict | None:
|
|
135
|
+
"""One JSON-RPC message in, the response out (None for a notification)."""
|
|
136
|
+
mid, method, params = msg.get("id"), msg.get("method"), msg.get("params") or {}
|
|
137
|
+
if mid is None:
|
|
138
|
+
return None # notifications/initialized and friends
|
|
139
|
+
if method == "initialize":
|
|
140
|
+
wanted = params.get("protocolVersion")
|
|
141
|
+
result = {"protocolVersion": wanted if wanted in PROTOCOLS else PROTOCOLS[0],
|
|
142
|
+
"capabilities": {"tools": {}},
|
|
143
|
+
"serverInfo": {"name": "proxy2model", "version": __version__},
|
|
144
|
+
"instructions": "Models of web apps built from captured traffic. Start with list_models, then "
|
|
145
|
+
"query <app> help. Every answer cites evidence ids (ev_N); absence means the "
|
|
146
|
+
"capture did not show it, not that it does not exist. Never call anything a "
|
|
147
|
+
"vulnerability: these are leads to verify in an authorized session."}
|
|
148
|
+
elif method == "ping":
|
|
149
|
+
result = {}
|
|
150
|
+
elif method == "tools/list":
|
|
151
|
+
result = {"tools": TOOLS}
|
|
152
|
+
elif method == "tools/call":
|
|
153
|
+
try:
|
|
154
|
+
ok, text = call(params.get("name"), params.get("arguments") or {}, root)
|
|
155
|
+
except ToolError as e:
|
|
156
|
+
ok, text = False, str(e)
|
|
157
|
+
result = {"content": [{"type": "text", "text": text}], "isError": not ok}
|
|
158
|
+
else:
|
|
159
|
+
return {"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": f"unknown method {method}"}}
|
|
160
|
+
return {"jsonrpc": "2.0", "id": mid, "result": result}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def serve_stdio(root: str, stdin=None, stdout=None) -> int:
|
|
164
|
+
stdin, stdout = stdin or sys.stdin, stdout or sys.stdout
|
|
165
|
+
for line in stdin:
|
|
166
|
+
if not line.strip():
|
|
167
|
+
continue
|
|
168
|
+
try:
|
|
169
|
+
msg = json.loads(line)
|
|
170
|
+
except ValueError:
|
|
171
|
+
reply = {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error"}}
|
|
172
|
+
else:
|
|
173
|
+
reply = handle(msg, root) if isinstance(msg, dict) else \
|
|
174
|
+
{"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "invalid request"}}
|
|
175
|
+
if reply is not None:
|
|
176
|
+
stdout.write(json.dumps(reply) + "\n")
|
|
177
|
+
stdout.flush()
|
|
178
|
+
return 0
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def cmd_mcp(args) -> int:
|
|
182
|
+
root = os.path.abspath(args.out)
|
|
183
|
+
print(f"proxy2model mcp: serving models in {root} over stdio", file=sys.stderr)
|
|
184
|
+
return serve_stdio(root)
|
proxy2model/cli/ui.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Terminal output: a little colour on a terminal, plain text when piped or NO_COLOR is set."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
_CODES = {"bold": "1", "dim": "2", "accent": "38;5;208", "green": "32", "yellow": "33", "red": "31", "cyan": "36"}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def color_on(stream=None) -> bool:
|
|
12
|
+
stream = stream or sys.stdout
|
|
13
|
+
if os.environ.get("NO_COLOR"):
|
|
14
|
+
return False
|
|
15
|
+
if os.environ.get("FORCE_COLOR"):
|
|
16
|
+
return True
|
|
17
|
+
return hasattr(stream, "isatty") and stream.isatty() and os.environ.get("TERM") != "dumb"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def style(text, *names: str, stream=None) -> str:
|
|
21
|
+
if not names or not color_on(stream):
|
|
22
|
+
return str(text)
|
|
23
|
+
return "\033[" + ";".join(_CODES[n] for n in names) + "m" + str(text) + "\033[0m"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def title(command: str, subject: str = "") -> str:
|
|
27
|
+
"""`proxy2model · build · shop`"""
|
|
28
|
+
parts = [style("proxy2model", "bold", "accent"), style(command, "bold")]
|
|
29
|
+
if subject:
|
|
30
|
+
parts.append(subject)
|
|
31
|
+
return style(" · ", "dim").join(parts)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def row(label: str, value: str, width: int = 8) -> str:
|
|
35
|
+
"""One ` label value` line; continuation lines of `value` line up under it."""
|
|
36
|
+
pad = " " * (width + 4)
|
|
37
|
+
lines = str(value).split("\n")
|
|
38
|
+
out = " " + style(label.ljust(width), "dim") + " " + lines[0]
|
|
39
|
+
return "\n".join([out] + [pad + ln for ln in lines[1:]])
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def stat(n, noun: str) -> str:
|
|
43
|
+
"""`3 routes`, `1 host`: the count bold, the noun plural when it should be."""
|
|
44
|
+
return f"{style(n, 'bold')} {noun if n == 1 else noun + 's'}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def files(entries: list[tuple[str, str]]) -> str:
|
|
48
|
+
"""Aligned `name what it is` lines."""
|
|
49
|
+
w = max(len(n) for n, _ in entries)
|
|
50
|
+
return "\n".join(f"{n.ljust(w)} {style(d, 'dim')}" for n, d in entries)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def ok(msg: str) -> str:
|
|
54
|
+
return style("✓", "green", stream=sys.stderr) + " " + msg
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def warn(msg: str) -> str:
|
|
58
|
+
return style("warning:", "yellow", stream=sys.stderr) + " " + msg
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def error(msg: str) -> str:
|
|
62
|
+
return style("error:", "red", "bold", stream=sys.stderr) + " " + msg
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The model: building it, reasoning over it, and comparing builds."""
|