command-vault-mcp 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. command_vault_mcp-0.8.0/.analyze/P1-implementation.md +108 -0
  2. command_vault_mcp-0.8.0/.analyze/analyze_boxes.py +215 -0
  3. command_vault_mcp-0.8.0/.analyze/analyze_challenges.py +232 -0
  4. command_vault_mcp-0.8.0/.analyze/analyze_corpus.py +254 -0
  5. command_vault_mcp-0.8.0/.analyze/analyze_sherlocks.py +221 -0
  6. command_vault_mcp-0.8.0/.analyze/improvements.md +402 -0
  7. command_vault_mcp-0.8.0/.claude/settings.local.json +13 -0
  8. command_vault_mcp-0.8.0/.gitignore +48 -0
  9. command_vault_mcp-0.8.0/AGENTS.md +141 -0
  10. command_vault_mcp-0.8.0/LICENSE +21 -0
  11. command_vault_mcp-0.8.0/PKG-INFO +279 -0
  12. command_vault_mcp-0.8.0/README.md +269 -0
  13. command_vault_mcp-0.8.0/_convert_/CONVERT.md +61 -0
  14. command_vault_mcp-0.8.0/_convert_/scripts/convert_writeup.py +241 -0
  15. command_vault_mcp-0.8.0/pyproject.toml +26 -0
  16. command_vault_mcp-0.8.0/src/command_vault/__init__.py +3 -0
  17. command_vault_mcp-0.8.0/src/command_vault/categories.py +427 -0
  18. command_vault_mcp-0.8.0/src/command_vault/cli.py +453 -0
  19. command_vault_mcp-0.8.0/src/command_vault/database.py +1557 -0
  20. command_vault_mcp-0.8.0/src/command_vault/history_parser.py +452 -0
  21. command_vault_mcp-0.8.0/src/command_vault/indexer.py +304 -0
  22. command_vault_mcp-0.8.0/src/command_vault/models.py +176 -0
  23. command_vault_mcp-0.8.0/src/command_vault/parser.py +901 -0
  24. command_vault_mcp-0.8.0/src/command_vault/security.py +190 -0
  25. command_vault_mcp-0.8.0/src/command_vault/server.py +623 -0
  26. command_vault_mcp-0.8.0/src/command_vault/techniques.py +228 -0
  27. command_vault_mcp-0.8.0/src/command_vault/tools.py +545 -0
  28. command_vault_mcp-0.8.0/tests/test_chunks.py +126 -0
  29. command_vault_mcp-0.8.0/tests/test_parser.py +416 -0
@@ -0,0 +1,108 @@
1
+ # P1: Index Writeup Prose as Searchable Chunks
2
+
3
+ ## Context
4
+
5
+ Currently command-vault-mcp only indexes **commands** and **scripts** from writeups. The rich prose — methodology explanations, attack chain reasoning, forensic analysis — is discarded during parsing. This is ~307K words of invisible knowledge. P1 stores prose paragraphs as chunks with FTS5 search, making concepts like "ADCS misconfiguration ESC8" or "NTLM relay to LDAP" searchable.
6
+
7
+ ## Changes
8
+
9
+ ### 1. Schema — `database.py`
10
+
11
+ Add to `SCHEMA` string:
12
+ ```sql
13
+ CREATE TABLE IF NOT EXISTS writeup_chunks (
14
+ id INTEGER PRIMARY KEY,
15
+ writeup_id INTEGER NOT NULL,
16
+ section TEXT,
17
+ content TEXT NOT NULL,
18
+ chunk_index INTEGER NOT NULL,
19
+ FOREIGN KEY (writeup_id) REFERENCES writeups(id)
20
+ );
21
+ CREATE INDEX IF NOT EXISTS idx_chunks_writeup ON writeup_chunks(writeup_id);
22
+ ```
23
+
24
+ Add to `FTS_SCHEMA` string:
25
+ ```sql
26
+ CREATE VIRTUAL TABLE IF NOT EXISTS writeup_chunks_fts USING fts5(
27
+ content, section,
28
+ content='writeup_chunks', content_rowid='id'
29
+ );
30
+ -- Sync triggers (INSERT/DELETE/UPDATE pattern matching existing ones)
31
+ ```
32
+
33
+ ### 2. Database methods — `database.py`
34
+
35
+ - `insert_chunk(writeup_id, section, content, chunk_index)` — insert a single chunk
36
+ - `clear_writeup_chunks(writeup_id)` — delete chunks for re-indexing (called from `clear_writeup_data`)
37
+ - `search_chunks(query, writeup_type, limit)` — FTS search returning chunk + writeup metadata
38
+ - Update `clear_writeup_data()` to also delete from `writeup_chunks`
39
+ - Update `reset()` to drop `writeup_chunks_fts` table and its triggers
40
+ - Update `get_stats()` to include chunk count
41
+
42
+ ### 3. Prose extraction — `parser.py`
43
+
44
+ Add `extract_prose_chunks(content) -> list[dict]` method to `WriteupParser`:
45
+ - Walk the content line by line
46
+ - Track current section (H2/H3 headings)
47
+ - Skip: code blocks (```...```), image embeds (![[...]]), the Tags: line, blank lines
48
+ - Collect consecutive prose lines into paragraphs
49
+ - Skip paragraphs shorter than ~30 chars (noise like "Let's check:" or single words)
50
+ - Return `[{section, content, chunk_index}]`
51
+
52
+ ### 4. Return chunks from parser — `parser.py`
53
+
54
+ - `parse_file()` currently returns `(Writeup, commands, scripts)` — change to also extract and return chunks
55
+ - Return type becomes `(Writeup, commands, scripts, chunks)` where chunks is a list of dicts
56
+
57
+ ### 5. Indexer — `indexer.py`
58
+
59
+ - Update `index_file()` to receive chunks from parser and call `db.insert_chunk()` for each
60
+ - Update `clear_writeup_data` call path to also clear chunks
61
+ - Add `chunks_extracted` counter to `IndexResult` tracking
62
+
63
+ ### 6. Model — `models.py`
64
+
65
+ - Add `ChunkResult` pydantic model: `id, section, content, source (dict with filename, writeup_type, title)`
66
+ - Add `chunks_extracted` field to `IndexResult` (default 0 for backward compat)
67
+
68
+ ### 7. MCP tool — `server.py`
69
+
70
+ - Add `search_writeup_prose` tool in `list_tools()` with params: `query` (required), `writeup_type`, `tags`, `limit`
71
+ - Add dispatch in `call_tool()`
72
+ - Implement in `VaultTools`
73
+
74
+ ### 8. CLI — `cli.py`
75
+
76
+ - Add `prose` subcommand: `vault prose "NTLM relay"` — searches writeup chunks
77
+ - Display format:
78
+ ```
79
+ ============================================================
80
+ Source: Authority.md [389/tcp - LDAP]
81
+
82
+ <prose content, truncated to ~300 chars>
83
+ ```
84
+
85
+ ### 9. Tests
86
+
87
+ - Test `extract_prose_chunks()` with sample markdown
88
+ - Test chunk insertion and FTS search
89
+ - Test that chunks are cleared on re-index
90
+
91
+ ## Files to modify
92
+
93
+ 1. `src/command_vault/database.py` — schema, CRUD, search, reset, stats
94
+ 2. `src/command_vault/parser.py` — `extract_prose_chunks()`, update `parse_file()` return
95
+ 3. `src/command_vault/indexer.py` — wire chunks through indexing pipeline
96
+ 4. `src/command_vault/models.py` — `ChunkResult`, update `IndexResult`
97
+ 5. `src/command_vault/server.py` — new MCP tool
98
+ 6. `src/command_vault/cli.py` — new `prose` subcommand
99
+ 7. `tests/test_parser.py` — prose extraction tests
100
+
101
+ ## Verification
102
+
103
+ 1. `uv run python -m pytest tests/ -q` — all tests pass
104
+ 2. `vault index --rebuild` — reindex all writeups (required for chunks)
105
+ 3. `vault stats` — shows chunk count
106
+ 4. `vault prose "NTLM relay"` — returns relevant prose from writeups
107
+ 5. `vault prose "ADCS ESC8"` — returns AD CS methodology text
108
+ 6. MCP `search_writeup_prose` tool returns results
@@ -0,0 +1,215 @@
1
+ #!/usr/bin/env python3
2
+ """Comprehensive box writeup analysis"""
3
+ import os, re, statistics
4
+ from pathlib import Path
5
+ from collections import defaultdict, Counter
6
+
7
+ BOXES_DIR = Path.home() / "writeups" / "boxes"
8
+
9
+ TECHNIQUES = {
10
+ "AD": ["Kerberoasting", "AS-REP", "DCSync", "Pass-the-Hash", "Pass-the-Ticket", "Silver Ticket",
11
+ "Golden Ticket", "ADCS", "ESC1", "ESC2", "ESC3", "ESC4", "ESC5", "ESC6", "ESC7", "ESC8",
12
+ "ESC9", "ESC10", "Shadow Credentials", "Constrained Delegation", "Unconstrained Delegation",
13
+ "RBCD", "GenericWrite", "WriteDacl", "WriteOwner", "AddMember", "ForceChangePassword",
14
+ "BloodHound", "DACL", "ACL", "GPO", "LAPS", "gMSA", "DPAPI", "Certipy", "NTLM relay",
15
+ "PetitPotam", "Coercion"],
16
+ "Web": ["CVE", "exploit", "vulnerability", "LFI", "RFI", "SSRF", "SQLi", "SQL injection", "XSS",
17
+ "SSTI", "RCE", "deserialization", "upload", "IDOR", "JWT", "cookie", "XXE", "CRLF",
18
+ "Prototype Pollution", "Command Injection", "Path Traversal", "GraphQL", "OAuth", "CORS"],
19
+ "Linux privesc": ["SUID", "sudo", "cron", "capabilities", "docker", "container escape",
20
+ "kernel exploit", "path hijack", "LD_PRELOAD", "NFS", "wildcard", "polkit"],
21
+ "Windows privesc": ["token", "impersonation", "SeImpersonate", "JuicyPotato", "PrintSpoofer",
22
+ "GodPotato", "service", "AlwaysInstallElevated", "DLL hijack", "UAC bypass"],
23
+ "Pivoting": ["chisel", "ligolo", "sshuttle", "proxychains", "tunnel", "port forward", "pivot"],
24
+ "Credentials": ["hashcat", "john", "hydra", "credential", "password spray", "brute force",
25
+ "NTLM", "Kerberos", "relay"]
26
+ }
27
+
28
+ TOOL_NAMES = ["nmap", "rustscan", "ffuf", "feroxbuster", "gobuster", "dirsearch", "nikto", "wpscan",
29
+ "sqlmap", "burp", "nuclei", "httpx", "curl", "wget", "bloodhound", "certipy", "rubeus",
30
+ "mimikatz", "impacket", "secretsdump", "smbclient", "smbmap", "crackmapexec", "netexec",
31
+ "nxc", "evil-winrm", "psexec", "wmiexec", "chisel", "ligolo", "proxychains", "hashcat",
32
+ "john", "hydra", "linpeas", "winpeas", "pspy", "seatbelt", "ghidra", "ida", "gdb",
33
+ "frida", "metasploit", "msfvenom", "responder", "ntlmrelayx"]
34
+
35
+ def analyze_file(filepath):
36
+ content = filepath.read_text(encoding='utf-8', errors='ignore')
37
+ lines = content.split('\n')
38
+ total = len(lines)
39
+
40
+ # Headers
41
+ headers = Counter()
42
+ sections = []
43
+ for line in lines:
44
+ m = re.match(r'^(#{1,4})\s+(.+)$', line)
45
+ if m:
46
+ headers[len(m.group(1))] += 1
47
+ sections.append(m.group(2).strip().lower())
48
+
49
+ # Code blocks & prose
50
+ in_code = False
51
+ code_lines = 0
52
+ code_blocks = 0
53
+ code_langs = Counter()
54
+ for line in lines:
55
+ if line.strip().startswith('```'):
56
+ if not in_code:
57
+ code_blocks += 1
58
+ lang = re.match(r'^```(\w+)', line.strip())
59
+ code_langs[lang.group(1) if lang else 'plain'] += 1
60
+ in_code = not in_code
61
+ elif in_code:
62
+ code_lines += 1
63
+
64
+ prose_lines = total - code_lines
65
+ prose_ratio = prose_lines / total if total > 0 else 0
66
+
67
+ # Images
68
+ images = len(re.findall(r'!\[\[.*?\]\]', content))
69
+
70
+ # Tags
71
+ tags = re.findall(r'#(\w+)', '\n'.join(lines[:10]))
72
+ tags = [t.lower() for t in tags]
73
+
74
+ # Tables (credential tables)
75
+ tables = sum(1 for line in lines if '|' in line and not line.strip().startswith('```'))
76
+
77
+ # Technique keywords in full content
78
+ content_lower = content.lower()
79
+ tech_hits = {}
80
+ for cat, keywords in TECHNIQUES.items():
81
+ hits = 0
82
+ for kw in keywords:
83
+ hits += len(re.findall(r'\b' + re.escape(kw.lower()) + r'\b', content_lower))
84
+ if hits > 0:
85
+ tech_hits[cat] = hits
86
+
87
+ # Tool mentions in prose (outside code blocks)
88
+ prose_content = re.sub(r'```.*?```', '', content, flags=re.DOTALL).lower()
89
+ tool_hits = Counter()
90
+ for tool in TOOL_NAMES:
91
+ count = len(re.findall(r'\b' + re.escape(tool.lower()) + r'\b', prose_content))
92
+ if count > 0:
93
+ tool_hits[tool] = count
94
+
95
+ # Word count
96
+ words = len(re.sub(r'```.*?```', '', content, flags=re.DOTALL).split())
97
+
98
+ return {
99
+ 'name': filepath.name, 'total': total, 'prose_lines': prose_lines,
100
+ 'code_lines': code_lines, 'prose_ratio': prose_ratio, 'headers': headers,
101
+ 'sections': sections, 'code_blocks': code_blocks, 'code_langs': code_langs,
102
+ 'images': images, 'tags': tags, 'tables': tables, 'tech_hits': tech_hits,
103
+ 'tool_hits': tool_hits, 'words': words
104
+ }
105
+
106
+ def main():
107
+ files = list(BOXES_DIR.glob("*.md"))
108
+ print(f"Analyzing {len(files)} box writeups...\n")
109
+ results = [analyze_file(f) for f in sorted(files)]
110
+
111
+ print("=" * 80)
112
+ print("BOX WRITEUP ANALYSIS")
113
+ print("=" * 80)
114
+
115
+ # 1. Structure
116
+ print("\n--- STRUCTURE ---")
117
+ totals = sum(r['total'] for r in results)
118
+ print(f"Total lines: {totals:,}")
119
+ print(f"Avg lines/writeup: {totals/len(results):.1f}")
120
+ print(f"Min: {min(r['total'] for r in results)}, Max: {max(r['total'] for r in results)}")
121
+ print(f"Median: {statistics.median(r['total'] for r in results):.0f}")
122
+
123
+ print(f"\nHeaders: H1={sum(r['headers'][1] for r in results)}, H2={sum(r['headers'][2] for r in results)}, "
124
+ f"H3={sum(r['headers'][3] for r in results)}, H4={sum(r['headers'][4] for r in results)}")
125
+
126
+ all_sections = Counter()
127
+ for r in results:
128
+ all_sections.update(r['sections'])
129
+ print("\nTop 20 section names:")
130
+ for name, count in all_sections.most_common(20):
131
+ print(f" {count:3d}x {name}")
132
+
133
+ print(f"\nTotal code blocks: {sum(r['code_blocks'] for r in results)}")
134
+ print(f"Avg code blocks/writeup: {sum(r['code_blocks'] for r in results)/len(results):.1f}")
135
+ print(f"Total images: {sum(r['images'] for r in results)}")
136
+ print(f"Avg images/writeup: {sum(r['images'] for r in results)/len(results):.1f}")
137
+ print(f"Writeups with images: {sum(1 for r in results if r['images'] > 0)}")
138
+ print(f"Total table lines: {sum(r['tables'] for r in results)}")
139
+
140
+ # 2. Prose ratio
141
+ print("\n--- PROSE RATIO ---")
142
+ avg_pr = statistics.mean(r['prose_ratio'] for r in results)
143
+ print(f"Average prose ratio: {avg_pr*100:.1f}%")
144
+ print(f"Median prose ratio: {statistics.median(r['prose_ratio'] for r in results)*100:.1f}%")
145
+ bins = [(0, 0.2, "<20%"), (0.2, 0.4, "20-40%"), (0.4, 0.6, "40-60%"), (0.6, 1.01, ">60%")]
146
+ for lo, hi, label in bins:
147
+ n = sum(1 for r in results if lo <= r['prose_ratio'] < hi)
148
+ print(f" {label}: {n} ({n/len(results)*100:.1f}%)")
149
+
150
+ # 3. Tags
151
+ print("\n--- TAGS ---")
152
+ all_tags = Counter()
153
+ tagged = 0
154
+ for r in results:
155
+ if r['tags']:
156
+ tagged += 1
157
+ all_tags.update(r['tags'])
158
+ print(f"Writeups with tags: {tagged}/{len(results)} ({tagged/len(results)*100:.1f}%)")
159
+ print(f"Unique tags: {len(all_tags)}")
160
+ print("\nTop 30 tags:")
161
+ for tag, count in all_tags.most_common(30):
162
+ print(f" {count:3d}x #{tag}")
163
+
164
+ # 4. Technique mentions
165
+ print("\n--- TECHNIQUE MENTIONS ---")
166
+ for cat in TECHNIQUES:
167
+ total_mentions = sum(r['tech_hits'].get(cat, 0) for r in results)
168
+ writeups_with = sum(1 for r in results if cat in r['tech_hits'])
169
+ print(f" {cat:20s}: {total_mentions:5d} mentions across {writeups_with:3d} writeups")
170
+
171
+ # 5. Tool mentions in prose
172
+ print("\n--- TOOL MENTIONS IN PROSE ---")
173
+ all_tools = Counter()
174
+ for r in results:
175
+ all_tools.update(r['tool_hits'])
176
+ print("Top 30 tools mentioned in prose:")
177
+ for tool, count in all_tools.most_common(30):
178
+ print(f" {tool:20s}: {count:4d}")
179
+
180
+ # 6. Code languages
181
+ print("\n--- CODE BLOCK LANGUAGES ---")
182
+ all_langs = Counter()
183
+ for r in results:
184
+ all_langs.update(r['code_langs'])
185
+ for lang, count in all_langs.most_common(15):
186
+ print(f" {lang:15s}: {count:4d} blocks")
187
+
188
+ # 7. Word count
189
+ total_words = sum(r['words'] for r in results)
190
+ print(f"\n--- WORD COUNT ---")
191
+ print(f"Total prose words: {total_words:,}")
192
+ print(f"Avg words/writeup: {total_words/len(results):.0f}")
193
+
194
+ # 8. Length distribution
195
+ print("\n--- LENGTH DISTRIBUTION ---")
196
+ for lo, hi, label in [(0,50,"<50"), (50,100,"50-100"), (100,200,"100-200"),
197
+ (200,400,"200-400"), (400,800,"400-800"), (800,9999,">800")]:
198
+ n = sum(1 for r in results if lo <= r['total'] < hi)
199
+ bar = "█" * (n // 3)
200
+ print(f" {label:>8s}: {n:3d} {bar}")
201
+
202
+ # Longest
203
+ by_len = sorted(results, key=lambda r: r['total'], reverse=True)
204
+ print("\n5 longest:")
205
+ for r in by_len[:5]:
206
+ print(f" {r['total']:5d} lines - {r['name']}")
207
+ print("5 shortest:")
208
+ for r in by_len[-5:]:
209
+ print(f" {r['total']:5d} lines - {r['name']}")
210
+
211
+ print("\n" + "=" * 80)
212
+ print("DONE")
213
+
214
+ if __name__ == "__main__":
215
+ main()
@@ -0,0 +1,232 @@
1
+ #!/usr/bin/env python3
2
+ """Comprehensive challenge writeup analysis"""
3
+ import os, re, statistics
4
+ from pathlib import Path
5
+ from collections import defaultdict, Counter
6
+
7
+ DIR = Path.home() / "writeups" / "challenges"
8
+
9
+ TECHNIQUES = [
10
+ "XSS", "SQLi", "SQL injection", "SSTI", "SSRF", "RCE", "LFI", "RFI", "deserialization",
11
+ "prototype pollution", "JWT", "cookie", "session", "CORS", "CSP", "WAF bypass",
12
+ "buffer overflow", "heap", "stack overflow", "ROP", "format string", "race condition",
13
+ "use-after-free", "shellcode", "GOT overwrite", "ret2libc", "canary", "PIE", "ASLR", "NX",
14
+ "reverse engineering", "decompile", "disassemble", "IDA", "Ghidra", "Frida", "debug",
15
+ "breakpoint", "crypto", "RSA", "AES", "XOR", "padding oracle", "hash", "ECDSA",
16
+ "elliptic curve", "CBC", "ECB", "GCM", "HMAC", "command injection", "path traversal",
17
+ "XXE", "CRLF", "WebSocket", "GraphQL", "OAuth", "IDOR", "file upload", "SUID",
18
+ "ptrace", "anti-debug", "packing", "obfuscation", "virtual machine", "sandbox escape"
19
+ ]
20
+
21
+ def extract_category(name):
22
+ m = re.search(r'\(([^)]+)\)\.md$', name)
23
+ return m.group(1).strip().lower() if m else 'unknown'
24
+
25
+ def analyze_file(filepath):
26
+ content = filepath.read_text(encoding='utf-8', errors='ignore')
27
+ lines = content.split('\n')
28
+ total = len(lines)
29
+ category = extract_category(filepath.name)
30
+
31
+ # Code blocks
32
+ in_code = False
33
+ code_lines = 0
34
+ code_blocks = 0
35
+ code_langs = Counter()
36
+ for line in lines:
37
+ if line.strip().startswith('```'):
38
+ if not in_code:
39
+ code_blocks += 1
40
+ lang = re.match(r'^```(\w+)', line.strip())
41
+ code_langs[lang.group(1) if lang else 'plain'] += 1
42
+ in_code = not in_code
43
+ elif in_code:
44
+ code_lines += 1
45
+
46
+ prose_lines = total - code_lines
47
+ prose_ratio = prose_lines / total if total > 0 else 0
48
+ images = len(re.findall(r'!\[\[.*?\]\]', content))
49
+
50
+ # Sections
51
+ sections = []
52
+ for line in lines:
53
+ m = re.match(r'^(#{1,4})\s+(.+)$', line)
54
+ if m:
55
+ sections.append(m.group(2).strip().lower())
56
+
57
+ # Techniques
58
+ content_lower = content.lower()
59
+ tech_hits = Counter()
60
+ for t in TECHNIQUES:
61
+ count = len(re.findall(r'\b' + re.escape(t.lower()) + r'\b', content_lower))
62
+ if count > 0:
63
+ tech_hits[t] = count
64
+
65
+ # Scripts
66
+ has_pwn = bool(re.search(r'from pwn import', content))
67
+ has_requests = bool(re.search(r'import requests', content))
68
+ has_exploit = bool(re.search(r'def exploit\(', content))
69
+ has_frida = bool(re.search(r'frida|Java\.perform|Interceptor\.attach', content, re.I))
70
+
71
+ # Python lines
72
+ py_lines = 0
73
+ in_py = False
74
+ for line in lines:
75
+ if re.match(r'^```(?:python|py)\b', line.strip()):
76
+ in_py = True
77
+ elif line.strip() == '```' and in_py:
78
+ in_py = False
79
+ elif in_py:
80
+ py_lines += 1
81
+
82
+ words = len(re.sub(r'```.*?```', '', content, flags=re.DOTALL).split())
83
+
84
+ return {
85
+ 'name': filepath.name, 'category': category, 'total': total,
86
+ 'prose_ratio': prose_ratio, 'code_blocks': code_blocks, 'code_langs': code_langs,
87
+ 'images': images, 'sections': sections, 'tech_hits': tech_hits,
88
+ 'has_pwn': has_pwn, 'has_requests': has_requests, 'has_exploit': has_exploit,
89
+ 'has_frida': has_frida, 'py_lines': py_lines, 'words': words,
90
+ 'prose_lines': prose_lines, 'code_lines': code_lines
91
+ }
92
+
93
+ def main():
94
+ files = list(DIR.glob("*.md"))
95
+ print(f"Analyzing {len(files)} challenge writeups...\n")
96
+ results = [analyze_file(f) for f in sorted(files)]
97
+
98
+ print("=" * 80)
99
+ print("CHALLENGE WRITEUP ANALYSIS")
100
+ print("=" * 80)
101
+
102
+ # Category breakdown
103
+ cats = Counter(r['category'] for r in results)
104
+ print("\n--- CATEGORIES ---")
105
+ for cat, count in cats.most_common():
106
+ avg_len = statistics.mean(r['total'] for r in results if r['category'] == cat)
107
+ avg_cb = statistics.mean(r['code_blocks'] for r in results if r['category'] == cat)
108
+ avg_pr = statistics.mean(r['prose_ratio'] for r in results if r['category'] == cat)
109
+ print(f" {cat:20s}: {count:3d} writeups | avg {avg_len:5.0f} lines | "
110
+ f"avg {avg_cb:4.1f} code blocks | {avg_pr*100:4.1f}% prose")
111
+
112
+ # Structure
113
+ print("\n--- STRUCTURE ---")
114
+ totals = sum(r['total'] for r in results)
115
+ print(f"Total lines: {totals:,}, Avg: {totals/len(results):.1f}")
116
+ print(f"Total code blocks: {sum(r['code_blocks'] for r in results)}")
117
+ print(f"Total images: {sum(r['images'] for r in results)}")
118
+ print(f"Avg images/writeup: {sum(r['images'] for r in results)/len(results):.1f}")
119
+
120
+ # Prose ratio
121
+ print("\n--- PROSE RATIO ---")
122
+ avg_pr = statistics.mean(r['prose_ratio'] for r in results)
123
+ print(f"Average: {avg_pr*100:.1f}%")
124
+ for lo, hi, label in [(0,0.2,"<20%"), (0.2,0.4,"20-40%"), (0.4,0.6,"40-60%"), (0.6,1.01,">60%")]:
125
+ n = sum(1 for r in results if lo <= r['prose_ratio'] < hi)
126
+ print(f" {label}: {n} ({n/len(results)*100:.1f}%)")
127
+
128
+ print("\nProse ratio by category:")
129
+ for cat, _ in cats.most_common():
130
+ ratios = [r['prose_ratio'] for r in results if r['category'] == cat]
131
+ if len(ratios) >= 3:
132
+ print(f" {cat:20s}: {statistics.mean(ratios)*100:5.1f}% (n={len(ratios)})")
133
+
134
+ # Code languages
135
+ print("\n--- CODE LANGUAGES ---")
136
+ all_langs = Counter()
137
+ for r in results:
138
+ all_langs.update(r['code_langs'])
139
+ for lang, count in all_langs.most_common(15):
140
+ print(f" {lang:15s}: {count:4d} blocks")
141
+
142
+ print("\nTop languages by category:")
143
+ for cat, _ in cats.most_common(10):
144
+ cat_langs = Counter()
145
+ for r in results:
146
+ if r['category'] == cat:
147
+ cat_langs.update(r['code_langs'])
148
+ top3 = ", ".join(f"{l}({c})" for l, c in cat_langs.most_common(3))
149
+ print(f" {cat:20s}: {top3}")
150
+
151
+ # Techniques
152
+ print("\n--- TECHNIQUE MENTIONS ---")
153
+ all_tech = Counter()
154
+ for r in results:
155
+ all_tech.update(r['tech_hits'])
156
+ print("Top 30:")
157
+ for tech, count in all_tech.most_common(30):
158
+ files_with = sum(1 for r in results if tech in r['tech_hits'])
159
+ print(f" {tech:25s}: {count:4d} mentions in {files_with:3d} files")
160
+
161
+ print("\nTechniques by category (top 5 per):")
162
+ for cat, _ in cats.most_common(10):
163
+ cat_tech = Counter()
164
+ for r in results:
165
+ if r['category'] == cat:
166
+ cat_tech.update(r['tech_hits'])
167
+ if cat_tech:
168
+ print(f"\n {cat.upper()}:")
169
+ for tech, count in cat_tech.most_common(5):
170
+ print(f" {tech:20s}: {count:3d}")
171
+
172
+ # Scripts
173
+ print("\n--- EXPLOIT SCRIPTS ---")
174
+ print(f"with pwntools: {sum(1 for r in results if r['has_pwn'])} ({sum(1 for r in results if r['has_pwn'])/len(results)*100:.1f}%)")
175
+ print(f"with requests: {sum(1 for r in results if r['has_requests'])} ({sum(1 for r in results if r['has_requests'])/len(results)*100:.1f}%)")
176
+ print(f"with def exploit(): {sum(1 for r in results if r['has_exploit'])} ({sum(1 for r in results if r['has_exploit'])/len(results)*100:.1f}%)")
177
+ print(f"with Frida: {sum(1 for r in results if r['has_frida'])} ({sum(1 for r in results if r['has_frida'])/len(results)*100:.1f}%)")
178
+
179
+ py_writeups = [r for r in results if r['py_lines'] > 0]
180
+ print(f"\nWriteups with Python code: {len(py_writeups)} ({len(py_writeups)/len(results)*100:.1f}%)")
181
+ if py_writeups:
182
+ total_py = sum(r['py_lines'] for r in py_writeups)
183
+ print(f"Total Python lines: {total_py:,}")
184
+ print(f"Avg Python lines (when present): {total_py/len(py_writeups):.1f}")
185
+ top_py = max(py_writeups, key=lambda r: r['py_lines'])
186
+ print(f"Most Python: {top_py['name']} ({top_py['py_lines']} lines)")
187
+
188
+ print(f"\nScript usage by category:")
189
+ for cat, _ in cats.most_common(10):
190
+ cat_results = [r for r in results if r['category'] == cat]
191
+ if len(cat_results) >= 3:
192
+ n = len(cat_results)
193
+ pwn = sum(1 for r in cat_results if r['has_pwn'])
194
+ req = sum(1 for r in cat_results if r['has_requests'])
195
+ fri = sum(1 for r in cat_results if r['has_frida'])
196
+ print(f" {cat:20s} (n={n:3d}): pwn={pwn:2d} requests={req:2d} frida={fri:2d}")
197
+
198
+ # Words
199
+ total_words = sum(r['words'] for r in results)
200
+ print(f"\n--- WORD COUNT ---")
201
+ print(f"Total prose words: {total_words:,}")
202
+ print(f"Avg words/writeup: {total_words/len(results):.0f}")
203
+
204
+ # Length
205
+ print("\n--- LENGTH DISTRIBUTION ---")
206
+ for lo, hi, label in [(0,50,"<50"), (50,100,"50-100"), (100,200,"100-200"),
207
+ (200,400,"200-400"), (400,800,"400-800"), (800,9999,">800")]:
208
+ n = sum(1 for r in results if lo <= r['total'] < hi)
209
+ bar = "█" * (n // 3)
210
+ print(f" {label:>8s}: {n:3d} {bar}")
211
+
212
+ by_len = sorted(results, key=lambda r: r['total'], reverse=True)
213
+ print("\n5 longest:")
214
+ for r in by_len[:5]:
215
+ print(f" {r['total']:5d} lines - {r['name']}")
216
+ print("5 shortest:")
217
+ for r in by_len[-5:]:
218
+ print(f" {r['total']:5d} lines - {r['name']}")
219
+
220
+ # Common sections
221
+ print("\n--- COMMON SECTIONS ---")
222
+ all_sections = Counter()
223
+ for r in results:
224
+ all_sections.update(r['sections'])
225
+ for name, count in all_sections.most_common(15):
226
+ print(f" {count:3d}x {name}")
227
+
228
+ print("\n" + "=" * 80)
229
+ print("DONE")
230
+
231
+ if __name__ == "__main__":
232
+ main()