tokenzero 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcp_server.py +346 -0
- proxy.py +205 -0
- scanner.py +398 -0
- squeeze.py +887 -0
- tokenzero-0.1.0.dist-info/METADATA +180 -0
- tokenzero-0.1.0.dist-info/RECORD +10 -0
- tokenzero-0.1.0.dist-info/WHEEL +5 -0
- tokenzero-0.1.0.dist-info/entry_points.txt +3 -0
- tokenzero-0.1.0.dist-info/licenses/LICENSE +22 -0
- tokenzero-0.1.0.dist-info/top_level.txt +4 -0
scanner.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TokenSqueeze - Multi-Language AST & Codebase Scanner
|
|
3
|
+
Supports Python (.py), JavaScript/TypeScript (.js, .jsx, .ts, .tsx, .mjs), and Go (.go)
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import ast
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Dict, Any, List
|
|
11
|
+
|
|
12
|
+
IGNORE_DIRS = {
|
|
13
|
+
".git", ".venv", "venv", "__pycache__", "node_modules",
|
|
14
|
+
".pytest_cache", ".tokenzero", ".tokensqueeze", "dist", "build", ".egg-info",
|
|
15
|
+
".next", ".turbo", "coverage", ".cache"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
SUPPORTED_EXTENSIONS = {
|
|
19
|
+
".py": "python",
|
|
20
|
+
".js": "javascript",
|
|
21
|
+
".jsx": "javascript",
|
|
22
|
+
".ts": "typescript",
|
|
23
|
+
".tsx": "typescript",
|
|
24
|
+
".mjs": "javascript",
|
|
25
|
+
".cjs": "javascript",
|
|
26
|
+
".go": "go"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
# --- PYTHON PARSER ---
|
|
30
|
+
|
|
31
|
+
def format_arg(arg: ast.arg) -> str:
|
|
32
|
+
name = arg.arg
|
|
33
|
+
if arg.annotation:
|
|
34
|
+
try:
|
|
35
|
+
return f"{name}: {ast.unparse(arg.annotation)}"
|
|
36
|
+
except Exception:
|
|
37
|
+
return name
|
|
38
|
+
return name
|
|
39
|
+
|
|
40
|
+
def format_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
|
|
41
|
+
prefix = "async def " if isinstance(node, ast.AsyncFunctionDef) else "def "
|
|
42
|
+
args = [format_arg(a) for a in node.args.args]
|
|
43
|
+
if node.args.vararg:
|
|
44
|
+
args.append(f"*{node.args.vararg.arg}")
|
|
45
|
+
if node.args.kwarg:
|
|
46
|
+
args.append(f"**{node.args.kwarg.arg}")
|
|
47
|
+
args_str = ", ".join(args)
|
|
48
|
+
|
|
49
|
+
ret = ""
|
|
50
|
+
if node.returns:
|
|
51
|
+
try:
|
|
52
|
+
ret = f" -> {ast.unparse(node.returns)}"
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
return f"{prefix}{node.name}({args_str}){ret}"
|
|
57
|
+
|
|
58
|
+
def extract_quick_summary(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, lines: List[str]) -> str:
|
|
59
|
+
doc = ast.get_docstring(node)
|
|
60
|
+
if doc:
|
|
61
|
+
first_line = [l.strip() for l in doc.splitlines() if l.strip()]
|
|
62
|
+
if first_line:
|
|
63
|
+
return first_line[0]
|
|
64
|
+
|
|
65
|
+
start = node.lineno
|
|
66
|
+
end = min(node.lineno + 5, getattr(node, "end_lineno", node.lineno))
|
|
67
|
+
for line in lines[start:end]:
|
|
68
|
+
stripped = line.strip()
|
|
69
|
+
if stripped.startswith("#"):
|
|
70
|
+
return stripped.lstrip("# ").strip()
|
|
71
|
+
|
|
72
|
+
calls = []
|
|
73
|
+
for subnode in ast.walk(node):
|
|
74
|
+
if isinstance(subnode, ast.Call):
|
|
75
|
+
if isinstance(subnode.func, ast.Name):
|
|
76
|
+
calls.append(subnode.func.id)
|
|
77
|
+
elif isinstance(subnode.func, ast.Attribute):
|
|
78
|
+
calls.append(subnode.func.attr)
|
|
79
|
+
|
|
80
|
+
if calls:
|
|
81
|
+
unique_calls = list(dict.fromkeys(calls))[:3]
|
|
82
|
+
return f"Calls: {', '.join(unique_calls)}()"
|
|
83
|
+
|
|
84
|
+
return "Executes internal logic"
|
|
85
|
+
|
|
86
|
+
def parse_python_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
|
|
87
|
+
try:
|
|
88
|
+
content = filepath.read_text(encoding="utf-8")
|
|
89
|
+
except Exception:
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
lines = content.splitlines()
|
|
93
|
+
try:
|
|
94
|
+
tree = ast.parse(content, filename=str(filepath))
|
|
95
|
+
except Exception:
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
rel_path = filepath.relative_to(base_dir)
|
|
99
|
+
file_info = {
|
|
100
|
+
"file": str(rel_path),
|
|
101
|
+
"total_lines": len(lines),
|
|
102
|
+
"classes": [],
|
|
103
|
+
"functions": []
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for node in tree.body:
|
|
107
|
+
if isinstance(node, ast.ClassDef):
|
|
108
|
+
class_dict = {
|
|
109
|
+
"name": node.name,
|
|
110
|
+
"start": node.lineno,
|
|
111
|
+
"end": getattr(node, "end_lineno", node.lineno),
|
|
112
|
+
"summary": extract_quick_summary(node, lines),
|
|
113
|
+
"methods": []
|
|
114
|
+
}
|
|
115
|
+
for item in node.body:
|
|
116
|
+
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
117
|
+
class_dict["methods"].append({
|
|
118
|
+
"name": item.name,
|
|
119
|
+
"sig": format_signature(item),
|
|
120
|
+
"start": item.lineno,
|
|
121
|
+
"end": getattr(item, "end_lineno", item.lineno),
|
|
122
|
+
"summary": extract_quick_summary(item, lines)
|
|
123
|
+
})
|
|
124
|
+
file_info["classes"].append(class_dict)
|
|
125
|
+
|
|
126
|
+
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
127
|
+
file_info["functions"].append({
|
|
128
|
+
"name": node.name,
|
|
129
|
+
"sig": format_signature(node),
|
|
130
|
+
"start": node.lineno,
|
|
131
|
+
"end": getattr(node, "end_lineno", node.lineno),
|
|
132
|
+
"summary": extract_quick_summary(node, lines)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
if file_info["classes"] or file_info["functions"]:
|
|
136
|
+
return file_info
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
# --- JAVASCRIPT & TYPESCRIPT PARSER ---
|
|
140
|
+
|
|
141
|
+
def extract_jsdoc(lines: List[str], line_idx: int) -> str:
|
|
142
|
+
"""Extracts preceding JSDoc comment or // comment."""
|
|
143
|
+
comments = []
|
|
144
|
+
idx = line_idx - 1
|
|
145
|
+
while idx >= 0:
|
|
146
|
+
raw = lines[idx].strip()
|
|
147
|
+
if raw.endswith("*/"):
|
|
148
|
+
idx -= 1
|
|
149
|
+
while idx >= 0:
|
|
150
|
+
inner = lines[idx].strip()
|
|
151
|
+
if inner.startswith("/*") or inner.startswith("/**"):
|
|
152
|
+
break
|
|
153
|
+
cleaned = inner.lstrip("* ").strip()
|
|
154
|
+
if cleaned:
|
|
155
|
+
comments.insert(0, cleaned)
|
|
156
|
+
idx -= 1
|
|
157
|
+
break
|
|
158
|
+
elif raw.startswith("//"):
|
|
159
|
+
cleaned = raw.lstrip("/ ").strip()
|
|
160
|
+
if cleaned:
|
|
161
|
+
comments.insert(0, cleaned)
|
|
162
|
+
idx -= 1
|
|
163
|
+
else:
|
|
164
|
+
break
|
|
165
|
+
return comments[0] if comments else "Executes logic"
|
|
166
|
+
|
|
167
|
+
def find_block_end(lines: List[str], start_idx: int) -> int:
|
|
168
|
+
"""Finds the closing brace line for a code block."""
|
|
169
|
+
brace_count = 0
|
|
170
|
+
found_first = False
|
|
171
|
+
for i in range(start_idx, len(lines)):
|
|
172
|
+
line = lines[i]
|
|
173
|
+
# Ignore comments
|
|
174
|
+
line_clean = re.sub(r'//.*', '', line)
|
|
175
|
+
for char in line_clean:
|
|
176
|
+
if char == '{':
|
|
177
|
+
brace_count += 1
|
|
178
|
+
found_first = True
|
|
179
|
+
elif char == '}':
|
|
180
|
+
brace_count -= 1
|
|
181
|
+
if found_first and brace_count == 0:
|
|
182
|
+
return i + 1
|
|
183
|
+
return min(start_idx + 30, len(lines))
|
|
184
|
+
|
|
185
|
+
def parse_js_ts_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
|
|
186
|
+
try:
|
|
187
|
+
content = filepath.read_text(encoding="utf-8", errors="ignore")
|
|
188
|
+
except Exception:
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
lines = content.splitlines()
|
|
192
|
+
rel_path = filepath.relative_to(base_dir)
|
|
193
|
+
|
|
194
|
+
file_info = {
|
|
195
|
+
"file": str(rel_path),
|
|
196
|
+
"total_lines": len(lines),
|
|
197
|
+
"classes": [],
|
|
198
|
+
"functions": []
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
# Regex patterns for JS/TS
|
|
202
|
+
class_pattern = re.compile(r'^\s*(?:export\s+)?class\s+([A-Za-z0-9_$]+)')
|
|
203
|
+
fn_pattern = re.compile(r'^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+([A-Za-z0-9_$]+)\s*\(([^)]*)\)')
|
|
204
|
+
arrow_pattern = re.compile(r'^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z0-9_$]+)\s*=\s*(?:async\s*)?\(([^)]*)\)\s*(?::\s*[^=]+)?\s*=>')
|
|
205
|
+
|
|
206
|
+
i = 0
|
|
207
|
+
while i < len(lines):
|
|
208
|
+
line = lines[i]
|
|
209
|
+
|
|
210
|
+
# Check Class
|
|
211
|
+
cm = class_pattern.match(line)
|
|
212
|
+
if cm:
|
|
213
|
+
class_name = cm.group(1)
|
|
214
|
+
start_line = i + 1
|
|
215
|
+
end_line = find_block_end(lines, i)
|
|
216
|
+
summary = extract_jsdoc(lines, i)
|
|
217
|
+
|
|
218
|
+
class_dict = {
|
|
219
|
+
"name": class_name,
|
|
220
|
+
"start": start_line,
|
|
221
|
+
"end": end_line,
|
|
222
|
+
"summary": summary,
|
|
223
|
+
"methods": []
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
# Scan methods inside class
|
|
227
|
+
for j in range(start_line, end_line):
|
|
228
|
+
subline = lines[j].strip()
|
|
229
|
+
method_match = re.match(r'^(?:async\s+)?([A-Za-z0-9_$]+)\s*\(([^)]*)\)\s*(?::\s*[^{]+)?\s*\{', subline)
|
|
230
|
+
if method_match and not method_match.group(1).startswith(("if", "for", "while", "switch")):
|
|
231
|
+
m_name = method_match.group(1)
|
|
232
|
+
m_args = method_match.group(2).strip()
|
|
233
|
+
m_end = find_block_end(lines, j)
|
|
234
|
+
class_dict["methods"].append({
|
|
235
|
+
"name": m_name,
|
|
236
|
+
"sig": f"{m_name}({m_args})",
|
|
237
|
+
"start": j + 1,
|
|
238
|
+
"end": m_end,
|
|
239
|
+
"summary": extract_jsdoc(lines, j)
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
file_info["classes"].append(class_dict)
|
|
243
|
+
i = end_line
|
|
244
|
+
continue
|
|
245
|
+
|
|
246
|
+
# Check Function
|
|
247
|
+
fm = fn_pattern.match(line)
|
|
248
|
+
if fm:
|
|
249
|
+
fn_name = fm.group(1)
|
|
250
|
+
fn_args = fm.group(2).strip()
|
|
251
|
+
start_line = i + 1
|
|
252
|
+
end_line = find_block_end(lines, i)
|
|
253
|
+
file_info["functions"].append({
|
|
254
|
+
"name": fn_name,
|
|
255
|
+
"sig": f"function {fn_name}({fn_args})",
|
|
256
|
+
"start": start_line,
|
|
257
|
+
"end": end_line,
|
|
258
|
+
"summary": extract_jsdoc(lines, i)
|
|
259
|
+
})
|
|
260
|
+
i = end_line
|
|
261
|
+
continue
|
|
262
|
+
|
|
263
|
+
# Check Arrow Function
|
|
264
|
+
am = arrow_pattern.match(line)
|
|
265
|
+
if am:
|
|
266
|
+
fn_name = am.group(1)
|
|
267
|
+
fn_args = am.group(2).strip()
|
|
268
|
+
start_line = i + 1
|
|
269
|
+
end_line = find_block_end(lines, i)
|
|
270
|
+
file_info["functions"].append({
|
|
271
|
+
"name": fn_name,
|
|
272
|
+
"sig": f"const {fn_name} = ({fn_args}) =>",
|
|
273
|
+
"start": start_line,
|
|
274
|
+
"end": end_line,
|
|
275
|
+
"summary": extract_jsdoc(lines, i)
|
|
276
|
+
})
|
|
277
|
+
i = end_line
|
|
278
|
+
continue
|
|
279
|
+
|
|
280
|
+
i += 1
|
|
281
|
+
|
|
282
|
+
if file_info["classes"] or file_info["functions"]:
|
|
283
|
+
return file_info
|
|
284
|
+
return None
|
|
285
|
+
|
|
286
|
+
# --- GO PARSER ---
|
|
287
|
+
|
|
288
|
+
def parse_go_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
|
|
289
|
+
try:
|
|
290
|
+
content = filepath.read_text(encoding="utf-8", errors="ignore")
|
|
291
|
+
except Exception:
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
lines = content.splitlines()
|
|
295
|
+
rel_path = filepath.relative_to(base_dir)
|
|
296
|
+
|
|
297
|
+
file_info = {
|
|
298
|
+
"file": str(rel_path),
|
|
299
|
+
"total_lines": len(lines),
|
|
300
|
+
"classes": [],
|
|
301
|
+
"functions": []
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
struct_pattern = re.compile(r'^\s*type\s+([A-Za-z0-9_$]+)\s+struct\s*\{')
|
|
305
|
+
fn_pattern = re.compile(r'^\s*func\s+(?:\(([^)]+)\)\s+)?([A-Za-z0-9_$]+)\s*\(([^)]*)\)')
|
|
306
|
+
|
|
307
|
+
i = 0
|
|
308
|
+
while i < len(lines):
|
|
309
|
+
line = lines[i]
|
|
310
|
+
|
|
311
|
+
sm = struct_pattern.match(line)
|
|
312
|
+
if sm:
|
|
313
|
+
struct_name = sm.group(1)
|
|
314
|
+
start_line = i + 1
|
|
315
|
+
end_line = find_block_end(lines, i)
|
|
316
|
+
file_info["classes"].append({
|
|
317
|
+
"name": struct_name,
|
|
318
|
+
"start": start_line,
|
|
319
|
+
"end": end_line,
|
|
320
|
+
"summary": extract_jsdoc(lines, i),
|
|
321
|
+
"methods": []
|
|
322
|
+
})
|
|
323
|
+
i = end_line
|
|
324
|
+
continue
|
|
325
|
+
|
|
326
|
+
fm = fn_pattern.match(line)
|
|
327
|
+
if fm:
|
|
328
|
+
receiver = fm.group(1)
|
|
329
|
+
fn_name = fm.group(2)
|
|
330
|
+
fn_args = fm.group(3).strip()
|
|
331
|
+
start_line = i + 1
|
|
332
|
+
end_line = find_block_end(lines, i)
|
|
333
|
+
|
|
334
|
+
sig = f"func ({receiver}) {fn_name}({fn_args})" if receiver else f"func {fn_name}({fn_args})"
|
|
335
|
+
file_info["functions"].append({
|
|
336
|
+
"name": fn_name,
|
|
337
|
+
"sig": sig,
|
|
338
|
+
"start": start_line,
|
|
339
|
+
"end": end_line,
|
|
340
|
+
"summary": extract_jsdoc(lines, i)
|
|
341
|
+
})
|
|
342
|
+
i = end_line
|
|
343
|
+
continue
|
|
344
|
+
|
|
345
|
+
i += 1
|
|
346
|
+
|
|
347
|
+
if file_info["classes"] or file_info["functions"]:
|
|
348
|
+
return file_info
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
# --- UNIFIED SCANNER ---
|
|
352
|
+
|
|
353
|
+
def scan_project(root_dir: Path) -> List[Dict[str, Any]]:
|
|
354
|
+
"""Walks the project directory and extracts skeletons from all supported files."""
|
|
355
|
+
results = []
|
|
356
|
+
for root, dirs, files in os.walk(root_dir):
|
|
357
|
+
dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not d.startswith(".")]
|
|
358
|
+
|
|
359
|
+
for file in sorted(files):
|
|
360
|
+
ext = Path(file).suffix.lower()
|
|
361
|
+
if ext in SUPPORTED_EXTENSIONS:
|
|
362
|
+
full_path = Path(root) / file
|
|
363
|
+
lang = SUPPORTED_EXTENSIONS[ext]
|
|
364
|
+
|
|
365
|
+
parsed = None
|
|
366
|
+
if lang == "python":
|
|
367
|
+
parsed = parse_python_file(full_path, root_dir)
|
|
368
|
+
elif lang in ("javascript", "typescript"):
|
|
369
|
+
parsed = parse_js_ts_file(full_path, root_dir)
|
|
370
|
+
elif lang == "go":
|
|
371
|
+
parsed = parse_go_file(full_path, root_dir)
|
|
372
|
+
|
|
373
|
+
if parsed:
|
|
374
|
+
results.append(parsed)
|
|
375
|
+
return results
|
|
376
|
+
|
|
377
|
+
def generate_markdown_map(scan_data: List[Dict[str, Any]]) -> str:
|
|
378
|
+
"""Converts the parsed AST data into an ultra-clean, dense markdown map."""
|
|
379
|
+
md = [
|
|
380
|
+
"# πΊοΈ Project Architecture & Symbol Map",
|
|
381
|
+
"> Auto-generated by **TokenSqueeze**. Consult this map for line ranges before reading full files.",
|
|
382
|
+
""
|
|
383
|
+
]
|
|
384
|
+
|
|
385
|
+
for item in scan_data:
|
|
386
|
+
md.append(f"### π `{item['file']}` ({item['total_lines']} lines)")
|
|
387
|
+
|
|
388
|
+
for cls in item["classes"]:
|
|
389
|
+
md.append(f"- **`class {cls['name']}`** (Lines {cls['start']}-{cls['end']}) β *{cls['summary']}*")
|
|
390
|
+
for m in cls["methods"]:
|
|
391
|
+
md.append(f" - `{m['sig']}` (L{m['start']}-L{m['end']}) β {m['summary']}")
|
|
392
|
+
|
|
393
|
+
for fn in item["functions"]:
|
|
394
|
+
md.append(f"- `{fn['sig']}` (Lines {fn['start']}-{fn['end']}) β *{fn['summary']}*")
|
|
395
|
+
|
|
396
|
+
md.append("")
|
|
397
|
+
|
|
398
|
+
return "\n".join(md)
|