tokenzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scanner.py ADDED
@@ -0,0 +1,398 @@
1
+ """
2
+ TokenSqueeze - Multi-Language AST & Codebase Scanner
3
+ Supports Python (.py), JavaScript/TypeScript (.js, .jsx, .ts, .tsx, .mjs), and Go (.go)
4
+ """
5
+
6
+ import os
7
+ import re
8
+ import ast
9
+ from pathlib import Path
10
+ from typing import Dict, Any, List
11
+
12
+ IGNORE_DIRS = {
13
+ ".git", ".venv", "venv", "__pycache__", "node_modules",
14
+ ".pytest_cache", ".tokenzero", ".tokensqueeze", "dist", "build", ".egg-info",
15
+ ".next", ".turbo", "coverage", ".cache"
16
+ }
17
+
18
+ SUPPORTED_EXTENSIONS = {
19
+ ".py": "python",
20
+ ".js": "javascript",
21
+ ".jsx": "javascript",
22
+ ".ts": "typescript",
23
+ ".tsx": "typescript",
24
+ ".mjs": "javascript",
25
+ ".cjs": "javascript",
26
+ ".go": "go"
27
+ }
28
+
29
+ # --- PYTHON PARSER ---
30
+
31
+ def format_arg(arg: ast.arg) -> str:
32
+ name = arg.arg
33
+ if arg.annotation:
34
+ try:
35
+ return f"{name}: {ast.unparse(arg.annotation)}"
36
+ except Exception:
37
+ return name
38
+ return name
39
+
40
+ def format_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
41
+ prefix = "async def " if isinstance(node, ast.AsyncFunctionDef) else "def "
42
+ args = [format_arg(a) for a in node.args.args]
43
+ if node.args.vararg:
44
+ args.append(f"*{node.args.vararg.arg}")
45
+ if node.args.kwarg:
46
+ args.append(f"**{node.args.kwarg.arg}")
47
+ args_str = ", ".join(args)
48
+
49
+ ret = ""
50
+ if node.returns:
51
+ try:
52
+ ret = f" -> {ast.unparse(node.returns)}"
53
+ except Exception:
54
+ pass
55
+
56
+ return f"{prefix}{node.name}({args_str}){ret}"
57
+
58
+ def extract_quick_summary(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, lines: List[str]) -> str:
59
+ doc = ast.get_docstring(node)
60
+ if doc:
61
+ first_line = [l.strip() for l in doc.splitlines() if l.strip()]
62
+ if first_line:
63
+ return first_line[0]
64
+
65
+ start = node.lineno
66
+ end = min(node.lineno + 5, getattr(node, "end_lineno", node.lineno))
67
+ for line in lines[start:end]:
68
+ stripped = line.strip()
69
+ if stripped.startswith("#"):
70
+ return stripped.lstrip("# ").strip()
71
+
72
+ calls = []
73
+ for subnode in ast.walk(node):
74
+ if isinstance(subnode, ast.Call):
75
+ if isinstance(subnode.func, ast.Name):
76
+ calls.append(subnode.func.id)
77
+ elif isinstance(subnode.func, ast.Attribute):
78
+ calls.append(subnode.func.attr)
79
+
80
+ if calls:
81
+ unique_calls = list(dict.fromkeys(calls))[:3]
82
+ return f"Calls: {', '.join(unique_calls)}()"
83
+
84
+ return "Executes internal logic"
85
+
86
+ def parse_python_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
87
+ try:
88
+ content = filepath.read_text(encoding="utf-8")
89
+ except Exception:
90
+ return None
91
+
92
+ lines = content.splitlines()
93
+ try:
94
+ tree = ast.parse(content, filename=str(filepath))
95
+ except Exception:
96
+ return None
97
+
98
+ rel_path = filepath.relative_to(base_dir)
99
+ file_info = {
100
+ "file": str(rel_path),
101
+ "total_lines": len(lines),
102
+ "classes": [],
103
+ "functions": []
104
+ }
105
+
106
+ for node in tree.body:
107
+ if isinstance(node, ast.ClassDef):
108
+ class_dict = {
109
+ "name": node.name,
110
+ "start": node.lineno,
111
+ "end": getattr(node, "end_lineno", node.lineno),
112
+ "summary": extract_quick_summary(node, lines),
113
+ "methods": []
114
+ }
115
+ for item in node.body:
116
+ if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
117
+ class_dict["methods"].append({
118
+ "name": item.name,
119
+ "sig": format_signature(item),
120
+ "start": item.lineno,
121
+ "end": getattr(item, "end_lineno", item.lineno),
122
+ "summary": extract_quick_summary(item, lines)
123
+ })
124
+ file_info["classes"].append(class_dict)
125
+
126
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
127
+ file_info["functions"].append({
128
+ "name": node.name,
129
+ "sig": format_signature(node),
130
+ "start": node.lineno,
131
+ "end": getattr(node, "end_lineno", node.lineno),
132
+ "summary": extract_quick_summary(node, lines)
133
+ })
134
+
135
+ if file_info["classes"] or file_info["functions"]:
136
+ return file_info
137
+ return None
138
+
139
+ # --- JAVASCRIPT & TYPESCRIPT PARSER ---
140
+
141
+ def extract_jsdoc(lines: List[str], line_idx: int) -> str:
142
+ """Extracts preceding JSDoc comment or // comment."""
143
+ comments = []
144
+ idx = line_idx - 1
145
+ while idx >= 0:
146
+ raw = lines[idx].strip()
147
+ if raw.endswith("*/"):
148
+ idx -= 1
149
+ while idx >= 0:
150
+ inner = lines[idx].strip()
151
+ if inner.startswith("/*") or inner.startswith("/**"):
152
+ break
153
+ cleaned = inner.lstrip("* ").strip()
154
+ if cleaned:
155
+ comments.insert(0, cleaned)
156
+ idx -= 1
157
+ break
158
+ elif raw.startswith("//"):
159
+ cleaned = raw.lstrip("/ ").strip()
160
+ if cleaned:
161
+ comments.insert(0, cleaned)
162
+ idx -= 1
163
+ else:
164
+ break
165
+ return comments[0] if comments else "Executes logic"
166
+
167
+ def find_block_end(lines: List[str], start_idx: int) -> int:
168
+ """Finds the closing brace line for a code block."""
169
+ brace_count = 0
170
+ found_first = False
171
+ for i in range(start_idx, len(lines)):
172
+ line = lines[i]
173
+ # Ignore comments
174
+ line_clean = re.sub(r'//.*', '', line)
175
+ for char in line_clean:
176
+ if char == '{':
177
+ brace_count += 1
178
+ found_first = True
179
+ elif char == '}':
180
+ brace_count -= 1
181
+ if found_first and brace_count == 0:
182
+ return i + 1
183
+ return min(start_idx + 30, len(lines))
184
+
185
+ def parse_js_ts_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
186
+ try:
187
+ content = filepath.read_text(encoding="utf-8", errors="ignore")
188
+ except Exception:
189
+ return None
190
+
191
+ lines = content.splitlines()
192
+ rel_path = filepath.relative_to(base_dir)
193
+
194
+ file_info = {
195
+ "file": str(rel_path),
196
+ "total_lines": len(lines),
197
+ "classes": [],
198
+ "functions": []
199
+ }
200
+
201
+ # Regex patterns for JS/TS
202
+ class_pattern = re.compile(r'^\s*(?:export\s+)?class\s+([A-Za-z0-9_$]+)')
203
+ fn_pattern = re.compile(r'^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+([A-Za-z0-9_$]+)\s*\(([^)]*)\)')
204
+ arrow_pattern = re.compile(r'^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z0-9_$]+)\s*=\s*(?:async\s*)?\(([^)]*)\)\s*(?::\s*[^=]+)?\s*=>')
205
+
206
+ i = 0
207
+ while i < len(lines):
208
+ line = lines[i]
209
+
210
+ # Check Class
211
+ cm = class_pattern.match(line)
212
+ if cm:
213
+ class_name = cm.group(1)
214
+ start_line = i + 1
215
+ end_line = find_block_end(lines, i)
216
+ summary = extract_jsdoc(lines, i)
217
+
218
+ class_dict = {
219
+ "name": class_name,
220
+ "start": start_line,
221
+ "end": end_line,
222
+ "summary": summary,
223
+ "methods": []
224
+ }
225
+
226
+ # Scan methods inside class
227
+ for j in range(start_line, end_line):
228
+ subline = lines[j].strip()
229
+ method_match = re.match(r'^(?:async\s+)?([A-Za-z0-9_$]+)\s*\(([^)]*)\)\s*(?::\s*[^{]+)?\s*\{', subline)
230
+ if method_match and not method_match.group(1).startswith(("if", "for", "while", "switch")):
231
+ m_name = method_match.group(1)
232
+ m_args = method_match.group(2).strip()
233
+ m_end = find_block_end(lines, j)
234
+ class_dict["methods"].append({
235
+ "name": m_name,
236
+ "sig": f"{m_name}({m_args})",
237
+ "start": j + 1,
238
+ "end": m_end,
239
+ "summary": extract_jsdoc(lines, j)
240
+ })
241
+
242
+ file_info["classes"].append(class_dict)
243
+ i = end_line
244
+ continue
245
+
246
+ # Check Function
247
+ fm = fn_pattern.match(line)
248
+ if fm:
249
+ fn_name = fm.group(1)
250
+ fn_args = fm.group(2).strip()
251
+ start_line = i + 1
252
+ end_line = find_block_end(lines, i)
253
+ file_info["functions"].append({
254
+ "name": fn_name,
255
+ "sig": f"function {fn_name}({fn_args})",
256
+ "start": start_line,
257
+ "end": end_line,
258
+ "summary": extract_jsdoc(lines, i)
259
+ })
260
+ i = end_line
261
+ continue
262
+
263
+ # Check Arrow Function
264
+ am = arrow_pattern.match(line)
265
+ if am:
266
+ fn_name = am.group(1)
267
+ fn_args = am.group(2).strip()
268
+ start_line = i + 1
269
+ end_line = find_block_end(lines, i)
270
+ file_info["functions"].append({
271
+ "name": fn_name,
272
+ "sig": f"const {fn_name} = ({fn_args}) =>",
273
+ "start": start_line,
274
+ "end": end_line,
275
+ "summary": extract_jsdoc(lines, i)
276
+ })
277
+ i = end_line
278
+ continue
279
+
280
+ i += 1
281
+
282
+ if file_info["classes"] or file_info["functions"]:
283
+ return file_info
284
+ return None
285
+
286
+ # --- GO PARSER ---
287
+
288
+ def parse_go_file(filepath: Path, base_dir: Path) -> Dict[str, Any] | None:
289
+ try:
290
+ content = filepath.read_text(encoding="utf-8", errors="ignore")
291
+ except Exception:
292
+ return None
293
+
294
+ lines = content.splitlines()
295
+ rel_path = filepath.relative_to(base_dir)
296
+
297
+ file_info = {
298
+ "file": str(rel_path),
299
+ "total_lines": len(lines),
300
+ "classes": [],
301
+ "functions": []
302
+ }
303
+
304
+ struct_pattern = re.compile(r'^\s*type\s+([A-Za-z0-9_$]+)\s+struct\s*\{')
305
+ fn_pattern = re.compile(r'^\s*func\s+(?:\(([^)]+)\)\s+)?([A-Za-z0-9_$]+)\s*\(([^)]*)\)')
306
+
307
+ i = 0
308
+ while i < len(lines):
309
+ line = lines[i]
310
+
311
+ sm = struct_pattern.match(line)
312
+ if sm:
313
+ struct_name = sm.group(1)
314
+ start_line = i + 1
315
+ end_line = find_block_end(lines, i)
316
+ file_info["classes"].append({
317
+ "name": struct_name,
318
+ "start": start_line,
319
+ "end": end_line,
320
+ "summary": extract_jsdoc(lines, i),
321
+ "methods": []
322
+ })
323
+ i = end_line
324
+ continue
325
+
326
+ fm = fn_pattern.match(line)
327
+ if fm:
328
+ receiver = fm.group(1)
329
+ fn_name = fm.group(2)
330
+ fn_args = fm.group(3).strip()
331
+ start_line = i + 1
332
+ end_line = find_block_end(lines, i)
333
+
334
+ sig = f"func ({receiver}) {fn_name}({fn_args})" if receiver else f"func {fn_name}({fn_args})"
335
+ file_info["functions"].append({
336
+ "name": fn_name,
337
+ "sig": sig,
338
+ "start": start_line,
339
+ "end": end_line,
340
+ "summary": extract_jsdoc(lines, i)
341
+ })
342
+ i = end_line
343
+ continue
344
+
345
+ i += 1
346
+
347
+ if file_info["classes"] or file_info["functions"]:
348
+ return file_info
349
+ return None
350
+
351
+ # --- UNIFIED SCANNER ---
352
+
353
+ def scan_project(root_dir: Path) -> List[Dict[str, Any]]:
354
+ """Walks the project directory and extracts skeletons from all supported files."""
355
+ results = []
356
+ for root, dirs, files in os.walk(root_dir):
357
+ dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not d.startswith(".")]
358
+
359
+ for file in sorted(files):
360
+ ext = Path(file).suffix.lower()
361
+ if ext in SUPPORTED_EXTENSIONS:
362
+ full_path = Path(root) / file
363
+ lang = SUPPORTED_EXTENSIONS[ext]
364
+
365
+ parsed = None
366
+ if lang == "python":
367
+ parsed = parse_python_file(full_path, root_dir)
368
+ elif lang in ("javascript", "typescript"):
369
+ parsed = parse_js_ts_file(full_path, root_dir)
370
+ elif lang == "go":
371
+ parsed = parse_go_file(full_path, root_dir)
372
+
373
+ if parsed:
374
+ results.append(parsed)
375
+ return results
376
+
377
+ def generate_markdown_map(scan_data: List[Dict[str, Any]]) -> str:
378
+ """Converts the parsed AST data into an ultra-clean, dense markdown map."""
379
+ md = [
380
+ "# πŸ—ΊοΈ Project Architecture & Symbol Map",
381
+ "> Auto-generated by **TokenSqueeze**. Consult this map for line ranges before reading full files.",
382
+ ""
383
+ ]
384
+
385
+ for item in scan_data:
386
+ md.append(f"### πŸ“„ `{item['file']}` ({item['total_lines']} lines)")
387
+
388
+ for cls in item["classes"]:
389
+ md.append(f"- **`class {cls['name']}`** (Lines {cls['start']}-{cls['end']}) β€” *{cls['summary']}*")
390
+ for m in cls["methods"]:
391
+ md.append(f" - `{m['sig']}` (L{m['start']}-L{m['end']}) β€” {m['summary']}")
392
+
393
+ for fn in item["functions"]:
394
+ md.append(f"- `{fn['sig']}` (Lines {fn['start']}-{fn['end']}) β€” *{fn['summary']}*")
395
+
396
+ md.append("")
397
+
398
+ return "\n".join(md)