tokenzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
squeeze.py ADDED
@@ -0,0 +1,887 @@
1
+ """
2
+ TokenZero CLI
3
+ Zero Token Waste — AST Code Skeletonizer & Live Real-Time AI Monitor.
4
+ """
5
+
6
+ import os
7
+ import sys
8
+ import time
9
+ import json
10
+ import shutil
11
+ import re
12
+ from datetime import datetime
13
+ from pathlib import Path
14
+ from rich.console import Console
15
+ from rich.panel import Panel
16
+ from rich.table import Table
17
+ from rich.live import Live
18
+ from rich.layout import Layout
19
+
20
+ from scanner import scan_project, generate_markdown_map
21
+
22
+ console = Console()
23
+
24
+ AGENT_CONFIGS = [
25
+ {
26
+ "file": "GEMINI.md",
27
+ "name": "Google Antigravity",
28
+ "rule": """<!-- BEGIN_TOKENZERO_RULE -->
29
+ # ⚡ TokenZero Optimization Guide (Active)
30
+
31
+ 1. **Check the Map First**: Before reading full source files, consult `.tokenzero/REPO_MAP.md` (or `.tokensqueeze/REPO_MAP.md`) to see the project structure, function signatures, and exact line ranges.
32
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading (`StartLine` and `EndLine`) based on the line numbers in `REPO_MAP.md`.
33
+ 3. **Targeted Edits**: Only inspect the exact function or block being edited to preserve context window and reduce token consumption.
34
+ 4. **Do NOT Edit audit.jsonl**: The TokenZero background hook manages audit logging automatically. Never propose edits to `audit.jsonl`.
35
+ 5. **Response Badge**: End your response with a 1-line badge:
36
+ `⚡ [TokenZero: Saved ~<N> tokens using REPO_MAP]`
37
+ <!-- END_TOKENZERO_RULE -->"""
38
+ },
39
+ {
40
+ "file": "CLAUDE.md",
41
+ "name": "Claude Code",
42
+ "rule": """<!-- BEGIN_TOKENZERO_RULE -->
43
+ # ⚡ TokenZero Optimization Guide (Active)
44
+
45
+ 1. **Check the Map First**: Before reading full source files, consult `.tokenzero/REPO_MAP.md` to see the project structure, function signatures, and exact line ranges.
46
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading based on the line numbers in `REPO_MAP.md`.
47
+ 3. **Targeted Edits**: Only inspect the exact function or block being edited to preserve context window and reduce token consumption.
48
+ 4. **Do NOT Edit audit.jsonl**: TokenZero manages audit logging automatically.
49
+ <!-- END_TOKENZERO_RULE -->"""
50
+ },
51
+ {
52
+ "file": ".cursorrules",
53
+ "name": "Cursor IDE",
54
+ "rule": """<!-- BEGIN_TOKENZERO_RULE -->
55
+ # ⚡ TokenZero Optimization Guide (Active)
56
+
57
+ 1. **Check the Map First**: Before reading full source files, consult `.tokenzero/REPO_MAP.md` to see the project structure, function signatures, and exact line ranges.
58
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading based on the line numbers in `REPO_MAP.md`.
59
+ 3. **Targeted Edits**: Only inspect the exact function or block being edited to preserve context window and reduce token consumption.
60
+ <!-- END_TOKENZERO_RULE -->"""
61
+ },
62
+ {
63
+ "file": ".windsurfrules",
64
+ "name": "Windsurf IDE",
65
+ "rule": """<!-- BEGIN_TOKENZERO_RULE -->
66
+ # ⚡ TokenZero Optimization Guide (Active)
67
+
68
+ 1. **Check the Map First**: Before reading full source files, consult `.tokenzero/REPO_MAP.md` to see the project structure, function signatures, and exact line ranges.
69
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading based on the line numbers in `REPO_MAP.md`.
70
+ 3. **Targeted Edits**: Only inspect the exact function or block being edited to preserve context window and reduce token consumption.
71
+ <!-- END_TOKENZERO_RULE -->"""
72
+ },
73
+ {
74
+ "file": ".github/copilot-instructions.md",
75
+ "name": "GitHub Copilot",
76
+ "rule": """<!-- BEGIN_TOKENZERO_RULE -->
77
+ # ⚡ TokenZero Optimization Guide (Active)
78
+
79
+ 1. **Check the Map First**: Before reading full source files, consult `.tokenzero/REPO_MAP.md` to see the project structure, function signatures, and exact line ranges.
80
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading based on the line numbers in `REPO_MAP.md`.
81
+ <!-- END_TOKENZERO_RULE -->"""
82
+ }
83
+ ]
84
+
85
+ HOOK_CONFIG = {
86
+ "tokenzero-live": {
87
+ "PreToolUse": [
88
+ {
89
+ "matcher": "read_file|edit_file|write_file|run_command",
90
+ "hooks": [
91
+ {
92
+ "type": "command",
93
+ "command": "PATH=\"$HOME/.local/bin:$PATH\" tokenzero hook",
94
+ "timeout": 5
95
+ }
96
+ ]
97
+ }
98
+ ]
99
+ }
100
+ }
101
+
102
+ INTERNAL_PATTERNS = (
103
+ ".tokenzero",
104
+ ".tokensqueeze",
105
+ "repo_map.md",
106
+ "audit.jsonl",
107
+ "hooks.json",
108
+ "gemini.md",
109
+ "claude.md",
110
+ ".cursorrules",
111
+ ".windsurfrules",
112
+ "copilot-instructions",
113
+ ".agents"
114
+ )
115
+
116
+ def is_internal_file(file_path: str) -> bool:
117
+ """Returns True if the file is an internal TokenZero file to keep secret."""
118
+ if not file_path:
119
+ return False
120
+ lower = str(file_path).lower().replace("\\", "/")
121
+ return any(pat in lower for pat in INTERNAL_PATTERNS)
122
+
123
+ GLOBAL_DIR = Path.home() / ".tokenzero"
124
+ GLOBAL_CACHE_DIR = GLOBAL_DIR / "cache"
125
+
126
+ def get_project_hash(target_dir: Path) -> str:
127
+ """Returns a deterministic safe hash/folder name for a project path."""
128
+ import hashlib
129
+ resolved = str(target_dir.resolve())
130
+ h = hashlib.sha256(resolved.encode("utf-8")).hexdigest()[:12]
131
+ safe_name = re.sub(r"[^a-zA-Z0-9_-]", "_", target_dir.name)
132
+ return f"{safe_name}_{h}"
133
+
134
+ def get_project_cache_dir(target_dir: Path) -> Path:
135
+ """Returns the global cache directory for this project: ~/.tokenzero/cache/<safe_name_hash>/"""
136
+ p_dir = GLOBAL_CACHE_DIR / get_project_hash(target_dir)
137
+ p_dir.mkdir(parents=True, exist_ok=True)
138
+ return p_dir
139
+
140
+ def get_repo_map_path(target_dir: Path) -> Path:
141
+ """Returns the REPO_MAP path: checks local first if exists, otherwise global cache."""
142
+ local_map = target_dir / ".tokenzero" / "REPO_MAP.md"
143
+ if local_map.exists():
144
+ return local_map
145
+ local_ts = target_dir / ".tokensqueeze" / "REPO_MAP.md"
146
+ if local_ts.exists():
147
+ return local_ts
148
+ return get_project_cache_dir(target_dir) / "REPO_MAP.md"
149
+
150
+ def get_config_dir(target_dir: Path) -> Path:
151
+ """Returns local .tokenzero dir if exists, else global cache."""
152
+ tz = target_dir / ".tokenzero"
153
+ ts = target_dir / ".tokensqueeze"
154
+ if tz.exists():
155
+ return tz
156
+ if ts.exists():
157
+ return ts
158
+ return get_project_cache_dir(target_dir)
159
+
160
+ def record_activity(target_dir: Path, action: str, file: str, function: str, lines_used: int, total_lines: int):
161
+ """Appends an event to global ~/.tokenzero/audit.jsonl (Zero files in project directory!)"""
162
+ if is_internal_file(file):
163
+ return
164
+
165
+ GLOBAL_DIR.mkdir(parents=True, exist_ok=True)
166
+ audit_file = GLOBAL_DIR / "audit.jsonl"
167
+
168
+ tokens_used = lines_used * 7
169
+ lines_saved = max(0, total_lines - lines_used)
170
+ tokens_saved = lines_saved * 7
171
+
172
+ entry = {
173
+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
174
+ "project": str(target_dir.resolve()),
175
+ "project_name": target_dir.name,
176
+ "action": action.upper(),
177
+ "file": file,
178
+ "function": function,
179
+ "lines_used": lines_used,
180
+ "total_file_lines": total_lines,
181
+ "tokens_used": tokens_used,
182
+ "tokens_saved": tokens_saved
183
+ }
184
+
185
+ with open(audit_file, "a", encoding="utf-8") as f:
186
+ f.write(json.dumps(entry) + "\n")
187
+
188
+ def handle_hook():
189
+ """
190
+ Real-Time Lifecycle Hook handler:
191
+ 1. Intercepts read_file / edit_file / run_command in real-time.
192
+ 2. Blocks manual edits to internal TokenZero files (protecting stealth & popup free).
193
+ 3. Logs on-the-spot user code activity to terminal (hides internal files).
194
+ """
195
+ decision = "allow"
196
+
197
+ try:
198
+ raw_input = sys.stdin.read()
199
+ if raw_input:
200
+ payload = json.loads(raw_input)
201
+ tool_call = payload.get("toolCall", {})
202
+ tool_name = tool_call.get("name", "")
203
+ args = tool_call.get("args", {})
204
+ cwd = Path(payload.get("cwd", ".")).resolve()
205
+
206
+ # 1. Block manual edits to audit.jsonl (kill IDE diff review popup)
207
+ if tool_name in ("edit_file", "write_file"):
208
+ file_path = args.get("Path", "")
209
+ if "audit.jsonl" in file_path:
210
+ print(json.dumps({
211
+ "decision": "deny",
212
+ "reason": "TokenZero manages audit.jsonl automatically. No manual edit needed."
213
+ }))
214
+ return
215
+
216
+ # 2. Record read_file / edit_file (silently hide internal TokenZero files)
217
+ if tool_name in ("read_file", "edit_file"):
218
+ file_path = args.get("Path", "")
219
+ if file_path and not is_internal_file(file_path):
220
+ try:
221
+ p = Path(file_path).resolve()
222
+ rel_file = str(p.relative_to(cwd)) if p.is_relative_to(cwd) else p.name
223
+ total_lines = len(p.read_text(encoding="utf-8", errors="ignore").splitlines()) if p.exists() else 100
224
+ except Exception:
225
+ rel_file = Path(file_path).name
226
+ total_lines = 100
227
+
228
+ start_line = args.get("StartLine", 1)
229
+ end_line = args.get("EndLine", total_lines)
230
+ is_partial = ("StartLine" in args) or ("EndLine" in args)
231
+ lines_used = max(1, end_line - start_line + 1) if is_partial else total_lines
232
+
233
+ action = "READ" if tool_name == "read_file" else "EDIT"
234
+ func_tag = f"L{start_line}-L{end_line}" if is_partial else "Full File"
235
+
236
+ record_activity(
237
+ target_dir=cwd,
238
+ action=action,
239
+ file=rel_file,
240
+ function=func_tag,
241
+ lines_used=lines_used,
242
+ total_lines=total_lines
243
+ )
244
+
245
+ # 3. Record run_command so user sees commands in real-time too
246
+ elif tool_name == "run_command":
247
+ cmd = args.get("CommandLine", "bash")
248
+ summary = args.get("toolSummary", cmd[:30])
249
+ if not is_internal_file(cmd) and not is_internal_file(summary):
250
+ record_activity(
251
+ target_dir=cwd,
252
+ action="EXEC",
253
+ file=summary[:30],
254
+ function=cmd[:30],
255
+ lines_used=0,
256
+ total_lines=0
257
+ )
258
+
259
+ except Exception:
260
+ pass
261
+
262
+ print(json.dumps({"decision": decision}))
263
+
264
+ def get_audit_entries(target_dir: Path):
265
+ """Retrieves audit entries for target_dir from global ~/.tokenzero/audit.jsonl + local fallback."""
266
+ target_resolved = str(target_dir.resolve())
267
+ target_name = target_dir.name
268
+ entries = []
269
+
270
+ # 1. Global audit log
271
+ global_audit = GLOBAL_DIR / "audit.jsonl"
272
+ if global_audit.exists():
273
+ try:
274
+ for line in global_audit.read_text(encoding="utf-8").splitlines():
275
+ if line.strip():
276
+ try:
277
+ item = json.loads(line)
278
+ if item.get("project") == target_resolved or item.get("project_name") == target_name:
279
+ entries.append(item)
280
+ except Exception:
281
+ pass
282
+ except Exception:
283
+ pass
284
+
285
+ # 2. Local fallback if legacy audit exists
286
+ for folder in (".tokenzero", ".tokensqueeze"):
287
+ local_audit = target_dir / folder / "audit.jsonl"
288
+ if local_audit.exists():
289
+ try:
290
+ for line in local_audit.read_text(encoding="utf-8").splitlines():
291
+ if line.strip():
292
+ try:
293
+ entries.append(json.loads(line))
294
+ except Exception:
295
+ pass
296
+ except Exception:
297
+ pass
298
+
299
+ return entries
300
+
301
+ def render_dashboard_layout(target_dir: Path, spinner_frame: str) -> Layout:
302
+ """Builds the clean rich layout for live watch mode (Direct Table without top box)."""
303
+ layout = Layout()
304
+ layout.split_column(
305
+ Layout(name="body", ratio=1),
306
+ Layout(name="footer", size=7)
307
+ )
308
+
309
+ entries = get_audit_entries(target_dir)
310
+
311
+ # Activity Table (Body) — Completely pure and minimal without any top title
312
+ table = Table(
313
+ header_style="bold magenta",
314
+ expand=True
315
+ )
316
+ table.add_column("Time", style="dim", width=19)
317
+ table.add_column("Action", style="bold white", width=8)
318
+ table.add_column("File & Function", style="cyan", ratio=2)
319
+ table.add_column("Lines Read", style="yellow", width=16)
320
+ table.add_column("Tokens Used", style="bold yellow", width=16)
321
+
322
+ total_tokens_used = 0
323
+ total_tokens_saved = 0
324
+ files_touched = set()
325
+ funcs_touched = set()
326
+
327
+ for item in entries:
328
+ used = item.get("tokens_used", item.get("lines_used", 0) * 7)
329
+ saved = item.get("tokens_saved", 0)
330
+ total_tokens_used += used
331
+ total_tokens_saved += saved
332
+
333
+ if item.get("action") in ("READ", "EDIT"):
334
+ files_touched.add(item.get("file", ""))
335
+ funcs_touched.add(item.get("function", ""))
336
+
337
+ if not entries:
338
+ table.add_row(
339
+ datetime.now().strftime("%H:%M:%S"),
340
+ "[dim]WAIT[/dim]",
341
+ "[dim]Waiting for Antigravity to inspect or edit code...[/dim]",
342
+ "-",
343
+ "-"
344
+ )
345
+ else:
346
+ for item in entries[-9:]:
347
+ action = item.get("action", "READ")
348
+ if action == "READ":
349
+ action_badge = "[green]READ[/green]"
350
+ elif action == "EDIT":
351
+ action_badge = "[yellow]EDIT[/yellow]"
352
+ else:
353
+ action_badge = "[cyan]EXEC[/cyan]"
354
+
355
+ func_str = item.get('function', '')
356
+ if action == "EXEC":
357
+ target_name = f"{item.get('file', '')} ({func_str})" if func_str else item.get('file', '')
358
+ else:
359
+ if func_str and not func_str.endswith('()') and not func_str.startswith('L'):
360
+ func_str = f"{func_str}()"
361
+ target_name = f"{item.get('file', '')}:{func_str}" if func_str else item.get('file', '')
362
+
363
+ lines_used = item.get('lines_used', 0)
364
+ total_lines = item.get('total_file_lines', 0)
365
+ lines_info = f"{lines_used} / {total_lines} lines" if total_lines > 0 else "-"
366
+
367
+ used = item.get("tokens_used", lines_used * 7)
368
+ used_str = f"~{used:,} tokens" if used > 0 else "-"
369
+
370
+ table.add_row(
371
+ item.get("timestamp", ""),
372
+ action_badge,
373
+ target_name,
374
+ lines_info,
375
+ used_str
376
+ )
377
+
378
+ layout["body"].update(table)
379
+
380
+ # Footer Summary — Clear Before vs After Comparison
381
+ without_tokens = total_tokens_used + total_tokens_saved
382
+ dollars_without = (without_tokens / 1_000_000) * 2.50
383
+ dollars_with = (total_tokens_used / 1_000_000) * 2.50
384
+ dollars_saved = (total_tokens_saved / 1_000_000) * 2.50
385
+ pct_saved = (total_tokens_saved / without_tokens * 100) if without_tokens > 0 else 0
386
+
387
+ summary_table = Table(show_header=False, expand=True, box=None)
388
+ summary_table.add_column("C1", justify="center")
389
+
390
+ summary_table.add_row(
391
+ f"[dim]Activity:[/dim] [bold white]{len(entries)} events[/bold white] across [bold white]{len(files_touched)} files[/bold white] | "
392
+ f"🔴 [red]Actual Cost (Without TokenZero):[/red] [bold white]{without_tokens:,}[/bold white] tokens (${dollars_without:.4f}) ➜ "
393
+ f"🟢 [green]Used (With TokenZero):[/green] [bold yellow]{total_tokens_used:,}[/bold yellow] tokens (${dollars_with:.4f})"
394
+ )
395
+ summary_table.add_row(
396
+ f"💰 [bold green]Net Impact: Saved +{total_tokens_saved:,} tokens ({pct_saved:.1f}% reduction) • You Saved ${dollars_saved:.4f} USD![/bold green]"
397
+ )
398
+
399
+ layout["footer"].update(Panel(summary_table, title="📊 TokenZero ROI & Savings Comparison", border_style="cyan"))
400
+
401
+ return layout
402
+
403
+ def watch_dashboard(target_dir: Path):
404
+ """Runs a continuous real-time live watcher (Zero files in project directory!)."""
405
+ setup_tokenzero(silent=True)
406
+ map_file = get_repo_map_path(target_dir)
407
+ if not map_file.exists():
408
+ console.print(f"[yellow]⚠️ TokenZero cache not found for {target_dir.name}. Initializing in Zero-File mode...[/yellow]")
409
+ init_tokenzero(target_dir, local=False)
410
+
411
+ console.clear()
412
+ spinner_chars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
413
+ idx = 0
414
+
415
+ try:
416
+ with Live(render_dashboard_layout(target_dir, spinner_chars[0]), refresh_per_second=5, console=console) as live:
417
+ while True:
418
+ time.sleep(0.2)
419
+ idx = (idx + 1) % len(spinner_chars)
420
+ live.update(render_dashboard_layout(target_dir, spinner_chars[idx]))
421
+ except KeyboardInterrupt:
422
+ console.print("\n[yellow]👋 TokenZero Live Monitor stopped.[/yellow]")
423
+
424
+ def show_stats_once(target_dir: Path):
425
+ """Static one-time stats print."""
426
+ console.print(render_dashboard_layout(target_dir, "●"))
427
+
428
+ def init_tokenzero(target_dir: Path, local: bool = False):
429
+ console.print(f"\n[bold cyan]🔍 Activating TokenZero in:[/bold cyan] {target_dir}")
430
+
431
+ # 1. Scan codebase (Python, JavaScript, TypeScript, Go)
432
+ scan_results = scan_project(target_dir)
433
+ if not scan_results:
434
+ console.print("[yellow]⚠️ No supported source files (.py, .js, .ts, .jsx, .tsx, .go) with functions/classes found.[/yellow]")
435
+ return
436
+
437
+ map_content = generate_markdown_map(scan_results)
438
+
439
+ if local:
440
+ tokenzero_dir = target_dir / ".tokenzero"
441
+ tokenzero_dir.mkdir(exist_ok=True)
442
+ map_path = tokenzero_dir / "REPO_MAP.md"
443
+ map_path.write_text(map_content, encoding="utf-8")
444
+
445
+ # Auto-append to .gitignore so it remains invisible to git!
446
+ gitignore_path = target_dir / ".gitignore"
447
+ existing_gi = gitignore_path.read_text(encoding="utf-8") if gitignore_path.exists() else ""
448
+ if ".tokenzero" not in existing_gi:
449
+ with open(gitignore_path, "a", encoding="utf-8") as f:
450
+ if existing_gi and not existing_gi.endswith("\n"):
451
+ f.write("\n")
452
+ f.write("\n# TokenZero (Auto-ignored)\n.tokenzero/\n.tokensqueeze/\n")
453
+
454
+ # Inject Rules into supported agent files
455
+ for cfg in AGENT_CONFIGS:
456
+ rule_path = target_dir / cfg["file"]
457
+ rule_path.parent.mkdir(parents=True, exist_ok=True)
458
+ existing_rule = rule_path.read_text(encoding="utf-8") if rule_path.exists() else ""
459
+
460
+ existing_rule = re.sub(
461
+ r"\n*<!-- BEGIN_TOKENSQUEEZE_RULE -->.*?<!-- END_TOKENSQUEEZE_RULE -->\n*",
462
+ "",
463
+ existing_rule,
464
+ flags=re.DOTALL
465
+ ).strip()
466
+
467
+ if "<!-- BEGIN_TOKENZERO_RULE -->" not in existing_rule:
468
+ final_content = (existing_rule + "\n\n" if existing_rule else "") + cfg["rule"].strip() + "\n"
469
+ rule_path.write_text(final_content, encoding="utf-8")
470
+ else:
471
+ updated = re.sub(
472
+ r"<!-- BEGIN_TOKENZERO_RULE -->.*?<!-- END_TOKENZERO_RULE -->",
473
+ cfg["rule"].strip(),
474
+ existing_rule,
475
+ flags=re.DOTALL
476
+ )
477
+ rule_path.write_text(updated.strip() + "\n", encoding="utf-8")
478
+
479
+ # Inject hook
480
+ agents_dir = target_dir / ".agents"
481
+ agents_dir.mkdir(exist_ok=True)
482
+ hooks_file = agents_dir / "hooks.json"
483
+ hooks_file.write_text(json.dumps(HOOK_CONFIG, indent=2), encoding="utf-8")
484
+ else:
485
+ # 100% Zero-File Stealth Mode: Saves in global cache ~/.tokenzero/cache/
486
+ cache_dir = get_project_cache_dir(target_dir)
487
+ map_path = cache_dir / "REPO_MAP.md"
488
+ map_path.write_text(map_content, encoding="utf-8")
489
+ setup_tokenzero(silent=True)
490
+
491
+ # 3. Calculate Stats
492
+ total_files = len(scan_results)
493
+ total_classes = sum(len(f["classes"]) for f in scan_results)
494
+ total_methods = sum(sum(len(c["methods"]) for c in f["classes"]) for f in scan_results)
495
+ total_funcs = sum(len(f["functions"]) for f in scan_results)
496
+ raw_lines = sum(f["total_lines"] for f in scan_results)
497
+ map_lines = len(map_content.splitlines())
498
+
499
+ raw_tokens = raw_lines * 7
500
+ map_tokens = map_lines * 5
501
+ saved_tokens = max(0, raw_tokens - map_tokens)
502
+ pct = (saved_tokens / raw_tokens * 100) if raw_tokens else 0
503
+
504
+ mode_label = "Local (Auto-.gitignore)" if local else "[bold green]Zero-File Stealth Mode (0 files in project)[/bold green]"
505
+
506
+ table = Table(title="⚡ [bold green]TokenZero Optimization Engine Activated[/bold green]", header_style="bold magenta")
507
+ table.add_column("Component", style="cyan")
508
+ table.add_column("Status / Impact", style="bold white")
509
+
510
+ table.add_row("Codebase Analysis", f"{total_files} files, {total_classes + total_funcs + total_methods} symbols indexed")
511
+ table.add_row("Project Directory", mode_label)
512
+ table.add_row("TokenZero Core Engine", "[bold green]Online & Synchronized[/bold green]")
513
+ table.add_row("Context Compression", f"[bold green]{pct:.1f}% reduction[/bold green]")
514
+ table.add_row("Multi-Agent Guard", "[bold green]Active (Antigravity, Claude, Cursor, Copilot, Windsurf)[/bold green]")
515
+ table.add_row("Real-Time Telemetry", "[bold green]Connected (~/.tokenzero/audit.jsonl)[/bold green]")
516
+
517
+ console.print()
518
+ console.print(table)
519
+ console.print("\n[bold green]🚀 Zero token waste active![/bold green] Run '[cyan]tokenzero watch[/cyan]' to monitor live AI tool calls!")
520
+
521
+ def ensure_antigravity_plugin() -> tuple:
522
+ """Installs or updates the global Antigravity plugin in ~/.gemini/config/plugins/tokenzero/"""
523
+ plugin_dir = Path.home() / ".gemini" / "config" / "plugins" / "tokenzero"
524
+ try:
525
+ plugin_dir.mkdir(parents=True, exist_ok=True)
526
+ (plugin_dir / "plugin.json").write_text(json.dumps({
527
+ "name": "tokenzero",
528
+ "displayName": "TokenZero",
529
+ "version": "0.1.0",
530
+ "description": "Real-time AI token optimizer and AST code mapper"
531
+ }, indent=2), encoding="utf-8")
532
+
533
+ (plugin_dir / "hooks.json").write_text(json.dumps(HOOK_CONFIG, indent=2), encoding="utf-8")
534
+
535
+ rules_dir = plugin_dir / "rules"
536
+ rules_dir.mkdir(exist_ok=True)
537
+ (rules_dir / "AGENTS.md").write_text("""<!-- BEGIN_TOKENZERO_RULE -->
538
+ # ⚡ TokenZero Optimization Guide (Active)
539
+
540
+ 1. **Check the Map First**: Before reading full source files, consult TokenZero REPO_MAP to see the project structure, function signatures, and exact line ranges.
541
+ 2. **Partial File Reads**: Do NOT read whole files (>100 lines) if you only need one class or function. Always use line-range reading (`StartLine` and `EndLine`).
542
+ 3. **Targeted Edits**: Only inspect the exact function or block being edited to preserve context window and reduce token consumption.
543
+ 4. **Do NOT Edit audit.jsonl**: The TokenZero background hook manages audit logging automatically. Never propose edits to `audit.jsonl`.
544
+ <!-- END_TOKENZERO_RULE -->
545
+ """, encoding="utf-8")
546
+ return True, str(plugin_dir)
547
+ except Exception as e:
548
+ return False, str(e)
549
+
550
+ def setup_tokenzero(silent: bool = False) -> list:
551
+ """
552
+ Automatically configures installed AI editors and CLI agents with TokenZero MCP server.
553
+ Configures Claude Desktop, Claude Code, Cursor IDE, etc.
554
+ """
555
+ if not silent:
556
+ console.print("\n[bold cyan]🔧 Setting up TokenZero Universal MCP Integration...[/bold cyan]\n")
557
+ results = []
558
+
559
+ mcp_config_entry = {
560
+ "command": "tokenzero",
561
+ "args": ["mcp"]
562
+ }
563
+
564
+ # 1. Cursor IDE (~/.cursor/mcp.json)
565
+ cursor_dir = Path.home() / ".cursor"
566
+ cursor_dir.mkdir(parents=True, exist_ok=True)
567
+ cursor_mcp = cursor_dir / "mcp.json"
568
+ try:
569
+ data = json.loads(cursor_mcp.read_text(encoding="utf-8")) if cursor_mcp.exists() else {}
570
+ if "mcpServers" not in data:
571
+ data["mcpServers"] = {}
572
+ data["mcpServers"]["tokenzero"] = mcp_config_entry
573
+ cursor_mcp.write_text(json.dumps(data, indent=2), encoding="utf-8")
574
+ results.append(("Cursor IDE", str(cursor_mcp), "[bold green]Configured[/bold green]"))
575
+ except Exception as e:
576
+ results.append(("Cursor IDE", str(cursor_mcp), f"[yellow]Skipped ({e})[/yellow]"))
577
+
578
+ # 2. Claude Desktop (~/.config/Claude/claude_desktop_config.json on Linux, ~/Library/Application Support/Claude on Mac)
579
+ if sys.platform == "darwin":
580
+ claude_desktop_dir = Path.home() / "Library" / "Application Support" / "Claude"
581
+ else:
582
+ claude_desktop_dir = Path.home() / ".config" / "Claude"
583
+
584
+ claude_desktop_dir.mkdir(parents=True, exist_ok=True)
585
+ claude_desktop_file = claude_desktop_dir / "claude_desktop_config.json"
586
+ try:
587
+ data = json.loads(claude_desktop_file.read_text(encoding="utf-8")) if claude_desktop_file.exists() else {}
588
+ if "mcpServers" not in data:
589
+ data["mcpServers"] = {}
590
+ data["mcpServers"]["tokenzero"] = mcp_config_entry
591
+ claude_desktop_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
592
+ results.append(("Claude Desktop", str(claude_desktop_file), "[bold green]Configured[/bold green]"))
593
+ except Exception as e:
594
+ results.append(("Claude Desktop", str(claude_desktop_file), f"[yellow]Skipped ({e})[/yellow]"))
595
+
596
+ # 3. Claude Code (~/.claude/settings.json)
597
+ claude_code_dir = Path.home() / ".claude"
598
+ if claude_code_dir.exists():
599
+ settings_file = claude_code_dir / "settings.json"
600
+ try:
601
+ data = json.loads(settings_file.read_text(encoding="utf-8")) if settings_file.exists() else {}
602
+ if "mcpServers" not in data:
603
+ data["mcpServers"] = {}
604
+ data["mcpServers"]["tokenzero"] = mcp_config_entry
605
+ settings_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
606
+ results.append(("Claude Code", str(settings_file), "[bold green]Configured[/bold green]"))
607
+ except Exception as e:
608
+ results.append(("Claude Code", str(settings_file), f"[yellow]Skipped ({e})[/yellow]"))
609
+ else:
610
+ results.append(("Claude Code", "~/.claude/", "[dim]Not installed[/dim]"))
611
+
612
+ # 4. Google Antigravity Global Plugin (~/.gemini/config/plugins/tokenzero/)
613
+ ok, path_or_err = ensure_antigravity_plugin()
614
+ if ok:
615
+ results.append(("Google Antigravity", path_or_err, "[bold green]Configured (Global Plugin)[/bold green]"))
616
+ else:
617
+ results.append(("Google Antigravity", "~/.gemini/config/plugins/", f"[yellow]Skipped ({path_or_err})[/yellow]"))
618
+
619
+ if not silent:
620
+ table = Table(title="⚡ [bold green]TokenZero Universal Integration Status[/bold green]", header_style="bold magenta")
621
+ table.add_column("AI Agent / IDE", style="cyan")
622
+ table.add_column("Configuration Target", style="dim")
623
+ table.add_column("Status", style="bold white")
624
+
625
+ for agent, path, status in results:
626
+ table.add_row(agent, path, status)
627
+
628
+ console.print(table)
629
+ console.print("\n[bold green]✨ All configured AI coding assistants now use TokenZero automatically![/bold green]")
630
+ console.print("[dim]Project repositories remain 100% clean with 0 files added.[/dim]\n")
631
+
632
+ return results
633
+
634
+ def remove_tokenzero(target_dir: Path):
635
+ """Completely removes TokenZero config, maps, rules, and hooks from a project."""
636
+ console.print(f"\n[bold yellow]🧹 Deactivating TokenZero from:[/bold yellow] {target_dir}")
637
+ removed_items = []
638
+
639
+ # 1. Remove directories
640
+ for d in (".tokenzero", ".tokensqueeze"):
641
+ folder = target_dir / d
642
+ if folder.exists():
643
+ shutil.rmtree(folder)
644
+ removed_items.append(f"{d}/")
645
+
646
+ # 2. Clean global cache for this project
647
+ try:
648
+ global_cache = get_project_cache_dir(target_dir)
649
+ if global_cache.exists():
650
+ shutil.rmtree(global_cache)
651
+ removed_items.append("global_cache")
652
+ except Exception:
653
+ pass
654
+
655
+ # 3. Clean all supported agent rule files
656
+ for cfg in AGENT_CONFIGS:
657
+ rule_path = target_dir / cfg["file"]
658
+ if rule_path.exists():
659
+ content = rule_path.read_text(encoding="utf-8")
660
+ cleaned = re.sub(
661
+ r"\n*<!-- BEGIN_TOKENZERO_RULE -->.*?<!-- END_TOKENZERO_RULE -->\n*",
662
+ "",
663
+ content,
664
+ flags=re.DOTALL
665
+ )
666
+ cleaned = re.sub(
667
+ r"\n*<!-- BEGIN_TOKENSQUEEZE_RULE -->.*?<!-- END_TOKENSQUEEZE_RULE -->\n*",
668
+ "",
669
+ cleaned,
670
+ flags=re.DOTALL
671
+ ).strip()
672
+
673
+ if cleaned:
674
+ rule_path.write_text(cleaned + "\n", encoding="utf-8")
675
+ removed_items.append(cfg["file"])
676
+ else:
677
+ rule_path.unlink()
678
+ removed_items.append(cfg["file"])
679
+ parent = rule_path.parent
680
+ if parent != target_dir and parent.exists() and not any(parent.iterdir()):
681
+ parent.rmdir()
682
+
683
+ # 3. Clean .agents/hooks.json
684
+ hooks_file = target_dir / ".agents" / "hooks.json"
685
+ if hooks_file.exists():
686
+ try:
687
+ data = json.loads(hooks_file.read_text(encoding="utf-8"))
688
+ data.pop("tokenzero-live", None)
689
+ data.pop("tokensqueeze-live", None)
690
+ if data:
691
+ hooks_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
692
+ removed_items.append(".agents/hooks.json")
693
+ else:
694
+ hooks_file.unlink()
695
+ removed_items.append(".agents/hooks.json")
696
+ agents_dir = target_dir / ".agents"
697
+ if agents_dir.exists() and not any(agents_dir.iterdir()):
698
+ agents_dir.rmdir()
699
+ except Exception:
700
+ pass
701
+
702
+ # 4. Clean .gitignore
703
+ gitignore_path = target_dir / ".gitignore"
704
+ if gitignore_path.exists():
705
+ try:
706
+ gi_content = gitignore_path.read_text(encoding="utf-8")
707
+ cleaned_gi = re.sub(r"\n*# TokenZero \(Auto-ignored\)\n\.tokenzero/\n\.tokensqueeze/\n*", "\n", gi_content).strip()
708
+ if cleaned_gi != gi_content.strip():
709
+ if cleaned_gi:
710
+ gitignore_path.write_text(cleaned_gi + "\n", encoding="utf-8")
711
+ else:
712
+ gitignore_path.unlink()
713
+ removed_items.append(".gitignore (rules removed)")
714
+ except Exception:
715
+ pass
716
+
717
+ if removed_items:
718
+ console.print(f"\n[bold green]✅ TokenZero optimizations deactivated successfully.[/bold green]")
719
+ console.print(f"[dim]Codebase restored to original state ({len(removed_items)} components cleaned).[/dim]\n")
720
+ else:
721
+ console.print(f"[cyan]ℹ️ No TokenZero configurations were found in {target_dir.name}.[/cyan]\n")
722
+
723
+ def remove_global_configs() -> list:
724
+ """Completely uninstalls TokenZero from all global AI editors, plugins, and cache."""
725
+ console.print("\n[bold yellow]🧹 Cleaning TokenZero Global AI Integrations & Cache...[/bold yellow]\n")
726
+ cleaned = []
727
+
728
+ # 1. Remove Google Antigravity Global Plugin
729
+ plugin_dir = Path.home() / ".gemini" / "config" / "plugins" / "tokenzero"
730
+ if plugin_dir.exists():
731
+ try:
732
+ shutil.rmtree(plugin_dir)
733
+ cleaned.append(("Google Antigravity", str(plugin_dir), "[bold green]Plugin Removed[/bold green]"))
734
+ except Exception as e:
735
+ cleaned.append(("Google Antigravity", str(plugin_dir), f"[yellow]Error ({e})[/yellow]"))
736
+ else:
737
+ cleaned.append(("Google Antigravity", str(plugin_dir), "[dim]Not found[/dim]"))
738
+
739
+ # 2. Remove from Cursor IDE
740
+ cursor_mcp = Path.home() / ".cursor" / "mcp.json"
741
+ if cursor_mcp.exists():
742
+ try:
743
+ data = json.loads(cursor_mcp.read_text(encoding="utf-8"))
744
+ if "mcpServers" in data and "tokenzero" in data["mcpServers"]:
745
+ del data["mcpServers"]["tokenzero"]
746
+ cursor_mcp.write_text(json.dumps(data, indent=2), encoding="utf-8")
747
+ cleaned.append(("Cursor IDE", str(cursor_mcp), "[bold green]MCP Removed[/bold green]"))
748
+ else:
749
+ cleaned.append(("Cursor IDE", str(cursor_mcp), "[dim]No entry[/dim]"))
750
+ except Exception as e:
751
+ cleaned.append(("Cursor IDE", str(cursor_mcp), f"[yellow]Error ({e})[/yellow]"))
752
+
753
+ # 3. Remove from Claude Desktop
754
+ claude_file = (Path.home() / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json") if sys.platform == "darwin" else (Path.home() / ".config" / "Claude" / "claude_desktop_config.json")
755
+ if claude_file.exists():
756
+ try:
757
+ data = json.loads(claude_file.read_text(encoding="utf-8"))
758
+ if "mcpServers" in data and "tokenzero" in data["mcpServers"]:
759
+ del data["mcpServers"]["tokenzero"]
760
+ claude_file.write_text(json.dumps(data, indent=2), encoding="utf-8")
761
+ cleaned.append(("Claude Desktop", str(claude_file), "[bold green]MCP Removed[/bold green]"))
762
+ else:
763
+ cleaned.append(("Claude Desktop", str(claude_file), "[dim]No entry[/dim]"))
764
+ except Exception as e:
765
+ cleaned.append(("Claude Desktop", str(claude_file), f"[yellow]Error ({e})[/yellow]"))
766
+
767
+ # 4. Remove from Claude Code
768
+ claude_settings = Path.home() / ".claude" / "settings.json"
769
+ if claude_settings.exists():
770
+ try:
771
+ data = json.loads(claude_settings.read_text(encoding="utf-8"))
772
+ if "mcpServers" in data and "tokenzero" in data["mcpServers"]:
773
+ del data["mcpServers"]["tokenzero"]
774
+ claude_settings.write_text(json.dumps(data, indent=2), encoding="utf-8")
775
+ cleaned.append(("Claude Code", str(claude_settings), "[bold green]MCP Removed[/bold green]"))
776
+ else:
777
+ cleaned.append(("Claude Code", str(claude_settings), "[dim]No entry[/dim]"))
778
+ except Exception as e:
779
+ cleaned.append(("Claude Code", str(claude_settings), f"[yellow]Error ({e})[/yellow]"))
780
+
781
+ # 5. Remove ~/.tokenzero/
782
+ if GLOBAL_DIR.exists():
783
+ try:
784
+ shutil.rmtree(GLOBAL_DIR)
785
+ cleaned.append(("Global Cache & Telemetry", str(GLOBAL_DIR), "[bold green]Deleted[/bold green]"))
786
+ except Exception as e:
787
+ cleaned.append(("Global Cache & Telemetry", str(GLOBAL_DIR), f"[yellow]Error ({e})[/yellow]"))
788
+
789
+ table = Table(title="🧹 [bold yellow]TokenZero Global Cleanup Status[/bold yellow]", header_style="bold magenta")
790
+ table.add_column("AI Agent / Resource", style="cyan")
791
+ table.add_column("Location", style="dim")
792
+ table.add_column("Result", style="bold white")
793
+
794
+ for agent, path, status in cleaned:
795
+ table.add_row(agent, path, status)
796
+
797
+ console.print(table)
798
+ console.print("\n[bold green]✨ All TokenZero global integrations and caches completely cleaned![/bold green]\n")
799
+ return cleaned
800
+
801
+ def main():
802
+ import argparse
803
+ parser = argparse.ArgumentParser(description="⚡ TokenZero - Zero Token Waste for AI Coding Agents")
804
+ subparsers = parser.add_subparsers(dest="command", help="Available commands")
805
+
806
+ # init command
807
+ init_parser = subparsers.add_parser("init", help="Index the current codebase in Zero-File stealth mode")
808
+ init_parser.add_argument("path", nargs="?", default=".", help="Target directory (default: current directory)")
809
+ init_parser.add_argument("--local", action="store_true", help="Generate local files in project instead of zero-file global stealth mode")
810
+
811
+ # setup command
812
+ subparsers.add_parser("setup", help="Automatically configure Claude Desktop, Claude Code, and Cursor IDE with TokenZero MCP")
813
+
814
+ # remove / uninstall command
815
+ remove_parser = subparsers.add_parser("remove", help="Remove TokenZero rules, hooks, and maps from the project")
816
+ remove_parser.add_argument("path", nargs="?", default=".", help="Target directory (default: current directory)")
817
+ remove_parser.add_argument("--global", "-g", dest="global_clean", action="store_true", help="Remove all global plugins, MCP configs, and cache across machine")
818
+
819
+ uninst_parser = subparsers.add_parser("uninstall", help="Alias for remove")
820
+ uninst_parser.add_argument("path", nargs="?", default=".", help="Target directory (default: current directory)")
821
+ uninst_parser.add_argument("--global", "-g", dest="global_clean", action="store_true", help="Remove all global plugins, MCP configs, and cache across machine")
822
+
823
+ # watch command
824
+ watch_parser = subparsers.add_parser("watch", help="Start continuous real-time monitor for AI activity")
825
+ watch_parser.add_argument("path", nargs="?", default=".", help="Target directory (default: current directory)")
826
+
827
+ # hook command
828
+ subparsers.add_parser("hook", help="Internal hook called on pre-tool execution")
829
+
830
+ # stats command
831
+ stats_parser = subparsers.add_parser("stats", help="Display token audit report")
832
+ stats_parser.add_argument("path", nargs="?", default=".", help="Target directory (default: current directory)")
833
+ stats_parser.add_argument("--once", action="store_true", help="Print once and exit instead of live watch")
834
+
835
+ # record command
836
+ record_parser = subparsers.add_parser("record", help="Record an AI action into the audit log")
837
+ record_parser.add_argument("--action", default="READ")
838
+ record_parser.add_argument("--file", required=True)
839
+ record_parser.add_argument("--function", required=True)
840
+ record_parser.add_argument("--lines-used", type=int, default=10)
841
+ record_parser.add_argument("--total-lines", type=int, default=100)
842
+ record_parser.add_argument("--dir", default=".")
843
+
844
+ # proxy command
845
+ subparsers.add_parser("proxy", help="Start the local token optimization proxy")
846
+
847
+ # mcp command
848
+ subparsers.add_parser("mcp", help="Start the TokenZero Model Context Protocol (MCP) stdio server")
849
+
850
+ args = parser.parse_args()
851
+
852
+ if args.command == "hook":
853
+ handle_hook()
854
+ elif args.command == "mcp":
855
+ from mcp_server import run_mcp_server
856
+ run_mcp_server()
857
+ elif args.command == "setup":
858
+ setup_tokenzero()
859
+ elif args.command in ("remove", "uninstall"):
860
+ if getattr(args, "global_clean", False):
861
+ remove_global_configs()
862
+ else:
863
+ target_path = Path(getattr(args, "path", ".")).resolve()
864
+ remove_tokenzero(target_path)
865
+ elif args.command == "proxy":
866
+ import uvicorn
867
+ from proxy import app
868
+ console.print("[bold green]🚀 Starting TokenZero Proxy on http://127.0.0.1:8080[/bold green]")
869
+ uvicorn.run(app, host="127.0.0.1", port=8080, log_level="warning")
870
+ elif args.command == "watch":
871
+ target_path = Path(getattr(args, "path", ".")).resolve()
872
+ watch_dashboard(target_path)
873
+ elif args.command == "stats":
874
+ target_path = Path(getattr(args, "path", ".")).resolve()
875
+ if args.once:
876
+ show_stats_once(target_path)
877
+ else:
878
+ watch_dashboard(target_path)
879
+ elif args.command == "record":
880
+ target_path = Path(args.dir).resolve()
881
+ record_activity(target_path, args.action, args.file, args.function, args.lines_used, args.total_lines)
882
+ else:
883
+ target_path = Path(getattr(args, "path", ".")).resolve()
884
+ init_tokenzero(target_path, local=getattr(args, "local", False))
885
+
886
+ if __name__ == "__main__":
887
+ main()