tokenzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mcp_server.py ADDED
@@ -0,0 +1,346 @@
1
+ """
2
+ TokenZero MCP Server
3
+ Provides Model Context Protocol (MCP) tools over stdio for Claude Desktop, Claude Code, Cursor, and other AI agents.
4
+ """
5
+
6
+ import sys
7
+ import json
8
+ from pathlib import Path
9
+ from datetime import datetime
10
+ from scanner import scan_project, generate_markdown_map
11
+
12
+ INTERNAL_PATTERNS = (
13
+ ".tokenzero",
14
+ ".tokensqueeze",
15
+ "repo_map.md",
16
+ "audit.jsonl",
17
+ "hooks.json",
18
+ "gemini.md",
19
+ "claude.md",
20
+ ".cursorrules",
21
+ ".windsurfrules",
22
+ "copilot-instructions",
23
+ ".agents"
24
+ )
25
+
26
+ def is_internal_file(file_path: str) -> bool:
27
+ if not file_path:
28
+ return False
29
+ lower = str(file_path).lower().replace("\\", "/")
30
+ return any(pat in lower for pat in INTERNAL_PATTERNS)
31
+
32
+ GLOBAL_DIR = Path.home() / ".tokenzero"
33
+ GLOBAL_CACHE_DIR = GLOBAL_DIR / "cache"
34
+
35
+ def get_project_hash(target_dir: Path) -> str:
36
+ import hashlib
37
+ resolved = str(target_dir.resolve())
38
+ h = hashlib.sha256(resolved.encode("utf-8")).hexdigest()[:12]
39
+ safe_name = "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in target_dir.name)
40
+ return f"{safe_name}_{h}"
41
+
42
+ def get_project_cache_dir(target_dir: Path) -> Path:
43
+ p_dir = GLOBAL_CACHE_DIR / get_project_hash(target_dir)
44
+ p_dir.mkdir(parents=True, exist_ok=True)
45
+ return p_dir
46
+
47
+ def get_repo_map_path(target_dir: Path) -> Path:
48
+ local_map = target_dir / ".tokenzero" / "REPO_MAP.md"
49
+ if local_map.exists():
50
+ return local_map
51
+ local_ts = target_dir / ".tokensqueeze" / "REPO_MAP.md"
52
+ if local_ts.exists():
53
+ return local_ts
54
+ return get_project_cache_dir(target_dir) / "REPO_MAP.md"
55
+
56
+ def record_activity(target_dir: Path, action: str, file: str, function: str, lines_used: int, total_lines: int):
57
+ if is_internal_file(file):
58
+ return
59
+
60
+ GLOBAL_DIR.mkdir(parents=True, exist_ok=True)
61
+ audit_file = GLOBAL_DIR / "audit.jsonl"
62
+ tokens_used = lines_used * 7
63
+ lines_saved = max(0, total_lines - lines_used)
64
+ tokens_saved = lines_saved * 7
65
+
66
+ entry = {
67
+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
68
+ "project": str(target_dir.resolve()),
69
+ "project_name": target_dir.name,
70
+ "action": action.upper(),
71
+ "file": file,
72
+ "function": function,
73
+ "lines_used": lines_used,
74
+ "total_file_lines": total_lines,
75
+ "tokens_used": tokens_used,
76
+ "tokens_saved": tokens_saved
77
+ }
78
+ with open(audit_file, "a", encoding="utf-8") as f:
79
+ f.write(json.dumps(entry) + "\n")
80
+
81
+ TOOLS = [
82
+ {
83
+ "name": "tokenzero_get_repo_map",
84
+ "description": "Returns the high-density AST skeleton map of the repository showing all classes, functions, and line ranges. Call this first before reading files to identify exact line ranges.",
85
+ "inputSchema": {
86
+ "type": "object",
87
+ "properties": {
88
+ "project_path": {
89
+ "type": "string",
90
+ "description": "Path to the project root directory (default: current directory)"
91
+ }
92
+ }
93
+ }
94
+ },
95
+ {
96
+ "name": "tokenzero_read_lines",
97
+ "description": "Reads specific line ranges of a file (StartLine to EndLine) instead of loading the full file, saving up to 95% of tokens. Automatically records token savings into the live audit monitor.",
98
+ "inputSchema": {
99
+ "type": "object",
100
+ "properties": {
101
+ "file_path": {
102
+ "type": "string",
103
+ "description": "Relative or absolute path to the file to inspect"
104
+ },
105
+ "start_line": {
106
+ "type": "integer",
107
+ "description": "Starting line number (1-based, inclusive)"
108
+ },
109
+ "end_line": {
110
+ "type": "integer",
111
+ "description": "Ending line number (inclusive)"
112
+ }
113
+ },
114
+ "required": ["file_path", "start_line", "end_line"]
115
+ }
116
+ },
117
+ {
118
+ "name": "tokenzero_get_stats",
119
+ "description": "Returns cumulative TokenZero metrics: total tokens used, tokens saved, net reduction %, and dollar ROI.",
120
+ "inputSchema": {
121
+ "type": "object",
122
+ "properties": {
123
+ "project_path": {
124
+ "type": "string",
125
+ "description": "Path to the project root directory (default: current directory)"
126
+ }
127
+ }
128
+ }
129
+ }
130
+ ]
131
+
132
+ def handle_get_repo_map(args: dict) -> str:
133
+ path_str = args.get("project_path", ".")
134
+ target_dir = Path(path_str).resolve()
135
+ map_file = get_repo_map_path(target_dir)
136
+
137
+ if map_file.exists():
138
+ content = map_file.read_text(encoding="utf-8")
139
+ return f"# TokenZero AST Map (Cached)\n\n{content}"
140
+
141
+ # Generate on the fly into global cache (Zero files in project directory!)
142
+ scan_results = scan_project(target_dir)
143
+ if not scan_results:
144
+ return "⚠️ No supported source files (.py, .js, .ts, .go) found in project."
145
+
146
+ map_content = generate_markdown_map(scan_results)
147
+ cache_dir = get_project_cache_dir(target_dir)
148
+ cache_map = cache_dir / "REPO_MAP.md"
149
+ cache_map.write_text(map_content, encoding="utf-8")
150
+ return f"# TokenZero AST Map (Generated)\n\n{map_content}"
151
+
152
+ def handle_read_lines(args: dict) -> str:
153
+ file_path_str = args.get("file_path", "")
154
+ start_line = int(args.get("start_line", 1))
155
+ end_line = int(args.get("end_line", 100))
156
+
157
+ target_file = Path(file_path_str).resolve()
158
+ if not target_file.exists():
159
+ return f"Error: File '{file_path_str}' does not exist."
160
+
161
+ try:
162
+ all_lines = target_file.read_text(encoding="utf-8", errors="replace").splitlines()
163
+ except Exception as e:
164
+ return f"Error reading file: {e}"
165
+
166
+ total_lines = len(all_lines)
167
+ clamped_start = max(1, start_line)
168
+ clamped_end = min(total_lines, end_line)
169
+
170
+ lines_slice = all_lines[clamped_start - 1 : clamped_end]
171
+ lines_used = len(lines_slice)
172
+
173
+ # Format numbered lines
174
+ output_lines = [f"{i + clamped_start:4d} | {line}" for i, line in enumerate(lines_slice)]
175
+ result_text = "\n".join(output_lines)
176
+
177
+ # Log to audit
178
+ project_root = target_file.parent
179
+ for parent in [target_file.parent] + list(target_file.parents):
180
+ if (parent / ".tokenzero").exists() or (parent / ".git").exists():
181
+ project_root = parent
182
+ break
183
+
184
+ try:
185
+ rel_path = str(target_file.relative_to(project_root))
186
+ except Exception:
187
+ rel_path = target_file.name
188
+
189
+ record_activity(
190
+ target_dir=project_root,
191
+ action="READ",
192
+ file=rel_path,
193
+ function=f"L{clamped_start}-L{clamped_end}",
194
+ lines_used=lines_used,
195
+ total_lines=total_lines
196
+ )
197
+
198
+ tokens_used = lines_used * 7
199
+ tokens_saved = max(0, total_lines - lines_used) * 7
200
+
201
+ header = f"[{rel_path} | Lines {clamped_start}-{clamped_end} of {total_lines} | Tokens: ~{tokens_used} used, ~{tokens_saved} saved]\n"
202
+ return header + result_text
203
+
204
+ def handle_get_stats(args: dict) -> str:
205
+ path_str = args.get("project_path", ".")
206
+ target_dir = Path(path_str).resolve()
207
+ target_resolved = str(target_dir)
208
+ target_name = target_dir.name
209
+
210
+ entries = []
211
+
212
+ # 1. Global audit log
213
+ global_audit = GLOBAL_DIR / "audit.jsonl"
214
+ if global_audit.exists():
215
+ for line in global_audit.read_text(encoding="utf-8").splitlines():
216
+ if line.strip():
217
+ try:
218
+ e = json.loads(line)
219
+ if e.get("project") == target_resolved or e.get("project_name") == target_name:
220
+ entries.append(e)
221
+ except Exception:
222
+ pass
223
+
224
+ # 2. Local fallback
225
+ for folder in (".tokenzero", ".tokensqueeze"):
226
+ local_audit = target_dir / folder / "audit.jsonl"
227
+ if local_audit.exists():
228
+ for line in local_audit.read_text(encoding="utf-8").splitlines():
229
+ if line.strip():
230
+ try:
231
+ entries.append(json.loads(line))
232
+ except Exception:
233
+ pass
234
+
235
+ if not entries:
236
+ return "No audit logs found for this project yet. Start coding with your AI agent to see live token savings!"
237
+
238
+ total_used = sum(e.get("tokens_used", 0) for e in entries)
239
+ total_saved = sum(e.get("tokens_saved", 0) for e in entries)
240
+ without_tokens = total_used + total_saved
241
+ pct = (total_saved / without_tokens * 100) if without_tokens > 0 else 0
242
+ dollars_saved = (total_saved / 1_000_000) * 2.50
243
+
244
+ return (
245
+ f"📊 TokenZero Session Stats:\n"
246
+ f"• Total Events: {len(entries)}\n"
247
+ f"• Cost Without TokenZero: {without_tokens:,} tokens\n"
248
+ f"• Used With TokenZero: {total_used:,} tokens\n"
249
+ f"• Net Saved: +{total_saved:,} tokens ({pct:.1f}% reduction)\n"
250
+ f"• Estimated Money Saved: ${dollars_saved:.4f} USD"
251
+ )
252
+
253
+ def run_mcp_server():
254
+ """Runs standard MCP stdio JSON-RPC loop."""
255
+ while True:
256
+ line = sys.stdin.readline()
257
+ if not line:
258
+ break
259
+ line = line.strip()
260
+ if not line:
261
+ continue
262
+
263
+ try:
264
+ req = json.loads(line)
265
+ except Exception:
266
+ continue
267
+
268
+ req_id = req.get("id")
269
+ method = req.get("method")
270
+ params = req.get("params", {})
271
+
272
+ if method == "initialize":
273
+ res = {
274
+ "jsonrpc": "2.0",
275
+ "id": req_id,
276
+ "result": {
277
+ "protocolVersion": "2024-11-05",
278
+ "capabilities": {
279
+ "tools": {}
280
+ },
281
+ "serverInfo": {
282
+ "name": "tokenzero",
283
+ "version": "0.1.0"
284
+ }
285
+ }
286
+ }
287
+ sys.stdout.write(json.dumps(res) + "\n")
288
+ sys.stdout.flush()
289
+
290
+ elif method == "notifications/initialized":
291
+ pass
292
+
293
+ elif method == "tools/list":
294
+ res = {
295
+ "jsonrpc": "2.0",
296
+ "id": req_id,
297
+ "result": {
298
+ "tools": TOOLS
299
+ }
300
+ }
301
+ sys.stdout.write(json.dumps(res) + "\n")
302
+ sys.stdout.flush()
303
+
304
+ elif method == "tools/call":
305
+ tool_name = params.get("name")
306
+ tool_args = params.get("arguments", {})
307
+
308
+ if tool_name == "tokenzero_get_repo_map":
309
+ text = handle_get_repo_map(tool_args)
310
+ elif tool_name == "tokenzero_read_lines":
311
+ text = handle_read_lines(tool_args)
312
+ elif tool_name == "tokenzero_get_stats":
313
+ text = handle_get_stats(tool_args)
314
+ else:
315
+ text = f"Unknown tool: {tool_name}"
316
+
317
+ res = {
318
+ "jsonrpc": "2.0",
319
+ "id": req_id,
320
+ "result": {
321
+ "content": [
322
+ {
323
+ "type": "text",
324
+ "text": text
325
+ }
326
+ ]
327
+ }
328
+ }
329
+ sys.stdout.write(json.dumps(res) + "\n")
330
+ sys.stdout.flush()
331
+
332
+ elif req_id is not None:
333
+ res = {
334
+ "jsonrpc": "2.0",
335
+ "id": req_id,
336
+ "error": {
337
+ "code": -32601,
338
+ "message": f"Method '{method}' not found"
339
+ }
340
+ }
341
+ sys.stdout.write(json.dumps(res) + "\n")
342
+ sys.stdout.flush()
343
+
344
+ if __name__ == "__main__":
345
+ run_mcp_server()
346
+
proxy.py ADDED
@@ -0,0 +1,205 @@
1
+ """
2
+ TokenSqueeze - The Drop-In AI Token Optimizer & Context Garbage Collector
3
+ Run this proxy locally, point your OpenAI client to http://localhost:8080/v1,
4
+ and watch your prompt token bills drop by 50% to 80%!
5
+ """
6
+
7
+ import os
8
+ import re
9
+ import json
10
+ import time
11
+ import httpx
12
+ import tiktoken
13
+ from typing import Dict, Any, List
14
+ from fastapi import FastAPI, Request, Response
15
+ from fastapi.responses import JSONResponse
16
+ from rich.console import Console
17
+ from rich.panel import Panel
18
+ from rich.table import Table
19
+
20
+ app = FastAPI(title="TokenSqueeze Proxy", version="0.1.0")
21
+ console = Console()
22
+
23
+ # Cache tokenizer encoding
24
+ try:
25
+ default_encoder = tiktoken.get_encoding("cl100k_base")
26
+ except Exception:
27
+ default_encoder = None
28
+
29
+ def count_tokens(messages: List[Dict[str, Any]], model: str = "gpt-4o") -> int:
30
+ """Accurately count tokens across a messages array."""
31
+ if not default_encoder:
32
+ # Fallback heuristic: ~4 chars per token
33
+ total_chars = sum(len(str(m.get("content", ""))) for m in messages)
34
+ return max(1, total_chars // 4)
35
+
36
+ num_tokens = 0
37
+ for message in messages:
38
+ num_tokens += 3 # every message follows <|start|>{role/name}\n{content}<|end|>\n
39
+ for key, value in message.items():
40
+ if isinstance(value, str):
41
+ num_tokens += len(default_encoder.encode(value))
42
+ elif isinstance(value, list) or isinstance(value, dict):
43
+ num_tokens += len(default_encoder.encode(json.dumps(value)))
44
+ num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
45
+ return num_tokens
46
+
47
+ def clean_whitespace(text: str) -> str:
48
+ """Strips useless blank lines, trailing spaces, and excessive tabs."""
49
+ lines = [line.rstrip() for line in text.splitlines()]
50
+ joined = "\n".join(lines)
51
+ # Collapse 3 or more consecutive newlines into 2
52
+ return re.sub(r'\n{3,}', '\n\n', joined).strip()
53
+
54
+ def compact_json_in_text(text: str) -> str:
55
+ """If text contains pretty-printed JSON, compacts it to single-line JSON."""
56
+ if text.strip().startswith(("{", "[")) and text.strip().endswith(("}", "]")):
57
+ try:
58
+ parsed = json.loads(text)
59
+ return json.dumps(parsed, separators=(',', ':'))
60
+ except Exception:
61
+ pass
62
+ return text
63
+
64
+ def prune_agent_history(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
65
+ """
66
+ Garbage Collects old tool outputs & intermediate thoughts:
67
+ If a tool output or verbose assistant log is older than 3 turns,
68
+ prunes the massive payload and replaces it with a compact summary.
69
+ """
70
+ total = len(messages)
71
+ optimized = []
72
+
73
+ for i, msg in enumerate(messages):
74
+ msg_copy = dict(msg)
75
+ role = msg_copy.get("role")
76
+ content = msg_copy.get("content", "")
77
+ turns_ago = total - 1 - i
78
+
79
+ if isinstance(content, str):
80
+ # 1. Clean whitespace & compact any JSON
81
+ content = clean_whitespace(content)
82
+ content = compact_json_in_text(content)
83
+
84
+ # 2. Tool GC: If 2 or more turns in the past and longer than 250 chars, prune!
85
+ if (role == "tool" or role == "function") and turns_ago >= 2 and len(content) > 250:
86
+ original_len = len(content)
87
+ preview = content[:80].replace("\n", " ")
88
+ content = f"[Pruned tool output ({original_len} chars): '{preview}...' -> outcome processed in subsequent turns]"
89
+
90
+ msg_copy["content"] = content
91
+
92
+ optimized.append(msg_copy)
93
+
94
+ return optimized
95
+
96
+ def align_kv_cache(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
97
+ """
98
+ Ensures system instructions stay strictly at index 0 for maximum
99
+ provider KV-cache hits (50-80% discount on OpenAI / Claude / DeepSeek).
100
+ """
101
+ system_msgs = [m for m in messages if m.get("role") == "system"]
102
+ other_msgs = [m for m in messages if m.get("role") != "system"]
103
+ return system_msgs + other_msgs
104
+
105
+ def print_savings_banner(before: int, after: int, model: str, duration_ms: float):
106
+ """Prints a beautiful rich banner to the terminal."""
107
+ saved = max(0, before - after)
108
+ pct = (saved / before * 100) if before > 0 else 0.0
109
+
110
+ # Pricing estimate ($2.50 per 1M tokens for gpt-4o input)
111
+ dollars = (saved / 1_000_000) * 2.50
112
+
113
+ table = Table(title="⚡ [bold green]TokenSqueeze Optimization[/bold green]", show_header=True, header_style="bold magenta")
114
+ table.add_column("Metric", style="cyan", width=20)
115
+ table.add_column("Value", style="bold white", width=25)
116
+
117
+ table.add_row("Model", f"[yellow]{model}[/yellow]")
118
+ table.add_row("Original Tokens", f"[red]{before:,}[/red]")
119
+ table.add_row("Optimized Tokens", f"[green]{after:,}[/green]")
120
+ table.add_row("Tokens Saved", f"[bold green]-{saved:,} (-{pct:.1f}%)[/bold green]")
121
+ table.add_row("Estimated Savings", f"[bold green]${dollars:.5f}[/bold green]")
122
+ table.add_row("Optimizer Latency", f"[blue]{duration_ms:.2f} ms[/blue]")
123
+
124
+ console.print()
125
+ console.print(table)
126
+ console.print()
127
+
128
+ @app.post("/v1/chat/completions")
129
+ async def chat_completions(request: Request):
130
+ start_time = time.perf_counter()
131
+ body = await request.json()
132
+
133
+ model = body.get("model", "gpt-4o")
134
+ raw_messages = body.get("messages", [])
135
+
136
+ # Count tokens before optimization
137
+ before_tokens = count_tokens(raw_messages, model=model)
138
+
139
+ # Apply optimizations
140
+ optimized_messages = prune_agent_history(raw_messages)
141
+ optimized_messages = align_kv_cache(optimized_messages)
142
+
143
+ # Count tokens after optimization
144
+ after_tokens = count_tokens(optimized_messages, model=model)
145
+ duration_ms = (time.perf_counter() - start_time) * 1000
146
+
147
+ # Print hype stats
148
+ print_savings_banner(before_tokens, after_tokens, model, duration_ms)
149
+
150
+ # Update body with optimized messages
151
+ body["messages"] = optimized_messages
152
+
153
+ # Check for OpenAI Auth header or environment variable
154
+ auth_header = request.headers.get("Authorization", "")
155
+ api_key = os.environ.get("OPENAI_API_KEY", "")
156
+ if auth_header.startswith("Bearer "):
157
+ api_key = auth_header.replace("Bearer ", "").strip()
158
+
159
+ # If no real API key is supplied, return a realistic mock response for zero-cost testing!
160
+ if not api_key or api_key == "sk-mock-key":
161
+ return JSONResponse({
162
+ "id": "chatcmpl-tokensqueeze-mock",
163
+ "object": "chat.completion",
164
+ "created": int(time.time()),
165
+ "model": model,
166
+ "choices": [{
167
+ "index": 0,
168
+ "message": {
169
+ "role": "assistant",
170
+ "content": f"Hello! TokenSqueeze intercepted and optimized your prompt. Saved {before_tokens - after_tokens:,} tokens ({((before_tokens - after_tokens)/before_tokens)*100:.1f}%) in {duration_ms:.2f}ms!"
171
+ },
172
+ "finish_reason": "stop"
173
+ }],
174
+ "usage": {
175
+ "prompt_tokens": after_tokens,
176
+ "completion_tokens": 35,
177
+ "total_tokens": after_tokens + 35
178
+ }
179
+ })
180
+
181
+ # Forward to real OpenAI API
182
+ headers = {
183
+ "Authorization": f"Bearer {api_key}",
184
+ "Content-Type": "application/json"
185
+ }
186
+
187
+ async with httpx.AsyncClient(timeout=60.0) as client:
188
+ try:
189
+ resp = await client.post(
190
+ "https://api.openai.com/v1/chat/completions",
191
+ json=body,
192
+ headers=headers
193
+ )
194
+ return Response(content=resp.content, status_code=resp.status_code, media_type=resp.headers.get("content-type"))
195
+ except Exception as e:
196
+ return JSONResponse({"error": str(e)}, status_code=500)
197
+
198
+ @app.get("/")
199
+ def root():
200
+ return {"status": "online", "message": "TokenSqueeze Proxy is running! Point your OpenAI client base_url to /v1"}
201
+
202
+ if __name__ == "__main__":
203
+ import uvicorn
204
+ console.print("[bold green]🚀 Starting TokenSqueeze Proxy on http://127.0.0.1:8080[/bold green]")
205
+ uvicorn.run(app, host="127.0.0.1", port=8080, log_level="warning")