tokenjar 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokenjar-1.0.1/.gitignore +50 -0
- tokenjar-1.0.1/LICENSE +41 -0
- tokenjar-1.0.1/PKG-INFO +474 -0
- tokenjar-1.0.1/README.md +441 -0
- tokenjar-1.0.1/pyproject.toml +77 -0
- tokenjar-1.0.1/src/tokenjar/__init__.py +3 -0
- tokenjar-1.0.1/src/tokenjar/__main__.py +629 -0
- tokenjar-1.0.1/src/tokenjar/cache/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/cache/persistent_cache.py +371 -0
- tokenjar-1.0.1/src/tokenjar/cache/session_cache.py +259 -0
- tokenjar-1.0.1/src/tokenjar/config.py +195 -0
- tokenjar-1.0.1/src/tokenjar/filters/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/filters/ansi.py +11 -0
- tokenjar-1.0.1/src/tokenjar/filters/build_tools.py +95 -0
- tokenjar-1.0.1/src/tokenjar/filters/git.py +23 -0
- tokenjar-1.0.1/src/tokenjar/filters/lockfile.py +275 -0
- tokenjar-1.0.1/src/tokenjar/filters/test_runners.py +137 -0
- tokenjar-1.0.1/src/tokenjar/hooks/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/hooks/manager.py +870 -0
- tokenjar-1.0.1/src/tokenjar/parsers/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/parsers/languages.py +66 -0
- tokenjar-1.0.1/src/tokenjar/rules/manager.py +297 -0
- tokenjar-1.0.1/src/tokenjar/server.py +81 -0
- tokenjar-1.0.1/src/tokenjar/telemetry/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/telemetry/stats.py +293 -0
- tokenjar-1.0.1/src/tokenjar/tools/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/tools/output_pruner.py +178 -0
- tokenjar-1.0.1/src/tokenjar/tools/repo_map.py +493 -0
- tokenjar-1.0.1/src/tokenjar/tools/skeleton.py +248 -0
- tokenjar-1.0.1/src/tokenjar/tools/smart_reader.py +171 -0
- tokenjar-1.0.1/src/tokenjar/tools/symbol_index.py +501 -0
- tokenjar-1.0.1/src/tokenjar/ui/__init__.py +11 -0
- tokenjar-1.0.1/src/tokenjar/ui/server.py +408 -0
- tokenjar-1.0.1/src/tokenjar/ui/static/index.html +773 -0
- tokenjar-1.0.1/src/tokenjar/utils/__init__.py +1 -0
- tokenjar-1.0.1/src/tokenjar/utils/file_utils.py +250 -0
- tokenjar-1.0.1/src/tokenjar/utils/token_counter.py +87 -0
- tokenjar-1.0.1/tests/conftest.py +6 -0
- tokenjar-1.0.1/tests/test_config.py +107 -0
- tokenjar-1.0.1/tests/test_hooks.py +174 -0
- tokenjar-1.0.1/tests/test_lockfile_shield.py +248 -0
- tokenjar-1.0.1/tests/test_output_pruner.py +196 -0
- tokenjar-1.0.1/tests/test_real_life_50_steps.py +424 -0
- tokenjar-1.0.1/tests/test_repo_map.py +76 -0
- tokenjar-1.0.1/tests/test_resources_prompts.py +29 -0
- tokenjar-1.0.1/tests/test_rules.py +110 -0
- tokenjar-1.0.1/tests/test_semantic_diff.py +25 -0
- tokenjar-1.0.1/tests/test_skeleton.py +95 -0
- tokenjar-1.0.1/tests/test_slash_commands.py +76 -0
- tokenjar-1.0.1/tests/test_smart_reader.py +158 -0
- tokenjar-1.0.1/tests/test_symbol_index.py +207 -0
- tokenjar-1.0.1/tests/test_telemetry.py +65 -0
- tokenjar-1.0.1/tests/test_token_counter.py +28 -0
- tokenjar-1.0.1/tests/test_ui_server.py +127 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
*.egg-info/
|
|
7
|
+
*.egg
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
.eggs/
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
env/
|
|
16
|
+
|
|
17
|
+
# IDE
|
|
18
|
+
.idea/
|
|
19
|
+
.vscode/
|
|
20
|
+
*.swp
|
|
21
|
+
*.swo
|
|
22
|
+
*~
|
|
23
|
+
|
|
24
|
+
# Testing
|
|
25
|
+
.pytest_cache/
|
|
26
|
+
.coverage
|
|
27
|
+
htmlcov/
|
|
28
|
+
.mypy_cache/
|
|
29
|
+
.ruff_cache/
|
|
30
|
+
|
|
31
|
+
# Local Scratch and Backups
|
|
32
|
+
scratch/
|
|
33
|
+
ROADMAP.md
|
|
34
|
+
*.ts_bak
|
|
35
|
+
*.db
|
|
36
|
+
*.db-wal
|
|
37
|
+
*.db-shm
|
|
38
|
+
telemetry.json
|
|
39
|
+
|
|
40
|
+
# Rust
|
|
41
|
+
target/
|
|
42
|
+
**/*.rs.bk
|
|
43
|
+
|
|
44
|
+
# OS
|
|
45
|
+
.DS_Store
|
|
46
|
+
Thumbs.db
|
|
47
|
+
|
|
48
|
+
# Assets (Local design prototypes)
|
|
49
|
+
assets/
|
|
50
|
+
|
tokenjar-1.0.1/LICENSE
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Business Source License 1.1
|
|
2
|
+
|
|
3
|
+
Parameters
|
|
4
|
+
|
|
5
|
+
Licensor:
|
|
6
|
+
Ömer Faruk Eskitürk
|
|
7
|
+
|
|
8
|
+
Software:
|
|
9
|
+
TokenJar
|
|
10
|
+
|
|
11
|
+
Change Date:
|
|
12
|
+
2030-01-01
|
|
13
|
+
|
|
14
|
+
Change License:
|
|
15
|
+
Apache License, Version 2.0
|
|
16
|
+
|
|
17
|
+
Additional Use Grant:
|
|
18
|
+
You may make use of the Software, unmodified or modified, for all personal, educational, research, evaluation, and internal business operations without charge, provided that you do not provide the Software, or any substantial portion of its core token optimization functionality, as a commercial hosted service, cloud service, or paid software distribution that competes directly with the Licensor.
|
|
19
|
+
|
|
20
|
+
Notice:
|
|
21
|
+
The Business Source License 1.1 (the "License") is not an Open Source Initiative (OSI) approved license. However, the Licensed Work will automatically transition to an OSI-approved license (Apache License, Version 2.0) on the Change Date.
|
|
22
|
+
|
|
23
|
+
================================================================================
|
|
24
|
+
|
|
25
|
+
License Text
|
|
26
|
+
|
|
27
|
+
The Licensor hereby grants you the right to copy, modify, create derivative works, redistribute, and make non-production and production use of the Software, but only for the purposes and subject to the restrictions in the Additional Use Grant.
|
|
28
|
+
|
|
29
|
+
Effective on the Change Date, or the fourth anniversary of the first publicly available distribution of a specific version of the Software under this License, whichever comes first, the Licensor hereby grants you rights under the terms of the Change License, and the restrictions of the Additional Use Grant shall no longer apply.
|
|
30
|
+
|
|
31
|
+
1. COPYING, MODIFICATION, AND REDISTRIBUTION:
|
|
32
|
+
You may copy, modify, and redistribute the Software in source code or binary form, provided that you comply with the terms of the Additional Use Grant, and that you retain all copyright, trademark, and other proprietary notices in the Software, as well as this License text.
|
|
33
|
+
|
|
34
|
+
2. CHANGE LICENSE TRANSITION:
|
|
35
|
+
On the Change Date, the Software shall automatically and irrevocably transition to the Change License (Apache License, Version 2.0). All copies and derivative works made on or after the Change Date shall be governed exclusively by the Change License.
|
|
36
|
+
|
|
37
|
+
3. TRADEMARKS:
|
|
38
|
+
The name "TokenJar", associated logos, and brand identity are trademarks of Ömer Faruk Eskitürk. This License does not grant any rights to use these trademarks in connection with any distribution, service, or commercial offering without prior written authorization.
|
|
39
|
+
|
|
40
|
+
4. DISCLAIMER OF WARRANTY:
|
|
41
|
+
TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED "AS IS" AND "AS AVAILABLE", WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL THE LICENSOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
tokenjar-1.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tokenjar
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: MCP server that saves 70-95% tokens for AI coding assistants without losing functionality
|
|
5
|
+
Project-URL: Homepage, https://github.com/Farukes/TokenJar
|
|
6
|
+
Project-URL: Repository, https://github.com/Farukes/TokenJar
|
|
7
|
+
Project-URL: Issues, https://github.com/Farukes/TokenJar/issues
|
|
8
|
+
Author-email: Ömer Faruk Eskitürk <omereskiturk2@gmail.com>
|
|
9
|
+
License: BSL-1.1
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai,coding-assistant,llm,mcp,optimization,token,tree-sitter
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: Other/Proprietary License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: fastmcp>=2.0
|
|
23
|
+
Requires-Dist: tomli>=1.1.0; python_version < '3.11'
|
|
24
|
+
Requires-Dist: tree-sitter-languages>=1.10
|
|
25
|
+
Requires-Dist: tree-sitter<0.22,>=0.21
|
|
26
|
+
Provides-Extra: counting
|
|
27
|
+
Requires-Dist: tiktoken>=0.7; extra == 'counting'
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
<p align="right">
|
|
35
|
+
<a href="README.md"><b>English</b></a> | <a href="README.tr.md"><b>Türkçe</b></a>
|
|
36
|
+
</p>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<img src="docs/images/tokenjar_logo.jpg" alt="TokenJar Logo" width="220" style="border-radius: 16px;" />
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
<h1 align="center">🍯 TokenJar</h1>
|
|
43
|
+
<p align="center"><b>Put tokens back in your jar. Save 70-95% tokens for AI coding assistants without losing functionality.</b></p>
|
|
44
|
+
|
|
45
|
+
[](https://github.com/Farukes/TokenJar/releases/latest)
|
|
46
|
+
[](https://github.com/Farukes/TokenJar/actions/workflows/ci.yml)
|
|
47
|
+
[](https://www.python.org/downloads/)
|
|
48
|
+
[](#-enterprise--high-performance-native-engine-rust-edition)
|
|
49
|
+
[](https://crates.io/crates/tokenjar)
|
|
50
|
+
[](https://pypi.org/project/tokenjar/)
|
|
51
|
+
[](#-proven-performance--stress-test-benchmark)
|
|
52
|
+
[](LICENSE)
|
|
53
|
+
[-success.svg)](#-enterprise-privacy--security-guarantee)
|
|
54
|
+
|
|
55
|
+
**MCP server that saves 70-95% tokens for AI coding assistants — without losing functionality.**
|
|
56
|
+
|
|
57
|
+
TokenJar sits between your AI coding assistant and your codebase, intelligently compressing code reads, terminal outputs, and file operations to dramatically reduce token consumption, context compaction, and latency.
|
|
58
|
+
|
|
59
|
+
Works with **Claude Code**, **Cursor**, **Antigravity (AGY)**, **Windsurf**, **Continue.dev**, and any MCP-compatible AI assistant.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## ✨ Features & Architecture
|
|
64
|
+
|
|
65
|
+
| Module | What It Does | Token Savings |
|
|
66
|
+
|:---|:---|:---|
|
|
67
|
+
| 🦴 **Code Skeletonizer** | Extracts structural skeleton (signatures, types, docstrings) via Tree-sitter AST | **80-95%** |
|
|
68
|
+
| 📖 **Smart File Reader** | L1 RAM + L2 Persistent SQLite cache with differential reads & diff headers | **90-99%** |
|
|
69
|
+
| 🛡️ **Lockfile & Asset Shield** | Intercepts massive lockfiles & minified bundles with surgical version queries (`query="react"`) | **99%** |
|
|
70
|
+
| 🎯 **Blast Radius & Symbols** | Instant global symbol lookup & cross-file reference caller tracking (`find_symbol_references`) | **85-95%** |
|
|
71
|
+
| 🖥️ **Terminal Pruner** | Compresses test/build/git terminal streams, keeps errors and summary info | **60-90%** |
|
|
72
|
+
| 🗺️ **Repo Map** | PageRank & Graph Centrality codebase overview fitted into custom token budgets | **Budget-fitted** |
|
|
73
|
+
| 🎨 **On-Demand UI Dashboard** | Lightweight standalone control panel (`tokenjar ui`) with **Zero Background RAM** | **Instant** |
|
|
74
|
+
| ⚡ **1-Click IDE Configuration** | Automatic configuration & non-destructive rollback for Cursor, Windsurf, Claude, VS Code | **Zero-effort** |
|
|
75
|
+
|
|
76
|
+
### 🛡️ Built-in Guardrails & Reliability
|
|
77
|
+
- **Lockfile & Giant Asset Shield:** Prevents context window destruction from 50,000-line lockfiles; supports 5-line surgical version queries.
|
|
78
|
+
- **L1 RAM + L2 SQLite Persistent Cache:** Survives MCP server restarts and IDE reboots (`~/.tokenjar/cache.db` with WAL mode).
|
|
79
|
+
- **Fallback Safety Guard:** If a test or command fails (`exit_code != 0`), TokenJar guarantees tracebacks and error contexts are preserved intact.
|
|
80
|
+
- **Tiny File Anomaly Guard:** If a diff header would consume more tokens than the file itself, the full content is returned to prevent token inflation.
|
|
81
|
+
- **Runaway Stream Protection:** Protects host memory from infinite loops by capping raw terminal buffers at 2MB with graceful truncation.
|
|
82
|
+
- **SQLite Database Bloat Guard:** Files larger than 5MB are cached by hash reference without bloating disk space.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## 🚀 Quick Start & Installation (v1.0.1 GA)
|
|
87
|
+
|
|
88
|
+
TokenJar is distributed in two official editions:
|
|
89
|
+
1. **🦀 Rust Native Engine (Recommended):** High-performance, self-contained single binary with microsecond AST, 14 MB RAM, and zero Python dependencies.
|
|
90
|
+
2. **🐍 Python Edition:** Pure Python FastMCP package for pip and virtual environments.
|
|
91
|
+
|
|
92
|
+
### 📥 1-Click Direct Downloads (Precompiled Binaries)
|
|
93
|
+
|
|
94
|
+
Click your operating system below to download the latest v1.0.1 release:
|
|
95
|
+
|
|
96
|
+
| Platform | Architecture | Click to Download | Format |
|
|
97
|
+
|:---|:---|:---|:---|
|
|
98
|
+
| 🪟 **Windows** | x86_64 (64-bit) | [**⬇️ Download tokenjar-windows-x64.zip**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-windows-x64.zip) | Standalone `.exe` + Installer |
|
|
99
|
+
| 🐧 **Linux** | x86_64 (64-bit) | [**⬇️ Download tokenjar-linux-x64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-linux-x64.tar.gz) | Standalone Binary |
|
|
100
|
+
| 🍏 **macOS** | Apple Silicon (M1/M2/M3/M4) | [**⬇️ Download tokenjar-macos-arm64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-macos-arm64.tar.gz) | Standalone Binary |
|
|
101
|
+
| 🍏 **macOS** | Intel x86_64 | [**⬇️ Download tokenjar-macos-x64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-macos-x64.tar.gz) | Standalone Binary |
|
|
102
|
+
| 🐍 **Python** | Cross-platform | [**⬇️ Download tokenjar-python.zip**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-python.zip) | Python Wheel (.whl) |
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
### ⚡ Option 1: Rust Native Engine (1-Click Terminal Install)
|
|
107
|
+
> **Best for:** Highest speed, 14 MB RAM, microsecond tree-sitter AST, and zero Python dependency.
|
|
108
|
+
|
|
109
|
+
Copy and paste one line into your terminal to install and add `tokenjar` to your PATH automatically:
|
|
110
|
+
|
|
111
|
+
**Windows (PowerShell):**
|
|
112
|
+
```powershell
|
|
113
|
+
iwr -useb https://raw.githubusercontent.com/Farukes/TokenJar/main/install.ps1 | iex
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
# Linux & macOS (Bash):
|
|
118
|
+
curl -fsSL https://raw.githubusercontent.com/Farukes/TokenJar/main/install.sh | bash
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
**Or install via Cargo (crates.io):**
|
|
122
|
+
```bash
|
|
123
|
+
cargo install tokenjar
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
### 🐍 Option 2: Python Edition (pip)
|
|
129
|
+
> **Best for:** Python-centric environments, custom script integration, or pip workflows.
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
# Install from PyPI
|
|
133
|
+
pip install tokenjar
|
|
134
|
+
|
|
135
|
+
# Or install directly from GitHub main:
|
|
136
|
+
pip install git+https://github.com/Farukes/TokenJar.git
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## 📊 Proven Performance & Stress Test Benchmark
|
|
142
|
+
|
|
143
|
+
Empirical results from our rigorous **100-Step Real-Life Developer Stress Test** and **50-Cycle MCP Head-to-Head Benchmark** comparing Standard Raw AI vs TokenJar Python vs TokenJar Rust Native Engine:
|
|
144
|
+
|
|
145
|
+
| Metric | 1. Raw AI (No TokenJar) | 2. TokenJar Python | 3. TokenJar Rust (v1.0.1) | Rust Advantage |
|
|
146
|
+
|:---|:---|:---|:---|:---|
|
|
147
|
+
| **Consumed Tokens (100 Steps)** | 622,892 tokens | 95,492 tokens | **68,641 tokens** | **89.0% net savings (554k tokens saved)** |
|
|
148
|
+
| **End-to-End Coding Savings** | 166,513 tokens | 12,400 tokens | **6,585 tokens** | **🚀 96.0% net savings (Surgical edits)** |
|
|
149
|
+
| **API Cost (per 100 Steps)** | $1.8687 | $0.2865 | **$0.2059** | **$1.66 saved per 100 steps** |
|
|
150
|
+
| **Total Runtime (100 Steps)** | 0.357 s (raw disk) | 2.618 s | **0.985 s** | **2.7x faster than Python** |
|
|
151
|
+
| **Warm Cycle Latency** | N/A | 23.6 ms | **8.1 ms** | **3.0x faster execution** |
|
|
152
|
+
| **RAM / Memory Footprint** | ~30.0 MB | 49.1 MB | **15.0 MB** | **70% to 84% less RAM** |
|
|
153
|
+
| **Quality & Accuracy Score** | 100.0% | 100.0% | **100.0% (100/100)** | **100% functional completeness** |
|
|
154
|
+
| **Syntax Integrity & Zero Truncation** | Ham (Unverified) | ✅ Enforced | ✅ **Enforced** | **Zero placeholder comments** |
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
### Auto-Configure Agent Steering Rules
|
|
159
|
+
|
|
160
|
+
Automatically inject TokenJar optimization instructions into your repository rules:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
# Injects rules into AGENTS.md, .cursorrules, .windsurfrules, and CLAUDE.md
|
|
164
|
+
tokenjar init-rules
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### 🎛️ Output Optimization Controls (CLI & Terminals)
|
|
168
|
+
|
|
169
|
+
Switch between compact surgical output and default unrestricted output with crystal-clear commands:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
# 🟢 Enable compact surgical diffs & zero-truncation quality mandate
|
|
173
|
+
tokenjar output on
|
|
174
|
+
|
|
175
|
+
# ⚪ Revert AI assistant to default unrestricted output settings
|
|
176
|
+
tokenjar output off
|
|
177
|
+
|
|
178
|
+
# 📊 Check current output configuration status
|
|
179
|
+
tokenjar output
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Slash commands are also supported in your AI assistant chat (`/tokenjar output on`, `/tokenjar output off`).
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## 🔌 Setup with Your AI Assistant
|
|
187
|
+
|
|
188
|
+
### ⚡ 1-Click Automatic Setup (Recommended)
|
|
189
|
+
|
|
190
|
+
Automatically detects and configures TokenJar MCP server in Claude Desktop, Cursor, Windsurf, Claude Code, and VS Code with automated backups:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
# 🟢 Configure all detected IDEs in one command
|
|
194
|
+
tokenjar install-mcp
|
|
195
|
+
|
|
196
|
+
# ⚪ Cleanly revert at any time (preserves all other servers you added!)
|
|
197
|
+
tokenjar uninstall-mcp
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### Manual Configuration
|
|
201
|
+
|
|
202
|
+
If you prefer to configure manually or use other clients:
|
|
203
|
+
|
|
204
|
+
<details>
|
|
205
|
+
<summary><b>Claude Code</b></summary>
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
claude mcp add tokenjar -- python -m tokenjar
|
|
209
|
+
```
|
|
210
|
+
</details>
|
|
211
|
+
|
|
212
|
+
<details>
|
|
213
|
+
<summary><b>Cursor</b></summary>
|
|
214
|
+
|
|
215
|
+
Create or update `.cursor/mcp.json`:
|
|
216
|
+
```json
|
|
217
|
+
{
|
|
218
|
+
"mcpServers": {
|
|
219
|
+
"tokenjar": {
|
|
220
|
+
"command": "python",
|
|
221
|
+
"args": ["-m", "tokenjar"],
|
|
222
|
+
"env": { "PYTHONUNBUFFERED": "1" }
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
```
|
|
227
|
+
</details>
|
|
228
|
+
|
|
229
|
+
<details>
|
|
230
|
+
<summary><b>Antigravity (AGY)</b></summary>
|
|
231
|
+
|
|
232
|
+
Add to `~/.gemini/config/mcp_config.json`:
|
|
233
|
+
```json
|
|
234
|
+
{
|
|
235
|
+
"mcpServers": {
|
|
236
|
+
"tokenjar": {
|
|
237
|
+
"command": "python",
|
|
238
|
+
"args": ["-m", "tokenjar"],
|
|
239
|
+
"env": { "PYTHONUNBUFFERED": "1" }
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
```
|
|
244
|
+
</details>
|
|
245
|
+
|
|
246
|
+
<details>
|
|
247
|
+
<summary><b>Windsurf / Cascade</b></summary>
|
|
248
|
+
|
|
249
|
+
Add to `~/.codeium/windsurf/mcp_config.json`:
|
|
250
|
+
```json
|
|
251
|
+
{
|
|
252
|
+
"mcpServers": {
|
|
253
|
+
"tokenjar": {
|
|
254
|
+
"command": "python",
|
|
255
|
+
"args": ["-m", "tokenjar"]
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
```
|
|
260
|
+
</details>
|
|
261
|
+
|
|
262
|
+
<details>
|
|
263
|
+
<summary><b>Continue.dev</b></summary>
|
|
264
|
+
|
|
265
|
+
Add to `.continue/config.yaml`:
|
|
266
|
+
```yaml
|
|
267
|
+
mcpServers:
|
|
268
|
+
- name: tokenjar
|
|
269
|
+
command: python
|
|
270
|
+
args: ["-m", "tokenjar"]
|
|
271
|
+
```
|
|
272
|
+
</details>
|
|
273
|
+
|
|
274
|
+
---
|
|
275
|
+
|
|
276
|
+
## 🛠️ Available MCP Tools
|
|
277
|
+
|
|
278
|
+
- **`find_symbol_global(query, root_path=".", exact=False)`**: Search for functions, methods, or classes across the entire codebase by name without reading multiple files.
|
|
279
|
+
- **`find_symbol_references(symbol_name, root_path=".", max_results=25)`**: Blast radius reference analyzer. Finds all callers, imports, and usages across the entire codebase before editing or refactoring code.
|
|
280
|
+
- **`tool_get_code_skeleton(file_path)`**: Extract structural skeleton of a file — classes, function signatures, docstrings, and type annotations with bodies replaced by `...`. (Supports Python, JS/TS, Go, Rust, Java, C/C++, C#, Ruby, PHP, Kotlin).
|
|
281
|
+
- **`tool_get_symbol(file_path, symbol_name)`**: Extract the full implementation of a specific class or function by name after inspecting its skeleton.
|
|
282
|
+
- **`read_file_smart(file_path, force_full=False, query="", start_line=None, end_line=None)`**: Differential file reader with session caching, targeted line range slicing, and Lockfile Shield. Supports `start_line` and `end_line` (1-indexed, inclusive) to surgically inspect specific line ranges with line numbers instead of loading entire large files. Returns `[CACHED] unchanged` (~3 tokens) or unified diffs on edits. For lockfiles (`package-lock.json`, `Cargo.lock`, etc.), pass `query="package-name"` for surgical 5-line version blocks instead of 50,000 lines.
|
|
283
|
+
- **`run_command_smart(command, cwd=".")`**: Executes shell commands and prunes verbose logs from pytest, jest, npm, cargo, and git.
|
|
284
|
+
- **`filter_output(output, output_type="auto")`**: Pure text filter for test runners, build pipelines, and version control logs without executing commands.
|
|
285
|
+
- **`get_repo_map_tool(root_path=".", max_tokens=1000)`**: Graph centrality codebase map prioritized by cross-file import relationships.
|
|
286
|
+
- **`get_directory_tree_tool(root_path=".", max_depth=4)`**: Lightweight directory tree honoring `.gitignore` and skipping binary folders.
|
|
287
|
+
- **`cache_stats()`**: Inspect session read hits, misses, diffs, and aggregate token savings.
|
|
288
|
+
|
|
289
|
+
### 📦 MCP Resources & Prompts
|
|
290
|
+
|
|
291
|
+
- **Resources:**
|
|
292
|
+
- `tokenjar://stats`: Live cumulative token and financial savings dashboard.
|
|
293
|
+
- `tokenjar://guide`: AI assistant best-practice optimization guidelines.
|
|
294
|
+
- `tokenjar://config`: Active project configuration and ignore settings.
|
|
295
|
+
- **Prompts:**
|
|
296
|
+
- `optimize_coding_task(task_description)`: System prompt template steering assistants toward token-efficient workflows.
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## ⚙️ Project Configuration (`tokenjar.toml`)
|
|
301
|
+
|
|
302
|
+
Create an optional `tokenjar.toml` in your repository root to customize exclusions and budgets:
|
|
303
|
+
|
|
304
|
+
```toml
|
|
305
|
+
[general]
|
|
306
|
+
ignore_patterns = ["tests/fixtures/*", "legacy/*", "*.bak"]
|
|
307
|
+
max_cacheable_bytes = 5242880 # 5 MB
|
|
308
|
+
|
|
309
|
+
[cache]
|
|
310
|
+
ttl_days = 30
|
|
311
|
+
max_entries = 5000
|
|
312
|
+
|
|
313
|
+
[repo_map]
|
|
314
|
+
default_budget = 1000
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
---
|
|
318
|
+
|
|
319
|
+
## 💻 CLI Commands & Shell Hooks
|
|
320
|
+
|
|
321
|
+
TokenJar also functions as an interactive command-line utility for human developers and local shell automation:
|
|
322
|
+
|
|
323
|
+
```bash
|
|
324
|
+
# 🎨 Launch On-Demand Control Dashboard (Zero Background RAM UI)
|
|
325
|
+
tokenjar ui
|
|
326
|
+
|
|
327
|
+
# 📊 Check comprehensive live operational status of TokenJar across IDEs
|
|
328
|
+
tokenjar status
|
|
329
|
+
|
|
330
|
+
# ⚡ 1-Click auto-configure MCP across Claude Desktop, Cursor, Windsurf, VS Code
|
|
331
|
+
tokenjar install-mcp
|
|
332
|
+
|
|
333
|
+
# ⚪ Safely remove TokenJar MCP configuration and restore exact original state
|
|
334
|
+
tokenjar uninstall-mcp
|
|
335
|
+
|
|
336
|
+
# View cumulative savings dashboard (tokens saved, money saved, operations)
|
|
337
|
+
tokenjar stats
|
|
338
|
+
|
|
339
|
+
# Run any shell command through intelligent filtering
|
|
340
|
+
tokenjar run "pytest tests/ -v"
|
|
341
|
+
tokenjar run "npm test"
|
|
342
|
+
|
|
343
|
+
# Temporary bypass: see 100% of raw output when you need full logs
|
|
344
|
+
RAW=1 tokenjar run "pytest"
|
|
345
|
+
tokenjar run "pytest --raw"
|
|
346
|
+
|
|
347
|
+
# Prune expired or excess entries from L2 SQLite cache
|
|
348
|
+
tokenjar cache-prune --ttl-days 30 --max-entries 5000
|
|
349
|
+
|
|
350
|
+
# Install transparent shell hooks (so pytest/npm are automatically filtered)
|
|
351
|
+
tokenjar hook
|
|
352
|
+
|
|
353
|
+
# Cleanly and safely uninstall all shell hooks
|
|
354
|
+
tokenjar unhook
|
|
355
|
+
|
|
356
|
+
# 🟢 Enable TokenJar for THIS project (default)
|
|
357
|
+
tokenjar on
|
|
358
|
+
|
|
359
|
+
# ⚪ Disable TokenJar for THIS project (keeps other projects active)
|
|
360
|
+
tokenjar off
|
|
361
|
+
|
|
362
|
+
# 🌐 Enable TokenJar MCP globally across all detected IDEs
|
|
363
|
+
tokenjar on --global
|
|
364
|
+
|
|
365
|
+
# 🔴 Disable TokenJar MCP globally and cleanly revert IDE settings
|
|
366
|
+
tokenjar off --global
|
|
367
|
+
|
|
368
|
+
# 📝 Alias: Inject steering rules into the current project
|
|
369
|
+
tokenjar init
|
|
370
|
+
tokenjar init --clean
|
|
371
|
+
|
|
372
|
+
# 🎨 Open interactive Web Dashboard (Zero Background RAM)
|
|
373
|
+
tokenjar ui
|
|
374
|
+
|
|
375
|
+
# 🧹 Completely clear L2 SQLite cache
|
|
376
|
+
tokenjar cache-clear
|
|
377
|
+
|
|
378
|
+
# ⚠️ Completely uninstall TokenJar from host (IDEs, project rules, hooks, cache, and PATH)
|
|
379
|
+
tokenjar uninstall
|
|
380
|
+
# or skip confirmation prompt:
|
|
381
|
+
tokenjar uninstall --yes
|
|
382
|
+
|
|
383
|
+
# Install /tokenjar slash commands for AGY CLI and Claude Code
|
|
384
|
+
tokenjar setup-commands
|
|
385
|
+
|
|
386
|
+
# Reset metrics counter
|
|
387
|
+
tokenjar reset-stats
|
|
388
|
+
```
|
|
389
|
+
|
|
390
|
+
---
|
|
391
|
+
|
|
392
|
+
## 🔒 Enterprise Privacy & Security Guarantee
|
|
393
|
+
|
|
394
|
+
TokenJar is built strictly under a **Zero-Telemetry, 100% Localhost** design philosophy:
|
|
395
|
+
|
|
396
|
+
- **100% Local Execution:** All parsing (Tree-sitter), caching (SQLite), and output filtering happen locally in-process on your CPU.
|
|
397
|
+
- **Zero External Network Calls:** No telemetry servers, no analytical trackers, no outbound pings, and no cloud dependencies whatsoever.
|
|
398
|
+
- **Air-Gapped Compatible:** Safely operates in classified, offline, or air-gapped corporate enterprise environments.
|
|
399
|
+
- **Local Data Isolation:** Persistent cache (`~/.tokenjar/cache.db`) and statistics (`~/.tokenjar/telemetry.json`) reside exclusively in your user directory and can be purged at any time with `tokenjar reset-stats` or by deleting the directory.
|
|
400
|
+
- **Non-Invasive Architecture:** Never modifies your project code without explicit assistant direction.
|
|
401
|
+
|
|
402
|
+
---
|
|
403
|
+
|
|
404
|
+
## 🦀 Enterprise & High-Performance Native Engine (Rust Edition)
|
|
405
|
+
|
|
406
|
+
For enterprise environments, massive monorepos (50,000+ files), CI/CD pipelines, or developer systems without a Python runtime, TokenJar provides an ultra-fast, zero-dependency native Rust binary (`tokenjar.exe` / standalone executable).
|
|
407
|
+
|
|
408
|
+
### Why the Enterprise Native Engine?
|
|
409
|
+
- **Zero Runtime Dependencies:** No Python, pip, Node.js, or virtual environments required. Single standalone binary.
|
|
410
|
+
- **Ultra-Low Latency:** Instant startup (~3ms cold start vs 300ms Python startup) for zero-delay MCP tool responses.
|
|
411
|
+
- **High-Concurrency Indexing:** True multithreaded (Rayon + Tokio) parallel code parsing and symbol extraction.
|
|
412
|
+
- **Embedded 15-Language AST Engine:** Built-in Tree-sitter parsers for Rust, C, C++, Go, C#, Java, Python, JavaScript, TypeScript, PHP, Ruby, Bash, HTML, CSS, JSON statically linked inside the binary.
|
|
413
|
+
- **Minimal Memory Footprint:** Consumes only ~8-15 MB RAM under active load.
|
|
414
|
+
|
|
415
|
+
### Enterprise Quick Start (Standalone Binary)
|
|
416
|
+
|
|
417
|
+
Download the precompiled binary from [GitHub Releases](https://github.com/Farukes/TokenJar/releases) or build directly with Cargo:
|
|
418
|
+
|
|
419
|
+
```bash
|
|
420
|
+
# Build optimized native release binary from source
|
|
421
|
+
cargo build --release --workspace
|
|
422
|
+
|
|
423
|
+
# The standalone binary is ready:
|
|
424
|
+
./target/release/tokenjar.exe status
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
### Enterprise MCP Configuration (`claude_desktop_config.json` / Cursor)
|
|
428
|
+
Point directly to the native binary without any Python wrapper:
|
|
429
|
+
|
|
430
|
+
```json
|
|
431
|
+
{
|
|
432
|
+
"mcpServers": {
|
|
433
|
+
"tokenjar": {
|
|
434
|
+
"command": "C:\\path\\to\\tokenjar.exe"
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
---
|
|
441
|
+
|
|
442
|
+
## 🌍 Supported Languages
|
|
443
|
+
|
|
444
|
+
TokenJar uses Tree-sitter for AST parsing and supports **130+ programming languages** out of the box, including:
|
|
445
|
+
|
|
446
|
+
Python · TypeScript · JavaScript · Go · Rust · Java · C# · C / C++ · Ruby · PHP · Swift · Kotlin · Scala · Dart · Lua · Elixir · Haskell · and more.
|
|
447
|
+
|
|
448
|
+
---
|
|
449
|
+
|
|
450
|
+
## 🧪 Development & Quality Assurance
|
|
451
|
+
|
|
452
|
+
TokenJar maintains dual test suites ensuring 100% parity across both implementations:
|
|
453
|
+
|
|
454
|
+
```bash
|
|
455
|
+
# Python (Community Edition & MCP SDK)
|
|
456
|
+
pip install -e ".[dev]"
|
|
457
|
+
pytest tests/ -v # 65 tests passing
|
|
458
|
+
|
|
459
|
+
# Rust (Enterprise Native Engine)
|
|
460
|
+
cargo test --workspace # 35 tests passing
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
---
|
|
464
|
+
|
|
465
|
+
## 📄 License & Intellectual Property
|
|
466
|
+
|
|
467
|
+
Copyright © 2026 Ömer Faruk Eskitürk. All rights reserved.
|
|
468
|
+
|
|
469
|
+
Licensed under the **Business Source License 1.1 (BSL 1.1)** with an automatic transition to the **Apache License, Version 2.0**.
|
|
470
|
+
- **Free Use:** Free for all personal, educational, research, evaluation, and internal business use.
|
|
471
|
+
- **Commercial Restrictions:** Cannot be hosted or provided as a paid commercial service or SaaS competing with the Licensor.
|
|
472
|
+
- **Sunset to Apache 2.0:** Converts automatically to 100% open-source Apache 2.0 on 2030-01-01.
|
|
473
|
+
|
|
474
|
+
See [LICENSE](LICENSE) for full legal terms.
|