tokenjar 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. tokenjar-1.0.1/.gitignore +50 -0
  2. tokenjar-1.0.1/LICENSE +41 -0
  3. tokenjar-1.0.1/PKG-INFO +474 -0
  4. tokenjar-1.0.1/README.md +441 -0
  5. tokenjar-1.0.1/pyproject.toml +77 -0
  6. tokenjar-1.0.1/src/tokenjar/__init__.py +3 -0
  7. tokenjar-1.0.1/src/tokenjar/__main__.py +629 -0
  8. tokenjar-1.0.1/src/tokenjar/cache/__init__.py +1 -0
  9. tokenjar-1.0.1/src/tokenjar/cache/persistent_cache.py +371 -0
  10. tokenjar-1.0.1/src/tokenjar/cache/session_cache.py +259 -0
  11. tokenjar-1.0.1/src/tokenjar/config.py +195 -0
  12. tokenjar-1.0.1/src/tokenjar/filters/__init__.py +1 -0
  13. tokenjar-1.0.1/src/tokenjar/filters/ansi.py +11 -0
  14. tokenjar-1.0.1/src/tokenjar/filters/build_tools.py +95 -0
  15. tokenjar-1.0.1/src/tokenjar/filters/git.py +23 -0
  16. tokenjar-1.0.1/src/tokenjar/filters/lockfile.py +275 -0
  17. tokenjar-1.0.1/src/tokenjar/filters/test_runners.py +137 -0
  18. tokenjar-1.0.1/src/tokenjar/hooks/__init__.py +1 -0
  19. tokenjar-1.0.1/src/tokenjar/hooks/manager.py +870 -0
  20. tokenjar-1.0.1/src/tokenjar/parsers/__init__.py +1 -0
  21. tokenjar-1.0.1/src/tokenjar/parsers/languages.py +66 -0
  22. tokenjar-1.0.1/src/tokenjar/rules/manager.py +297 -0
  23. tokenjar-1.0.1/src/tokenjar/server.py +81 -0
  24. tokenjar-1.0.1/src/tokenjar/telemetry/__init__.py +1 -0
  25. tokenjar-1.0.1/src/tokenjar/telemetry/stats.py +293 -0
  26. tokenjar-1.0.1/src/tokenjar/tools/__init__.py +1 -0
  27. tokenjar-1.0.1/src/tokenjar/tools/output_pruner.py +178 -0
  28. tokenjar-1.0.1/src/tokenjar/tools/repo_map.py +493 -0
  29. tokenjar-1.0.1/src/tokenjar/tools/skeleton.py +248 -0
  30. tokenjar-1.0.1/src/tokenjar/tools/smart_reader.py +171 -0
  31. tokenjar-1.0.1/src/tokenjar/tools/symbol_index.py +501 -0
  32. tokenjar-1.0.1/src/tokenjar/ui/__init__.py +11 -0
  33. tokenjar-1.0.1/src/tokenjar/ui/server.py +408 -0
  34. tokenjar-1.0.1/src/tokenjar/ui/static/index.html +773 -0
  35. tokenjar-1.0.1/src/tokenjar/utils/__init__.py +1 -0
  36. tokenjar-1.0.1/src/tokenjar/utils/file_utils.py +250 -0
  37. tokenjar-1.0.1/src/tokenjar/utils/token_counter.py +87 -0
  38. tokenjar-1.0.1/tests/conftest.py +6 -0
  39. tokenjar-1.0.1/tests/test_config.py +107 -0
  40. tokenjar-1.0.1/tests/test_hooks.py +174 -0
  41. tokenjar-1.0.1/tests/test_lockfile_shield.py +248 -0
  42. tokenjar-1.0.1/tests/test_output_pruner.py +196 -0
  43. tokenjar-1.0.1/tests/test_real_life_50_steps.py +424 -0
  44. tokenjar-1.0.1/tests/test_repo_map.py +76 -0
  45. tokenjar-1.0.1/tests/test_resources_prompts.py +29 -0
  46. tokenjar-1.0.1/tests/test_rules.py +110 -0
  47. tokenjar-1.0.1/tests/test_semantic_diff.py +25 -0
  48. tokenjar-1.0.1/tests/test_skeleton.py +95 -0
  49. tokenjar-1.0.1/tests/test_slash_commands.py +76 -0
  50. tokenjar-1.0.1/tests/test_smart_reader.py +158 -0
  51. tokenjar-1.0.1/tests/test_symbol_index.py +207 -0
  52. tokenjar-1.0.1/tests/test_telemetry.py +65 -0
  53. tokenjar-1.0.1/tests/test_token_counter.py +28 -0
  54. tokenjar-1.0.1/tests/test_ui_server.py +127 -0
@@ -0,0 +1,50 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ *.egg-info/
7
+ *.egg
8
+ dist/
9
+ build/
10
+ .eggs/
11
+
12
+ # Virtual environments
13
+ .venv/
14
+ venv/
15
+ env/
16
+
17
+ # IDE
18
+ .idea/
19
+ .vscode/
20
+ *.swp
21
+ *.swo
22
+ *~
23
+
24
+ # Testing
25
+ .pytest_cache/
26
+ .coverage
27
+ htmlcov/
28
+ .mypy_cache/
29
+ .ruff_cache/
30
+
31
+ # Local Scratch and Backups
32
+ scratch/
33
+ ROADMAP.md
34
+ *.ts_bak
35
+ *.db
36
+ *.db-wal
37
+ *.db-shm
38
+ telemetry.json
39
+
40
+ # Rust
41
+ target/
42
+ **/*.rs.bk
43
+
44
+ # OS
45
+ .DS_Store
46
+ Thumbs.db
47
+
48
+ # Assets (Local design prototypes)
49
+ assets/
50
+
tokenjar-1.0.1/LICENSE ADDED
@@ -0,0 +1,41 @@
1
+ Business Source License 1.1
2
+
3
+ Parameters
4
+
5
+ Licensor:
6
+ Ömer Faruk Eskitürk
7
+
8
+ Software:
9
+ TokenJar
10
+
11
+ Change Date:
12
+ 2030-01-01
13
+
14
+ Change License:
15
+ Apache License, Version 2.0
16
+
17
+ Additional Use Grant:
18
+ You may make use of the Software, unmodified or modified, for all personal, educational, research, evaluation, and internal business operations without charge, provided that you do not provide the Software, or any substantial portion of its core token optimization functionality, as a commercial hosted service, cloud service, or paid software distribution that competes directly with the Licensor.
19
+
20
+ Notice:
21
+ The Business Source License 1.1 (the "License") is not an Open Source Initiative (OSI) approved license. However, the Licensed Work will automatically transition to an OSI-approved license (Apache License, Version 2.0) on the Change Date.
22
+
23
+ ================================================================================
24
+
25
+ License Text
26
+
27
+ The Licensor hereby grants you the right to copy, modify, create derivative works, redistribute, and make non-production and production use of the Software, but only for the purposes and subject to the restrictions in the Additional Use Grant.
28
+
29
+ Effective on the Change Date, or the fourth anniversary of the first publicly available distribution of a specific version of the Software under this License, whichever comes first, the Licensor hereby grants you rights under the terms of the Change License, and the restrictions of the Additional Use Grant shall no longer apply.
30
+
31
+ 1. COPYING, MODIFICATION, AND REDISTRIBUTION:
32
+ You may copy, modify, and redistribute the Software in source code or binary form, provided that you comply with the terms of the Additional Use Grant, and that you retain all copyright, trademark, and other proprietary notices in the Software, as well as this License text.
33
+
34
+ 2. CHANGE LICENSE TRANSITION:
35
+ On the Change Date, the Software shall automatically and irrevocably transition to the Change License (Apache License, Version 2.0). All copies and derivative works made on or after the Change Date shall be governed exclusively by the Change License.
36
+
37
+ 3. TRADEMARKS:
38
+ The name "TokenJar", associated logos, and brand identity are trademarks of Ömer Faruk Eskitürk. This License does not grant any rights to use these trademarks in connection with any distribution, service, or commercial offering without prior written authorization.
39
+
40
+ 4. DISCLAIMER OF WARRANTY:
41
+ TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE IS PROVIDED "AS IS" AND "AS AVAILABLE", WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL THE LICENSOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,474 @@
1
+ Metadata-Version: 2.5
2
+ Name: tokenjar
3
+ Version: 1.0.1
4
+ Summary: MCP server that saves 70-95% tokens for AI coding assistants without losing functionality
5
+ Project-URL: Homepage, https://github.com/Farukes/TokenJar
6
+ Project-URL: Repository, https://github.com/Farukes/TokenJar
7
+ Project-URL: Issues, https://github.com/Farukes/TokenJar/issues
8
+ Author-email: Ömer Faruk Eskitürk <omereskiturk2@gmail.com>
9
+ License: BSL-1.1
10
+ License-File: LICENSE
11
+ Keywords: ai,coding-assistant,llm,mcp,optimization,token,tree-sitter
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: Other/Proprietary License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Software Development :: Libraries
20
+ Classifier: Topic :: Software Development :: Quality Assurance
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: fastmcp>=2.0
23
+ Requires-Dist: tomli>=1.1.0; python_version < '3.11'
24
+ Requires-Dist: tree-sitter-languages>=1.10
25
+ Requires-Dist: tree-sitter<0.22,>=0.21
26
+ Provides-Extra: counting
27
+ Requires-Dist: tiktoken>=0.7; extra == 'counting'
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
30
+ Requires-Dist: pytest>=8.0; extra == 'dev'
31
+ Requires-Dist: ruff>=0.6; extra == 'dev'
32
+ Description-Content-Type: text/markdown
33
+
34
+ <p align="right">
35
+ <a href="README.md"><b>English</b></a> | <a href="README.tr.md"><b>Türkçe</b></a>
36
+ </p>
37
+
38
+ <p align="center">
39
+ <img src="docs/images/tokenjar_logo.jpg" alt="TokenJar Logo" width="220" style="border-radius: 16px;" />
40
+ </p>
41
+
42
+ <h1 align="center">🍯 TokenJar</h1>
43
+ <p align="center"><b>Put tokens back in your jar. Save 70-95% tokens for AI coding assistants without losing functionality.</b></p>
44
+
45
+ [![Release: v1.0.1](https://img.shields.io/badge/Release-v1.0.1%20GA-green.svg)](https://github.com/Farukes/TokenJar/releases/latest)
46
+ [![CI](https://github.com/Farukes/TokenJar/actions/workflows/ci.yml/badge.svg)](https://github.com/Farukes/TokenJar/actions/workflows/ci.yml)
47
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://www.python.org/downloads/)
48
+ [![Enterprise Native: Rust](https://img.shields.io/badge/Enterprise%20Native-Rust%20v1.0.1-orange.svg)](#-enterprise--high-performance-native-engine-rust-edition)
49
+ [![Crates.io](https://img.shields.io/crates/v/tokenjar.svg?color=orange)](https://crates.io/crates/tokenjar)
50
+ [![PyPI](https://img.shields.io/pypi/v/tokenjar.svg?color=blue)](https://pypi.org/project/tokenjar/)
51
+ [![Token Reduction](https://img.shields.io/badge/Token%20Savings-89%25%20to%2096%25-brightgreen.svg)](#-proven-performance--stress-test-benchmark)
52
+ [![License: BSL 1.1](https://img.shields.io/badge/License-BSL%201.1-blue.svg)](LICENSE)
53
+ [![Zero Telemetry](https://img.shields.io/badge/telemetry-0%25%20(100%25%20local)-success.svg)](#-enterprise-privacy--security-guarantee)
54
+
55
+ **MCP server that saves 70-95% tokens for AI coding assistants — without losing functionality.**
56
+
57
+ TokenJar sits between your AI coding assistant and your codebase, intelligently compressing code reads, terminal outputs, and file operations to dramatically reduce token consumption, context compaction, and latency.
58
+
59
+ Works with **Claude Code**, **Cursor**, **Antigravity (AGY)**, **Windsurf**, **Continue.dev**, and any MCP-compatible AI assistant.
60
+
61
+ ---
62
+
63
+ ## ✨ Features & Architecture
64
+
65
+ | Module | What It Does | Token Savings |
66
+ |:---|:---|:---|
67
+ | 🦴 **Code Skeletonizer** | Extracts structural skeleton (signatures, types, docstrings) via Tree-sitter AST | **80-95%** |
68
+ | 📖 **Smart File Reader** | L1 RAM + L2 Persistent SQLite cache with differential reads & diff headers | **90-99%** |
69
+ | 🛡️ **Lockfile & Asset Shield** | Intercepts massive lockfiles & minified bundles with surgical version queries (`query="react"`) | **99%** |
70
+ | 🎯 **Blast Radius & Symbols** | Instant global symbol lookup & cross-file reference caller tracking (`find_symbol_references`) | **85-95%** |
71
+ | 🖥️ **Terminal Pruner** | Compresses test/build/git terminal streams, keeps errors and summary info | **60-90%** |
72
+ | 🗺️ **Repo Map** | PageRank & Graph Centrality codebase overview fitted into custom token budgets | **Budget-fitted** |
73
+ | 🎨 **On-Demand UI Dashboard** | Lightweight standalone control panel (`tokenjar ui`) with **Zero Background RAM** | **Instant** |
74
+ | ⚡ **1-Click IDE Configuration** | Automatic configuration & non-destructive rollback for Cursor, Windsurf, Claude, VS Code | **Zero-effort** |
75
+
76
+ ### 🛡️ Built-in Guardrails & Reliability
77
+ - **Lockfile & Giant Asset Shield:** Prevents context window destruction from 50,000-line lockfiles; supports 5-line surgical version queries.
78
+ - **L1 RAM + L2 SQLite Persistent Cache:** Survives MCP server restarts and IDE reboots (`~/.tokenjar/cache.db` with WAL mode).
79
+ - **Fallback Safety Guard:** If a test or command fails (`exit_code != 0`), TokenJar guarantees tracebacks and error contexts are preserved intact.
80
+ - **Tiny File Anomaly Guard:** If a diff header would consume more tokens than the file itself, the full content is returned to prevent token inflation.
81
+ - **Runaway Stream Protection:** Protects host memory from infinite loops by capping raw terminal buffers at 2MB with graceful truncation.
82
+ - **SQLite Database Bloat Guard:** Files larger than 5MB are cached by hash reference without bloating disk space.
83
+
84
+ ---
85
+
86
+ ## 🚀 Quick Start & Installation (v1.0.1 GA)
87
+
88
+ TokenJar is distributed in two official editions:
89
+ 1. **🦀 Rust Native Engine (Recommended):** High-performance, self-contained single binary with microsecond AST, 14 MB RAM, and zero Python dependencies.
90
+ 2. **🐍 Python Edition:** Pure Python FastMCP package for pip and virtual environments.
91
+
92
+ ### 📥 1-Click Direct Downloads (Precompiled Binaries)
93
+
94
+ Click your operating system below to download the latest v1.0.1 release:
95
+
96
+ | Platform | Architecture | Click to Download | Format |
97
+ |:---|:---|:---|:---|
98
+ | 🪟 **Windows** | x86_64 (64-bit) | [**⬇️ Download tokenjar-windows-x64.zip**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-windows-x64.zip) | Standalone `.exe` + Installer |
99
+ | 🐧 **Linux** | x86_64 (64-bit) | [**⬇️ Download tokenjar-linux-x64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-linux-x64.tar.gz) | Standalone Binary |
100
+ | 🍏 **macOS** | Apple Silicon (M1/M2/M3/M4) | [**⬇️ Download tokenjar-macos-arm64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-macos-arm64.tar.gz) | Standalone Binary |
101
+ | 🍏 **macOS** | Intel x86_64 | [**⬇️ Download tokenjar-macos-x64.tar.gz**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-macos-x64.tar.gz) | Standalone Binary |
102
+ | 🐍 **Python** | Cross-platform | [**⬇️ Download tokenjar-python.zip**](https://github.com/Farukes/TokenJar/releases/latest/download/tokenjar-python.zip) | Python Wheel (.whl) |
103
+
104
+ ---
105
+
106
+ ### ⚡ Option 1: Rust Native Engine (1-Click Terminal Install)
107
+ > **Best for:** Highest speed, 14 MB RAM, microsecond tree-sitter AST, and zero Python dependency.
108
+
109
+ Copy and paste one line into your terminal to install and add `tokenjar` to your PATH automatically:
110
+
111
+ **Windows (PowerShell):**
112
+ ```powershell
113
+ iwr -useb https://raw.githubusercontent.com/Farukes/TokenJar/main/install.ps1 | iex
114
+ ```
115
+
116
+ ```bash
117
+ # Linux & macOS (Bash):
118
+ curl -fsSL https://raw.githubusercontent.com/Farukes/TokenJar/main/install.sh | bash
119
+ ```
120
+
121
+ **Or install via Cargo (crates.io):**
122
+ ```bash
123
+ cargo install tokenjar
124
+ ```
125
+
126
+ ---
127
+
128
+ ### 🐍 Option 2: Python Edition (pip)
129
+ > **Best for:** Python-centric environments, custom script integration, or pip workflows.
130
+
131
+ ```bash
132
+ # Install from PyPI
133
+ pip install tokenjar
134
+
135
+ # Or install directly from GitHub main:
136
+ pip install git+https://github.com/Farukes/TokenJar.git
137
+ ```
138
+
139
+ ---
140
+
141
+ ## 📊 Proven Performance & Stress Test Benchmark
142
+
143
+ Empirical results from our rigorous **100-Step Real-Life Developer Stress Test** and **50-Cycle MCP Head-to-Head Benchmark** comparing Standard Raw AI vs TokenJar Python vs TokenJar Rust Native Engine:
144
+
145
+ | Metric | 1. Raw AI (No TokenJar) | 2. TokenJar Python | 3. TokenJar Rust (v1.0.1) | Rust Advantage |
146
+ |:---|:---|:---|:---|:---|
147
+ | **Consumed Tokens (100 Steps)** | 622,892 tokens | 95,492 tokens | **68,641 tokens** | **89.0% net savings (554k tokens saved)** |
148
+ | **End-to-End Coding Savings** | 166,513 tokens | 12,400 tokens | **6,585 tokens** | **🚀 96.0% net savings (Surgical edits)** |
149
+ | **API Cost (per 100 Steps)** | $1.8687 | $0.2865 | **$0.2059** | **$1.66 saved per 100 steps** |
150
+ | **Total Runtime (100 Steps)** | 0.357 s (raw disk) | 2.618 s | **0.985 s** | **2.7x faster than Python** |
151
+ | **Warm Cycle Latency** | N/A | 23.6 ms | **8.1 ms** | **3.0x faster execution** |
152
+ | **RAM / Memory Footprint** | ~30.0 MB | 49.1 MB | **15.0 MB** | **70% to 84% less RAM** |
153
+ | **Quality & Accuracy Score** | 100.0% | 100.0% | **100.0% (100/100)** | **100% functional completeness** |
154
+ | **Syntax Integrity & Zero Truncation** | Ham (Unverified) | ✅ Enforced | ✅ **Enforced** | **Zero placeholder comments** |
155
+
156
+ ---
157
+
158
+ ### Auto-Configure Agent Steering Rules
159
+
160
+ Automatically inject TokenJar optimization instructions into your repository rules:
161
+
162
+ ```bash
163
+ # Injects rules into AGENTS.md, .cursorrules, .windsurfrules, and CLAUDE.md
164
+ tokenjar init-rules
165
+ ```
166
+
167
+ ### 🎛️ Output Optimization Controls (CLI & Terminals)
168
+
169
+ Switch between compact surgical output and default unrestricted output with crystal-clear commands:
170
+
171
+ ```bash
172
+ # 🟢 Enable compact surgical diffs & zero-truncation quality mandate
173
+ tokenjar output on
174
+
175
+ # ⚪ Revert AI assistant to default unrestricted output settings
176
+ tokenjar output off
177
+
178
+ # 📊 Check current output configuration status
179
+ tokenjar output
180
+ ```
181
+
182
+ Slash commands are also supported in your AI assistant chat (`/tokenjar output on`, `/tokenjar output off`).
183
+
184
+ ---
185
+
186
+ ## 🔌 Setup with Your AI Assistant
187
+
188
+ ### ⚡ 1-Click Automatic Setup (Recommended)
189
+
190
+ Automatically detects and configures TokenJar MCP server in Claude Desktop, Cursor, Windsurf, Claude Code, and VS Code with automated backups:
191
+
192
+ ```bash
193
+ # 🟢 Configure all detected IDEs in one command
194
+ tokenjar install-mcp
195
+
196
+ # ⚪ Cleanly revert at any time (preserves all other servers you added!)
197
+ tokenjar uninstall-mcp
198
+ ```
199
+
200
+ ### Manual Configuration
201
+
202
+ If you prefer to configure manually or use other clients:
203
+
204
+ <details>
205
+ <summary><b>Claude Code</b></summary>
206
+
207
+ ```bash
208
+ claude mcp add tokenjar -- python -m tokenjar
209
+ ```
210
+ </details>
211
+
212
+ <details>
213
+ <summary><b>Cursor</b></summary>
214
+
215
+ Create or update `.cursor/mcp.json`:
216
+ ```json
217
+ {
218
+ "mcpServers": {
219
+ "tokenjar": {
220
+ "command": "python",
221
+ "args": ["-m", "tokenjar"],
222
+ "env": { "PYTHONUNBUFFERED": "1" }
223
+ }
224
+ }
225
+ }
226
+ ```
227
+ </details>
228
+
229
+ <details>
230
+ <summary><b>Antigravity (AGY)</b></summary>
231
+
232
+ Add to `~/.gemini/config/mcp_config.json`:
233
+ ```json
234
+ {
235
+ "mcpServers": {
236
+ "tokenjar": {
237
+ "command": "python",
238
+ "args": ["-m", "tokenjar"],
239
+ "env": { "PYTHONUNBUFFERED": "1" }
240
+ }
241
+ }
242
+ }
243
+ ```
244
+ </details>
245
+
246
+ <details>
247
+ <summary><b>Windsurf / Cascade</b></summary>
248
+
249
+ Add to `~/.codeium/windsurf/mcp_config.json`:
250
+ ```json
251
+ {
252
+ "mcpServers": {
253
+ "tokenjar": {
254
+ "command": "python",
255
+ "args": ["-m", "tokenjar"]
256
+ }
257
+ }
258
+ }
259
+ ```
260
+ </details>
261
+
262
+ <details>
263
+ <summary><b>Continue.dev</b></summary>
264
+
265
+ Add to `.continue/config.yaml`:
266
+ ```yaml
267
+ mcpServers:
268
+ - name: tokenjar
269
+ command: python
270
+ args: ["-m", "tokenjar"]
271
+ ```
272
+ </details>
273
+
274
+ ---
275
+
276
+ ## 🛠️ Available MCP Tools
277
+
278
+ - **`find_symbol_global(query, root_path=".", exact=False)`**: Search for functions, methods, or classes across the entire codebase by name without reading multiple files.
279
+ - **`find_symbol_references(symbol_name, root_path=".", max_results=25)`**: Blast radius reference analyzer. Finds all callers, imports, and usages across the entire codebase before editing or refactoring code.
280
+ - **`tool_get_code_skeleton(file_path)`**: Extract structural skeleton of a file — classes, function signatures, docstrings, and type annotations with bodies replaced by `...`. (Supports Python, JS/TS, Go, Rust, Java, C/C++, C#, Ruby, PHP, Kotlin).
281
+ - **`tool_get_symbol(file_path, symbol_name)`**: Extract the full implementation of a specific class or function by name after inspecting its skeleton.
282
+ - **`read_file_smart(file_path, force_full=False, query="", start_line=None, end_line=None)`**: Differential file reader with session caching, targeted line range slicing, and Lockfile Shield. Supports `start_line` and `end_line` (1-indexed, inclusive) to surgically inspect specific line ranges with line numbers instead of loading entire large files. Returns `[CACHED] unchanged` (~3 tokens) or unified diffs on edits. For lockfiles (`package-lock.json`, `Cargo.lock`, etc.), pass `query="package-name"` for surgical 5-line version blocks instead of 50,000 lines.
283
+ - **`run_command_smart(command, cwd=".")`**: Executes shell commands and prunes verbose logs from pytest, jest, npm, cargo, and git.
284
+ - **`filter_output(output, output_type="auto")`**: Pure text filter for test runners, build pipelines, and version control logs without executing commands.
285
+ - **`get_repo_map_tool(root_path=".", max_tokens=1000)`**: Graph centrality codebase map prioritized by cross-file import relationships.
286
+ - **`get_directory_tree_tool(root_path=".", max_depth=4)`**: Lightweight directory tree honoring `.gitignore` and skipping binary folders.
287
+ - **`cache_stats()`**: Inspect session read hits, misses, diffs, and aggregate token savings.
288
+
289
+ ### 📦 MCP Resources & Prompts
290
+
291
+ - **Resources:**
292
+ - `tokenjar://stats`: Live cumulative token and financial savings dashboard.
293
+ - `tokenjar://guide`: AI assistant best-practice optimization guidelines.
294
+ - `tokenjar://config`: Active project configuration and ignore settings.
295
+ - **Prompts:**
296
+ - `optimize_coding_task(task_description)`: System prompt template steering assistants toward token-efficient workflows.
297
+
298
+ ---
299
+
300
+ ## ⚙️ Project Configuration (`tokenjar.toml`)
301
+
302
+ Create an optional `tokenjar.toml` in your repository root to customize exclusions and budgets:
303
+
304
+ ```toml
305
+ [general]
306
+ ignore_patterns = ["tests/fixtures/*", "legacy/*", "*.bak"]
307
+ max_cacheable_bytes = 5242880 # 5 MB
308
+
309
+ [cache]
310
+ ttl_days = 30
311
+ max_entries = 5000
312
+
313
+ [repo_map]
314
+ default_budget = 1000
315
+ ```
316
+
317
+ ---
318
+
319
+ ## 💻 CLI Commands & Shell Hooks
320
+
321
+ TokenJar also functions as an interactive command-line utility for human developers and local shell automation:
322
+
323
+ ```bash
324
+ # 🎨 Launch On-Demand Control Dashboard (Zero Background RAM UI)
325
+ tokenjar ui
326
+
327
+ # 📊 Check comprehensive live operational status of TokenJar across IDEs
328
+ tokenjar status
329
+
330
+ # ⚡ 1-Click auto-configure MCP across Claude Desktop, Cursor, Windsurf, VS Code
331
+ tokenjar install-mcp
332
+
333
+ # ⚪ Safely remove TokenJar MCP configuration and restore exact original state
334
+ tokenjar uninstall-mcp
335
+
336
+ # View cumulative savings dashboard (tokens saved, money saved, operations)
337
+ tokenjar stats
338
+
339
+ # Run any shell command through intelligent filtering
340
+ tokenjar run "pytest tests/ -v"
341
+ tokenjar run "npm test"
342
+
343
+ # Temporary bypass: see 100% of raw output when you need full logs
344
+ RAW=1 tokenjar run "pytest"
345
+ tokenjar run "pytest --raw"
346
+
347
+ # Prune expired or excess entries from L2 SQLite cache
348
+ tokenjar cache-prune --ttl-days 30 --max-entries 5000
349
+
350
+ # Install transparent shell hooks (so pytest/npm are automatically filtered)
351
+ tokenjar hook
352
+
353
+ # Cleanly and safely uninstall all shell hooks
354
+ tokenjar unhook
355
+
356
+ # 🟢 Enable TokenJar for THIS project (default)
357
+ tokenjar on
358
+
359
+ # ⚪ Disable TokenJar for THIS project (keeps other projects active)
360
+ tokenjar off
361
+
362
+ # 🌐 Enable TokenJar MCP globally across all detected IDEs
363
+ tokenjar on --global
364
+
365
+ # 🔴 Disable TokenJar MCP globally and cleanly revert IDE settings
366
+ tokenjar off --global
367
+
368
+ # 📝 Alias: Inject steering rules into the current project
369
+ tokenjar init
370
+ tokenjar init --clean
371
+
372
+ # 🎨 Open interactive Web Dashboard (Zero Background RAM)
373
+ tokenjar ui
374
+
375
+ # 🧹 Completely clear L2 SQLite cache
376
+ tokenjar cache-clear
377
+
378
+ # ⚠️ Completely uninstall TokenJar from host (IDEs, project rules, hooks, cache, and PATH)
379
+ tokenjar uninstall
380
+ # or skip confirmation prompt:
381
+ tokenjar uninstall --yes
382
+
383
+ # Install /tokenjar slash commands for AGY CLI and Claude Code
384
+ tokenjar setup-commands
385
+
386
+ # Reset metrics counter
387
+ tokenjar reset-stats
388
+ ```
389
+
390
+ ---
391
+
392
+ ## 🔒 Enterprise Privacy & Security Guarantee
393
+
394
+ TokenJar is built strictly under a **Zero-Telemetry, 100% Localhost** design philosophy:
395
+
396
+ - **100% Local Execution:** All parsing (Tree-sitter), caching (SQLite), and output filtering happen locally in-process on your CPU.
397
+ - **Zero External Network Calls:** No telemetry servers, no analytical trackers, no outbound pings, and no cloud dependencies whatsoever.
398
+ - **Air-Gapped Compatible:** Safely operates in classified, offline, or air-gapped corporate enterprise environments.
399
+ - **Local Data Isolation:** Persistent cache (`~/.tokenjar/cache.db`) and statistics (`~/.tokenjar/telemetry.json`) reside exclusively in your user directory and can be purged at any time with `tokenjar reset-stats` or by deleting the directory.
400
+ - **Non-Invasive Architecture:** Never modifies your project code without explicit assistant direction.
401
+
402
+ ---
403
+
404
+ ## 🦀 Enterprise & High-Performance Native Engine (Rust Edition)
405
+
406
+ For enterprise environments, massive monorepos (50,000+ files), CI/CD pipelines, or developer systems without a Python runtime, TokenJar provides an ultra-fast, zero-dependency native Rust binary (`tokenjar.exe` / standalone executable).
407
+
408
+ ### Why the Enterprise Native Engine?
409
+ - **Zero Runtime Dependencies:** No Python, pip, Node.js, or virtual environments required. Single standalone binary.
410
+ - **Ultra-Low Latency:** Instant startup (~3ms cold start vs 300ms Python startup) for zero-delay MCP tool responses.
411
+ - **High-Concurrency Indexing:** True multithreaded (Rayon + Tokio) parallel code parsing and symbol extraction.
412
+ - **Embedded 15-Language AST Engine:** Built-in Tree-sitter parsers for Rust, C, C++, Go, C#, Java, Python, JavaScript, TypeScript, PHP, Ruby, Bash, HTML, CSS, JSON statically linked inside the binary.
413
+ - **Minimal Memory Footprint:** Consumes only ~8-15 MB RAM under active load.
414
+
415
+ ### Enterprise Quick Start (Standalone Binary)
416
+
417
+ Download the precompiled binary from [GitHub Releases](https://github.com/Farukes/TokenJar/releases) or build directly with Cargo:
418
+
419
+ ```bash
420
+ # Build optimized native release binary from source
421
+ cargo build --release --workspace
422
+
423
+ # The standalone binary is ready:
424
+ ./target/release/tokenjar.exe status
425
+ ```
426
+
427
+ ### Enterprise MCP Configuration (`claude_desktop_config.json` / Cursor)
428
+ Point directly to the native binary without any Python wrapper:
429
+
430
+ ```json
431
+ {
432
+ "mcpServers": {
433
+ "tokenjar": {
434
+ "command": "C:\\path\\to\\tokenjar.exe"
435
+ }
436
+ }
437
+ }
438
+ ```
439
+
440
+ ---
441
+
442
+ ## 🌍 Supported Languages
443
+
444
+ TokenJar uses Tree-sitter for AST parsing and supports **130+ programming languages** out of the box, including:
445
+
446
+ Python · TypeScript · JavaScript · Go · Rust · Java · C# · C / C++ · Ruby · PHP · Swift · Kotlin · Scala · Dart · Lua · Elixir · Haskell · and more.
447
+
448
+ ---
449
+
450
+ ## 🧪 Development & Quality Assurance
451
+
452
+ TokenJar maintains dual test suites ensuring 100% parity across both implementations:
453
+
454
+ ```bash
455
+ # Python (Community Edition & MCP SDK)
456
+ pip install -e ".[dev]"
457
+ pytest tests/ -v # 65 tests passing
458
+
459
+ # Rust (Enterprise Native Engine)
460
+ cargo test --workspace # 35 tests passing
461
+ ```
462
+
463
+ ---
464
+
465
+ ## 📄 License & Intellectual Property
466
+
467
+ Copyright © 2026 Ömer Faruk Eskitürk. All rights reserved.
468
+
469
+ Licensed under the **Business Source License 1.1 (BSL 1.1)** with an automatic transition to the **Apache License, Version 2.0**.
470
+ - **Free Use:** Free for all personal, educational, research, evaluation, and internal business use.
471
+ - **Commercial Restrictions:** Cannot be hosted or provided as a paid commercial service or SaaS competing with the Licensor.
472
+ - **Sunset to Apache 2.0:** Converts automatically to 100% open-source Apache 2.0 on 2030-01-01.
473
+
474
+ See [LICENSE](LICENSE) for full legal terms.