rlm-tools 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rlm_tools-0.1.0/.claude/settings.local.json +18 -0
- rlm_tools-0.1.0/.env.example +12 -0
- rlm_tools-0.1.0/.github/workflows/publish.yml +105 -0
- rlm_tools-0.1.0/.gitignore +11 -0
- rlm_tools-0.1.0/.mcp.json +11 -0
- rlm_tools-0.1.0/LICENSE +21 -0
- rlm_tools-0.1.0/PKG-INFO +221 -0
- rlm_tools-0.1.0/README.md +184 -0
- rlm_tools-0.1.0/docs/benchmarks.md +71 -0
- rlm_tools-0.1.0/evals/conftest.py +214 -0
- rlm_tools-0.1.0/evals/metrics.py +113 -0
- rlm_tools-0.1.0/evals/run_ab_eval.sh +257 -0
- rlm_tools-0.1.0/evals/test_comparative.py +523 -0
- rlm_tools-0.1.0/evals/test_correctness.py +142 -0
- rlm_tools-0.1.0/evals/test_task_completion.py +127 -0
- rlm_tools-0.1.0/pyproject.toml +33 -0
- rlm_tools-0.1.0/server.json +23 -0
- rlm_tools-0.1.0/src/rlm_tools/__init__.py +1 -0
- rlm_tools-0.1.0/src/rlm_tools/__main__.py +5 -0
- rlm_tools-0.1.0/src/rlm_tools/helpers.py +212 -0
- rlm_tools-0.1.0/src/rlm_tools/llm_bridge.py +69 -0
- rlm_tools-0.1.0/src/rlm_tools/sandbox.py +136 -0
- rlm_tools-0.1.0/src/rlm_tools/server.py +331 -0
- rlm_tools-0.1.0/src/rlm_tools/session.py +78 -0
- rlm_tools-0.1.0/src/rlm_tools/setup_cli.py +179 -0
- rlm_tools-0.1.0/tests/test_helpers.py +163 -0
- rlm_tools-0.1.0/tests/test_llm_bridge.py +45 -0
- rlm_tools-0.1.0/tests/test_sandbox.py +41 -0
- rlm_tools-0.1.0/tests/test_server.py +183 -0
- rlm_tools-0.1.0/tests/test_session.py +24 -0
- rlm_tools-0.1.0/tests/test_setup_cli.py +60 -0
- rlm_tools-0.1.0/uv.lock +1104 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"permissions": {
|
|
3
|
+
"allow": [
|
|
4
|
+
"Bash(git status:*)",
|
|
5
|
+
"Bash(uv run pytest:*)",
|
|
6
|
+
"Bash(uv run:*)",
|
|
7
|
+
"Bash(PATH=/usr/bin:/bin uv run:*)",
|
|
8
|
+
"Bash(echo:*)",
|
|
9
|
+
"Bash(git add:*)",
|
|
10
|
+
"Bash(git reset:*)",
|
|
11
|
+
"Bash(git commit:*)"
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
"enableAllProjectMcpServers": true,
|
|
15
|
+
"enabledMcpjsonServers": [
|
|
16
|
+
"rlm-tools"
|
|
17
|
+
]
|
|
18
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Required for llm_query() / llm_query_batched() sub-LLM features
|
|
2
|
+
# Optional — server works without it, just no LLM query capability
|
|
3
|
+
ANTHROPIC_API_KEY=sk-ant-...
|
|
4
|
+
|
|
5
|
+
# Optional: override the sub-LLM model (default: claude-haiku-4-5-20251001)
|
|
6
|
+
# RLM_SUB_MODEL=claude-haiku-4-5-20251001
|
|
7
|
+
|
|
8
|
+
# Optional: max concurrent sessions (default: 5)
|
|
9
|
+
# RLM_MAX_SESSIONS=5
|
|
10
|
+
|
|
11
|
+
# Optional: session inactivity timeout in minutes (default: 10)
|
|
12
|
+
# RLM_SESSION_TIMEOUT=10
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
name: Publish rlm-tools
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
build:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
outputs:
|
|
15
|
+
version: ${{ steps.version.outputs.version }}
|
|
16
|
+
steps:
|
|
17
|
+
- name: Check out code
|
|
18
|
+
uses: actions/checkout@v5
|
|
19
|
+
|
|
20
|
+
- name: Resolve release version
|
|
21
|
+
id: version
|
|
22
|
+
run: |
|
|
23
|
+
if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
|
|
24
|
+
echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
|
|
25
|
+
else
|
|
26
|
+
VERSION="$(python3 - <<'PY'
|
|
27
|
+
import tomllib
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
data = tomllib.loads(Path('pyproject.toml').read_text(encoding='utf-8'))
|
|
30
|
+
print(data['project']['version'])
|
|
31
|
+
PY
|
|
32
|
+
)"
|
|
33
|
+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
|
34
|
+
fi
|
|
35
|
+
|
|
36
|
+
- name: Set up Python
|
|
37
|
+
uses: actions/setup-python@v5
|
|
38
|
+
with:
|
|
39
|
+
python-version: "3.12"
|
|
40
|
+
|
|
41
|
+
- name: Set up uv
|
|
42
|
+
uses: astral-sh/setup-uv@v4
|
|
43
|
+
|
|
44
|
+
- name: Install dependencies
|
|
45
|
+
run: uv sync --dev
|
|
46
|
+
|
|
47
|
+
- name: Run tests
|
|
48
|
+
run: uv run pytest tests
|
|
49
|
+
|
|
50
|
+
- name: Build package
|
|
51
|
+
run: uv build
|
|
52
|
+
|
|
53
|
+
- name: Upload package artifacts
|
|
54
|
+
uses: actions/upload-artifact@v4
|
|
55
|
+
with:
|
|
56
|
+
name: python-dist
|
|
57
|
+
path: dist/*
|
|
58
|
+
|
|
59
|
+
publish-pypi:
|
|
60
|
+
runs-on: ubuntu-latest
|
|
61
|
+
needs: build
|
|
62
|
+
permissions:
|
|
63
|
+
id-token: write
|
|
64
|
+
contents: read
|
|
65
|
+
steps:
|
|
66
|
+
- name: Download package artifacts
|
|
67
|
+
uses: actions/download-artifact@v4
|
|
68
|
+
with:
|
|
69
|
+
name: python-dist
|
|
70
|
+
path: dist
|
|
71
|
+
|
|
72
|
+
- name: Publish package to PyPI
|
|
73
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
74
|
+
|
|
75
|
+
publish-mcp-registry:
|
|
76
|
+
runs-on: ubuntu-latest
|
|
77
|
+
needs:
|
|
78
|
+
- build
|
|
79
|
+
- publish-pypi
|
|
80
|
+
permissions:
|
|
81
|
+
id-token: write
|
|
82
|
+
contents: read
|
|
83
|
+
steps:
|
|
84
|
+
- name: Check out code
|
|
85
|
+
uses: actions/checkout@v5
|
|
86
|
+
|
|
87
|
+
- name: Update server.json version
|
|
88
|
+
run: |
|
|
89
|
+
VERSION="${{ needs.build.outputs.version }}"
|
|
90
|
+
jq --arg v "$VERSION" '
|
|
91
|
+
.version = $v
|
|
92
|
+
| (.packages[] | select(.registryType == "pypi" and .identifier == "rlm-tools") | .version) = $v
|
|
93
|
+
' server.json > server.json.tmp
|
|
94
|
+
mv server.json.tmp server.json
|
|
95
|
+
|
|
96
|
+
- name: Install mcp-publisher
|
|
97
|
+
run: |
|
|
98
|
+
curl -L "https://github.com/modelcontextprotocol/registry/releases/latest/download/mcp-publisher_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/').tar.gz" \
|
|
99
|
+
| tar xz mcp-publisher
|
|
100
|
+
|
|
101
|
+
- name: Authenticate to MCP Registry
|
|
102
|
+
run: ./mcp-publisher login github-oidc
|
|
103
|
+
|
|
104
|
+
- name: Publish server to MCP Registry
|
|
105
|
+
run: ./mcp-publisher publish
|
rlm_tools-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stefan O'Shea
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rlm_tools-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rlm-tools
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server providing a persistent Python sandbox for token-efficient codebase exploration across MCP clients
|
|
5
|
+
Project-URL: Homepage, https://github.com/stefanoshea/rlm-tools
|
|
6
|
+
Project-URL: Repository, https://github.com/stefanoshea/rlm-tools
|
|
7
|
+
Project-URL: Issues, https://github.com/stefanoshea/rlm-tools/issues
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Stefan O'Shea
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Requires-Python: >=3.10
|
|
31
|
+
Requires-Dist: anthropic>=0.40.0
|
|
32
|
+
Requires-Dist: mcp[cli]>=1.2.0
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# RLM Tools
|
|
39
|
+
|
|
40
|
+
<!-- mcp-name: io.github.stefanoshea/rlm-tools -->
|
|
41
|
+
|
|
42
|
+
Your AI coding agent spends most of its token budget just *reading* your code — not reasoning about it. Every grep, file read, and glob result gets dumped into the conversation. On a large codebase, that's 25-35% of your context (and cost) burned on raw data the model never needed to see.
|
|
43
|
+
|
|
44
|
+
RLM Tools gives your agent a persistent sandbox to explore code in. Data stays server-side. Only the conclusions come back.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# For Codex / generic MCP clients
|
|
48
|
+
uvx rlm-tools-setup --write-policy AGENTS.md
|
|
49
|
+
|
|
50
|
+
# For Claude Code
|
|
51
|
+
uvx rlm-tools-setup --write-policy CLAUDE.md
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
This auto-detects Claude/Codex on your PATH, registers the MCP server, and writes a policy block that nudges agents toward `rlm-tools` for repo exploration. Use `CLAUDE.md` for Claude Code (it reads this file automatically) or `AGENTS.md` for other clients.
|
|
55
|
+
|
|
56
|
+
If you prefer manual setup:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Claude Code
|
|
60
|
+
claude mcp add rlm-tools -- uvx rlm-tools
|
|
61
|
+
|
|
62
|
+
# Codex
|
|
63
|
+
codex mcp add rlm-tools -- uvx rlm-tools
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## What Changes
|
|
67
|
+
|
|
68
|
+
**Without RLM Tools** — agent greps for `import UIKit`, gets 500 matches dumped into context. Reads 10 files, burns all their content as tokens. Context window fills up. Agent forgets what it was doing.
|
|
69
|
+
|
|
70
|
+
**With RLM Tools** — agent runs the same exploration in a server-side Python sandbox. Data stays in sandbox memory. Only the `print()` output enters context:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
matches = grep("import UIKit")
|
|
74
|
+
by_module = {}
|
|
75
|
+
for m in matches:
|
|
76
|
+
module = m["file"].split("/")[0]
|
|
77
|
+
by_module.setdefault(module, []).append(m)
|
|
78
|
+
for module, ms in sorted(by_module.items(), key=lambda x: -len(x[1]))[:5]:
|
|
79
|
+
print(f"{module}: {len(ms)} files")
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
500 lines of grep results become 5 lines of summary. The agent sees what it needs, nothing more.
|
|
83
|
+
|
|
84
|
+
## Real-World Impact
|
|
85
|
+
|
|
86
|
+
In typical coding workflows: **25-35% context reduction.** That means your agent can explore roughly 40-50% more code before hitting context limits.
|
|
87
|
+
|
|
88
|
+
In heavy exploration tasks (reading many files, broad searches), savings go much further:
|
|
89
|
+
|
|
90
|
+
| Scenario | Standard Tools | RLM Tools | Saved |
|
|
91
|
+
|---|---:|---:|---:|
|
|
92
|
+
| Grep across full app | 40,045 chars | 1,644 chars | 95.9% |
|
|
93
|
+
| Read 10 large files | 1,493,720 chars | 13,588 chars | 99.1% |
|
|
94
|
+
| Multi-step exploration | 136,102 chars | 5,285 chars | 96.1% |
|
|
95
|
+
| Grep then read matches | 340,408 chars | 6,022 chars | 98.2% |
|
|
96
|
+
| Find all usages of a pattern | 13,478 chars | 3,691 chars | 72.6% |
|
|
97
|
+
| Understand a module | 94,745 chars | 16,925 chars | 82.1% |
|
|
98
|
+
|
|
99
|
+
Full benchmark methodology and reproduction steps: [`docs/benchmarks.md`](docs/benchmarks.md)
|
|
100
|
+
|
|
101
|
+
## How It Works
|
|
102
|
+
|
|
103
|
+
Three MCP tools. That's the entire API:
|
|
104
|
+
|
|
105
|
+
| Tool | Purpose |
|
|
106
|
+
|---|---|
|
|
107
|
+
| `rlm_start(path, query)` | Open a session on a directory |
|
|
108
|
+
| `rlm_execute(session_id, code)` | Run Python in the sandbox |
|
|
109
|
+
| `rlm_end(session_id)` | Close session, free resources |
|
|
110
|
+
|
|
111
|
+
The sandbox provides built-in helpers:
|
|
112
|
+
|
|
113
|
+
- `read_file(path)` / `read_files(paths)` — Read files into variables (cached across calls)
|
|
114
|
+
- `grep(pattern)` / `grep_summary(pattern)` / `grep_read(pattern)` — Search
|
|
115
|
+
- `glob_files(pattern)` — Find files by pattern
|
|
116
|
+
- `tree(path, max_depth)` — Directory structure
|
|
117
|
+
- `llm_query(prompt, context)` — Sub-LLM analysis (optional, requires API key)
|
|
118
|
+
|
|
119
|
+
Variables persist across `rlm_execute` calls within a session. The agent can build up understanding incrementally — search, filter, read, analyze — without any intermediate data touching the context window.
|
|
120
|
+
|
|
121
|
+
## Works With
|
|
122
|
+
|
|
123
|
+
RLM Tools is a standard [MCP](https://modelcontextprotocol.io) server. It works with any MCP-compatible client: **Claude Code**, **Codex**, **Cursor**, and others.
|
|
124
|
+
|
|
125
|
+
### Optional Strict Mode
|
|
126
|
+
|
|
127
|
+
If your MCP client supports tool permissions or hooks, disable/block default read/grep/glob tools for repository exploration and keep `rlm-tools` enabled. This is the strongest way to force the token-efficient path.
|
|
128
|
+
|
|
129
|
+
<details>
|
|
130
|
+
<summary><strong>Other installation methods</strong></summary>
|
|
131
|
+
|
|
132
|
+
### JSON MCP config (Cursor, Windsurf, etc.)
|
|
133
|
+
|
|
134
|
+
```json
|
|
135
|
+
{
|
|
136
|
+
"mcpServers": {
|
|
137
|
+
"rlm-tools": {
|
|
138
|
+
"command": "uvx",
|
|
139
|
+
"args": ["rlm-tools"]
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Setup helper command
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
# Auto-detect Claude/Codex CLIs and configure rlm-tools
|
|
149
|
+
uvx rlm-tools-setup
|
|
150
|
+
|
|
151
|
+
# Print JSON config snippet for other clients
|
|
152
|
+
uvx rlm-tools-setup --print-json
|
|
153
|
+
|
|
154
|
+
# Print policy text instead of writing it
|
|
155
|
+
uvx rlm-tools-setup --print-policy
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Direct run
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
uvx rlm-tools
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### From source
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
git clone https://github.com/stefanoshea/rlm-tools.git
|
|
168
|
+
cd rlm-tools
|
|
169
|
+
uv sync
|
|
170
|
+
uv run rlm-tools
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Then point your MCP client to `command: uv`, `args: ["--directory", "/path/to/rlm-tools", "run", "rlm-tools"]`.
|
|
174
|
+
|
|
175
|
+
</details>
|
|
176
|
+
|
|
177
|
+
## Configuration
|
|
178
|
+
|
|
179
|
+
Copy `.env.example` to `.env` to customize. All settings are optional — RLM Tools works out of the box with zero config.
|
|
180
|
+
|
|
181
|
+
The core exploration features (read, grep, glob, tree) require no API key. The optional `llm_query()` helper calls the Anthropic API for semantic analysis within the sandbox — this is the only feature that requires a key.
|
|
182
|
+
|
|
183
|
+
| Variable | Default | Description |
|
|
184
|
+
|----------|---------|-------------|
|
|
185
|
+
| `ANTHROPIC_API_KEY` | — | Required for `llm_query()` only. Uses Anthropic's API (Claude). |
|
|
186
|
+
| `RLM_SUB_MODEL` | `claude-haiku-4-5-20251001` | Claude model used for `llm_query()` |
|
|
187
|
+
| `RLM_MAX_SESSIONS` | `5` | Max concurrent sessions |
|
|
188
|
+
| `RLM_SESSION_TIMEOUT` | `10` | Session timeout in minutes |
|
|
189
|
+
|
|
190
|
+
## Security
|
|
191
|
+
|
|
192
|
+
The sandbox is read-only and restricted:
|
|
193
|
+
|
|
194
|
+
- **Imports**: Safe stdlib only (re, json, collections, math, etc.)
|
|
195
|
+
- **Builtins**: Blocks exec, eval, compile, `__import__`, breakpoint
|
|
196
|
+
- **File access**: Read-only, scoped to session directory, path traversal blocked
|
|
197
|
+
- **Execution**: Configurable per-call timeout (default 30s)
|
|
198
|
+
- **Rate limits**: Configurable max calls per session
|
|
199
|
+
|
|
200
|
+
## Background
|
|
201
|
+
|
|
202
|
+
RLM Tools implements an [RLM-style](https://arxiv.org/abs/2512.24601) exploration loop: keep raw data in tool-side memory, send only compact outputs to the model. Built on the [Model Context Protocol](https://modelcontextprotocol.io).
|
|
203
|
+
|
|
204
|
+
## Development
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
git clone https://github.com/stefanoshea/rlm-tools.git
|
|
208
|
+
cd rlm-tools
|
|
209
|
+
uv sync --dev
|
|
210
|
+
pytest tests
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Run comparative benchmarks (requires a local project checkout):
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
RLM_EVAL_PROJECT_PATH=/path/to/project pytest evals -q -s
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## License
|
|
220
|
+
|
|
221
|
+
MIT
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# RLM Tools
|
|
2
|
+
|
|
3
|
+
<!-- mcp-name: io.github.stefanoshea/rlm-tools -->
|
|
4
|
+
|
|
5
|
+
Your AI coding agent spends most of its token budget just *reading* your code — not reasoning about it. Every grep, file read, and glob result gets dumped into the conversation. On a large codebase, that's 25-35% of your context (and cost) burned on raw data the model never needed to see.
|
|
6
|
+
|
|
7
|
+
RLM Tools gives your agent a persistent sandbox to explore code in. Data stays server-side. Only the conclusions come back.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# For Codex / generic MCP clients
|
|
11
|
+
uvx rlm-tools-setup --write-policy AGENTS.md
|
|
12
|
+
|
|
13
|
+
# For Claude Code
|
|
14
|
+
uvx rlm-tools-setup --write-policy CLAUDE.md
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
This auto-detects Claude/Codex on your PATH, registers the MCP server, and writes a policy block that nudges agents toward `rlm-tools` for repo exploration. Use `CLAUDE.md` for Claude Code (it reads this file automatically) or `AGENTS.md` for other clients.
|
|
18
|
+
|
|
19
|
+
If you prefer manual setup:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
# Claude Code
|
|
23
|
+
claude mcp add rlm-tools -- uvx rlm-tools
|
|
24
|
+
|
|
25
|
+
# Codex
|
|
26
|
+
codex mcp add rlm-tools -- uvx rlm-tools
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## What Changes
|
|
30
|
+
|
|
31
|
+
**Without RLM Tools** — agent greps for `import UIKit`, gets 500 matches dumped into context. Reads 10 files, burns all their content as tokens. Context window fills up. Agent forgets what it was doing.
|
|
32
|
+
|
|
33
|
+
**With RLM Tools** — agent runs the same exploration in a server-side Python sandbox. Data stays in sandbox memory. Only the `print()` output enters context:
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
matches = grep("import UIKit")
|
|
37
|
+
by_module = {}
|
|
38
|
+
for m in matches:
|
|
39
|
+
module = m["file"].split("/")[0]
|
|
40
|
+
by_module.setdefault(module, []).append(m)
|
|
41
|
+
for module, ms in sorted(by_module.items(), key=lambda x: -len(x[1]))[:5]:
|
|
42
|
+
print(f"{module}: {len(ms)} files")
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
500 lines of grep results become 5 lines of summary. The agent sees what it needs, nothing more.
|
|
46
|
+
|
|
47
|
+
## Real-World Impact
|
|
48
|
+
|
|
49
|
+
In typical coding workflows: **25-35% context reduction.** That means your agent can explore roughly 40-50% more code before hitting context limits.
|
|
50
|
+
|
|
51
|
+
In heavy exploration tasks (reading many files, broad searches), savings go much further:
|
|
52
|
+
|
|
53
|
+
| Scenario | Standard Tools | RLM Tools | Saved |
|
|
54
|
+
|---|---:|---:|---:|
|
|
55
|
+
| Grep across full app | 40,045 chars | 1,644 chars | 95.9% |
|
|
56
|
+
| Read 10 large files | 1,493,720 chars | 13,588 chars | 99.1% |
|
|
57
|
+
| Multi-step exploration | 136,102 chars | 5,285 chars | 96.1% |
|
|
58
|
+
| Grep then read matches | 340,408 chars | 6,022 chars | 98.2% |
|
|
59
|
+
| Find all usages of a pattern | 13,478 chars | 3,691 chars | 72.6% |
|
|
60
|
+
| Understand a module | 94,745 chars | 16,925 chars | 82.1% |
|
|
61
|
+
|
|
62
|
+
Full benchmark methodology and reproduction steps: [`docs/benchmarks.md`](docs/benchmarks.md)
|
|
63
|
+
|
|
64
|
+
## How It Works
|
|
65
|
+
|
|
66
|
+
Three MCP tools. That's the entire API:
|
|
67
|
+
|
|
68
|
+
| Tool | Purpose |
|
|
69
|
+
|---|---|
|
|
70
|
+
| `rlm_start(path, query)` | Open a session on a directory |
|
|
71
|
+
| `rlm_execute(session_id, code)` | Run Python in the sandbox |
|
|
72
|
+
| `rlm_end(session_id)` | Close session, free resources |
|
|
73
|
+
|
|
74
|
+
The sandbox provides built-in helpers:
|
|
75
|
+
|
|
76
|
+
- `read_file(path)` / `read_files(paths)` — Read files into variables (cached across calls)
|
|
77
|
+
- `grep(pattern)` / `grep_summary(pattern)` / `grep_read(pattern)` — Search
|
|
78
|
+
- `glob_files(pattern)` — Find files by pattern
|
|
79
|
+
- `tree(path, max_depth)` — Directory structure
|
|
80
|
+
- `llm_query(prompt, context)` — Sub-LLM analysis (optional, requires API key)
|
|
81
|
+
|
|
82
|
+
Variables persist across `rlm_execute` calls within a session. The agent can build up understanding incrementally — search, filter, read, analyze — without any intermediate data touching the context window.
|
|
83
|
+
|
|
84
|
+
## Works With
|
|
85
|
+
|
|
86
|
+
RLM Tools is a standard [MCP](https://modelcontextprotocol.io) server. It works with any MCP-compatible client: **Claude Code**, **Codex**, **Cursor**, and others.
|
|
87
|
+
|
|
88
|
+
### Optional Strict Mode
|
|
89
|
+
|
|
90
|
+
If your MCP client supports tool permissions or hooks, disable/block default read/grep/glob tools for repository exploration and keep `rlm-tools` enabled. This is the strongest way to force the token-efficient path.
|
|
91
|
+
|
|
92
|
+
<details>
|
|
93
|
+
<summary><strong>Other installation methods</strong></summary>
|
|
94
|
+
|
|
95
|
+
### JSON MCP config (Cursor, Windsurf, etc.)
|
|
96
|
+
|
|
97
|
+
```json
|
|
98
|
+
{
|
|
99
|
+
"mcpServers": {
|
|
100
|
+
"rlm-tools": {
|
|
101
|
+
"command": "uvx",
|
|
102
|
+
"args": ["rlm-tools"]
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Setup helper command
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# Auto-detect Claude/Codex CLIs and configure rlm-tools
|
|
112
|
+
uvx rlm-tools-setup
|
|
113
|
+
|
|
114
|
+
# Print JSON config snippet for other clients
|
|
115
|
+
uvx rlm-tools-setup --print-json
|
|
116
|
+
|
|
117
|
+
# Print policy text instead of writing it
|
|
118
|
+
uvx rlm-tools-setup --print-policy
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Direct run
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
uvx rlm-tools
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### From source
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
git clone https://github.com/stefanoshea/rlm-tools.git
|
|
131
|
+
cd rlm-tools
|
|
132
|
+
uv sync
|
|
133
|
+
uv run rlm-tools
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Then point your MCP client to `command: uv`, `args: ["--directory", "/path/to/rlm-tools", "run", "rlm-tools"]`.
|
|
137
|
+
|
|
138
|
+
</details>
|
|
139
|
+
|
|
140
|
+
## Configuration
|
|
141
|
+
|
|
142
|
+
Copy `.env.example` to `.env` to customize. All settings are optional — RLM Tools works out of the box with zero config.
|
|
143
|
+
|
|
144
|
+
The core exploration features (read, grep, glob, tree) require no API key. The optional `llm_query()` helper calls the Anthropic API for semantic analysis within the sandbox — this is the only feature that requires a key.
|
|
145
|
+
|
|
146
|
+
| Variable | Default | Description |
|
|
147
|
+
|----------|---------|-------------|
|
|
148
|
+
| `ANTHROPIC_API_KEY` | — | Required for `llm_query()` only. Uses Anthropic's API (Claude). |
|
|
149
|
+
| `RLM_SUB_MODEL` | `claude-haiku-4-5-20251001` | Claude model used for `llm_query()` |
|
|
150
|
+
| `RLM_MAX_SESSIONS` | `5` | Max concurrent sessions |
|
|
151
|
+
| `RLM_SESSION_TIMEOUT` | `10` | Session timeout in minutes |
|
|
152
|
+
|
|
153
|
+
## Security
|
|
154
|
+
|
|
155
|
+
The sandbox is read-only and restricted:
|
|
156
|
+
|
|
157
|
+
- **Imports**: Safe stdlib only (re, json, collections, math, etc.)
|
|
158
|
+
- **Builtins**: Blocks exec, eval, compile, `__import__`, breakpoint
|
|
159
|
+
- **File access**: Read-only, scoped to session directory, path traversal blocked
|
|
160
|
+
- **Execution**: Configurable per-call timeout (default 30s)
|
|
161
|
+
- **Rate limits**: Configurable max calls per session
|
|
162
|
+
|
|
163
|
+
## Background
|
|
164
|
+
|
|
165
|
+
RLM Tools implements an [RLM-style](https://arxiv.org/abs/2512.24601) exploration loop: keep raw data in tool-side memory, send only compact outputs to the model. Built on the [Model Context Protocol](https://modelcontextprotocol.io).
|
|
166
|
+
|
|
167
|
+
## Development
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
git clone https://github.com/stefanoshea/rlm-tools.git
|
|
171
|
+
cd rlm-tools
|
|
172
|
+
uv sync --dev
|
|
173
|
+
pytest tests
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Run comparative benchmarks (requires a local project checkout):
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
RLM_EVAL_PROJECT_PATH=/path/to/project pytest evals -q -s
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Benchmarks
|
|
2
|
+
|
|
3
|
+
This document describes how to benchmark `rlm-tools` and includes a publishable baseline result snapshot.
|
|
4
|
+
|
|
5
|
+
## What Is Measured
|
|
6
|
+
|
|
7
|
+
The comparative evals measure **context payload size** for equivalent exploration tasks:
|
|
8
|
+
|
|
9
|
+
- **With RLM Tools:** data stays in sandbox memory; only `print()` output is returned.
|
|
10
|
+
- **Baseline:** equivalent `Read`/`Grep`/`Glob` style behavior returns full payloads.
|
|
11
|
+
|
|
12
|
+
Metric used in `evals/test_comparative.py`:
|
|
13
|
+
|
|
14
|
+
- `total_context_chars = agent_output_chars + tool_response_chars + per-turn overhead`
|
|
15
|
+
|
|
16
|
+
Lower is better.
|
|
17
|
+
|
|
18
|
+
## Reproduce Benchmarks
|
|
19
|
+
|
|
20
|
+
Prereq: local checkout of the `your-iOS-project` dataset.
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
export RLM_EVAL_PROJECT_PATH=/path/to/your-iOS-project
|
|
24
|
+
uv run pytest evals/test_comparative.py -q -s
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
For a live client-level A/B comparison (Claude CLI-specific helper script):
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
export RLM_EVAL_PROJECT_PATH=/path/to/your-iOS-project
|
|
31
|
+
./evals/run_ab_eval.sh
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Note: `evals/test_comparative.py` is MCP-client agnostic. `evals/run_ab_eval.sh` is currently tailored to Claude CLI.
|
|
35
|
+
|
|
36
|
+
Optional env vars for A/B script:
|
|
37
|
+
|
|
38
|
+
- `RLM_TOOLS_PATH` (default: current repo root)
|
|
39
|
+
- `RLM_AB_TARGET_PATH` (default: `$RLM_EVAL_PROJECT_PATH/app`)
|
|
40
|
+
|
|
41
|
+
## Snapshot (February 13, 2026)
|
|
42
|
+
|
|
43
|
+
Source:
|
|
44
|
+
|
|
45
|
+
- Command: `uv run pytest evals/test_comparative.py -q -s`
|
|
46
|
+
- Commit: `20765a0`
|
|
47
|
+
- Dataset: `your-iOS-project/app`
|
|
48
|
+
|
|
49
|
+
Interpretation note:
|
|
50
|
+
|
|
51
|
+
- The scenarios below are intentionally heavy payload stress tests.
|
|
52
|
+
- In typical day-to-day coding workflows, observed context/token savings are usually closer to **25-35%** (task- and prompting-dependent).
|
|
53
|
+
|
|
54
|
+
| Scenario | RLM Context Chars | Baseline Context Chars | Savings |
|
|
55
|
+
|---|---:|---:|---:|
|
|
56
|
+
| Grep full app (`import UIKit`) | 1,644 | 40,045 | 95.9% |
|
|
57
|
+
| Read 10 large files | 13,588 | 1,493,720 | 99.1% |
|
|
58
|
+
| Multi-step exploration | 5,285 | 136,102 | 96.1% |
|
|
59
|
+
| Grep then read | 6,022 | 340,408 | 98.2% |
|
|
60
|
+
| Find usages (`@objc func`) | 3,691 | 13,478 | 72.6% |
|
|
61
|
+
| Module understanding | 16,925 | 94,745 | 82.1% |
|
|
62
|
+
| **Weighted total (all scenarios above)** | **47,155** | **2,118,498** | **97.8%** |
|
|
63
|
+
|
|
64
|
+
## Reporting Guidance
|
|
65
|
+
|
|
66
|
+
When publishing benchmark results in GitHub:
|
|
67
|
+
|
|
68
|
+
1. Include the command(s), commit SHA, and benchmark date.
|
|
69
|
+
2. Include dataset path/shape so readers understand scope.
|
|
70
|
+
3. Report both raw counts and percentage savings.
|
|
71
|
+
4. Keep at least one "harder" scenario (where savings are smaller) to avoid cherry-picking.
|