rlm-tools 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Bash(git status:*)",
5
+ "Bash(uv run pytest:*)",
6
+ "Bash(uv run:*)",
7
+ "Bash(PATH=/usr/bin:/bin uv run:*)",
8
+ "Bash(echo:*)",
9
+ "Bash(git add:*)",
10
+ "Bash(git reset:*)",
11
+ "Bash(git commit:*)"
12
+ ]
13
+ },
14
+ "enableAllProjectMcpServers": true,
15
+ "enabledMcpjsonServers": [
16
+ "rlm-tools"
17
+ ]
18
+ }
@@ -0,0 +1,12 @@
1
+ # Required for llm_query() / llm_query_batched() sub-LLM features
2
+ # Optional — server works without it, just no LLM query capability
3
+ ANTHROPIC_API_KEY=sk-ant-...
4
+
5
+ # Optional: override the sub-LLM model (default: claude-haiku-4-5-20251001)
6
+ # RLM_SUB_MODEL=claude-haiku-4-5-20251001
7
+
8
+ # Optional: max concurrent sessions (default: 5)
9
+ # RLM_MAX_SESSIONS=5
10
+
11
+ # Optional: session inactivity timeout in minutes (default: 10)
12
+ # RLM_SESSION_TIMEOUT=10
@@ -0,0 +1,105 @@
1
+ name: Publish rlm-tools
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ build:
11
+ runs-on: ubuntu-latest
12
+ permissions:
13
+ contents: read
14
+ outputs:
15
+ version: ${{ steps.version.outputs.version }}
16
+ steps:
17
+ - name: Check out code
18
+ uses: actions/checkout@v5
19
+
20
+ - name: Resolve release version
21
+ id: version
22
+ run: |
23
+ if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
24
+ echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
25
+ else
26
+ VERSION="$(python3 - <<'PY'
27
+ import tomllib
28
+ from pathlib import Path
29
+ data = tomllib.loads(Path('pyproject.toml').read_text(encoding='utf-8'))
30
+ print(data['project']['version'])
31
+ PY
32
+ )"
33
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
34
+ fi
35
+
36
+ - name: Set up Python
37
+ uses: actions/setup-python@v5
38
+ with:
39
+ python-version: "3.12"
40
+
41
+ - name: Set up uv
42
+ uses: astral-sh/setup-uv@v4
43
+
44
+ - name: Install dependencies
45
+ run: uv sync --dev
46
+
47
+ - name: Run tests
48
+ run: uv run pytest tests
49
+
50
+ - name: Build package
51
+ run: uv build
52
+
53
+ - name: Upload package artifacts
54
+ uses: actions/upload-artifact@v4
55
+ with:
56
+ name: python-dist
57
+ path: dist/*
58
+
59
+ publish-pypi:
60
+ runs-on: ubuntu-latest
61
+ needs: build
62
+ permissions:
63
+ id-token: write
64
+ contents: read
65
+ steps:
66
+ - name: Download package artifacts
67
+ uses: actions/download-artifact@v4
68
+ with:
69
+ name: python-dist
70
+ path: dist
71
+
72
+ - name: Publish package to PyPI
73
+ uses: pypa/gh-action-pypi-publish@release/v1
74
+
75
+ publish-mcp-registry:
76
+ runs-on: ubuntu-latest
77
+ needs:
78
+ - build
79
+ - publish-pypi
80
+ permissions:
81
+ id-token: write
82
+ contents: read
83
+ steps:
84
+ - name: Check out code
85
+ uses: actions/checkout@v5
86
+
87
+ - name: Update server.json version
88
+ run: |
89
+ VERSION="${{ needs.build.outputs.version }}"
90
+ jq --arg v "$VERSION" '
91
+ .version = $v
92
+ | (.packages[] | select(.registryType == "pypi" and .identifier == "rlm-tools") | .version) = $v
93
+ ' server.json > server.json.tmp
94
+ mv server.json.tmp server.json
95
+
96
+ - name: Install mcp-publisher
97
+ run: |
98
+ curl -L "https://github.com/modelcontextprotocol/registry/releases/latest/download/mcp-publisher_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/').tar.gz" \
99
+ | tar xz mcp-publisher
100
+
101
+ - name: Authenticate to MCP Registry
102
+ run: ./mcp-publisher login github-oidc
103
+
104
+ - name: Publish server to MCP Registry
105
+ run: ./mcp-publisher publish
@@ -0,0 +1,11 @@
1
+ __pycache__/
2
+ *.pyc
3
+ *.pyo
4
+ .venv/
5
+ dist/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .env
9
+ .mcp.local.json
10
+ evals/ab_results/
11
+ docs/plans/
@@ -0,0 +1,11 @@
1
+ {
2
+ "mcpServers": {
3
+ "rlm-tools": {
4
+ "command": "uvx",
5
+ "args": ["rlm-tools"],
6
+ "env": {
7
+ "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}"
8
+ }
9
+ }
10
+ }
11
+ }
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Stefan O'Shea
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,221 @@
1
+ Metadata-Version: 2.4
2
+ Name: rlm-tools
3
+ Version: 0.1.0
4
+ Summary: MCP server providing a persistent Python sandbox for token-efficient codebase exploration across MCP clients
5
+ Project-URL: Homepage, https://github.com/stefanoshea/rlm-tools
6
+ Project-URL: Repository, https://github.com/stefanoshea/rlm-tools
7
+ Project-URL: Issues, https://github.com/stefanoshea/rlm-tools/issues
8
+ License: MIT License
9
+
10
+ Copyright (c) 2026 Stefan O'Shea
11
+
12
+ Permission is hereby granted, free of charge, to any person obtaining a copy
13
+ of this software and associated documentation files (the "Software"), to deal
14
+ in the Software without restriction, including without limitation the rights
15
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+ copies of the Software, and to permit persons to whom the Software is
17
+ furnished to do so, subject to the following conditions:
18
+
19
+ The above copyright notice and this permission notice shall be included in all
20
+ copies or substantial portions of the Software.
21
+
22
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+ SOFTWARE.
29
+ License-File: LICENSE
30
+ Requires-Python: >=3.10
31
+ Requires-Dist: anthropic>=0.40.0
32
+ Requires-Dist: mcp[cli]>=1.2.0
33
+ Provides-Extra: dev
34
+ Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
35
+ Requires-Dist: pytest>=8.0; extra == 'dev'
36
+ Description-Content-Type: text/markdown
37
+
38
+ # RLM Tools
39
+
40
+ <!-- mcp-name: io.github.stefanoshea/rlm-tools -->
41
+
42
+ Your AI coding agent spends most of its token budget just *reading* your code — not reasoning about it. Every grep, file read, and glob result gets dumped into the conversation. On a large codebase, that's 25-35% of your context (and cost) burned on raw data the model never needed to see.
43
+
44
+ RLM Tools gives your agent a persistent sandbox to explore code in. Data stays server-side. Only the conclusions come back.
45
+
46
+ ```bash
47
+ # For Codex / generic MCP clients
48
+ uvx rlm-tools-setup --write-policy AGENTS.md
49
+
50
+ # For Claude Code
51
+ uvx rlm-tools-setup --write-policy CLAUDE.md
52
+ ```
53
+
54
+ This auto-detects Claude/Codex on your PATH, registers the MCP server, and writes a policy block that nudges agents toward `rlm-tools` for repo exploration. Use `CLAUDE.md` for Claude Code (it reads this file automatically) or `AGENTS.md` for other clients.
55
+
56
+ If you prefer manual setup:
57
+
58
+ ```bash
59
+ # Claude Code
60
+ claude mcp add rlm-tools -- uvx rlm-tools
61
+
62
+ # Codex
63
+ codex mcp add rlm-tools -- uvx rlm-tools
64
+ ```
65
+
66
+ ## What Changes
67
+
68
+ **Without RLM Tools** — agent greps for `import UIKit`, gets 500 matches dumped into context. Reads 10 files, burns all their content as tokens. Context window fills up. Agent forgets what it was doing.
69
+
70
+ **With RLM Tools** — agent runs the same exploration in a server-side Python sandbox. Data stays in sandbox memory. Only the `print()` output enters context:
71
+
72
+ ```python
73
+ matches = grep("import UIKit")
74
+ by_module = {}
75
+ for m in matches:
76
+ module = m["file"].split("/")[0]
77
+ by_module.setdefault(module, []).append(m)
78
+ for module, ms in sorted(by_module.items(), key=lambda x: -len(x[1]))[:5]:
79
+ print(f"{module}: {len(ms)} files")
80
+ ```
81
+
82
+ 500 lines of grep results become 5 lines of summary. The agent sees what it needs, nothing more.
83
+
84
+ ## Real-World Impact
85
+
86
+ In typical coding workflows: **25-35% context reduction.** That means your agent can explore roughly 40-50% more code before hitting context limits.
87
+
88
+ In heavy exploration tasks (reading many files, broad searches), savings go much further:
89
+
90
+ | Scenario | Standard Tools | RLM Tools | Saved |
91
+ |---|---:|---:|---:|
92
+ | Grep across full app | 40,045 chars | 1,644 chars | 95.9% |
93
+ | Read 10 large files | 1,493,720 chars | 13,588 chars | 99.1% |
94
+ | Multi-step exploration | 136,102 chars | 5,285 chars | 96.1% |
95
+ | Grep then read matches | 340,408 chars | 6,022 chars | 98.2% |
96
+ | Find all usages of a pattern | 13,478 chars | 3,691 chars | 72.6% |
97
+ | Understand a module | 94,745 chars | 16,925 chars | 82.1% |
98
+
99
+ Full benchmark methodology and reproduction steps: [`docs/benchmarks.md`](docs/benchmarks.md)
100
+
101
+ ## How It Works
102
+
103
+ Three MCP tools. That's the entire API:
104
+
105
+ | Tool | Purpose |
106
+ |---|---|
107
+ | `rlm_start(path, query)` | Open a session on a directory |
108
+ | `rlm_execute(session_id, code)` | Run Python in the sandbox |
109
+ | `rlm_end(session_id)` | Close session, free resources |
110
+
111
+ The sandbox provides built-in helpers:
112
+
113
+ - `read_file(path)` / `read_files(paths)` — Read files into variables (cached across calls)
114
+ - `grep(pattern)` / `grep_summary(pattern)` / `grep_read(pattern)` — Search
115
+ - `glob_files(pattern)` — Find files by pattern
116
+ - `tree(path, max_depth)` — Directory structure
117
+ - `llm_query(prompt, context)` — Sub-LLM analysis (optional, requires API key)
118
+
119
+ Variables persist across `rlm_execute` calls within a session. The agent can build up understanding incrementally — search, filter, read, analyze — without any intermediate data touching the context window.
120
+
121
+ ## Works With
122
+
123
+ RLM Tools is a standard [MCP](https://modelcontextprotocol.io) server. It works with any MCP-compatible client: **Claude Code**, **Codex**, **Cursor**, and others.
124
+
125
+ ### Optional Strict Mode
126
+
127
+ If your MCP client supports tool permissions or hooks, disable/block default read/grep/glob tools for repository exploration and keep `rlm-tools` enabled. This is the strongest way to force the token-efficient path.
128
+
129
+ <details>
130
+ <summary><strong>Other installation methods</strong></summary>
131
+
132
+ ### JSON MCP config (Cursor, Windsurf, etc.)
133
+
134
+ ```json
135
+ {
136
+ "mcpServers": {
137
+ "rlm-tools": {
138
+ "command": "uvx",
139
+ "args": ["rlm-tools"]
140
+ }
141
+ }
142
+ }
143
+ ```
144
+
145
+ ### Setup helper command
146
+
147
+ ```bash
148
+ # Auto-detect Claude/Codex CLIs and configure rlm-tools
149
+ uvx rlm-tools-setup
150
+
151
+ # Print JSON config snippet for other clients
152
+ uvx rlm-tools-setup --print-json
153
+
154
+ # Print policy text instead of writing it
155
+ uvx rlm-tools-setup --print-policy
156
+ ```
157
+
158
+ ### Direct run
159
+
160
+ ```bash
161
+ uvx rlm-tools
162
+ ```
163
+
164
+ ### From source
165
+
166
+ ```bash
167
+ git clone https://github.com/stefanoshea/rlm-tools.git
168
+ cd rlm-tools
169
+ uv sync
170
+ uv run rlm-tools
171
+ ```
172
+
173
+ Then point your MCP client to `command: uv`, `args: ["--directory", "/path/to/rlm-tools", "run", "rlm-tools"]`.
174
+
175
+ </details>
176
+
177
+ ## Configuration
178
+
179
+ Copy `.env.example` to `.env` to customize. All settings are optional — RLM Tools works out of the box with zero config.
180
+
181
+ The core exploration features (read, grep, glob, tree) require no API key. The optional `llm_query()` helper calls the Anthropic API for semantic analysis within the sandbox — this is the only feature that requires a key.
182
+
183
+ | Variable | Default | Description |
184
+ |----------|---------|-------------|
185
+ | `ANTHROPIC_API_KEY` | — | Required for `llm_query()` only. Uses Anthropic's API (Claude). |
186
+ | `RLM_SUB_MODEL` | `claude-haiku-4-5-20251001` | Claude model used for `llm_query()` |
187
+ | `RLM_MAX_SESSIONS` | `5` | Max concurrent sessions |
188
+ | `RLM_SESSION_TIMEOUT` | `10` | Session timeout in minutes |
189
+
190
+ ## Security
191
+
192
+ The sandbox is read-only and restricted:
193
+
194
+ - **Imports**: Safe stdlib only (re, json, collections, math, etc.)
195
+ - **Builtins**: Blocks exec, eval, compile, `__import__`, breakpoint
196
+ - **File access**: Read-only, scoped to session directory, path traversal blocked
197
+ - **Execution**: Configurable per-call timeout (default 30s)
198
+ - **Rate limits**: Configurable max calls per session
199
+
200
+ ## Background
201
+
202
+ RLM Tools implements an [RLM-style](https://arxiv.org/abs/2512.24601) exploration loop: keep raw data in tool-side memory, send only compact outputs to the model. Built on the [Model Context Protocol](https://modelcontextprotocol.io).
203
+
204
+ ## Development
205
+
206
+ ```bash
207
+ git clone https://github.com/stefanoshea/rlm-tools.git
208
+ cd rlm-tools
209
+ uv sync --dev
210
+ pytest tests
211
+ ```
212
+
213
+ Run comparative benchmarks (requires a local project checkout):
214
+
215
+ ```bash
216
+ RLM_EVAL_PROJECT_PATH=/path/to/project pytest evals -q -s
217
+ ```
218
+
219
+ ## License
220
+
221
+ MIT
@@ -0,0 +1,184 @@
1
+ # RLM Tools
2
+
3
+ <!-- mcp-name: io.github.stefanoshea/rlm-tools -->
4
+
5
+ Your AI coding agent spends most of its token budget just *reading* your code — not reasoning about it. Every grep, file read, and glob result gets dumped into the conversation. On a large codebase, that's 25-35% of your context (and cost) burned on raw data the model never needed to see.
6
+
7
+ RLM Tools gives your agent a persistent sandbox to explore code in. Data stays server-side. Only the conclusions come back.
8
+
9
+ ```bash
10
+ # For Codex / generic MCP clients
11
+ uvx rlm-tools-setup --write-policy AGENTS.md
12
+
13
+ # For Claude Code
14
+ uvx rlm-tools-setup --write-policy CLAUDE.md
15
+ ```
16
+
17
+ This auto-detects Claude/Codex on your PATH, registers the MCP server, and writes a policy block that nudges agents toward `rlm-tools` for repo exploration. Use `CLAUDE.md` for Claude Code (it reads this file automatically) or `AGENTS.md` for other clients.
18
+
19
+ If you prefer manual setup:
20
+
21
+ ```bash
22
+ # Claude Code
23
+ claude mcp add rlm-tools -- uvx rlm-tools
24
+
25
+ # Codex
26
+ codex mcp add rlm-tools -- uvx rlm-tools
27
+ ```
28
+
29
+ ## What Changes
30
+
31
+ **Without RLM Tools** — agent greps for `import UIKit`, gets 500 matches dumped into context. Reads 10 files, burns all their content as tokens. Context window fills up. Agent forgets what it was doing.
32
+
33
+ **With RLM Tools** — agent runs the same exploration in a server-side Python sandbox. Data stays in sandbox memory. Only the `print()` output enters context:
34
+
35
+ ```python
36
+ matches = grep("import UIKit")
37
+ by_module = {}
38
+ for m in matches:
39
+ module = m["file"].split("/")[0]
40
+ by_module.setdefault(module, []).append(m)
41
+ for module, ms in sorted(by_module.items(), key=lambda x: -len(x[1]))[:5]:
42
+ print(f"{module}: {len(ms)} files")
43
+ ```
44
+
45
+ 500 lines of grep results become 5 lines of summary. The agent sees what it needs, nothing more.
46
+
47
+ ## Real-World Impact
48
+
49
+ In typical coding workflows: **25-35% context reduction.** That means your agent can explore roughly 40-50% more code before hitting context limits.
50
+
51
+ In heavy exploration tasks (reading many files, broad searches), savings go much further:
52
+
53
+ | Scenario | Standard Tools | RLM Tools | Saved |
54
+ |---|---:|---:|---:|
55
+ | Grep across full app | 40,045 chars | 1,644 chars | 95.9% |
56
+ | Read 10 large files | 1,493,720 chars | 13,588 chars | 99.1% |
57
+ | Multi-step exploration | 136,102 chars | 5,285 chars | 96.1% |
58
+ | Grep then read matches | 340,408 chars | 6,022 chars | 98.2% |
59
+ | Find all usages of a pattern | 13,478 chars | 3,691 chars | 72.6% |
60
+ | Understand a module | 94,745 chars | 16,925 chars | 82.1% |
61
+
62
+ Full benchmark methodology and reproduction steps: [`docs/benchmarks.md`](docs/benchmarks.md)
63
+
64
+ ## How It Works
65
+
66
+ Three MCP tools. That's the entire API:
67
+
68
+ | Tool | Purpose |
69
+ |---|---|
70
+ | `rlm_start(path, query)` | Open a session on a directory |
71
+ | `rlm_execute(session_id, code)` | Run Python in the sandbox |
72
+ | `rlm_end(session_id)` | Close session, free resources |
73
+
74
+ The sandbox provides built-in helpers:
75
+
76
+ - `read_file(path)` / `read_files(paths)` — Read files into variables (cached across calls)
77
+ - `grep(pattern)` / `grep_summary(pattern)` / `grep_read(pattern)` — Search
78
+ - `glob_files(pattern)` — Find files by pattern
79
+ - `tree(path, max_depth)` — Directory structure
80
+ - `llm_query(prompt, context)` — Sub-LLM analysis (optional, requires API key)
81
+
82
+ Variables persist across `rlm_execute` calls within a session. The agent can build up understanding incrementally — search, filter, read, analyze — without any intermediate data touching the context window.
83
+
84
+ ## Works With
85
+
86
+ RLM Tools is a standard [MCP](https://modelcontextprotocol.io) server. It works with any MCP-compatible client: **Claude Code**, **Codex**, **Cursor**, and others.
87
+
88
+ ### Optional Strict Mode
89
+
90
+ If your MCP client supports tool permissions or hooks, disable/block default read/grep/glob tools for repository exploration and keep `rlm-tools` enabled. This is the strongest way to force the token-efficient path.
91
+
92
+ <details>
93
+ <summary><strong>Other installation methods</strong></summary>
94
+
95
+ ### JSON MCP config (Cursor, Windsurf, etc.)
96
+
97
+ ```json
98
+ {
99
+ "mcpServers": {
100
+ "rlm-tools": {
101
+ "command": "uvx",
102
+ "args": ["rlm-tools"]
103
+ }
104
+ }
105
+ }
106
+ ```
107
+
108
+ ### Setup helper command
109
+
110
+ ```bash
111
+ # Auto-detect Claude/Codex CLIs and configure rlm-tools
112
+ uvx rlm-tools-setup
113
+
114
+ # Print JSON config snippet for other clients
115
+ uvx rlm-tools-setup --print-json
116
+
117
+ # Print policy text instead of writing it
118
+ uvx rlm-tools-setup --print-policy
119
+ ```
120
+
121
+ ### Direct run
122
+
123
+ ```bash
124
+ uvx rlm-tools
125
+ ```
126
+
127
+ ### From source
128
+
129
+ ```bash
130
+ git clone https://github.com/stefanoshea/rlm-tools.git
131
+ cd rlm-tools
132
+ uv sync
133
+ uv run rlm-tools
134
+ ```
135
+
136
+ Then point your MCP client to `command: uv`, `args: ["--directory", "/path/to/rlm-tools", "run", "rlm-tools"]`.
137
+
138
+ </details>
139
+
140
+ ## Configuration
141
+
142
+ Copy `.env.example` to `.env` to customize. All settings are optional — RLM Tools works out of the box with zero config.
143
+
144
+ The core exploration features (read, grep, glob, tree) require no API key. The optional `llm_query()` helper calls the Anthropic API for semantic analysis within the sandbox — this is the only feature that requires a key.
145
+
146
+ | Variable | Default | Description |
147
+ |----------|---------|-------------|
148
+ | `ANTHROPIC_API_KEY` | — | Required for `llm_query()` only. Uses Anthropic's API (Claude). |
149
+ | `RLM_SUB_MODEL` | `claude-haiku-4-5-20251001` | Claude model used for `llm_query()` |
150
+ | `RLM_MAX_SESSIONS` | `5` | Max concurrent sessions |
151
+ | `RLM_SESSION_TIMEOUT` | `10` | Session timeout in minutes |
152
+
153
+ ## Security
154
+
155
+ The sandbox is read-only and restricted:
156
+
157
+ - **Imports**: Safe stdlib only (re, json, collections, math, etc.)
158
+ - **Builtins**: Blocks exec, eval, compile, `__import__`, breakpoint
159
+ - **File access**: Read-only, scoped to session directory, path traversal blocked
160
+ - **Execution**: Configurable per-call timeout (default 30s)
161
+ - **Rate limits**: Configurable max calls per session
162
+
163
+ ## Background
164
+
165
+ RLM Tools implements an [RLM-style](https://arxiv.org/abs/2512.24601) exploration loop: keep raw data in tool-side memory, send only compact outputs to the model. Built on the [Model Context Protocol](https://modelcontextprotocol.io).
166
+
167
+ ## Development
168
+
169
+ ```bash
170
+ git clone https://github.com/stefanoshea/rlm-tools.git
171
+ cd rlm-tools
172
+ uv sync --dev
173
+ pytest tests
174
+ ```
175
+
176
+ Run comparative benchmarks (requires a local project checkout):
177
+
178
+ ```bash
179
+ RLM_EVAL_PROJECT_PATH=/path/to/project pytest evals -q -s
180
+ ```
181
+
182
+ ## License
183
+
184
+ MIT
@@ -0,0 +1,71 @@
1
+ # Benchmarks
2
+
3
+ This document describes how to benchmark `rlm-tools` and includes a publishable baseline result snapshot.
4
+
5
+ ## What Is Measured
6
+
7
+ The comparative evals measure **context payload size** for equivalent exploration tasks:
8
+
9
+ - **With RLM Tools:** data stays in sandbox memory; only `print()` output is returned.
10
+ - **Baseline:** equivalent `Read`/`Grep`/`Glob` style behavior returns full payloads.
11
+
12
+ Metric used in `evals/test_comparative.py`:
13
+
14
+ - `total_context_chars = agent_output_chars + tool_response_chars + per-turn overhead`
15
+
16
+ Lower is better.
17
+
18
+ ## Reproduce Benchmarks
19
+
20
+ Prereq: local checkout of the `your-iOS-project` dataset.
21
+
22
+ ```bash
23
+ export RLM_EVAL_PROJECT_PATH=/path/to/your-iOS-project
24
+ uv run pytest evals/test_comparative.py -q -s
25
+ ```
26
+
27
+ For a live client-level A/B comparison (Claude CLI-specific helper script):
28
+
29
+ ```bash
30
+ export RLM_EVAL_PROJECT_PATH=/path/to/your-iOS-project
31
+ ./evals/run_ab_eval.sh
32
+ ```
33
+
34
+ Note: `evals/test_comparative.py` is MCP-client agnostic. `evals/run_ab_eval.sh` is currently tailored to Claude CLI.
35
+
36
+ Optional env vars for A/B script:
37
+
38
+ - `RLM_TOOLS_PATH` (default: current repo root)
39
+ - `RLM_AB_TARGET_PATH` (default: `$RLM_EVAL_PROJECT_PATH/app`)
40
+
41
+ ## Snapshot (February 13, 2026)
42
+
43
+ Source:
44
+
45
+ - Command: `uv run pytest evals/test_comparative.py -q -s`
46
+ - Commit: `20765a0`
47
+ - Dataset: `your-iOS-project/app`
48
+
49
+ Interpretation note:
50
+
51
+ - The scenarios below are intentionally heavy payload stress tests.
52
+ - In typical day-to-day coding workflows, observed context/token savings are usually closer to **25-35%** (task- and prompting-dependent).
53
+
54
+ | Scenario | RLM Context Chars | Baseline Context Chars | Savings |
55
+ |---|---:|---:|---:|
56
+ | Grep full app (`import UIKit`) | 1,644 | 40,045 | 95.9% |
57
+ | Read 10 large files | 13,588 | 1,493,720 | 99.1% |
58
+ | Multi-step exploration | 5,285 | 136,102 | 96.1% |
59
+ | Grep then read | 6,022 | 340,408 | 98.2% |
60
+ | Find usages (`@objc func`) | 3,691 | 13,478 | 72.6% |
61
+ | Module understanding | 16,925 | 94,745 | 82.1% |
62
+ | **Weighted total (all scenarios above)** | **47,155** | **2,118,498** | **97.8%** |
63
+
64
+ ## Reporting Guidance
65
+
66
+ When publishing benchmark results in GitHub:
67
+
68
+ 1. Include the command(s), commit SHA, and benchmark date.
69
+ 2. Include dataset path/shape so readers understand scope.
70
+ 3. Report both raw counts and percentage savings.
71
+ 4. Keep at least one "harder" scenario (where savings are smaller) to avoid cherry-picking.