tinkerscope 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. tinkerscope-1.0.0/.githooks/README.md +18 -0
  2. tinkerscope-1.0.0/.gitignore +30 -0
  3. tinkerscope-1.0.0/LICENSE +27 -0
  4. tinkerscope-1.0.0/PKG-INFO +304 -0
  5. tinkerscope-1.0.0/README.md +266 -0
  6. tinkerscope-1.0.0/hatch_build.py +63 -0
  7. tinkerscope-1.0.0/pyproject.toml +105 -0
  8. tinkerscope-1.0.0/src/tinkerscope/__init__.py +3 -0
  9. tinkerscope-1.0.0/src/tinkerscope/api/__init__.py +0 -0
  10. tinkerscope-1.0.0/src/tinkerscope/api/discovery.py +381 -0
  11. tinkerscope-1.0.0/src/tinkerscope/api/main.py +118 -0
  12. tinkerscope-1.0.0/src/tinkerscope/api/openrouter.py +185 -0
  13. tinkerscope-1.0.0/src/tinkerscope/api/pack_models_store.py +58 -0
  14. tinkerscope-1.0.0/src/tinkerscope/api/raw_view.py +58 -0
  15. tinkerscope-1.0.0/src/tinkerscope/api/routes/__init__.py +0 -0
  16. tinkerscope-1.0.0/src/tinkerscope/api/routes/chat.py +798 -0
  17. tinkerscope-1.0.0/src/tinkerscope/api/routes/datasets.py +49 -0
  18. tinkerscope-1.0.0/src/tinkerscope/api/routes/highlights.py +123 -0
  19. tinkerscope-1.0.0/src/tinkerscope/api/routes/models.py +105 -0
  20. tinkerscope-1.0.0/src/tinkerscope/api/routes/openrouter_models.py +113 -0
  21. tinkerscope-1.0.0/src/tinkerscope/api/routes/packs.py +63 -0
  22. tinkerscope-1.0.0/src/tinkerscope/api/routes/pins.py +62 -0
  23. tinkerscope-1.0.0/src/tinkerscope/api/routes/prefs.py +48 -0
  24. tinkerscope-1.0.0/src/tinkerscope/api/routes/state.py +98 -0
  25. tinkerscope-1.0.0/src/tinkerscope/api/routes/workspaces.py +202 -0
  26. tinkerscope-1.0.0/src/tinkerscope/api/settings.py +138 -0
  27. tinkerscope-1.0.0/src/tinkerscope/api/state.py +309 -0
  28. tinkerscope-1.0.0/src/tinkerscope/api/store.py +62 -0
  29. tinkerscope-1.0.0/src/tinkerscope/api/tinker_oai.py +161 -0
  30. tinkerscope-1.0.0/src/tinkerscope/api/tinker_sampler.py +659 -0
  31. tinkerscope-1.0.0/src/tinkerscope/api/workspace_store.py +763 -0
  32. tinkerscope-1.0.0/src/tinkerscope/cli.py +2702 -0
  33. tinkerscope-1.0.0/src/tinkerscope/instances.py +159 -0
  34. tinkerscope-1.0.0/src/tinkerscope/pack.py +782 -0
  35. tinkerscope-1.0.0/src/tinkerscope/paths.py +18 -0
  36. tinkerscope-1.0.0/src/tinkerscope/serve.py +340 -0
  37. tinkerscope-1.0.0/src/tinkerscope/site_export.py +436 -0
  38. tinkerscope-1.0.0/web/package-lock.json +1723 -0
  39. tinkerscope-1.0.0/web/package.json +28 -0
  40. tinkerscope-1.0.0/web/src/app.css +172 -0
  41. tinkerscope-1.0.0/web/src/app.html +32 -0
  42. tinkerscope-1.0.0/web/src/lib/ActionMenu.svelte +114 -0
  43. tinkerscope-1.0.0/web/src/lib/ChartModal.svelte +969 -0
  44. tinkerscope-1.0.0/web/src/lib/ChatMessage.svelte +910 -0
  45. tinkerscope-1.0.0/web/src/lib/DatasetModal.svelte +39 -0
  46. tinkerscope-1.0.0/web/src/lib/DiffLabel.svelte +57 -0
  47. tinkerscope-1.0.0/web/src/lib/HelpModal.svelte +565 -0
  48. tinkerscope-1.0.0/web/src/lib/HighlightRules.svelte +703 -0
  49. tinkerscope-1.0.0/web/src/lib/Icon.svelte +97 -0
  50. tinkerscope-1.0.0/web/src/lib/Modal.svelte +71 -0
  51. tinkerscope-1.0.0/web/src/lib/OpenLocallyModal.svelte +107 -0
  52. tinkerscope-1.0.0/web/src/lib/OrManagerModal.svelte +88 -0
  53. tinkerscope-1.0.0/web/src/lib/OverflowRow.svelte +113 -0
  54. tinkerscope-1.0.0/web/src/lib/PackInstallModal.svelte +103 -0
  55. tinkerscope-1.0.0/web/src/lib/PackLoadingModal.svelte +86 -0
  56. tinkerscope-1.0.0/web/src/lib/PickerDropdown.svelte +171 -0
  57. tinkerscope-1.0.0/web/src/lib/SlideshowModal.svelte +129 -0
  58. tinkerscope-1.0.0/web/src/lib/SplitChip.svelte +73 -0
  59. tinkerscope-1.0.0/web/src/lib/TagModal.svelte +50 -0
  60. tinkerscope-1.0.0/web/src/lib/ThreadSwitcher.svelte +151 -0
  61. tinkerscope-1.0.0/web/src/lib/TinkerPickerModal.svelte +71 -0
  62. tinkerscope-1.0.0/web/src/lib/TokenHeatOverlay.svelte +405 -0
  63. tinkerscope-1.0.0/web/src/lib/TokenLogprobs.svelte +117 -0
  64. tinkerscope-1.0.0/web/src/lib/TokenPopover.svelte +155 -0
  65. tinkerscope-1.0.0/web/src/lib/TruncLabel.svelte +53 -0
  66. tinkerscope-1.0.0/web/src/lib/Typeahead.svelte +289 -0
  67. tinkerscope-1.0.0/web/src/lib/api-static.ts +478 -0
  68. tinkerscope-1.0.0/web/src/lib/api.ts +196 -0
  69. tinkerscope-1.0.0/web/src/lib/branch-ops.svelte.ts +405 -0
  70. tinkerscope-1.0.0/web/src/lib/bus-scope.test.ts +143 -0
  71. tinkerscope-1.0.0/web/src/lib/bus-scope.ts +84 -0
  72. tinkerscope-1.0.0/web/src/lib/chart-view.test.ts +230 -0
  73. tinkerscope-1.0.0/web/src/lib/chart-view.ts +252 -0
  74. tinkerscope-1.0.0/web/src/lib/chart.test.ts +540 -0
  75. tinkerscope-1.0.0/web/src/lib/chart.ts +547 -0
  76. tinkerscope-1.0.0/web/src/lib/chat.css +232 -0
  77. tinkerscope-1.0.0/web/src/lib/chat.svelte.ts +222 -0
  78. tinkerscope-1.0.0/web/src/lib/drag-reorder.svelte.ts +63 -0
  79. tinkerscope-1.0.0/web/src/lib/fuzzy.test.ts +194 -0
  80. tinkerscope-1.0.0/web/src/lib/fuzzy.ts +172 -0
  81. tinkerscope-1.0.0/web/src/lib/highlight-match.ts +144 -0
  82. tinkerscope-1.0.0/web/src/lib/highlight-render.ts +111 -0
  83. tinkerscope-1.0.0/web/src/lib/highlight.test.ts +168 -0
  84. tinkerscope-1.0.0/web/src/lib/highlights.svelte.ts +136 -0
  85. tinkerscope-1.0.0/web/src/lib/kbnav.test.ts +79 -0
  86. tinkerscope-1.0.0/web/src/lib/kbnav.ts +38 -0
  87. tinkerscope-1.0.0/web/src/lib/label-diff.fuzz.test.ts +76 -0
  88. tinkerscope-1.0.0/web/src/lib/label-diff.test.ts +258 -0
  89. tinkerscope-1.0.0/web/src/lib/label-diff.ts +168 -0
  90. tinkerscope-1.0.0/web/src/lib/label-split.test.ts +124 -0
  91. tinkerscope-1.0.0/web/src/lib/label-split.ts +74 -0
  92. tinkerscope-1.0.0/web/src/lib/logprobs.svelte.ts +139 -0
  93. tinkerscope-1.0.0/web/src/lib/model-catalog.svelte.ts +228 -0
  94. tinkerscope-1.0.0/web/src/lib/model-sel.test.ts +74 -0
  95. tinkerscope-1.0.0/web/src/lib/model-sel.ts +59 -0
  96. tinkerscope-1.0.0/web/src/lib/node-blobs.svelte.ts +115 -0
  97. tinkerscope-1.0.0/web/src/lib/node-split.ts +72 -0
  98. tinkerscope-1.0.0/web/src/lib/overlay-store.ts +191 -0
  99. tinkerscope-1.0.0/web/src/lib/pack-install.ts +287 -0
  100. tinkerscope-1.0.0/web/src/lib/pack-logprobs.test.ts +110 -0
  101. tinkerscope-1.0.0/web/src/lib/pack-logprobs.ts +33 -0
  102. tinkerscope-1.0.0/web/src/lib/pack-source.test.ts +150 -0
  103. tinkerscope-1.0.0/web/src/lib/pack-source.ts +77 -0
  104. tinkerscope-1.0.0/web/src/lib/panel-view.test.ts +196 -0
  105. tinkerscope-1.0.0/web/src/lib/panel-view.ts +184 -0
  106. tinkerscope-1.0.0/web/src/lib/picker.ts +19 -0
  107. tinkerscope-1.0.0/web/src/lib/render.ts +55 -0
  108. tinkerscope-1.0.0/web/src/lib/reorder.test.ts +113 -0
  109. tinkerscope-1.0.0/web/src/lib/reorder.ts +48 -0
  110. tinkerscope-1.0.0/web/src/lib/save-plan.test.ts +182 -0
  111. tinkerscope-1.0.0/web/src/lib/save-plan.ts +106 -0
  112. tinkerscope-1.0.0/web/src/lib/scroll.svelte.ts +120 -0
  113. tinkerscope-1.0.0/web/src/lib/state.svelte.ts +294 -0
  114. tinkerscope-1.0.0/web/src/lib/static-mode.ts +96 -0
  115. tinkerscope-1.0.0/web/src/lib/thinking-view.svelte.ts +35 -0
  116. tinkerscope-1.0.0/web/src/lib/token-align.test.ts +237 -0
  117. tinkerscope-1.0.0/web/src/lib/token-align.ts +161 -0
  118. tinkerscope-1.0.0/web/src/lib/token-edit.test.ts +218 -0
  119. tinkerscope-1.0.0/web/src/lib/token-edit.ts +115 -0
  120. tinkerscope-1.0.0/web/src/lib/token-logprob.test.ts +263 -0
  121. tinkerscope-1.0.0/web/src/lib/token-logprob.ts +221 -0
  122. tinkerscope-1.0.0/web/src/lib/token-prefill.test.ts +77 -0
  123. tinkerscope-1.0.0/web/src/lib/token-prefill.ts +28 -0
  124. tinkerscope-1.0.0/web/src/lib/token-search.test.ts +74 -0
  125. tinkerscope-1.0.0/web/src/lib/token-search.ts +66 -0
  126. tinkerscope-1.0.0/web/src/lib/tooltip.svelte.ts +83 -0
  127. tinkerscope-1.0.0/web/src/lib/tree.test.ts +625 -0
  128. tinkerscope-1.0.0/web/src/lib/tree.ts +732 -0
  129. tinkerscope-1.0.0/web/src/lib/types.ts +363 -0
  130. tinkerscope-1.0.0/web/src/lib/when.test.ts +59 -0
  131. tinkerscope-1.0.0/web/src/lib/when.ts +26 -0
  132. tinkerscope-1.0.0/web/src/lib/workspaces.svelte.ts +990 -0
  133. tinkerscope-1.0.0/web/src/routes/+layout.svelte +7 -0
  134. tinkerscope-1.0.0/web/src/routes/+layout.ts +4 -0
  135. tinkerscope-1.0.0/web/src/routes/+page.svelte +2771 -0
  136. tinkerscope-1.0.0/web/static/favicon.svg +5 -0
  137. tinkerscope-1.0.0/web/svelte.config.js +18 -0
  138. tinkerscope-1.0.0/web/tsconfig.json +15 -0
  139. tinkerscope-1.0.0/web/vite.config.ts +23 -0
@@ -0,0 +1,18 @@
1
+ # Git hooks
2
+
3
+ Version-controlled hooks for this repo. Enable them once per clone:
4
+
5
+ ```bash
6
+ git config core.hooksPath .githooks
7
+ ```
8
+
9
+ (This box's working tree already has it set.)
10
+
11
+ ## `pre-commit`
12
+
13
+ Runs `npm run build` (vite build) in `web/` before each commit, so a broken
14
+ build can't land. Skips automatically when no staged change touches `web/`
15
+ (doc-only or Python-only commits stay instant). On a build failure it prints the
16
+ tail of the build log and aborts.
17
+
18
+ Bypass for a deliberate WIP commit: `git commit --no-verify`.
@@ -0,0 +1,30 @@
1
+ # Pythonh
2
+ .venv/
3
+ __pycache__/
4
+ *.py[cod]
5
+ *.egg-info/
6
+ .pytest_cache/
7
+ .ruff_cache/
8
+
9
+ # uv
10
+ # (uv.lock IS committed)
11
+
12
+ # Env / secrets
13
+ .env
14
+ .env.*
15
+ !.env.example
16
+
17
+ # Web build + deps
18
+ web/node_modules/
19
+ web/dist/
20
+ web/.svelte-kit/
21
+ web/tsconfig.tsbuildinfo
22
+
23
+ # Packaged web build (produced by hatch_build.py at wheel time)
24
+ src/tinkerscope/web_dist/
25
+
26
+ # OS / editor
27
+ .DS_Store
28
+
29
+ # Backups written by scripts/repair_panel_layouts.py --apply
30
+ layout-backups/
@@ -0,0 +1,27 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Clément Dumas and tinkerscope contributors
4
+
5
+ tinkerscope is a fork of, and builds substantially on, Harry Mayne's model
6
+ playground (https://github.com/HarryMayne/negation_neglect_working_repo,
7
+ tools/playground). The chat UI and inference layer originate from his work;
8
+ see README.md for full credits. The upstream playground was published without
9
+ an explicit license — please retain this attribution.
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in all
19
+ copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ SOFTWARE.
@@ -0,0 +1,304 @@
1
+ Metadata-Version: 2.4
2
+ Name: tinkerscope
3
+ Version: 1.0.0
4
+ Summary: Auto-discover Tinker training runs under a directory tree and chat with / sample from their checkpoints in the browser — driveable from the terminal.
5
+ Project-URL: Homepage, https://github.com/Butanium/tinkerscope
6
+ Project-URL: Repository, https://github.com/Butanium/tinkerscope
7
+ Project-URL: Documentation, https://github.com/Butanium/tinkerscope/blob/main/README.md
8
+ Project-URL: Issues, https://github.com/Butanium/tinkerscope/issues
9
+ Author: Clément Dumas, Claude
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: checkpoints,fine-tuning,interpretability,llm,playground,sampling,tinker
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Environment :: Web Environment
15
+ Classifier: Framework :: FastAPI
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Requires-Python: >=3.11
25
+ Requires-Dist: fastapi>=0.115
26
+ Requires-Dist: httpx-sse>=0.4
27
+ Requires-Dist: httpx>=0.27
28
+ Requires-Dist: openai>=1.40
29
+ Requires-Dist: pydantic>=2
30
+ Requires-Dist: python-dotenv>=1.0
31
+ Requires-Dist: pyyaml>=6
32
+ Requires-Dist: sse-starlette>=2.1
33
+ Requires-Dist: tinker-cookbook[inkling]
34
+ Requires-Dist: tinker>=0.23
35
+ Requires-Dist: typer>=0.12
36
+ Requires-Dist: uvicorn[standard]>=0.30
37
+ Description-Content-Type: text/markdown
38
+
39
+ # tinkerscope
40
+
41
+ A browser playground for **Tinker-trained checkpoints**. Point it at your
42
+ training directory and it **auto-discovers every run** — no `models.yaml`, no
43
+ registration — then chat with them, fan out N samples to see what a model
44
+ *usually* says, branch like on claude.ai, and compare checkpoints side by side.
45
+ The whole thing is also drivable **live from your terminal** (`tinkpg`), so
46
+ your AI agent can fire samples into the browser you're watching.
47
+
48
+ ![The chat view with workspace branching](docs/img/chat-branching.png)
49
+
50
+ ## Quick start
51
+
52
+ ```bash
53
+ export TINKER_API_KEY=... # required to sample (discovery works without it)
54
+ uv tool install tinkerscope
55
+ tinkerscope ~/my-training-runs # scans the tree, auto-picks a port, prints the URL
56
+ ```
57
+
58
+ Open the printed URL and you're in. `tinkerscope DIR1 DIR2 …` scans several
59
+ trees at once. Set `OPENROUTER_API_KEY` too if you want OpenRouter reference
60
+ models next to your checkpoints. Lost in the UI? The **`?` button** in the
61
+ sidebar explains every control and keyboard shortcut — or just ask your agent
62
+ (guide skill below).
63
+
64
+ ### Agent skills
65
+
66
+ The repo doubles as a plugin marketplace shipping two skills:
67
+ `tinkerscope:guide` makes your agent a help desk for the UI — ask it "how do I
68
+ compare two checkpoints?" and it knows every button and shortcut — and
69
+ `tinkerscope:cli` teaches it to drive the playground from a terminal (more
70
+ below).
71
+
72
+ <details>
73
+ <summary><b>Claude Code</b></summary>
74
+
75
+ ```
76
+ /plugin marketplace add Butanium/tinkerscope
77
+ /plugin install tinkerscope@tinkerscope
78
+ ```
79
+
80
+ Claude Code treats the commit SHA as the plugin version, so the skills track
81
+ `main`. Third-party marketplaces are manual-update by default — toggle
82
+ auto-update from `/plugin` > Marketplaces, or run `/plugin marketplace update
83
+ tinkerscope` followed by `/reload-plugins`.
84
+
85
+ </details>
86
+
87
+ <details>
88
+ <summary><b>Codex</b></summary>
89
+
90
+ ```bash
91
+ codex plugin marketplace add Butanium/tinkerscope
92
+ codex plugin add tinkerscope@tinkerscope
93
+ ```
94
+
95
+ Codex caches by the `version` in `plugin/.codex-plugin/plugin.json`; update
96
+ with `codex plugin marketplace upgrade tinkerscope`.
97
+
98
+ </details>
99
+
100
+ <details>
101
+ <summary><b>Cursor / Copilot / Windsurf / Cline</b></summary>
102
+
103
+ ```bash
104
+ npx skills add Butanium/tinkerscope --agent <agent> -g -y
105
+ ```
106
+
107
+ </details>
108
+
109
+ ## Features
110
+
111
+ ### Auto-discovery — no config files
112
+
113
+ tinkerscope scans your directories for `checkpoints.jsonl` + `config.json`
114
+ (the two files every `tinker_cookbook` run drops) and surfaces one run per
115
+ directory with its whole checkpoint trajectory. Runs Tinker can no longer
116
+ sample are greyed out with the reason instead of failing on click. Every
117
+ picker is type-to-filter with typo-tolerant fuzzy matching, and every model
118
+ has a copy button for its `tinker://…` sampler path.
119
+
120
+ You can also add models straight from the UI: a raw **Tinker base model**, a
121
+ **loose checkpoint** by sampler path, or any **OpenRouter model** to sit next
122
+ to your checkpoints as a reference.
123
+
124
+ ### N-sample fan-out
125
+
126
+ Set **n > 1** and one send draws N samples, rendered as cards — a quick read
127
+ on what the model usually says. The usual knobs (temperature, max tokens,
128
+ top-p), a **thinking toggle** (Off / On / **Both** — Both draws n with and n
129
+ without, tagged), a system prompt, and an assistant **prefill** field.
130
+
131
+ ![n>1 sample cards](docs/img/n-samples.png)
132
+
133
+ ### Response distribution chart
134
+
135
+ The draws power a live chart. Define **highlight rules** (named colors +
136
+ patterns) and each sample is bucketed by the rules it matches — "define a
137
+ rule, see its prevalence per model" is one loop. A **first token** mode
138
+ charts the model's *own* probability distribution over the first generated
139
+ token next to what was actually sampled. Segments are clickable to inspect
140
+ the exact samples underneath, and the chart live-updates while a batch
141
+ streams.
142
+
143
+ ![The response distribution chart](docs/img/distribution-chart.png)
144
+
145
+ ### Token probabilities
146
+
147
+ Native Tinker sampling stores **per-token logprobs + top-5 alternatives** on
148
+ every turn. Flip the sidebar's **Token probs** toggle to paint a surprisal
149
+ heat under the prose (or show the raw token stream), and hover any token for
150
+ its probability and the alternatives the model weighed. **Color by match**
151
+ tints each token by how much probability mass a highlight rule *almost* got —
152
+ where did "red" nearly happen.
153
+
154
+ ![Token probabilities overlay](docs/img/token-probs.png)
155
+
156
+ ### Parallel conversations with different models
157
+
158
+ Have the same conversation with several models at once: **Add panel** puts
159
+ models side by side, and every message you send gets answered by each — same
160
+ user turns, one response per model. Panels generate concurrently and each
161
+ keeps its own branch tree.
162
+
163
+ ![The same conversation answered by two different checkpoints](docs/img/compare.png)
164
+
165
+ ### Branch everything
166
+
167
+ Nothing is ever destroyed: regenerating, editing a turn, or drawing N samples
168
+ creates sibling branches you can cycle through with **‹ k/N ›**, and you can
169
+ branch from the very start to keep several probe prompts in one workspace.
170
+ Workspaces are named, persisted per project, and restored on restart. Every
171
+ button has a tooltip, Shift/Ctrl unlock power variants, and the `?` modal (or
172
+ your agent, via the guide skill) covers the rest.
173
+
174
+ ### Share packs
175
+
176
+ Bundle checkpoints + params + workspaces into one portable YAML. Models are
177
+ addressed by public sampler paths, so a collaborator reproduces your exact
178
+ setup with no local run dirs:
179
+
180
+ ```bash
181
+ tinkerscope pack export demo.yaml # author from your current setup
182
+ tinkerscope --pack <file|url> # consume + serve
183
+ ```
184
+
185
+ A pack is also a **link**: open `?w=<pack url>` on a running instance and it
186
+ installs (after asking) with no restart. Full doc: [`docs/PACK.md`](docs/PACK.md).
187
+
188
+ ### Publish a read-only site
189
+
190
+ Export the playground as a static site — no backend, no API key, hostable on
191
+ GitHub Pages:
192
+
193
+ ```bash
194
+ tinkerscope site export ./site --workspace "the good one"
195
+ ```
196
+
197
+ Visitors browse the real thing: branches, threads, panels, the chart, token
198
+ probabilities. Per-token logprobs are ~97% of the bytes — export a subset, or
199
+ `--no-logprobs`. A published site also works as a **viewer for anyone's
200
+ pack** (`?w=<pack url>`, or drop the file on the page), and its read-only
201
+ badge hands readers the command to run everything locally. Full doc:
202
+ [`docs/STATIC_SITE.md`](docs/STATIC_SITE.md).
203
+
204
+ ## Bring your agent — `tinkpg`
205
+
206
+ The CLI hits the same API as the browser and shares its state, which makes the
207
+ playground a place you and your AI agent work in together:
208
+
209
+ - **It reads what you did.** Your agent can walk every branch of your
210
+ workspaces, pull full n-sample fan-outs, and `grep` across everything you
211
+ ever sampled ("which workspace had that self-description prompt?").
212
+ - **You watch what it does.** Probes the agent fires land live as new threads
213
+ in your open browser — its exploration stays auditable, thread by thread,
214
+ instead of buried in a script's stdout.
215
+ - **Its reads feed write-ups.** Every read command takes `--json`, so a
216
+ morning of browser sampling turns into an eval design or a report without
217
+ re-running anything.
218
+
219
+ ```bash
220
+ tinkpg ws [<id|name>] # browse saved workspaces / read one
221
+ tinkpg samples [<id|name>] # the full n-sample fan-out at a fork
222
+ tinkpg grep "<text>" # search every branch of every workspace
223
+ tinkpg threads # index root threads across workspaces
224
+ tinkpg node <id> # look up a node id → record + logprobs
225
+ tinkpg send "prompt" # fire a new thread at the current panels
226
+ tinkpg continue "follow-up" # add a turn to the current threads
227
+ tinkpg battery <dir> # fire a directory of probe files
228
+ tinkpg probe <run> "prompt" # sample off-workspace (no broadcast)
229
+ tinkpg ls / checkpoints <run> # discovered runs / a run's checkpoints
230
+ tinkpg open <run>[@<checkpoint>] # switch the browser to this model, live
231
+ tinkpg chat <run> "prompt" --n 50 # one-shot: select + sample + stream
232
+ tinkpg compare <runA> <runB> "..." # one-shot: two panels + a first turn
233
+ tinkpg params / state / refresh # sampling params / shared state / rescan
234
+ ```
235
+
236
+ Param flags on a fire are per-call — they never clobber your browser sidebar.
237
+ The `tinkerscope:cli` skill teaches your agent all of it, flags included
238
+ ([`plugin/skills/cli/SKILL.md`](plugin/skills/cli/SKILL.md)).
239
+
240
+ ## Tests
241
+
242
+ `uv run pytest -q` covers discovery and the API with zero remote calls. The
243
+ pure frontend logic has ~20 bare-Node unit suites (`npm test` from `web/`),
244
+ and `scripts/smoke.sh` runs the Playwright browser smokes against an isolated
245
+ throwaway instance.
246
+
247
+ ## Development
248
+
249
+ ```bash
250
+ ./run.sh [DIR ...] # dev mode: backend + vite dev server (hot reload)
251
+ ./run.sh --build [DIR ...] # packaged: build the web UI once, serve from one process
252
+ ```
253
+
254
+ Installing as a tool while hacking? Use `uv tool install -e .` (editable) so
255
+ the process runs your checkout. Backend changes need a process restart; web
256
+ changes need `npm run build` (a pre-commit hook runs it on `web/` commits)
257
+ plus a browser refresh.
258
+
259
+ <details>
260
+ <summary><b>Hacking on the skills</b></summary>
261
+
262
+ Don't install them — the install snapshots into a cache, so your edits won't
263
+ show. Symlink each skill into your skills dir instead and it reads your
264
+ working tree live:
265
+
266
+ ```bash
267
+ for s in cli guide; do
268
+ mkdir -p ~/.claude/skills/tinkerscope-$s
269
+ ln -sf "$PWD/plugin/skills/$s/SKILL.md" ~/.claude/skills/tinkerscope-$s/SKILL.md
270
+ done
271
+ ```
272
+
273
+ They load as `tinkerscope-cli` / `tinkerscope-guide` (directory name wins over
274
+ frontmatter). Symlinking `plugin/` as a directory also works — but then don't
275
+ *also* install it, or every skill registers twice.
276
+
277
+ </details>
278
+
279
+ For orientation and contracts: **`CLAUDE.md`** (where everything lives),
280
+ [`docs/API_CONTRACT.md`](docs/API_CONTRACT.md) (HTTP + SSE shapes),
281
+ [`docs/BRANCHING_DESIGN.md`](docs/BRANCHING_DESIGN.md) (the branching model),
282
+ [`docs/TODO.md`](docs/TODO.md) (roadmap).
283
+
284
+ ## Credits
285
+
286
+ The UI is forked from **Harry Mayne**'s `tools/playground` in
287
+ [`HarryMayne/negation_neglect_working_repo`](https://github.com/HarryMayne/negation_neglect_working_repo)
288
+ (commit `ec7da09`, Harry Mayne <harrymayne@gmail.com>). The core chat experience
289
+ — streaming, n-sample fan-out, the response-distribution chart, the thinking
290
+ toggle, the raw-text view, and the side-by-side compare — is his work.
291
+ tinkerscope adds run auto-discovery, workspace branching, named/persisted
292
+ workspaces, N-panel comparison, the token-probability views, share packs, the
293
+ static-site export, the terminal-driving CLI, and standalone packaging on top.
294
+
295
+ Renderer selection (chat templates / stop sequences / response parsing) uses
296
+ `tinker_cookbook` (Thinking Machines). An earlier iteration routed inference
297
+ through **James Chua**'s [`latteries`](https://github.com/thejaminator/latteries);
298
+ tinkerscope now calls the Tinker SDK directly, but the renderer-cache and
299
+ thinking-block-parsing lessons from that code carried over.
300
+
301
+ tinkerscope's own code is MIT-licensed (see `LICENSE`). The upstream playground
302
+ ships **without** a license; substantial portions of the UI and inference layer
303
+ are Harry Mayne's work, retained here with attribution. If you build on this,
304
+ keep that credit.
@@ -0,0 +1,266 @@
1
+ # tinkerscope
2
+
3
+ A browser playground for **Tinker-trained checkpoints**. Point it at your
4
+ training directory and it **auto-discovers every run** — no `models.yaml`, no
5
+ registration — then chat with them, fan out N samples to see what a model
6
+ *usually* says, branch like on claude.ai, and compare checkpoints side by side.
7
+ The whole thing is also drivable **live from your terminal** (`tinkpg`), so
8
+ your AI agent can fire samples into the browser you're watching.
9
+
10
+ ![The chat view with workspace branching](docs/img/chat-branching.png)
11
+
12
+ ## Quick start
13
+
14
+ ```bash
15
+ export TINKER_API_KEY=... # required to sample (discovery works without it)
16
+ uv tool install tinkerscope
17
+ tinkerscope ~/my-training-runs # scans the tree, auto-picks a port, prints the URL
18
+ ```
19
+
20
+ Open the printed URL and you're in. `tinkerscope DIR1 DIR2 …` scans several
21
+ trees at once. Set `OPENROUTER_API_KEY` too if you want OpenRouter reference
22
+ models next to your checkpoints. Lost in the UI? The **`?` button** in the
23
+ sidebar explains every control and keyboard shortcut — or just ask your agent
24
+ (guide skill below).
25
+
26
+ ### Agent skills
27
+
28
+ The repo doubles as a plugin marketplace shipping two skills:
29
+ `tinkerscope:guide` makes your agent a help desk for the UI — ask it "how do I
30
+ compare two checkpoints?" and it knows every button and shortcut — and
31
+ `tinkerscope:cli` teaches it to drive the playground from a terminal (more
32
+ below).
33
+
34
+ <details>
35
+ <summary><b>Claude Code</b></summary>
36
+
37
+ ```
38
+ /plugin marketplace add Butanium/tinkerscope
39
+ /plugin install tinkerscope@tinkerscope
40
+ ```
41
+
42
+ Claude Code treats the commit SHA as the plugin version, so the skills track
43
+ `main`. Third-party marketplaces are manual-update by default — toggle
44
+ auto-update from `/plugin` > Marketplaces, or run `/plugin marketplace update
45
+ tinkerscope` followed by `/reload-plugins`.
46
+
47
+ </details>
48
+
49
+ <details>
50
+ <summary><b>Codex</b></summary>
51
+
52
+ ```bash
53
+ codex plugin marketplace add Butanium/tinkerscope
54
+ codex plugin add tinkerscope@tinkerscope
55
+ ```
56
+
57
+ Codex caches by the `version` in `plugin/.codex-plugin/plugin.json`; update
58
+ with `codex plugin marketplace upgrade tinkerscope`.
59
+
60
+ </details>
61
+
62
+ <details>
63
+ <summary><b>Cursor / Copilot / Windsurf / Cline</b></summary>
64
+
65
+ ```bash
66
+ npx skills add Butanium/tinkerscope --agent <agent> -g -y
67
+ ```
68
+
69
+ </details>
70
+
71
+ ## Features
72
+
73
+ ### Auto-discovery — no config files
74
+
75
+ tinkerscope scans your directories for `checkpoints.jsonl` + `config.json`
76
+ (the two files every `tinker_cookbook` run drops) and surfaces one run per
77
+ directory with its whole checkpoint trajectory. Runs Tinker can no longer
78
+ sample are greyed out with the reason instead of failing on click. Every
79
+ picker is type-to-filter with typo-tolerant fuzzy matching, and every model
80
+ has a copy button for its `tinker://…` sampler path.
81
+
82
+ You can also add models straight from the UI: a raw **Tinker base model**, a
83
+ **loose checkpoint** by sampler path, or any **OpenRouter model** to sit next
84
+ to your checkpoints as a reference.
85
+
86
+ ### N-sample fan-out
87
+
88
+ Set **n > 1** and one send draws N samples, rendered as cards — a quick read
89
+ on what the model usually says. The usual knobs (temperature, max tokens,
90
+ top-p), a **thinking toggle** (Off / On / **Both** — Both draws n with and n
91
+ without, tagged), a system prompt, and an assistant **prefill** field.
92
+
93
+ ![n>1 sample cards](docs/img/n-samples.png)
94
+
95
+ ### Response distribution chart
96
+
97
+ The draws power a live chart. Define **highlight rules** (named colors +
98
+ patterns) and each sample is bucketed by the rules it matches — "define a
99
+ rule, see its prevalence per model" is one loop. A **first token** mode
100
+ charts the model's *own* probability distribution over the first generated
101
+ token next to what was actually sampled. Segments are clickable to inspect
102
+ the exact samples underneath, and the chart live-updates while a batch
103
+ streams.
104
+
105
+ ![The response distribution chart](docs/img/distribution-chart.png)
106
+
107
+ ### Token probabilities
108
+
109
+ Native Tinker sampling stores **per-token logprobs + top-5 alternatives** on
110
+ every turn. Flip the sidebar's **Token probs** toggle to paint a surprisal
111
+ heat under the prose (or show the raw token stream), and hover any token for
112
+ its probability and the alternatives the model weighed. **Color by match**
113
+ tints each token by how much probability mass a highlight rule *almost* got —
114
+ where did "red" nearly happen.
115
+
116
+ ![Token probabilities overlay](docs/img/token-probs.png)
117
+
118
+ ### Parallel conversations with different models
119
+
120
+ Have the same conversation with several models at once: **Add panel** puts
121
+ models side by side, and every message you send gets answered by each — same
122
+ user turns, one response per model. Panels generate concurrently and each
123
+ keeps its own branch tree.
124
+
125
+ ![The same conversation answered by two different checkpoints](docs/img/compare.png)
126
+
127
+ ### Branch everything
128
+
129
+ Nothing is ever destroyed: regenerating, editing a turn, or drawing N samples
130
+ creates sibling branches you can cycle through with **‹ k/N ›**, and you can
131
+ branch from the very start to keep several probe prompts in one workspace.
132
+ Workspaces are named, persisted per project, and restored on restart. Every
133
+ button has a tooltip, Shift/Ctrl unlock power variants, and the `?` modal (or
134
+ your agent, via the guide skill) covers the rest.
135
+
136
+ ### Share packs
137
+
138
+ Bundle checkpoints + params + workspaces into one portable YAML. Models are
139
+ addressed by public sampler paths, so a collaborator reproduces your exact
140
+ setup with no local run dirs:
141
+
142
+ ```bash
143
+ tinkerscope pack export demo.yaml # author from your current setup
144
+ tinkerscope --pack <file|url> # consume + serve
145
+ ```
146
+
147
+ A pack is also a **link**: open `?w=<pack url>` on a running instance and it
148
+ installs (after asking) with no restart. Full doc: [`docs/PACK.md`](docs/PACK.md).
149
+
150
+ ### Publish a read-only site
151
+
152
+ Export the playground as a static site — no backend, no API key, hostable on
153
+ GitHub Pages:
154
+
155
+ ```bash
156
+ tinkerscope site export ./site --workspace "the good one"
157
+ ```
158
+
159
+ Visitors browse the real thing: branches, threads, panels, the chart, token
160
+ probabilities. Per-token logprobs are ~97% of the bytes — export a subset, or
161
+ `--no-logprobs`. A published site also works as a **viewer for anyone's
162
+ pack** (`?w=<pack url>`, or drop the file on the page), and its read-only
163
+ badge hands readers the command to run everything locally. Full doc:
164
+ [`docs/STATIC_SITE.md`](docs/STATIC_SITE.md).
165
+
166
+ ## Bring your agent — `tinkpg`
167
+
168
+ The CLI hits the same API as the browser and shares its state, which makes the
169
+ playground a place you and your AI agent work in together:
170
+
171
+ - **It reads what you did.** Your agent can walk every branch of your
172
+ workspaces, pull full n-sample fan-outs, and `grep` across everything you
173
+ ever sampled ("which workspace had that self-description prompt?").
174
+ - **You watch what it does.** Probes the agent fires land live as new threads
175
+ in your open browser — its exploration stays auditable, thread by thread,
176
+ instead of buried in a script's stdout.
177
+ - **Its reads feed write-ups.** Every read command takes `--json`, so a
178
+ morning of browser sampling turns into an eval design or a report without
179
+ re-running anything.
180
+
181
+ ```bash
182
+ tinkpg ws [<id|name>] # browse saved workspaces / read one
183
+ tinkpg samples [<id|name>] # the full n-sample fan-out at a fork
184
+ tinkpg grep "<text>" # search every branch of every workspace
185
+ tinkpg threads # index root threads across workspaces
186
+ tinkpg node <id> # look up a node id → record + logprobs
187
+ tinkpg send "prompt" # fire a new thread at the current panels
188
+ tinkpg continue "follow-up" # add a turn to the current threads
189
+ tinkpg battery <dir> # fire a directory of probe files
190
+ tinkpg probe <run> "prompt" # sample off-workspace (no broadcast)
191
+ tinkpg ls / checkpoints <run> # discovered runs / a run's checkpoints
192
+ tinkpg open <run>[@<checkpoint>] # switch the browser to this model, live
193
+ tinkpg chat <run> "prompt" --n 50 # one-shot: select + sample + stream
194
+ tinkpg compare <runA> <runB> "..." # one-shot: two panels + a first turn
195
+ tinkpg params / state / refresh # sampling params / shared state / rescan
196
+ ```
197
+
198
+ Param flags on a fire are per-call — they never clobber your browser sidebar.
199
+ The `tinkerscope:cli` skill teaches your agent all of it, flags included
200
+ ([`plugin/skills/cli/SKILL.md`](plugin/skills/cli/SKILL.md)).
201
+
202
+ ## Tests
203
+
204
+ `uv run pytest -q` covers discovery and the API with zero remote calls. The
205
+ pure frontend logic has ~20 bare-Node unit suites (`npm test` from `web/`),
206
+ and `scripts/smoke.sh` runs the Playwright browser smokes against an isolated
207
+ throwaway instance.
208
+
209
+ ## Development
210
+
211
+ ```bash
212
+ ./run.sh [DIR ...] # dev mode: backend + vite dev server (hot reload)
213
+ ./run.sh --build [DIR ...] # packaged: build the web UI once, serve from one process
214
+ ```
215
+
216
+ Installing as a tool while hacking? Use `uv tool install -e .` (editable) so
217
+ the process runs your checkout. Backend changes need a process restart; web
218
+ changes need `npm run build` (a pre-commit hook runs it on `web/` commits)
219
+ plus a browser refresh.
220
+
221
+ <details>
222
+ <summary><b>Hacking on the skills</b></summary>
223
+
224
+ Don't install them — the install snapshots into a cache, so your edits won't
225
+ show. Symlink each skill into your skills dir instead and it reads your
226
+ working tree live:
227
+
228
+ ```bash
229
+ for s in cli guide; do
230
+ mkdir -p ~/.claude/skills/tinkerscope-$s
231
+ ln -sf "$PWD/plugin/skills/$s/SKILL.md" ~/.claude/skills/tinkerscope-$s/SKILL.md
232
+ done
233
+ ```
234
+
235
+ They load as `tinkerscope-cli` / `tinkerscope-guide` (directory name wins over
236
+ frontmatter). Symlinking `plugin/` as a directory also works — but then don't
237
+ *also* install it, or every skill registers twice.
238
+
239
+ </details>
240
+
241
+ For orientation and contracts: **`CLAUDE.md`** (where everything lives),
242
+ [`docs/API_CONTRACT.md`](docs/API_CONTRACT.md) (HTTP + SSE shapes),
243
+ [`docs/BRANCHING_DESIGN.md`](docs/BRANCHING_DESIGN.md) (the branching model),
244
+ [`docs/TODO.md`](docs/TODO.md) (roadmap).
245
+
246
+ ## Credits
247
+
248
+ The UI is forked from **Harry Mayne**'s `tools/playground` in
249
+ [`HarryMayne/negation_neglect_working_repo`](https://github.com/HarryMayne/negation_neglect_working_repo)
250
+ (commit `ec7da09`, Harry Mayne <harrymayne@gmail.com>). The core chat experience
251
+ — streaming, n-sample fan-out, the response-distribution chart, the thinking
252
+ toggle, the raw-text view, and the side-by-side compare — is his work.
253
+ tinkerscope adds run auto-discovery, workspace branching, named/persisted
254
+ workspaces, N-panel comparison, the token-probability views, share packs, the
255
+ static-site export, the terminal-driving CLI, and standalone packaging on top.
256
+
257
+ Renderer selection (chat templates / stop sequences / response parsing) uses
258
+ `tinker_cookbook` (Thinking Machines). An earlier iteration routed inference
259
+ through **James Chua**'s [`latteries`](https://github.com/thejaminator/latteries);
260
+ tinkerscope now calls the Tinker SDK directly, but the renderer-cache and
261
+ thinking-block-parsing lessons from that code carried over.
262
+
263
+ tinkerscope's own code is MIT-licensed (see `LICENSE`). The upstream playground
264
+ ships **without** a license; substantial portions of the UI and inference layer
265
+ are Harry Mayne's work, retained here with attribution. If you build on this,
266
+ keep that credit.