lemming-cli 0.1.1__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. lemming_cli-0.1.3/.github/workflows/screenshots.yml +51 -0
  2. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/PKG-INFO +54 -32
  3. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/README.md +53 -31
  4. lemming_cli-0.1.3/docs/EVALS.md +89 -0
  5. lemming_cli-0.1.3/docs/HOOKS.md +164 -0
  6. lemming_cli-0.1.3/docs/screenshots/dashboard-desktop.png +0 -0
  7. lemming_cli-0.1.3/docs/screenshots/dashboard-mobile.png +0 -0
  8. lemming_cli-0.1.3/docs/screenshots/task-log-desktop.png +0 -0
  9. lemming_cli-0.1.3/docs/screenshots/task-log-mobile.png +0 -0
  10. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/package-lock.json +13 -13
  11. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/package.json +4 -3
  12. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/playwright.config.js +7 -0
  13. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/pyproject.toml +1 -1
  14. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/config.py +4 -4
  15. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/config_test.py +12 -12
  16. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/conftest.py +2 -2
  17. lemming_cli-0.1.3/src/lemming/api/hooks.py +57 -0
  18. lemming_cli-0.1.3/src/lemming/api/hooks_test.py +92 -0
  19. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/main.py +1 -1
  20. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/tasks.py +1 -1
  21. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/tasks_test.py +1 -1
  22. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/__init__.py +1 -1
  23. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/config.py +5 -2
  24. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/config_test.py +4 -1
  25. lemming_cli-0.1.3/src/lemming/cli/goal.py +38 -0
  26. lemming_cli-0.1.1/src/lemming/cli/context_test.py → lemming_cli-0.1.3/src/lemming/cli/goal_test.py +9 -9
  27. lemming_cli-0.1.3/src/lemming/cli/hooks.py +85 -0
  28. lemming_cli-0.1.3/src/lemming/cli/hooks_test.py +123 -0
  29. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/main.py +2 -2
  30. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/operations_test.py +1 -1
  31. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/progress_test.py +1 -1
  32. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/tasks.py +6 -6
  33. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/tasks_test.py +1 -1
  34. lemming_cli-0.1.3/src/lemming/evals/__init__.py +1 -0
  35. lemming_cli-0.1.3/src/lemming/evals/__main__.py +6 -0
  36. lemming_cli-0.1.3/src/lemming/evals/cli.py +205 -0
  37. lemming_cli-0.1.3/src/lemming/evals/cli_test.py +136 -0
  38. lemming_cli-0.1.3/src/lemming/evals/container.py +145 -0
  39. lemming_cli-0.1.3/src/lemming/evals/container_test.py +86 -0
  40. lemming_cli-0.1.3/src/lemming/evals/fixtures.py +90 -0
  41. lemming_cli-0.1.3/src/lemming/evals/fixtures_test.py +60 -0
  42. lemming_cli-0.1.3/src/lemming/evals/harness.py +239 -0
  43. lemming_cli-0.1.3/src/lemming/evals/harness_test.py +187 -0
  44. lemming_cli-0.1.3/src/lemming/evals/readability.py +323 -0
  45. lemming_cli-0.1.3/src/lemming/evals/readability_test.py +174 -0
  46. lemming_cli-0.1.3/src/lemming/evals/roadmap.py +466 -0
  47. lemming_cli-0.1.3/src/lemming/evals/roadmap_test.py +224 -0
  48. lemming_cli-0.1.3/src/lemming/evals/scenarios.py +89 -0
  49. lemming_cli-0.1.3/src/lemming/evals/scenarios_test.py +27 -0
  50. lemming_cli-0.1.3/src/lemming/evals/suites.py +11 -0
  51. lemming_cli-0.1.3/src/lemming/evals/trial.py +86 -0
  52. lemming_cli-0.1.3/src/lemming/evals/trial_test.py +132 -0
  53. lemming_cli-0.1.3/src/lemming/hooks.py +238 -0
  54. lemming_cli-0.1.3/src/lemming/hooks_test.py +224 -0
  55. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/integration_test.py +1 -1
  56. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/main_test.py +1 -1
  57. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/models.py +2 -3
  58. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/models_test.py +1 -2
  59. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/orchestrator.py +150 -4
  60. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/orchestrator_test.py +249 -2
  61. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/paths.py +4 -2
  62. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/paths_test.py +34 -1
  63. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/persistence.py +4 -19
  64. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/persistence_test.py +2 -35
  65. lemming_cli-0.1.3/src/lemming/prompts/hooks/50-readability.md +62 -0
  66. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/prompts.py +18 -108
  67. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/prompts_test.py +26 -64
  68. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/runner.py +5 -3
  69. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/runner_test.py +5 -4
  70. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/__init__.py +3 -3
  71. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/operations.py +7 -7
  72. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/operations_test.py +3 -3
  73. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/queries.py +2 -2
  74. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/queries_test.py +1 -1
  75. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/dashboard.spec.js +10 -2
  76. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/dashboard.test.js +8 -9
  77. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/favicon.spec.js +11 -3
  78. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/index.html +13 -24
  79. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/index.js +27 -29
  80. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/screenshots.spec.js +53 -58
  81. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/uv.lock +1 -1
  82. lemming_cli-0.1.1/docs/HOOKS.md +0 -148
  83. lemming_cli-0.1.1/docs/screenshots/dashboard-desktop.png +0 -0
  84. lemming_cli-0.1.1/docs/screenshots/dashboard-mobile.png +0 -0
  85. lemming_cli-0.1.1/docs/screenshots/task-log-desktop.png +0 -0
  86. lemming_cli-0.1.1/docs/screenshots/task-log-mobile.png +0 -0
  87. lemming_cli-0.1.1/src/lemming/api/hooks.py +0 -15
  88. lemming_cli-0.1.1/src/lemming/api/hooks_test.py +0 -33
  89. lemming_cli-0.1.1/src/lemming/api.py +0 -5
  90. lemming_cli-0.1.1/src/lemming/cli/context.py +0 -40
  91. lemming_cli-0.1.1/src/lemming/cli/hooks.py +0 -147
  92. lemming_cli-0.1.1/src/lemming/cli/hooks_test.py +0 -50
  93. lemming_cli-0.1.1/src/lemming/cli.py +0 -5
  94. lemming_cli-0.1.1/src/lemming/hooks.py +0 -145
  95. lemming_cli-0.1.1/src/lemming/hooks_test.py +0 -180
  96. lemming_cli-0.1.1/src/lemming/prompts/hooks/readability.md +0 -42
  97. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/.github/workflows/ci.yml +0 -0
  98. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/.github/workflows/publish.yml +0 -0
  99. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/.gitignore +0 -0
  100. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/.prettierignore +0 -0
  101. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/.prettierrc +0 -0
  102. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/Dockerfile +0 -0
  103. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/LICENSE +0 -0
  104. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/biome.json +0 -0
  105. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/docker-compose.yml +0 -0
  106. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/__init__.py +0 -0
  107. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/__init__.py +0 -0
  108. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/auth.py +0 -0
  109. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/auth_test.py +0 -0
  110. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/context.py +0 -0
  111. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/context_test.py +0 -0
  112. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/directories.py +0 -0
  113. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/directories_test.py +0 -0
  114. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/files.py +0 -0
  115. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/files_test.py +0 -0
  116. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/logging.py +0 -0
  117. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/logging_test.py +0 -0
  118. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/loop.py +0 -0
  119. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/loop_test.py +0 -0
  120. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api/main_test.py +0 -0
  121. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/api_test.py +0 -0
  122. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/main_test.py +0 -0
  123. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/operations.py +0 -0
  124. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/progress.py +0 -0
  125. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/readability_cli.py +0 -0
  126. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli/readability_cli_test.py +0 -0
  127. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/cli_test.py +0 -0
  128. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/conftest.py +0 -0
  129. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/main.py +0 -0
  130. /lemming_cli-0.1.1/src/lemming/prompts/hooks/testing.md → /lemming_cli-0.1.3/src/lemming/prompts/hooks/60-testing.md +0 -0
  131. /lemming_cli-0.1.1/src/lemming/prompts/hooks/roadmap.md → /lemming_cli-0.1.3/src/lemming/prompts/hooks/90-roadmap.md +0 -0
  132. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/prompts/taskrunner.md +0 -0
  133. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/providers.py +0 -0
  134. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/providers_test.py +0 -0
  135. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/lifecycle.py +0 -0
  136. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/lifecycle_test.py +0 -0
  137. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/progress.py +0 -0
  138. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/tasks/progress_test.py +0 -0
  139. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/favicon.js +0 -0
  140. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/files.html +0 -0
  141. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/files.spec.js +0 -0
  142. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/logs.html +0 -0
  143. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/logs.test.js +0 -0
  144. {lemming_cli-0.1.1 → lemming_cli-0.1.3}/src/lemming/web/mancha.js +0 -0
@@ -0,0 +1,51 @@
1
+ name: Screenshots
2
+
3
+ # Regenerates the dashboard screenshots in docs/ whenever the web UI changes,
4
+ # so they can never go stale. Generation is deterministic (frozen clock and
5
+ # animations disabled), so this only commits when the UI actually changed.
6
+ # The commit touches only docs/screenshots, which is outside the paths filter
7
+ # below, so it cannot re-trigger this workflow.
8
+ on:
9
+ push:
10
+ branches: [main]
11
+ paths:
12
+ - 'src/lemming/web/**'
13
+ - 'playwright.config.js'
14
+ - '.github/workflows/screenshots.yml'
15
+
16
+ permissions:
17
+ contents: write
18
+
19
+ jobs:
20
+ regenerate:
21
+ runs-on: ubuntu-latest
22
+
23
+ steps:
24
+ - uses: actions/checkout@v6
25
+
26
+ - name: Set up Node.js
27
+ uses: actions/setup-node@v6
28
+ with:
29
+ node-version: '20'
30
+
31
+ - name: Install Node.js dependencies
32
+ run: npm install
33
+
34
+ - name: Install Playwright browser
35
+ run: npx playwright install --with-deps chromium
36
+
37
+ - name: Regenerate screenshots
38
+ run: npm run screenshots
39
+
40
+ - name: Commit updated screenshots
41
+ run: |
42
+ if git diff --quiet docs/screenshots; then
43
+ echo "Screenshots are up to date."
44
+ exit 0
45
+ fi
46
+ git config user.name "github-actions[bot]"
47
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
48
+ git add docs/screenshots
49
+ git commit -m "Regenerate dashboard screenshots"
50
+ git pull --rebase --autostash origin main
51
+ git push
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lemming-cli
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: An autonomous, iterative task runner for AI coding agents.
5
5
  Project-URL: Homepage, https://github.com/owahltinez/lemming
6
6
  Project-URL: Repository, https://github.com/owahltinez/lemming
@@ -32,7 +32,7 @@ human-readable `tasks.yml` file.
32
32
  ## Why Lemming?
33
33
 
34
34
  - **Zero Context Drift**: By breaking projects into discrete tasks, Lemming
35
- ensures agents stay focused. They only see the project context, relevant
35
+ ensures agents stay focused. They only see the long-term goal, relevant
36
36
  history, and the specific task at hand.
37
37
  - **Transparency & Control**: Every decision, technical finding, and progress
38
38
  update is recorded in a human-readable `tasks.yml` file. You can step in,
@@ -61,11 +61,11 @@ uv tool install lemming-cli
61
61
 
62
62
  ### 1. Scaffold the Roadmap
63
63
 
64
- Initialize your project context and define your goals.
64
+ Set the long-term goal and define the first tasks.
65
65
 
66
66
  ```bash
67
- # Set project-wide rules (e.g. tech stack, style guides)
68
- lemming context "Use React, TypeScript, and Tailwind. Follow TDD."
67
+ # Set the long-term goal that every task works toward
68
+ lemming goal "Build a habit-tracking web app with auth, offline support, and tests"
69
69
 
70
70
  # Add tasks to the queue
71
71
  lemming add "Initialize the project with Vite"
@@ -73,6 +73,11 @@ lemming add "Create the Button component"
73
73
  lemming add "Implement the authentication flow"
74
74
  ```
75
75
 
76
+ The goal is the one piece of state every task sees, no matter how far into the
77
+ roadmap it runs — describe what "done" looks like for the project. Durable
78
+ coding rules (tech stack, style guides) belong in your repo's agent files (e.g.
79
+ `AGENTS.md`, `CLAUDE.md`), which your agent already reads on its own.
80
+
76
81
  ### 2. Review and Refine
77
82
 
78
83
  See exactly what's pending and what the agent will see.
@@ -114,19 +119,19 @@ lemming serve --tunnel cloudflare
114
119
  folders directly from the UI.
115
120
  - **Browse Files**: Quickly open your workspace in a separate window to inspect
116
121
  files and directory structure alongside the roadmap.
117
- - **Interactive Controls**: Add tasks, edit context, and manage the execution
122
+ - **Interactive Controls**: Add tasks, edit the goal, and manage the execution
118
123
  loop from your browser.
119
124
 
120
125
  ---
121
126
 
122
127
  ## How it Works
123
128
 
124
- Lemming maintains a human-readable `tasks.yml` file containing your project
125
- context, a queue of tasks, and recorded progress. When you run `lemming run`, it
129
+ Lemming maintains a human-readable `tasks.yml` file containing your long-term
130
+ goal, a queue of tasks, and recorded progress. When you run `lemming run`, it
126
131
  loops through each pending task:
127
132
 
128
133
  1. **Build a scoped prompt**: Lemming assembles a prompt containing only the
129
- project context, a summary of completed tasks and their progress, and the
134
+ long-term goal, a summary of completed tasks and their progress, and the
130
135
  current task description.
131
136
  2. **Invoke the agent**: It launches your chosen agent CLI with that prompt,
132
137
  monitors it with heartbeats, and streams output to a log file.
@@ -153,19 +158,24 @@ tasks may not fully achieve the stated goal. **Orchestrator Hooks** address this
153
158
  by running custom agents or scripts after each task execution to evaluate
154
159
  results and adapt the roadmap.
155
160
 
156
- By default, Lemming runs all available hooks (including the built-in `roadmap`
157
- hook). You can customize this behavior via the `hooks` subcommand:
161
+ Lemming runs every hook it discovers on the filesystem (including the
162
+ built-in `roadmap` hook). Hooks are plain Markdown files, and udev-style
163
+ filename conventions control their behavior:
158
164
 
159
- ```bash
160
- # Enable or disable hooks for the project
161
- lemming hooks enable lint
162
- lemming hooks disable roadmap
165
+ - **Ordering**: A numeric prefix sets the execution order (e.g.
166
+ `10-lint.md` runs before `90-roadmap.md`); files without a prefix
167
+ default to priority 50.
168
+ - **Failure hooks**: Hooks at priority 90 and above also run when a task
169
+ fails; all others only run on success.
170
+ - **Disabling**: An empty file masks (disables) the hook of the same name
171
+ from a lower-precedence layer.
163
172
 
164
- # Set the exact list of active hooks
165
- lemming hooks set roadmap lint
173
+ ```bash
174
+ # Disable a hook for this project (writes an empty .lemming/hooks/50-lint.md)
175
+ lemming hooks disable lint
166
176
 
167
- # Reset to default (run all available hooks)
168
- lemming hooks reset
177
+ # Re-enable it (removes the mask file)
178
+ lemming hooks enable lint
169
179
  ```
170
180
 
171
181
  ### Built-in Hooks ⚓️
@@ -189,33 +199,41 @@ You can create your own hooks by adding Markdown files to:
189
199
  1. **Project-specific**: `.lemming/hooks/*.md`
190
200
  2. **Global**: `~/.local/lemming/hooks/*.md`
191
201
 
192
- Lemming's built-in hooks can be symlinked to the global directory using
193
- `lemming hooks install`. This allows you to easily **override** a built-in hook
194
- by replacing its symlink with a file, or **disable** a global override by
195
- deleting the symlink.
202
+ To **override** a built-in hook, create a file with the same logical name
203
+ (the numeric prefix is not part of the name, so `20-roadmap.md` overrides
204
+ the built-in `90-roadmap.md` and also moves it to priority 20); delete the
205
+ file to restore the built-in version.
196
206
 
197
207
  Hooks follow a specific discovery precedence: **Project > Global > Built-in**.
198
- See [DOCS/HOOKS.md](docs/HOOKS.md) for more details.
208
+ See [docs/HOOKS.md](docs/HOOKS.md) for more details.
199
209
 
200
210
  ### Managing Hooks and Configuration
201
211
 
202
212
  Use the `config` and `hooks` commands to manage your project's execution loop:
203
213
 
204
214
  ```bash
205
- # List all available hooks (built-in and local)
215
+ # List all hooks in execution order, with source and status
206
216
  lemming hooks list
207
217
 
208
- # Install (symlink) built-in hooks to the global directory
209
- lemming hooks install
210
-
211
- # View current project configuration
218
+ # View current project configuration (includes the active hooks)
212
219
  lemming config list
213
220
 
214
221
  # Persist configuration to tasks.yml
215
222
  lemming config set runner aider
216
- lemming hooks set roadmap lint
217
223
  ```
218
224
 
225
+ ### Evaluating Prompt Changes
226
+
227
+ Editing a hook prompt can regress behavior without any test failing. The
228
+ containerized eval harness replays realistic scenarios against the real hook
229
+ execution path and grades the outcome mechanically:
230
+
231
+ ```bash
232
+ uv run python -m lemming.evals run --suite roadmap
233
+ ```
234
+
235
+ See [docs/EVALS.md](docs/EVALS.md) for details.
236
+
219
237
  ---
220
238
 
221
239
  ## Command Reference
@@ -223,8 +241,8 @@ lemming hooks set roadmap lint
223
241
  ### Roadmap Management
224
242
 
225
243
  - **`status [<id>]`**: Roadmap overview or deep-dive into a specific task.
226
- - **`context [<text>]`**: Set or view project-wide instructions. Supports
227
- `-f/--file`.
244
+ - **`goal [<text>]`**: Set or view the long-term goal shared by all tasks.
245
+ Supports `-f/--file`.
228
246
  - **`add <desc>`**: Append a new task. Supports `--index` and `--runner`.
229
247
  - **`edit <id>`**: Modify a task's description, runner, or position.
230
248
  - **`delete <id>`**: Remove a task. Supports `--all` and `--completed` for bulk
@@ -321,6 +339,10 @@ git push origin v0.1.1
321
339
 
322
340
  ## Screenshots
323
341
 
342
+ These are regenerated automatically by CI whenever the web UI changes (see
343
+ `.github/workflows/screenshots.yml`); run `npm run screenshots` to preview them
344
+ locally.
345
+
324
346
  ### Dashboard
325
347
 
326
348
  ![Dashboard](docs/screenshots/dashboard-desktop.png)
@@ -10,7 +10,7 @@ human-readable `tasks.yml` file.
10
10
  ## Why Lemming?
11
11
 
12
12
  - **Zero Context Drift**: By breaking projects into discrete tasks, Lemming
13
- ensures agents stay focused. They only see the project context, relevant
13
+ ensures agents stay focused. They only see the long-term goal, relevant
14
14
  history, and the specific task at hand.
15
15
  - **Transparency & Control**: Every decision, technical finding, and progress
16
16
  update is recorded in a human-readable `tasks.yml` file. You can step in,
@@ -39,11 +39,11 @@ uv tool install lemming-cli
39
39
 
40
40
  ### 1. Scaffold the Roadmap
41
41
 
42
- Initialize your project context and define your goals.
42
+ Set the long-term goal and define the first tasks.
43
43
 
44
44
  ```bash
45
- # Set project-wide rules (e.g. tech stack, style guides)
46
- lemming context "Use React, TypeScript, and Tailwind. Follow TDD."
45
+ # Set the long-term goal that every task works toward
46
+ lemming goal "Build a habit-tracking web app with auth, offline support, and tests"
47
47
 
48
48
  # Add tasks to the queue
49
49
  lemming add "Initialize the project with Vite"
@@ -51,6 +51,11 @@ lemming add "Create the Button component"
51
51
  lemming add "Implement the authentication flow"
52
52
  ```
53
53
 
54
+ The goal is the one piece of state every task sees, no matter how far into the
55
+ roadmap it runs — describe what "done" looks like for the project. Durable
56
+ coding rules (tech stack, style guides) belong in your repo's agent files (e.g.
57
+ `AGENTS.md`, `CLAUDE.md`), which your agent already reads on its own.
58
+
54
59
  ### 2. Review and Refine
55
60
 
56
61
  See exactly what's pending and what the agent will see.
@@ -92,19 +97,19 @@ lemming serve --tunnel cloudflare
92
97
  folders directly from the UI.
93
98
  - **Browse Files**: Quickly open your workspace in a separate window to inspect
94
99
  files and directory structure alongside the roadmap.
95
- - **Interactive Controls**: Add tasks, edit context, and manage the execution
100
+ - **Interactive Controls**: Add tasks, edit the goal, and manage the execution
96
101
  loop from your browser.
97
102
 
98
103
  ---
99
104
 
100
105
  ## How it Works
101
106
 
102
- Lemming maintains a human-readable `tasks.yml` file containing your project
103
- context, a queue of tasks, and recorded progress. When you run `lemming run`, it
107
+ Lemming maintains a human-readable `tasks.yml` file containing your long-term
108
+ goal, a queue of tasks, and recorded progress. When you run `lemming run`, it
104
109
  loops through each pending task:
105
110
 
106
111
  1. **Build a scoped prompt**: Lemming assembles a prompt containing only the
107
- project context, a summary of completed tasks and their progress, and the
112
+ long-term goal, a summary of completed tasks and their progress, and the
108
113
  current task description.
109
114
  2. **Invoke the agent**: It launches your chosen agent CLI with that prompt,
110
115
  monitors it with heartbeats, and streams output to a log file.
@@ -131,19 +136,24 @@ tasks may not fully achieve the stated goal. **Orchestrator Hooks** address this
131
136
  by running custom agents or scripts after each task execution to evaluate
132
137
  results and adapt the roadmap.
133
138
 
134
- By default, Lemming runs all available hooks (including the built-in `roadmap`
135
- hook). You can customize this behavior via the `hooks` subcommand:
139
+ Lemming runs every hook it discovers on the filesystem (including the
140
+ built-in `roadmap` hook). Hooks are plain Markdown files, and udev-style
141
+ filename conventions control their behavior:
136
142
 
137
- ```bash
138
- # Enable or disable hooks for the project
139
- lemming hooks enable lint
140
- lemming hooks disable roadmap
143
+ - **Ordering**: A numeric prefix sets the execution order (e.g.
144
+ `10-lint.md` runs before `90-roadmap.md`); files without a prefix
145
+ default to priority 50.
146
+ - **Failure hooks**: Hooks at priority 90 and above also run when a task
147
+ fails; all others only run on success.
148
+ - **Disabling**: An empty file masks (disables) the hook of the same name
149
+ from a lower-precedence layer.
141
150
 
142
- # Set the exact list of active hooks
143
- lemming hooks set roadmap lint
151
+ ```bash
152
+ # Disable a hook for this project (writes an empty .lemming/hooks/50-lint.md)
153
+ lemming hooks disable lint
144
154
 
145
- # Reset to default (run all available hooks)
146
- lemming hooks reset
155
+ # Re-enable it (removes the mask file)
156
+ lemming hooks enable lint
147
157
  ```
148
158
 
149
159
  ### Built-in Hooks ⚓️
@@ -167,33 +177,41 @@ You can create your own hooks by adding Markdown files to:
167
177
  1. **Project-specific**: `.lemming/hooks/*.md`
168
178
  2. **Global**: `~/.local/lemming/hooks/*.md`
169
179
 
170
- Lemming's built-in hooks can be symlinked to the global directory using
171
- `lemming hooks install`. This allows you to easily **override** a built-in hook
172
- by replacing its symlink with a file, or **disable** a global override by
173
- deleting the symlink.
180
+ To **override** a built-in hook, create a file with the same logical name
181
+ (the numeric prefix is not part of the name, so `20-roadmap.md` overrides
182
+ the built-in `90-roadmap.md` and also moves it to priority 20); delete the
183
+ file to restore the built-in version.
174
184
 
175
185
  Hooks follow a specific discovery precedence: **Project > Global > Built-in**.
176
- See [DOCS/HOOKS.md](docs/HOOKS.md) for more details.
186
+ See [docs/HOOKS.md](docs/HOOKS.md) for more details.
177
187
 
178
188
  ### Managing Hooks and Configuration
179
189
 
180
190
  Use the `config` and `hooks` commands to manage your project's execution loop:
181
191
 
182
192
  ```bash
183
- # List all available hooks (built-in and local)
193
+ # List all hooks in execution order, with source and status
184
194
  lemming hooks list
185
195
 
186
- # Install (symlink) built-in hooks to the global directory
187
- lemming hooks install
188
-
189
- # View current project configuration
196
+ # View current project configuration (includes the active hooks)
190
197
  lemming config list
191
198
 
192
199
  # Persist configuration to tasks.yml
193
200
  lemming config set runner aider
194
- lemming hooks set roadmap lint
195
201
  ```
196
202
 
203
+ ### Evaluating Prompt Changes
204
+
205
+ Editing a hook prompt can regress behavior without any test failing. The
206
+ containerized eval harness replays realistic scenarios against the real hook
207
+ execution path and grades the outcome mechanically:
208
+
209
+ ```bash
210
+ uv run python -m lemming.evals run --suite roadmap
211
+ ```
212
+
213
+ See [docs/EVALS.md](docs/EVALS.md) for details.
214
+
197
215
  ---
198
216
 
199
217
  ## Command Reference
@@ -201,8 +219,8 @@ lemming hooks set roadmap lint
201
219
  ### Roadmap Management
202
220
 
203
221
  - **`status [<id>]`**: Roadmap overview or deep-dive into a specific task.
204
- - **`context [<text>]`**: Set or view project-wide instructions. Supports
205
- `-f/--file`.
222
+ - **`goal [<text>]`**: Set or view the long-term goal shared by all tasks.
223
+ Supports `-f/--file`.
206
224
  - **`add <desc>`**: Append a new task. Supports `--index` and `--runner`.
207
225
  - **`edit <id>`**: Modify a task's description, runner, or position.
208
226
  - **`delete <id>`**: Remove a task. Supports `--all` and `--completed` for bulk
@@ -299,6 +317,10 @@ git push origin v0.1.1
299
317
 
300
318
  ## Screenshots
301
319
 
320
+ These are regenerated automatically by CI whenever the web UI changes (see
321
+ `.github/workflows/screenshots.yml`); run `npm run screenshots` to preview them
322
+ locally.
323
+
302
324
  ### Dashboard
303
325
 
304
326
  ![Dashboard](docs/screenshots/dashboard-desktop.png)
@@ -0,0 +1,89 @@
1
+ # Prompt Evals 🧪
2
+
3
+ Lemming's behavior is driven by prompts (the task runner and the orchestrator
4
+ hooks), so a prompt edit can regress the system without any test failing. The
5
+ eval harness in `lemming.evals` catches that: it replays realistic "a task just
6
+ finished" situations against the real hook execution path and grades the outcome
7
+ with mechanical checks — no LLM judging involved.
8
+
9
+ ## How it Works
10
+
11
+ Each **scenario** seeds a hermetic fixture: a tiny git repository plus a
12
+ `tasks.yml` mid-flight (e.g. a task that just failed for the third time). A
13
+ **trial** runs the hook under eval against that fixture using the same code path
14
+ as the orchestrator, then a **grader** inspects the aftermath:
15
+
16
+ - Did the roadmap hook repair a task that failed at max attempts, or did it
17
+ naively reset it?
18
+ - Did it leave a healthy roadmap untouched?
19
+ - Did it keep its hands off source files (`git status` must stay clean)?
20
+
21
+ Every trial runs in its own container built from the repo `Dockerfile`, with
22
+ only the fixture workspace and a per-trial `LEMMING_HOME` mounted. The agent
23
+ under eval cannot touch the host, and concurrent trials share no state, so
24
+ trials run in parallel safely.
25
+
26
+ ## Running
27
+
28
+ Evals invoke real agents: expect minutes of wall clock and real token spend.
29
+ They are a manual gate for prompt changes, not part of the unit test suite.
30
+
31
+ ```bash
32
+ # List available suites and scenarios
33
+ uv run python -m lemming.evals list
34
+
35
+ # Run a suite (roadmap or readability): scenarios x 3 trials in parallel
36
+ uv run python -m lemming.evals run --suite roadmap
37
+ uv run python -m lemming.evals run --suite readability
38
+
39
+ # Iterate on a single scenario with fewer trials
40
+ uv run python -m lemming.evals run \
41
+ --scenario repair-exhausted-failure --trials 1 --skip-build
42
+
43
+ # Machine-readable results and a non-default pass threshold
44
+ uv run python -m lemming.evals run --json-report report.json --min-pass-rate 0.67
45
+ ```
46
+
47
+ The command prints per-scenario pass rates (agents are stochastic, so think in
48
+ rates, not booleans) and, for each failed trial, the failing checks plus the
49
+ kept workspace path so you can inspect exactly what the agent did. Runner logs
50
+ land in the trial's `home/` directory next to the workspace. The exit code is
51
+ non-zero when any scenario's pass rate drops below `--min-pass-rate`.
52
+
53
+ ### Required vs Advisory Checks
54
+
55
+ Most checks encode the prompt's hard contract mechanically (no source changes,
56
+ no naive task resets, tests stay green) — a red there is a defect, and the right
57
+ response is to improve the prompt. A few checks grade a semantic property
58
+ through a keyword proxy (e.g. "the added task mentions multiply") and are marked
59
+ **advisory**: they never fail a trial and print as yellow `inspect:` lines
60
+ instead. An advisory red means read the workspace and adjudicate — if the
61
+ agent's output was genuinely fine, widen the proxy; if it was vague, tighten the
62
+ prompt. Keeping proxies out of the pass/fail gate is what keeps hard reds
63
+ trustworthy.
64
+
65
+ ### Credentials
66
+
67
+ The default runner is `agy`. Each trial automatically receives a private,
68
+ disposable copy of the host's `~/.gemini` config (caches, history, and the
69
+ bundled CLI excluded) mounted at `/root/.gemini`, so containers can refresh
70
+ tokens and write state without ever touching the real agy home — and concurrent
71
+ trials stay fully isolated from each other.
72
+
73
+ For API-key runners, `ANTHROPIC_API_KEY`, `CLAUDE_CODE_OAUTH_TOKEN`,
74
+ `GEMINI_API_KEY`, and `GOOGLE_API_KEY` are forwarded into the container when set
75
+ on the host. Any other credential files can be mounted with `--volume`.
76
+
77
+ ## Adding Scenarios
78
+
79
+ Scenarios live in `src/lemming/evals/` (see `roadmap.py`):
80
+
81
+ 1. Write a `build` function that seeds the workspace via `fixtures.init_repo`
82
+ and `fixtures.save_roadmap`.
83
+ 2. Write a `grade` function returning `scenarios.Check` results. Prefer checks
84
+ that are mechanically verifiable: roadmap diffs, `fixtures.dirty_paths` for
85
+ source drift, task statuses.
86
+ 3. Register the scenario in the module's `SCENARIOS` list, and new suites in
87
+ `suites.all_suites`.
88
+ 4. Add unit tests that grade simulated good and bad agent behavior; the graders
89
+ themselves must stay fast and offline.
@@ -0,0 +1,164 @@
1
+ # Orchestrator Hooks ⚓️
2
+
3
+ Orchestrator hooks are a powerful way to extend the Lemming workflow. They allow
4
+ you to execute custom agents or scripts after each task, enabling automated
5
+ roadmap revision, code quality checks, or any other post-task validation.
6
+
7
+ ## How it Works
8
+
9
+ When a task finishes (whether it succeeded or failed), Lemming can run one or
10
+ more **orchestrator hooks**. Each hook:
11
+
12
+ 1. **Loads a prompt template**: Lemming looks for a `.md` file corresponding to
13
+ the hook name.
14
+ 2. **Renders the prompt**: It injects context about the current roadmap and the
15
+ task that just finished.
16
+ 3. **Invokes an agent**: It runs your configured runner with the rendered
17
+ prompt.
18
+ 4. **Captures the output**: The agent's output is appended to the task's
19
+ execution log, and any commands it runs (e.g. `lemming add`, `lemming edit`)
20
+ are executed against the roadmap.
21
+
22
+ ## Naming Convention
23
+
24
+ Hooks are Markdown files discovered from the filesystem, and udev-style
25
+ filename conventions control their behavior:
26
+
27
+ - **Ordering**: A numeric prefix determines execution order: `10-lint.md`
28
+ runs before `90-roadmap.md`. Files without a prefix default to priority
29
+ 50. The prefix is **not** part of the hook's name: `90-roadmap.md`
30
+ defines the hook `roadmap`.
31
+ - **Failure hooks**: Hooks at priority 90 and above also run when a task
32
+ fails. All other hooks only run after successful tasks. The built-in
33
+ `roadmap` hook ships as `90-roadmap.md` so it can react to failures.
34
+ - **Masking**: An empty file disables the hook of the same name from a
35
+ lower-precedence layer (see below). For example, an empty
36
+ `.lemming/hooks/readability.md` disables the built-in `readability` hook
37
+ for that project.
38
+ - **Overriding**: A non-empty file replaces the hook of the same name from
39
+ a lower-precedence layer. The winning filename also determines the
40
+ priority, so keep the `9x-` prefix when overriding a failure hook (e.g.
41
+ `90-roadmap.md`), or use a different prefix to deliberately reorder it.
42
+
43
+ ## Built-in Hooks
44
+
45
+ ### `roadmap` (Default)
46
+
47
+ The primary mechanism for autonomous project management. It analyzes the results
48
+ of the finished task and decides if the remaining roadmap needs to be adjusted
49
+ (e.g., adding a missing prerequisite, skipping obsolete tasks, or breaking down
50
+ a broad task).
51
+
52
+ ### `readability`
53
+
54
+ A code quality hook that reviews changes for adherence to the Google Style Guide
55
+ and general readability using the bundled `lemming readability` tool. It
56
+ provides feedback via task progress or suggests follow-up refactoring tasks.
57
+
58
+ ### Editing Built-in Hook Prompts
59
+
60
+ The built-in hook prompts are load-bearing: a wording change can regress
61
+ orchestration behavior without any unit test failing. When you edit a prompt
62
+ under `src/lemming/prompts/hooks/`, run its eval suite before shipping:
63
+
64
+ ```bash
65
+ uv run python -m lemming.evals run --suite roadmap
66
+ uv run python -m lemming.evals run --suite readability
67
+ ```
68
+
69
+ See [EVALS.md](EVALS.md) for how the suites work and how to interpret results.
70
+
71
+ ## Custom Hooks
72
+
73
+ You can define your own hooks by creating Markdown files in the following
74
+ locations:
75
+
76
+ 1. `.lemming/hooks/{name}.md` (Project-specific hooks)
77
+ 2. `~/.local/lemming/hooks/{name}.md` (Global hooks)
78
+
79
+ ### Precedence
80
+
81
+ When Lemming looks for a hook, it resolves each logical name through the
82
+ following layers, highest precedence first:
83
+
84
+ 1. **Project-specific**: `.lemming/hooks/`
85
+ 2. **Global**: `~/.local/lemming/hooks/`
86
+ 3. **Built-in**: Bundled with the Lemming package.
87
+
88
+ Because of this precedence order, overriding a built-in hook only requires
89
+ creating a Markdown file with the same logical name in the project or global
90
+ directory (e.g. `~/.local/lemming/hooks/90-roadmap.md`); delete the file to
91
+ restore the built-in version. Global hooks are available to all Lemming
92
+ projects on the system.
93
+
94
+ ### Example: `lint` hook
95
+
96
+ Create `.lemming/hooks/lint.md`:
97
+
98
+ ```markdown
99
+ You are a code quality orchestrator. A task has just finished. Review the files
100
+ changed in this task and ensure they follow the project's style guide. If there
101
+ are issues, add a new task to fix them.
102
+
103
+ ### Roadmap Context
104
+
105
+ {{roadmap}}
106
+
107
+ ### Finished Task
108
+
109
+ {{finished_task}}
110
+ ```
111
+
112
+ ## Using Hooks
113
+
114
+ ### Enabling and Disabling
115
+
116
+ Lemming runs every hook it discovers that is not masked. There is no hook
117
+ configuration in `tasks.yml`; the filesystem is the single source of truth.
118
+
119
+ The `hooks` command group provides shortcuts for managing project masks:
120
+
121
+ ```bash
122
+ # Disable a hook for this project (writes an empty .lemming/hooks/50-lint.md)
123
+ lemming hooks disable lint
124
+
125
+ # Re-enable it (removes the mask file)
126
+ lemming hooks enable lint
127
+ ```
128
+
129
+ These commands validate the hook names before changing anything and refuse
130
+ to touch files with content (a non-empty file is an override, not a mask).
131
+ An empty file created by hand works just as well; the commands additionally
132
+ keep the hook's priority in the mask filename so listings stay accurate.
133
+
134
+ Hooks are re-discovered on every task execution, so changes are picked up
135
+ dynamically by a running orchestrator loop.
136
+
137
+ ### Available Variables
138
+
139
+ Your hook template can use the following placeholders:
140
+
141
+ - `{{roadmap}}`: A structured summary of the long-term goal and all tasks.
142
+ - `{{finished_task}}`: Details about the task that just finished (ID,
143
+ description, progress, and the last 100 lines of its execution log).
144
+ - `{{finished_task_id}}`: The ID of the task that just finished.
145
+ - `{{tasks_file_name}}`: The filename of the tasks YAML file.
146
+ - `{{tasks_file_path}}`: The full path to the tasks YAML file.
147
+
148
+ ## Developer Ergonomics
149
+
150
+ ### Listing Hooks
151
+
152
+ You can see all hooks in execution order, with their priority, source layer,
153
+ and status (disabled, runs on failure) using the CLI:
154
+
155
+ ```bash
156
+ lemming hooks list
157
+ ```
158
+
159
+ ### Runner Selection
160
+
161
+ Hooks always use the same runner as your tasks. This ensures consistency and
162
+ simplifies configuration. Command-line arguments passed after `--` are
163
+ automatically forwarded to both the main task execution and any subsequent
164
+ hooks.