lemming-cli 0.1.0__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. lemming_cli-0.1.2/.github/workflows/screenshots.yml +51 -0
  2. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/PKG-INFO +50 -13
  3. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/README.md +49 -12
  4. lemming_cli-0.1.2/docs/EVALS.md +89 -0
  5. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/docs/HOOKS.md +14 -2
  6. lemming_cli-0.1.2/docs/screenshots/dashboard-desktop.png +0 -0
  7. lemming_cli-0.1.2/docs/screenshots/dashboard-mobile.png +0 -0
  8. lemming_cli-0.1.2/docs/screenshots/task-log-desktop.png +0 -0
  9. lemming_cli-0.1.2/docs/screenshots/task-log-mobile.png +0 -0
  10. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/package-lock.json +13 -13
  11. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/package.json +4 -3
  12. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/playwright.config.js +7 -0
  13. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/pyproject.toml +1 -1
  14. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/config.py +4 -4
  15. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/config_test.py +12 -12
  16. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/conftest.py +2 -2
  17. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/main.py +1 -1
  18. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/tasks.py +1 -1
  19. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/tasks_test.py +1 -1
  20. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/__init__.py +1 -1
  21. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/config_test.py +1 -1
  22. lemming_cli-0.1.2/src/lemming/cli/goal.py +38 -0
  23. lemming_cli-0.1.0/src/lemming/cli/context_test.py → lemming_cli-0.1.2/src/lemming/cli/goal_test.py +9 -9
  24. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/hooks_test.py +1 -1
  25. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/main.py +2 -2
  26. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/operations_test.py +1 -1
  27. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/progress_test.py +1 -1
  28. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/tasks.py +6 -6
  29. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/tasks_test.py +1 -1
  30. lemming_cli-0.1.2/src/lemming/evals/__init__.py +1 -0
  31. lemming_cli-0.1.2/src/lemming/evals/__main__.py +6 -0
  32. lemming_cli-0.1.2/src/lemming/evals/cli.py +205 -0
  33. lemming_cli-0.1.2/src/lemming/evals/cli_test.py +136 -0
  34. lemming_cli-0.1.2/src/lemming/evals/container.py +145 -0
  35. lemming_cli-0.1.2/src/lemming/evals/container_test.py +86 -0
  36. lemming_cli-0.1.2/src/lemming/evals/fixtures.py +90 -0
  37. lemming_cli-0.1.2/src/lemming/evals/fixtures_test.py +60 -0
  38. lemming_cli-0.1.2/src/lemming/evals/harness.py +239 -0
  39. lemming_cli-0.1.2/src/lemming/evals/harness_test.py +187 -0
  40. lemming_cli-0.1.2/src/lemming/evals/readability.py +324 -0
  41. lemming_cli-0.1.2/src/lemming/evals/readability_test.py +174 -0
  42. lemming_cli-0.1.2/src/lemming/evals/roadmap.py +466 -0
  43. lemming_cli-0.1.2/src/lemming/evals/roadmap_test.py +224 -0
  44. lemming_cli-0.1.2/src/lemming/evals/scenarios.py +89 -0
  45. lemming_cli-0.1.2/src/lemming/evals/scenarios_test.py +27 -0
  46. lemming_cli-0.1.2/src/lemming/evals/suites.py +11 -0
  47. lemming_cli-0.1.2/src/lemming/evals/trial.py +86 -0
  48. lemming_cli-0.1.2/src/lemming/evals/trial_test.py +132 -0
  49. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/hooks.py +13 -4
  50. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/hooks_test.py +48 -1
  51. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/integration_test.py +1 -1
  52. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/main_test.py +1 -1
  53. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/models.py +2 -2
  54. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/models_test.py +1 -1
  55. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/orchestrator_test.py +1 -1
  56. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/persistence.py +4 -19
  57. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/persistence_test.py +2 -35
  58. lemming_cli-0.1.2/src/lemming/prompts/hooks/readability.md +62 -0
  59. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/prompts.py +1 -3
  60. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/prompts_test.py +6 -6
  61. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/runner.py +67 -14
  62. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/runner_test.py +103 -0
  63. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/__init__.py +3 -3
  64. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/lifecycle.py +77 -28
  65. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/lifecycle_test.py +77 -1
  66. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/operations.py +7 -7
  67. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/operations_test.py +3 -3
  68. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/queries.py +2 -2
  69. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/queries_test.py +1 -1
  70. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/dashboard.spec.js +1 -1
  71. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/dashboard.test.js +8 -8
  72. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/favicon.spec.js +2 -2
  73. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/index.html +8 -8
  74. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/index.js +11 -14
  75. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/screenshots.spec.js +32 -58
  76. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/uv.lock +1 -1
  77. lemming_cli-0.1.0/docs/screenshots/dashboard-desktop.png +0 -0
  78. lemming_cli-0.1.0/docs/screenshots/dashboard-mobile.png +0 -0
  79. lemming_cli-0.1.0/docs/screenshots/task-log-desktop.png +0 -0
  80. lemming_cli-0.1.0/docs/screenshots/task-log-mobile.png +0 -0
  81. lemming_cli-0.1.0/src/lemming/api.py +0 -5
  82. lemming_cli-0.1.0/src/lemming/cli/context.py +0 -40
  83. lemming_cli-0.1.0/src/lemming/cli.py +0 -5
  84. lemming_cli-0.1.0/src/lemming/prompts/hooks/readability.md +0 -42
  85. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/.github/workflows/ci.yml +0 -0
  86. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/.github/workflows/publish.yml +0 -0
  87. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/.gitignore +0 -0
  88. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/.prettierignore +0 -0
  89. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/.prettierrc +0 -0
  90. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/Dockerfile +0 -0
  91. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/LICENSE +0 -0
  92. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/biome.json +0 -0
  93. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/docker-compose.yml +0 -0
  94. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/__init__.py +0 -0
  95. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/__init__.py +0 -0
  96. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/auth.py +0 -0
  97. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/auth_test.py +0 -0
  98. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/context.py +0 -0
  99. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/context_test.py +0 -0
  100. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/directories.py +0 -0
  101. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/directories_test.py +0 -0
  102. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/files.py +0 -0
  103. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/files_test.py +0 -0
  104. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/hooks.py +0 -0
  105. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/hooks_test.py +0 -0
  106. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/logging.py +0 -0
  107. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/logging_test.py +0 -0
  108. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/loop.py +0 -0
  109. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/loop_test.py +0 -0
  110. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api/main_test.py +0 -0
  111. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/api_test.py +0 -0
  112. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/config.py +0 -0
  113. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/hooks.py +0 -0
  114. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/main_test.py +0 -0
  115. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/operations.py +0 -0
  116. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/progress.py +0 -0
  117. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/readability_cli.py +0 -0
  118. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli/readability_cli_test.py +0 -0
  119. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/cli_test.py +0 -0
  120. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/conftest.py +0 -0
  121. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/main.py +0 -0
  122. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/orchestrator.py +0 -0
  123. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/paths.py +0 -0
  124. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/paths_test.py +0 -0
  125. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/prompts/hooks/roadmap.md +0 -0
  126. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/prompts/hooks/testing.md +0 -0
  127. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/prompts/taskrunner.md +0 -0
  128. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/providers.py +0 -0
  129. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/providers_test.py +0 -0
  130. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/progress.py +0 -0
  131. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/tasks/progress_test.py +0 -0
  132. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/favicon.js +0 -0
  133. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/files.html +0 -0
  134. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/files.spec.js +0 -0
  135. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/logs.html +0 -0
  136. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/logs.test.js +0 -0
  137. {lemming_cli-0.1.0 → lemming_cli-0.1.2}/src/lemming/web/mancha.js +0 -0
@@ -0,0 +1,51 @@
1
+ name: Screenshots
2
+
3
+ # Regenerates the dashboard screenshots in docs/ whenever the web UI changes,
4
+ # so they can never go stale. Generation is deterministic (frozen clock and
5
+ # animations disabled), so this only commits when the UI actually changed.
6
+ # The commit touches only docs/screenshots, which is outside the paths filter
7
+ # below, so it cannot re-trigger this workflow.
8
+ on:
9
+ push:
10
+ branches: [main]
11
+ paths:
12
+ - 'src/lemming/web/**'
13
+ - 'playwright.config.js'
14
+ - '.github/workflows/screenshots.yml'
15
+
16
+ permissions:
17
+ contents: write
18
+
19
+ jobs:
20
+ regenerate:
21
+ runs-on: ubuntu-latest
22
+
23
+ steps:
24
+ - uses: actions/checkout@v6
25
+
26
+ - name: Set up Node.js
27
+ uses: actions/setup-node@v6
28
+ with:
29
+ node-version: '20'
30
+
31
+ - name: Install Node.js dependencies
32
+ run: npm install
33
+
34
+ - name: Install Playwright browser
35
+ run: npx playwright install --with-deps chromium
36
+
37
+ - name: Regenerate screenshots
38
+ run: npm run screenshots
39
+
40
+ - name: Commit updated screenshots
41
+ run: |
42
+ if git diff --quiet docs/screenshots; then
43
+ echo "Screenshots are up to date."
44
+ exit 0
45
+ fi
46
+ git config user.name "github-actions[bot]"
47
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
48
+ git add docs/screenshots
49
+ git commit -m "Regenerate dashboard screenshots"
50
+ git pull --rebase --autostash origin main
51
+ git push
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lemming-cli
3
- Version: 0.1.0
3
+ Version: 0.1.2
4
4
  Summary: An autonomous, iterative task runner for AI coding agents.
5
5
  Project-URL: Homepage, https://github.com/owahltinez/lemming
6
6
  Project-URL: Repository, https://github.com/owahltinez/lemming
@@ -32,7 +32,7 @@ human-readable `tasks.yml` file.
32
32
  ## Why Lemming?
33
33
 
34
34
  - **Zero Context Drift**: By breaking projects into discrete tasks, Lemming
35
- ensures agents stay focused. They only see the project context, relevant
35
+ ensures agents stay focused. They only see the long-term goal, relevant
36
36
  history, and the specific task at hand.
37
37
  - **Transparency & Control**: Every decision, technical finding, and progress
38
38
  update is recorded in a human-readable `tasks.yml` file. You can step in,
@@ -54,18 +54,18 @@ human-readable `tasks.yml` file.
54
54
  Install globally using `uv`:
55
55
 
56
56
  ```bash
57
- uv tool install git+https://github.com/owahltinez/lemming.git
57
+ uv tool install lemming-cli
58
58
  ```
59
59
 
60
60
  ## Quick Start in 3 Steps
61
61
 
62
62
  ### 1. Scaffold the Roadmap
63
63
 
64
- Initialize your project context and define your goals.
64
+ Set the long-term goal and define the first tasks.
65
65
 
66
66
  ```bash
67
- # Set project-wide rules (e.g. tech stack, style guides)
68
- lemming context "Use React, TypeScript, and Tailwind. Follow TDD."
67
+ # Set the long-term goal that every task works toward
68
+ lemming goal "Build a habit-tracking web app with auth, offline support, and tests"
69
69
 
70
70
  # Add tasks to the queue
71
71
  lemming add "Initialize the project with Vite"
@@ -73,6 +73,11 @@ lemming add "Create the Button component"
73
73
  lemming add "Implement the authentication flow"
74
74
  ```
75
75
 
76
+ The goal is the one piece of state every task sees, no matter how far into the
77
+ roadmap it runs — describe what "done" looks like for the project. Durable
78
+ coding rules (tech stack, style guides) belong in your repo's agent files (e.g.
79
+ `AGENTS.md`, `CLAUDE.md`), which your agent already reads on its own.
80
+
76
81
  ### 2. Review and Refine
77
82
 
78
83
  See exactly what's pending and what the agent will see.
@@ -114,19 +119,19 @@ lemming serve --tunnel cloudflare
114
119
  folders directly from the UI.
115
120
  - **Browse Files**: Quickly open your workspace in a separate window to inspect
116
121
  files and directory structure alongside the roadmap.
117
- - **Interactive Controls**: Add tasks, edit context, and manage the execution
122
+ - **Interactive Controls**: Add tasks, edit the goal, and manage the execution
118
123
  loop from your browser.
119
124
 
120
125
  ---
121
126
 
122
127
  ## How it Works
123
128
 
124
- Lemming maintains a human-readable `tasks.yml` file containing your project
125
- context, a queue of tasks, and recorded progress. When you run `lemming run`, it
129
+ Lemming maintains a human-readable `tasks.yml` file containing your long-term
130
+ goal, a queue of tasks, and recorded progress. When you run `lemming run`, it
126
131
  loops through each pending task:
127
132
 
128
133
  1. **Build a scoped prompt**: Lemming assembles a prompt containing only the
129
- project context, a summary of completed tasks and their progress, and the
134
+ long-term goal, a summary of completed tasks and their progress, and the
130
135
  current task description.
131
136
  2. **Invoke the agent**: It launches your chosen agent CLI with that prompt,
132
137
  monitors it with heartbeats, and streams output to a log file.
@@ -195,7 +200,7 @@ by replacing its symlink with a file, or **disable** a global override by
195
200
  deleting the symlink.
196
201
 
197
202
  Hooks follow a specific discovery precedence: **Project > Global > Built-in**.
198
- See [DOCS/HOOKS.md](docs/HOOKS.md) for more details.
203
+ See [docs/HOOKS.md](docs/HOOKS.md) for more details.
199
204
 
200
205
  ### Managing Hooks and Configuration
201
206
 
@@ -216,6 +221,18 @@ lemming config set runner aider
216
221
  lemming hooks set roadmap lint
217
222
  ```
218
223
 
224
+ ### Evaluating Prompt Changes
225
+
226
+ Editing a hook prompt can regress behavior without any test failing. The
227
+ containerized eval harness replays realistic scenarios against the real hook
228
+ execution path and grades the outcome mechanically:
229
+
230
+ ```bash
231
+ uv run python -m lemming.evals run --suite roadmap
232
+ ```
233
+
234
+ See [docs/EVALS.md](docs/EVALS.md) for details.
235
+
219
236
  ---
220
237
 
221
238
  ## Command Reference
@@ -223,8 +240,8 @@ lemming hooks set roadmap lint
223
240
  ### Roadmap Management
224
241
 
225
242
  - **`status [<id>]`**: Roadmap overview or deep-dive into a specific task.
226
- - **`context [<text>]`**: Set or view project-wide instructions. Supports
227
- `-f/--file`.
243
+ - **`goal [<text>]`**: Set or view the long-term goal shared by all tasks.
244
+ Supports `-f/--file`.
228
245
  - **`add <desc>`**: Append a new task. Supports `--index` and `--runner`.
229
246
  - **`edit <id>`**: Modify a task's description, runner, or position.
230
247
  - **`delete <id>`**: Remove a task. Supports `--all` and `--completed` for bulk
@@ -303,8 +320,28 @@ prompt text and skips all default flag injection.
303
320
 
304
321
  ---
305
322
 
323
+ ## Releasing
324
+
325
+ Releases are published to PyPI as
326
+ [`lemming-cli`](https://pypi.org/project/lemming-cli/) via trusted publishing:
327
+ pushing a `v*` tag triggers the `publish.yml` GitHub Actions workflow, which
328
+ builds the package with `uv build` and uploads it.
329
+
330
+ ```bash
331
+ # 1. Bump the version in pyproject.toml, commit, and push
332
+ # 2. Tag the release and push the tag
333
+ git tag v0.1.1
334
+ git push origin v0.1.1
335
+ ```
336
+
337
+ ---
338
+
306
339
  ## Screenshots
307
340
 
341
+ These are regenerated automatically by CI whenever the web UI changes (see
342
+ `.github/workflows/screenshots.yml`); run `npm run screenshots` to preview them
343
+ locally.
344
+
308
345
  ### Dashboard
309
346
 
310
347
  ![Dashboard](docs/screenshots/dashboard-desktop.png)
@@ -10,7 +10,7 @@ human-readable `tasks.yml` file.
10
10
  ## Why Lemming?
11
11
 
12
12
  - **Zero Context Drift**: By breaking projects into discrete tasks, Lemming
13
- ensures agents stay focused. They only see the project context, relevant
13
+ ensures agents stay focused. They only see the long-term goal, relevant
14
14
  history, and the specific task at hand.
15
15
  - **Transparency & Control**: Every decision, technical finding, and progress
16
16
  update is recorded in a human-readable `tasks.yml` file. You can step in,
@@ -32,18 +32,18 @@ human-readable `tasks.yml` file.
32
32
  Install globally using `uv`:
33
33
 
34
34
  ```bash
35
- uv tool install git+https://github.com/owahltinez/lemming.git
35
+ uv tool install lemming-cli
36
36
  ```
37
37
 
38
38
  ## Quick Start in 3 Steps
39
39
 
40
40
  ### 1. Scaffold the Roadmap
41
41
 
42
- Initialize your project context and define your goals.
42
+ Set the long-term goal and define the first tasks.
43
43
 
44
44
  ```bash
45
- # Set project-wide rules (e.g. tech stack, style guides)
46
- lemming context "Use React, TypeScript, and Tailwind. Follow TDD."
45
+ # Set the long-term goal that every task works toward
46
+ lemming goal "Build a habit-tracking web app with auth, offline support, and tests"
47
47
 
48
48
  # Add tasks to the queue
49
49
  lemming add "Initialize the project with Vite"
@@ -51,6 +51,11 @@ lemming add "Create the Button component"
51
51
  lemming add "Implement the authentication flow"
52
52
  ```
53
53
 
54
+ The goal is the one piece of state every task sees, no matter how far into the
55
+ roadmap it runs — describe what "done" looks like for the project. Durable
56
+ coding rules (tech stack, style guides) belong in your repo's agent files (e.g.
57
+ `AGENTS.md`, `CLAUDE.md`), which your agent already reads on its own.
58
+
54
59
  ### 2. Review and Refine
55
60
 
56
61
  See exactly what's pending and what the agent will see.
@@ -92,19 +97,19 @@ lemming serve --tunnel cloudflare
92
97
  folders directly from the UI.
93
98
  - **Browse Files**: Quickly open your workspace in a separate window to inspect
94
99
  files and directory structure alongside the roadmap.
95
- - **Interactive Controls**: Add tasks, edit context, and manage the execution
100
+ - **Interactive Controls**: Add tasks, edit the goal, and manage the execution
96
101
  loop from your browser.
97
102
 
98
103
  ---
99
104
 
100
105
  ## How it Works
101
106
 
102
- Lemming maintains a human-readable `tasks.yml` file containing your project
103
- context, a queue of tasks, and recorded progress. When you run `lemming run`, it
107
+ Lemming maintains a human-readable `tasks.yml` file containing your long-term
108
+ goal, a queue of tasks, and recorded progress. When you run `lemming run`, it
104
109
  loops through each pending task:
105
110
 
106
111
  1. **Build a scoped prompt**: Lemming assembles a prompt containing only the
107
- project context, a summary of completed tasks and their progress, and the
112
+ long-term goal, a summary of completed tasks and their progress, and the
108
113
  current task description.
109
114
  2. **Invoke the agent**: It launches your chosen agent CLI with that prompt,
110
115
  monitors it with heartbeats, and streams output to a log file.
@@ -173,7 +178,7 @@ by replacing its symlink with a file, or **disable** a global override by
173
178
  deleting the symlink.
174
179
 
175
180
  Hooks follow a specific discovery precedence: **Project > Global > Built-in**.
176
- See [DOCS/HOOKS.md](docs/HOOKS.md) for more details.
181
+ See [docs/HOOKS.md](docs/HOOKS.md) for more details.
177
182
 
178
183
  ### Managing Hooks and Configuration
179
184
 
@@ -194,6 +199,18 @@ lemming config set runner aider
194
199
  lemming hooks set roadmap lint
195
200
  ```
196
201
 
202
+ ### Evaluating Prompt Changes
203
+
204
+ Editing a hook prompt can regress behavior without any test failing. The
205
+ containerized eval harness replays realistic scenarios against the real hook
206
+ execution path and grades the outcome mechanically:
207
+
208
+ ```bash
209
+ uv run python -m lemming.evals run --suite roadmap
210
+ ```
211
+
212
+ See [docs/EVALS.md](docs/EVALS.md) for details.
213
+
197
214
  ---
198
215
 
199
216
  ## Command Reference
@@ -201,8 +218,8 @@ lemming hooks set roadmap lint
201
218
  ### Roadmap Management
202
219
 
203
220
  - **`status [<id>]`**: Roadmap overview or deep-dive into a specific task.
204
- - **`context [<text>]`**: Set or view project-wide instructions. Supports
205
- `-f/--file`.
221
+ - **`goal [<text>]`**: Set or view the long-term goal shared by all tasks.
222
+ Supports `-f/--file`.
206
223
  - **`add <desc>`**: Append a new task. Supports `--index` and `--runner`.
207
224
  - **`edit <id>`**: Modify a task's description, runner, or position.
208
225
  - **`delete <id>`**: Remove a task. Supports `--all` and `--completed` for bulk
@@ -281,8 +298,28 @@ prompt text and skips all default flag injection.
281
298
 
282
299
  ---
283
300
 
301
+ ## Releasing
302
+
303
+ Releases are published to PyPI as
304
+ [`lemming-cli`](https://pypi.org/project/lemming-cli/) via trusted publishing:
305
+ pushing a `v*` tag triggers the `publish.yml` GitHub Actions workflow, which
306
+ builds the package with `uv build` and uploads it.
307
+
308
+ ```bash
309
+ # 1. Bump the version in pyproject.toml, commit, and push
310
+ # 2. Tag the release and push the tag
311
+ git tag v0.1.1
312
+ git push origin v0.1.1
313
+ ```
314
+
315
+ ---
316
+
284
317
  ## Screenshots
285
318
 
319
+ These are regenerated automatically by CI whenever the web UI changes (see
320
+ `.github/workflows/screenshots.yml`); run `npm run screenshots` to preview them
321
+ locally.
322
+
286
323
  ### Dashboard
287
324
 
288
325
  ![Dashboard](docs/screenshots/dashboard-desktop.png)
@@ -0,0 +1,89 @@
1
+ # Prompt Evals 🧪
2
+
3
+ Lemming's behavior is driven by prompts (the task runner and the orchestrator
4
+ hooks), so a prompt edit can regress the system without any test failing. The
5
+ eval harness in `lemming.evals` catches that: it replays realistic "a task just
6
+ finished" situations against the real hook execution path and grades the outcome
7
+ with mechanical checks — no LLM judging involved.
8
+
9
+ ## How it Works
10
+
11
+ Each **scenario** seeds a hermetic fixture: a tiny git repository plus a
12
+ `tasks.yml` mid-flight (e.g. a task that just failed for the third time). A
13
+ **trial** runs the hook under eval against that fixture using the same code path
14
+ as the orchestrator, then a **grader** inspects the aftermath:
15
+
16
+ - Did the roadmap hook repair a task that failed at max attempts, or did it
17
+ naively reset it?
18
+ - Did it leave a healthy roadmap untouched?
19
+ - Did it keep its hands off source files (`git status` must stay clean)?
20
+
21
+ Every trial runs in its own container built from the repo `Dockerfile`, with
22
+ only the fixture workspace and a per-trial `LEMMING_HOME` mounted. The agent
23
+ under eval cannot touch the host, and concurrent trials share no state, so
24
+ trials run in parallel safely.
25
+
26
+ ## Running
27
+
28
+ Evals invoke real agents: expect minutes of wall clock and real token spend.
29
+ They are a manual gate for prompt changes, not part of the unit test suite.
30
+
31
+ ```bash
32
+ # List available suites and scenarios
33
+ uv run python -m lemming.evals list
34
+
35
+ # Run a suite (roadmap or readability): scenarios x 3 trials in parallel
36
+ uv run python -m lemming.evals run --suite roadmap
37
+ uv run python -m lemming.evals run --suite readability
38
+
39
+ # Iterate on a single scenario with fewer trials
40
+ uv run python -m lemming.evals run \
41
+ --scenario repair-exhausted-failure --trials 1 --skip-build
42
+
43
+ # Machine-readable results and a non-default pass threshold
44
+ uv run python -m lemming.evals run --json-report report.json --min-pass-rate 0.67
45
+ ```
46
+
47
+ The command prints per-scenario pass rates (agents are stochastic, so think in
48
+ rates, not booleans) and, for each failed trial, the failing checks plus the
49
+ kept workspace path so you can inspect exactly what the agent did. Runner logs
50
+ land in the trial's `home/` directory next to the workspace. The exit code is
51
+ non-zero when any scenario's pass rate drops below `--min-pass-rate`.
52
+
53
+ ### Required vs Advisory Checks
54
+
55
+ Most checks encode the prompt's hard contract mechanically (no source changes,
56
+ no naive task resets, tests stay green) — a red there is a defect, and the right
57
+ response is to improve the prompt. A few checks grade a semantic property
58
+ through a keyword proxy (e.g. "the added task mentions multiply") and are marked
59
+ **advisory**: they never fail a trial and print as yellow `inspect:` lines
60
+ instead. An advisory red means read the workspace and adjudicate — if the
61
+ agent's output was genuinely fine, widen the proxy; if it was vague, tighten the
62
+ prompt. Keeping proxies out of the pass/fail gate is what keeps hard reds
63
+ trustworthy.
64
+
65
+ ### Credentials
66
+
67
+ The default runner is `agy`. Each trial automatically receives a private,
68
+ disposable copy of the host's `~/.gemini` config (caches, history, and the
69
+ bundled CLI excluded) mounted at `/root/.gemini`, so containers can refresh
70
+ tokens and write state without ever touching the real agy home — and concurrent
71
+ trials stay fully isolated from each other.
72
+
73
+ For API-key runners, `ANTHROPIC_API_KEY`, `CLAUDE_CODE_OAUTH_TOKEN`,
74
+ `GEMINI_API_KEY`, and `GOOGLE_API_KEY` are forwarded into the container when set
75
+ on the host. Any other credential files can be mounted with `--volume`.
76
+
77
+ ## Adding Scenarios
78
+
79
+ Scenarios live in `src/lemming/evals/` (see `roadmap.py`):
80
+
81
+ 1. Write a `build` function that seeds the workspace via `fixtures.init_repo`
82
+ and `fixtures.save_roadmap`.
83
+ 2. Write a `grade` function returning `scenarios.Check` results. Prefer checks
84
+ that are mechanically verifiable: roadmap diffs, `fixtures.dirty_paths` for
85
+ source drift, task statuses.
86
+ 3. Register the scenario in the module's `SCENARIOS` list, and new suites in
87
+ `suites.all_suites`.
88
+ 4. Add unit tests that grade simulated good and bad agent behavior; the graders
89
+ themselves must stay fast and offline.
@@ -34,6 +34,19 @@ A code quality hook that reviews changes for adherence to the Google Style Guide
34
34
  and general readability using the bundled `lemming readability` tool. It
35
35
  provides feedback via task progress or suggests follow-up refactoring tasks.
36
36
 
37
+ ### Editing Built-in Hook Prompts
38
+
39
+ The built-in hook prompts are load-bearing: a wording change can regress
40
+ orchestration behavior without any unit test failing. When you edit a prompt
41
+ under `src/lemming/prompts/hooks/`, run its eval suite before shipping:
42
+
43
+ ```bash
44
+ uv run python -m lemming.evals run --suite roadmap
45
+ uv run python -m lemming.evals run --suite readability
46
+ ```
47
+
48
+ See [EVALS.md](EVALS.md) for how the suites work and how to interpret results.
49
+
37
50
  ## Custom Hooks
38
51
 
39
52
  You can define your own hooks by creating Markdown files in the following
@@ -122,8 +135,7 @@ is running, and the next task execution will respect the updated settings.
122
135
 
123
136
  Your hook template can use the following placeholders:
124
137
 
125
- - `{{roadmap}}`: A structured summary of the entire project context and all
126
- tasks.
138
+ - `{{roadmap}}`: A structured summary of the long-term goal and all tasks.
127
139
  - `{{finished_task}}`: Details about the task that just finished (ID,
128
140
  description, progress, and the last 100 lines of its execution log).
129
141
  - `{{finished_task_id}}`: The ID of the task that just finished.
@@ -10,10 +10,10 @@
10
10
  "hasInstallScript": true,
11
11
  "devDependencies": {
12
12
  "@biomejs/biome": "1.9.4",
13
- "@playwright/test": "^1.58.2",
13
+ "@playwright/test": "^1.60.0",
14
14
  "jsdom": "latest",
15
15
  "mancha": "^0.22.4",
16
- "playwright": "^1.58.2"
16
+ "playwright": "^1.60.0"
17
17
  }
18
18
  },
19
19
  "node_modules/@acemir/cssom": {
@@ -430,13 +430,13 @@
430
430
  }
431
431
  },
432
432
  "node_modules/@playwright/test": {
433
- "version": "1.58.2",
434
- "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz",
435
- "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==",
433
+ "version": "1.60.0",
434
+ "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.60.0.tgz",
435
+ "integrity": "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==",
436
436
  "dev": true,
437
437
  "license": "Apache-2.0",
438
438
  "dependencies": {
439
- "playwright": "1.58.2"
439
+ "playwright": "1.60.0"
440
440
  },
441
441
  "bin": {
442
442
  "playwright": "cli.js"
@@ -1253,13 +1253,13 @@
1253
1253
  "license": "ISC"
1254
1254
  },
1255
1255
  "node_modules/playwright": {
1256
- "version": "1.58.2",
1257
- "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz",
1258
- "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
1256
+ "version": "1.60.0",
1257
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz",
1258
+ "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==",
1259
1259
  "dev": true,
1260
1260
  "license": "Apache-2.0",
1261
1261
  "dependencies": {
1262
- "playwright-core": "1.58.2"
1262
+ "playwright-core": "1.60.0"
1263
1263
  },
1264
1264
  "bin": {
1265
1265
  "playwright": "cli.js"
@@ -1272,9 +1272,9 @@
1272
1272
  }
1273
1273
  },
1274
1274
  "node_modules/playwright-core": {
1275
- "version": "1.58.2",
1276
- "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz",
1277
- "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
1275
+ "version": "1.60.0",
1276
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz",
1277
+ "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==",
1278
1278
  "dev": true,
1279
1279
  "license": "Apache-2.0",
1280
1280
  "bin": {
@@ -5,7 +5,8 @@
5
5
  "type": "module",
6
6
  "scripts": {
7
7
  "postinstall": "npm run vendor",
8
- "test": "node --test src/lemming/web/*.test.js && playwright test",
8
+ "test": "node --test src/lemming/web/*.test.js && playwright test --project=chromium",
9
+ "screenshots": "playwright test --project=screenshots",
9
10
  "lint": "biome check .",
10
11
  "format": "biome format --write .",
11
12
  "check": "mancha check src/lemming/web/*.html",
@@ -13,9 +14,9 @@
13
14
  },
14
15
  "devDependencies": {
15
16
  "@biomejs/biome": "1.9.4",
16
- "@playwright/test": "^1.58.2",
17
+ "@playwright/test": "^1.60.0",
17
18
  "jsdom": "latest",
18
19
  "mancha": "^0.22.4",
19
- "playwright": "^1.58.2"
20
+ "playwright": "^1.60.0"
20
21
  }
21
22
  }
@@ -10,6 +10,13 @@ export default defineConfig({
10
10
  {
11
11
  name: 'chromium',
12
12
  use: { ...devices['Desktop Chrome'] },
13
+ testIgnore: /screenshots\.spec\.js/,
14
+ },
15
+ // Regenerates docs/screenshots; run via `npm run screenshots`.
16
+ {
17
+ name: 'screenshots',
18
+ use: { ...devices['Desktop Chrome'] },
19
+ testMatch: /screenshots\.spec\.js/,
13
20
  },
14
21
  ],
15
22
  });
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "lemming-cli"
7
- version = "0.1.0"
7
+ version = "0.1.2"
8
8
  description = "An autonomous, iterative task runner for AI coding agents."
9
9
  readme = "README.md"
10
10
  authors = [
@@ -19,13 +19,13 @@ def get_runners():
19
19
  return list(models.KNOWN_RUNNERS)
20
20
 
21
21
 
22
- @router.post("/api/context")
23
- def update_context(
22
+ @router.post("/api/goal")
23
+ def update_goal(
24
24
  request: fastapi.Request, update: dict, project: str | None = None
25
25
  ):
26
- """Update the shared project context passed to task runners."""
26
+ """Update the long-term goal shared with all task runners."""
27
27
  tasks_file = context.resolve_tasks_file(request.app.state, project)
28
- tasks.update_context(tasks_file, update.get("context", ""))
28
+ tasks.update_goal(tasks_file, update.get("goal", ""))
29
29
  return {"status": "ok"}
30
30
 
31
31
 
@@ -23,29 +23,29 @@ def test_get_runners(client):
23
23
  assert response.json() == ["agy", "aider", "claude", "codex"]
24
24
 
25
25
 
26
- def test_project_context_isolation(client, test_tasks):
27
- """Context is isolated per project."""
26
+ def test_project_goal_isolation(client, test_tasks):
27
+ """The goal is isolated per project."""
28
28
  root = api.app.state.root
29
- subdir = root / "ctx_test"
29
+ subdir = root / "goal_test"
30
30
  subdir.mkdir(exist_ok=True)
31
31
 
32
- # Set context for sub-project
32
+ # Set the goal for the sub-project
33
33
  response = client.post(
34
- "/api/context",
35
- json={"context": "Sub-project context"},
36
- params={"project": "ctx_test"},
34
+ "/api/goal",
35
+ json={"goal": "Sub-project goal"},
36
+ params={"project": "goal_test"},
37
37
  )
38
38
  assert response.status_code == 200
39
39
 
40
- # Root context should be unchanged
40
+ # Root goal should be unchanged
41
41
  res = client.get("/api/data")
42
42
  assert res.status_code == 200
43
- assert res.json()["context"] == "Initial context"
43
+ assert res.json()["goal"] == "Initial goal"
44
44
 
45
- # Sub-project context should be set
46
- res = client.get("/api/data", params={"project": "ctx_test"})
45
+ # Sub-project goal should be set
46
+ res = client.get("/api/data", params={"project": "goal_test"})
47
47
  assert res.status_code == 200
48
- assert res.json()["context"] == "Sub-project context"
48
+ assert res.json()["goal"] == "Sub-project goal"
49
49
 
50
50
 
51
51
  def test_run_loop_with_project(client, test_tasks):
@@ -28,7 +28,7 @@ def test_tasks():
28
28
 
29
29
  # Scaffold a valid file
30
30
  data = tasks.Roadmap(
31
- context="Initial context",
31
+ goal="Initial goal",
32
32
  tasks=[
33
33
  tasks.Task(
34
34
  id="task1",
@@ -175,7 +175,7 @@ def test_workspace():
175
175
  # Set up some tasks in the subproject
176
176
  sub_tasks_file = subproject_dir / "tasks.yml"
177
177
  data = tasks.Roadmap(
178
- context="Subproject context",
178
+ goal="Subproject goal",
179
179
  tasks=[
180
180
  tasks.Task(
181
181
  id="sub1",
@@ -11,7 +11,7 @@ from .. import paths
11
11
  from . import auth, config, directories, files, hooks, tasks
12
12
  from . import logging as lemming_logging
13
13
 
14
- # Re-export for backward compatibility and tests
14
+ # Re-exported so logging configs can reference lemming.api.QuietPollFilter
15
15
  QuietPollFilter = lemming_logging.QuietPollFilter
16
16
 
17
17