devin-dream 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devin_dream-0.1.0/LICENSE +21 -0
- devin_dream-0.1.0/PKG-INFO +186 -0
- devin_dream-0.1.0/README.md +172 -0
- devin_dream-0.1.0/pyproject.toml +27 -0
- devin_dream-0.1.0/setup.cfg +4 -0
- devin_dream-0.1.0/src/devin_dream/__init__.py +3 -0
- devin_dream-0.1.0/src/devin_dream/cli.py +139 -0
- devin_dream-0.1.0/src/devin_dream/defects.py +266 -0
- devin_dream-0.1.0/src/devin_dream/generate.py +539 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/PKG-INFO +186 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/SOURCES.txt +15 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/dependency_links.txt +1 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/entry_points.txt +2 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/requires.txt +4 -0
- devin_dream-0.1.0/src/devin_dream.egg-info/top_level.txt +1 -0
- devin_dream-0.1.0/tests/test_dream.py +170 -0
- devin_dream-0.1.0/tests/test_offline_core.py +76 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Icaro0310
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: devin-dream
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Synthetic Devin sessions with known verdicts — regression and adversarial test data for the devin-* catalog.
|
|
5
|
+
Author: Icaro0310
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: devin-internals-spec<0.4.0,>=0.3.0
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
<div align="center">
|
|
16
|
+
|
|
17
|
+
# devin-dream — MOVED
|
|
18
|
+
|
|
19
|
+
**This repository was absorbed into
|
|
20
|
+
[`devin-evals`](https://github.com/Icaro0310/devin-evals).**
|
|
21
|
+
|
|
22
|
+
The code now lives at `src/devin_evals/dream/` and the CLI moved to
|
|
23
|
+
`devin-evals dream {unit,inject,fleet}` — same commands, same defect
|
|
24
|
+
catalogue (D01–D09), one package instead of two.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# before
|
|
28
|
+
devin-dream unit --out out/ --defect D01 D03
|
|
29
|
+
|
|
30
|
+
# after
|
|
31
|
+
devin-evals dream unit --out out/ --defect D01 D03
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The repository is archived; open issues were migrated to devin-evals.
|
|
35
|
+
History remains readable here for reference.
|
|
36
|
+
|
|
37
|
+
</div>
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
<details>
|
|
42
|
+
<summary>Original README (pre-archive)</summary>
|
|
43
|
+
|
|
44
|
+
<div align="center">
|
|
45
|
+
|
|
46
|
+
<a href="https://github.com/Icaro0310/devin-dream/actions/workflows/ci.yml"><img src="https://github.com/Icaro0310/devin-dream/actions/workflows/ci.yml/badge.svg" alt="ci"/></a>
|
|
47
|
+
|
|
48
|
+
<a href="https://scorecard.dev/viewer/?uri=github.com/Icaro0310/devin-dream"><img src="https://api.scorecard.dev/projects/github.com/Icaro0310/devin-dream/badge" alt="OpenSSF Scorecard"/></a>
|
|
49
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License: MIT"/></a>
|
|
50
|
+
<a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python 3.10+"/></a>
|
|
51
|
+
<a href="https://github.com/Icaro0310/devin-dream"><img src="https://img.shields.io/github/stars/Icaro0310/devin-dream" alt="GitHub stars"/></a>
|
|
52
|
+
<a href="https://github.com/Icaro0310/devin-dream/commits/main"><img src="https://img.shields.io/github/last-commit/Icaro0310/devin-dream" alt="Last commit"/></a>
|
|
53
|
+
<a href="https://github.com/Icaro0310/awesome-devin"><img src="https://img.shields.io/badge/part%20of-devin--*-ecosystem-7c3aed" alt="devin-* ecosystem"/></a>
|
|
54
|
+
<a href="https://github.com/Icaro0310/devin-dream/issues"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen" alt="PRs welcome"/></a>
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
# devin-dream
|
|
58
|
+
|
|
59
|
+
Synthetic Devin sessions with a **known verdict** — regression and
|
|
60
|
+
adversarial test data for the `devin-*` catalog.
|
|
61
|
+
|
|
62
|
+
> Unofficial community project; not affiliated with or endorsed by
|
|
63
|
+
**[Linux](README.linux.md)** · **[Personal Windows](README.windows.md)** · **[Corporate Windows](README.corporate-windows.md)**
|
|
64
|
+
|
|
65
|
+
Part of the [awesome-devin](https://github.com/Icaro0310/awesome-devin) ecosystem: the curated hub for the devin-* tools.
|
|
66
|
+
|
|
67
|
+
Every tool in the ecosystem reads Devin's local stores. Testing them needs
|
|
68
|
+
sessions — but real sessions are private. `devin-dream` generates
|
|
69
|
+
`sessions.db` files in the exact real DDL (reusing
|
|
70
|
+
[`devin-internals-spec`](https://github.com/Icaro0310/devin-internals-spec)
|
|
71
|
+
fixtures) with realistic content, labeled defects, and an `expected.json`
|
|
72
|
+
stating what each catalog tool *should* conclude.
|
|
73
|
+
|
|
74
|
+
## Defects
|
|
75
|
+
|
|
76
|
+
| ID | Injected defect | Expected verdict | Tool under test |
|
|
77
|
+
|-----|---------------------------------------------------|---------------------------|----------------------|
|
|
78
|
+
| D01 | "I ran the tests" claim with no tool call | UNVERIFIED | `devin-qa-pack` |
|
|
79
|
+
| D02 | claim with incomplete evidence | PARTIAL | `devin-qa-pack` |
|
|
80
|
+
| D03 | claim with complete evidence | PASS | `devin-qa-pack` |
|
|
81
|
+
| D04 | fake secret inside a tool output | masked / BLOCKED | `devin-redact` |
|
|
82
|
+
| D05 | fake PII inside a user prompt | masked / REVIEW | `devin-redact` |
|
|
83
|
+
| D06 | schema drift (version 18) | loud drift failure | `devin-internals-spec` |
|
|
84
|
+
| D07 | malicious instruction inside a tool result | denied | `devin-bridge` |
|
|
85
|
+
| D08 | agent tries to persist a false "decision" | quarantined | `devin-memory` |
|
|
86
|
+
| D09 | fake secret split across two tool payloads | masked (cross-chunk) | `devin-redact` |
|
|
87
|
+
|
|
88
|
+
Secrets and PII are always obviously fake public-documentation values —
|
|
89
|
+
nothing real is ever generated or read.
|
|
90
|
+
|
|
91
|
+
## Install
|
|
92
|
+
|
|
93
|
+
<!-- DIST-STATUS:BEGIN — generated from devin-powerups/registry.json -->
|
|
94
|
+
> **Source-only distribution.** This tool is not yet published to PyPI.
|
|
95
|
+
> Install from source:
|
|
96
|
+
>
|
|
97
|
+
> ```bash
|
|
98
|
+
> pipx install git+https://github.com/Icaro0310/devin-dream.git
|
|
99
|
+
> # or
|
|
100
|
+
> uv tool install git+https://github.com/Icaro0310/devin-dream.git
|
|
101
|
+
> ```
|
|
102
|
+
<!-- DIST-STATUS:END -->
|
|
103
|
+
|
|
104
|
+
## Usage
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
devin-dream unit --out out/ # one dir per defect, + expected.json
|
|
108
|
+
devin-dream unit --out out/ --defect D01 D03
|
|
109
|
+
devin-dream inject --out adv/ --n 5 # adversarial (D07/D08) + scorecard
|
|
110
|
+
devin-dream fleet --out big/ --sessions 2000 --seed 1 --manifest
|
|
111
|
+
devin-dream fleet --out big/ --sessions 500 --vscdb big/state.vscdb
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Each `unit` run writes `<out>/<defect>/sessions.db` — openable by any
|
|
115
|
+
catalog parser — and `expected.json` with the labeled verdicts. `inject`
|
|
116
|
+
adds a top-level scorecard (`expected_blocks` per defect).
|
|
117
|
+
|
|
118
|
+
## Fleet
|
|
119
|
+
|
|
120
|
+
`fleet` writes one seeded `sessions.db` holding a population of
|
|
121
|
+
archetype sessions (no labeled defect — `defect_id = "FLEET"`). Every
|
|
122
|
+
archetype is guaranteed to appear within the first 7 sessions; beyond
|
|
123
|
+
that the mix is weighted toward `short-task`:
|
|
124
|
+
|
|
125
|
+
| Archetype | Shape |
|
|
126
|
+
|--------------------|-----------------------------------------------------------|
|
|
127
|
+
| `short-task` | 2–6 messages, a few read/test tool calls |
|
|
128
|
+
| `long-refactor` | 20–40 messages, 30 min–2 h `last_activity_at` spread |
|
|
129
|
+
| `failed-attempt` | repeated `pytest` runs exiting 1, unresolved ending |
|
|
130
|
+
| `multi-project` | `workspace_dirs` lists 2–3 dirs, tool calls `cd` between |
|
|
131
|
+
| `heavy-churn` | the same file path touched by 8–14 tool calls |
|
|
132
|
+
| `commit-producing` | git add/commit/rev-parse calls carrying real 40-hex SHAs |
|
|
133
|
+
| `empty-trivial` | 1–2 messages, no tool calls |
|
|
134
|
+
|
|
135
|
+
`commit-producing` payloads (40-hex SHAs plus a
|
|
136
|
+
`github.com/dream-org/.../commit/<sha>` URL) are picked up by
|
|
137
|
+
`devin_internals.commits.commit_references`, so graph/janitor
|
|
138
|
+
commit-coverage tests can run against the fleet.
|
|
139
|
+
|
|
140
|
+
`--manifest` writes `fleet.json` next to `sessions.db` with the seed,
|
|
141
|
+
per-archetype counts, and one entry per session:
|
|
142
|
+
|
|
143
|
+
```json
|
|
144
|
+
{"session_id": "dream-fleet-00012", "archetype": "commit-producing",
|
|
145
|
+
"expected": {"produces_commit": true,
|
|
146
|
+
"commit_shas": ["<40-hex>"]},
|
|
147
|
+
"gui_slug": "canyon-violin"}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Every `expected` block carries the uniform flags `produces_commit`,
|
|
151
|
+
`heavy_churn`, `failed`, `trivial` plus archetype-specific keys
|
|
152
|
+
(`commit_shas`, `churned_file`, `projects`, `duration_ms`, …) for
|
|
153
|
+
downstream assertions.
|
|
154
|
+
|
|
155
|
+
`--vscdb PATH` additionally emits a synthetic `state.vscdb`
|
|
156
|
+
(`ItemTable`) binding ~35% of the sessions to a
|
|
157
|
+
`windsurfSpace.sessionWorkspace/<backend>/<slug>` entry — including the
|
|
158
|
+
`resourceToSpace` / `metadata` rollups — so `devin-graph` (GR-1) and
|
|
159
|
+
`devin-history` (HI-1) GUI coverage can be fleet-tested. Bound sessions
|
|
160
|
+
get `gui_slug` in the manifest.
|
|
161
|
+
|
|
162
|
+
All output is deterministic given `--seed`: same seed, same manifest,
|
|
163
|
+
byte-identical `sessions.db`.
|
|
164
|
+
|
|
165
|
+
## What it does not do
|
|
166
|
+
|
|
167
|
+
- It does not run the verdicts — checking that tools actually reach the
|
|
168
|
+
expected verdict is the catalog's job (e.g. `devin-evals` packs, manual
|
|
169
|
+
`qa-pack`/`redact` runs against the generated DBs).
|
|
170
|
+
- Fleet content is templated synthetic text, not realistic transcripts;
|
|
171
|
+
commit SHAs are seeded-`random` hex, never real objects.
|
|
172
|
+
- Fleet sessions populate `sessions` / `message_nodes` /
|
|
173
|
+
`tool_call_state` only — no `prompt_history`, `rendered_commits`, or
|
|
174
|
+
`subagent_heads` rows.
|
|
175
|
+
- The emitted `state.vscdb` covers only the
|
|
176
|
+
`windsurfSpace.sessionWorkspace`/`resourceToSpace`/`metadata` key
|
|
177
|
+
family — not the full Electron store.
|
|
178
|
+
- GUI `acp-messages` stores are not generated yet (planned after the typed
|
|
179
|
+
reader lands everywhere).
|
|
180
|
+
- `fleet` volume beyond a few thousand sessions is untested.
|
|
181
|
+
|
|
182
|
+
Requires Python ≥ 3.10, stdlib only beyond `devin-internals-spec`.
|
|
183
|
+
Read-only w.r.t. real Devin stores: it only writes its own output dir.
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
</details>
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# devin-dream — MOVED
|
|
4
|
+
|
|
5
|
+
**This repository was absorbed into
|
|
6
|
+
[`devin-evals`](https://github.com/Icaro0310/devin-evals).**
|
|
7
|
+
|
|
8
|
+
The code now lives at `src/devin_evals/dream/` and the CLI moved to
|
|
9
|
+
`devin-evals dream {unit,inject,fleet}` — same commands, same defect
|
|
10
|
+
catalogue (D01–D09), one package instead of two.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
# before
|
|
14
|
+
devin-dream unit --out out/ --defect D01 D03
|
|
15
|
+
|
|
16
|
+
# after
|
|
17
|
+
devin-evals dream unit --out out/ --defect D01 D03
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The repository is archived; open issues were migrated to devin-evals.
|
|
21
|
+
History remains readable here for reference.
|
|
22
|
+
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
<details>
|
|
28
|
+
<summary>Original README (pre-archive)</summary>
|
|
29
|
+
|
|
30
|
+
<div align="center">
|
|
31
|
+
|
|
32
|
+
<a href="https://github.com/Icaro0310/devin-dream/actions/workflows/ci.yml"><img src="https://github.com/Icaro0310/devin-dream/actions/workflows/ci.yml/badge.svg" alt="ci"/></a>
|
|
33
|
+
|
|
34
|
+
<a href="https://scorecard.dev/viewer/?uri=github.com/Icaro0310/devin-dream"><img src="https://api.scorecard.dev/projects/github.com/Icaro0310/devin-dream/badge" alt="OpenSSF Scorecard"/></a>
|
|
35
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="License: MIT"/></a>
|
|
36
|
+
<a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python 3.10+"/></a>
|
|
37
|
+
<a href="https://github.com/Icaro0310/devin-dream"><img src="https://img.shields.io/github/stars/Icaro0310/devin-dream" alt="GitHub stars"/></a>
|
|
38
|
+
<a href="https://github.com/Icaro0310/devin-dream/commits/main"><img src="https://img.shields.io/github/last-commit/Icaro0310/devin-dream" alt="Last commit"/></a>
|
|
39
|
+
<a href="https://github.com/Icaro0310/awesome-devin"><img src="https://img.shields.io/badge/part%20of-devin--*-ecosystem-7c3aed" alt="devin-* ecosystem"/></a>
|
|
40
|
+
<a href="https://github.com/Icaro0310/devin-dream/issues"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen" alt="PRs welcome"/></a>
|
|
41
|
+
</div>
|
|
42
|
+
|
|
43
|
+
# devin-dream
|
|
44
|
+
|
|
45
|
+
Synthetic Devin sessions with a **known verdict** — regression and
|
|
46
|
+
adversarial test data for the `devin-*` catalog.
|
|
47
|
+
|
|
48
|
+
> Unofficial community project; not affiliated with or endorsed by
|
|
49
|
+
**[Linux](README.linux.md)** · **[Personal Windows](README.windows.md)** · **[Corporate Windows](README.corporate-windows.md)**
|
|
50
|
+
|
|
51
|
+
Part of the [awesome-devin](https://github.com/Icaro0310/awesome-devin) ecosystem: the curated hub for the devin-* tools.
|
|
52
|
+
|
|
53
|
+
Every tool in the ecosystem reads Devin's local stores. Testing them needs
|
|
54
|
+
sessions — but real sessions are private. `devin-dream` generates
|
|
55
|
+
`sessions.db` files in the exact real DDL (reusing
|
|
56
|
+
[`devin-internals-spec`](https://github.com/Icaro0310/devin-internals-spec)
|
|
57
|
+
fixtures) with realistic content, labeled defects, and an `expected.json`
|
|
58
|
+
stating what each catalog tool *should* conclude.
|
|
59
|
+
|
|
60
|
+
## Defects
|
|
61
|
+
|
|
62
|
+
| ID | Injected defect | Expected verdict | Tool under test |
|
|
63
|
+
|-----|---------------------------------------------------|---------------------------|----------------------|
|
|
64
|
+
| D01 | "I ran the tests" claim with no tool call | UNVERIFIED | `devin-qa-pack` |
|
|
65
|
+
| D02 | claim with incomplete evidence | PARTIAL | `devin-qa-pack` |
|
|
66
|
+
| D03 | claim with complete evidence | PASS | `devin-qa-pack` |
|
|
67
|
+
| D04 | fake secret inside a tool output | masked / BLOCKED | `devin-redact` |
|
|
68
|
+
| D05 | fake PII inside a user prompt | masked / REVIEW | `devin-redact` |
|
|
69
|
+
| D06 | schema drift (version 18) | loud drift failure | `devin-internals-spec` |
|
|
70
|
+
| D07 | malicious instruction inside a tool result | denied | `devin-bridge` |
|
|
71
|
+
| D08 | agent tries to persist a false "decision" | quarantined | `devin-memory` |
|
|
72
|
+
| D09 | fake secret split across two tool payloads | masked (cross-chunk) | `devin-redact` |
|
|
73
|
+
|
|
74
|
+
Secrets and PII are always obviously fake public-documentation values —
|
|
75
|
+
nothing real is ever generated or read.
|
|
76
|
+
|
|
77
|
+
## Install
|
|
78
|
+
|
|
79
|
+
<!-- DIST-STATUS:BEGIN — generated from devin-powerups/registry.json -->
|
|
80
|
+
> **Source-only distribution.** This tool is not yet published to PyPI.
|
|
81
|
+
> Install from source:
|
|
82
|
+
>
|
|
83
|
+
> ```bash
|
|
84
|
+
> pipx install git+https://github.com/Icaro0310/devin-dream.git
|
|
85
|
+
> # or
|
|
86
|
+
> uv tool install git+https://github.com/Icaro0310/devin-dream.git
|
|
87
|
+
> ```
|
|
88
|
+
<!-- DIST-STATUS:END -->
|
|
89
|
+
|
|
90
|
+
## Usage
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
devin-dream unit --out out/ # one dir per defect, + expected.json
|
|
94
|
+
devin-dream unit --out out/ --defect D01 D03
|
|
95
|
+
devin-dream inject --out adv/ --n 5 # adversarial (D07/D08) + scorecard
|
|
96
|
+
devin-dream fleet --out big/ --sessions 2000 --seed 1 --manifest
|
|
97
|
+
devin-dream fleet --out big/ --sessions 500 --vscdb big/state.vscdb
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Each `unit` run writes `<out>/<defect>/sessions.db` — openable by any
|
|
101
|
+
catalog parser — and `expected.json` with the labeled verdicts. `inject`
|
|
102
|
+
adds a top-level scorecard (`expected_blocks` per defect).
|
|
103
|
+
|
|
104
|
+
## Fleet
|
|
105
|
+
|
|
106
|
+
`fleet` writes one seeded `sessions.db` holding a population of
|
|
107
|
+
archetype sessions (no labeled defect — `defect_id = "FLEET"`). Every
|
|
108
|
+
archetype is guaranteed to appear within the first 7 sessions; beyond
|
|
109
|
+
that the mix is weighted toward `short-task`:
|
|
110
|
+
|
|
111
|
+
| Archetype | Shape |
|
|
112
|
+
|--------------------|-----------------------------------------------------------|
|
|
113
|
+
| `short-task` | 2–6 messages, a few read/test tool calls |
|
|
114
|
+
| `long-refactor` | 20–40 messages, 30 min–2 h `last_activity_at` spread |
|
|
115
|
+
| `failed-attempt` | repeated `pytest` runs exiting 1, unresolved ending |
|
|
116
|
+
| `multi-project` | `workspace_dirs` lists 2–3 dirs, tool calls `cd` between |
|
|
117
|
+
| `heavy-churn` | the same file path touched by 8–14 tool calls |
|
|
118
|
+
| `commit-producing` | git add/commit/rev-parse calls carrying real 40-hex SHAs |
|
|
119
|
+
| `empty-trivial` | 1–2 messages, no tool calls |
|
|
120
|
+
|
|
121
|
+
`commit-producing` payloads (40-hex SHAs plus a
|
|
122
|
+
`github.com/dream-org/.../commit/<sha>` URL) are picked up by
|
|
123
|
+
`devin_internals.commits.commit_references`, so graph/janitor
|
|
124
|
+
commit-coverage tests can run against the fleet.
|
|
125
|
+
|
|
126
|
+
`--manifest` writes `fleet.json` next to `sessions.db` with the seed,
|
|
127
|
+
per-archetype counts, and one entry per session:
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{"session_id": "dream-fleet-00012", "archetype": "commit-producing",
|
|
131
|
+
"expected": {"produces_commit": true,
|
|
132
|
+
"commit_shas": ["<40-hex>"]},
|
|
133
|
+
"gui_slug": "canyon-violin"}
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Every `expected` block carries the uniform flags `produces_commit`,
|
|
137
|
+
`heavy_churn`, `failed`, `trivial` plus archetype-specific keys
|
|
138
|
+
(`commit_shas`, `churned_file`, `projects`, `duration_ms`, …) for
|
|
139
|
+
downstream assertions.
|
|
140
|
+
|
|
141
|
+
`--vscdb PATH` additionally emits a synthetic `state.vscdb`
|
|
142
|
+
(`ItemTable`) binding ~35% of the sessions to a
|
|
143
|
+
`windsurfSpace.sessionWorkspace/<backend>/<slug>` entry — including the
|
|
144
|
+
`resourceToSpace` / `metadata` rollups — so `devin-graph` (GR-1) and
|
|
145
|
+
`devin-history` (HI-1) GUI coverage can be fleet-tested. Bound sessions
|
|
146
|
+
get `gui_slug` in the manifest.
|
|
147
|
+
|
|
148
|
+
All output is deterministic given `--seed`: same seed, same manifest,
|
|
149
|
+
byte-identical `sessions.db`.
|
|
150
|
+
|
|
151
|
+
## What it does not do
|
|
152
|
+
|
|
153
|
+
- It does not run the verdicts — checking that tools actually reach the
|
|
154
|
+
expected verdict is the catalog's job (e.g. `devin-evals` packs, manual
|
|
155
|
+
`qa-pack`/`redact` runs against the generated DBs).
|
|
156
|
+
- Fleet content is templated synthetic text, not realistic transcripts;
|
|
157
|
+
commit SHAs are seeded-`random` hex, never real objects.
|
|
158
|
+
- Fleet sessions populate `sessions` / `message_nodes` /
|
|
159
|
+
`tool_call_state` only — no `prompt_history`, `rendered_commits`, or
|
|
160
|
+
`subagent_heads` rows.
|
|
161
|
+
- The emitted `state.vscdb` covers only the
|
|
162
|
+
`windsurfSpace.sessionWorkspace`/`resourceToSpace`/`metadata` key
|
|
163
|
+
family — not the full Electron store.
|
|
164
|
+
- GUI `acp-messages` stores are not generated yet (planned after the typed
|
|
165
|
+
reader lands everywhere).
|
|
166
|
+
- `fleet` volume beyond a few thousand sessions is untested.
|
|
167
|
+
|
|
168
|
+
Requires Python ≥ 3.10, stdlib only beyond `devin-internals-spec`.
|
|
169
|
+
Read-only w.r.t. real Devin stores: it only writes its own output dir.
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
</details>
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "devin-dream"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Synthetic Devin sessions with known verdicts — regression and adversarial test data for the devin-* catalog."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Icaro0310" }]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"devin-internals-spec>=0.3.0,<0.4.0",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
dev = ["pytest>=8"]
|
|
19
|
+
|
|
20
|
+
[project.scripts]
|
|
21
|
+
devin-dream = "devin_dream.cli:main"
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.packages.find]
|
|
24
|
+
where = ["src"]
|
|
25
|
+
|
|
26
|
+
[tool.pytest.ini_options]
|
|
27
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""devin-dream CLI: unit | inject | fleet."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from devin_dream.defects import ADVERSARIAL, DEFECTS, UNIT_IDS
|
|
11
|
+
from devin_dream.generate import (
|
|
12
|
+
build_fleet,
|
|
13
|
+
write_expected,
|
|
14
|
+
write_fleet_manifest,
|
|
15
|
+
write_sessions_db,
|
|
16
|
+
write_state_vscdb,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _cmd_unit(args: argparse.Namespace) -> int:
|
|
21
|
+
ids = list(UNIT_IDS) if args.defect == ["all"] else args.defect
|
|
22
|
+
unknown = [d for d in ids if d not in DEFECTS]
|
|
23
|
+
if unknown:
|
|
24
|
+
print(f"error: unknown defect(s): {', '.join(unknown)}",
|
|
25
|
+
file=sys.stderr)
|
|
26
|
+
return 2
|
|
27
|
+
out = Path(args.out)
|
|
28
|
+
for did in ids:
|
|
29
|
+
spec = DEFECTS[did]()
|
|
30
|
+
ddir = out / did.lower()
|
|
31
|
+
write_sessions_db(ddir / "sessions.db", [spec],
|
|
32
|
+
schema_version=args.schema_version, seed=args.seed)
|
|
33
|
+
ep = write_expected(ddir, spec)
|
|
34
|
+
print(f"{did}: {ddir / 'sessions.db'} (+ {ep.name})")
|
|
35
|
+
return 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _cmd_inject(args: argparse.Namespace) -> int:
|
|
39
|
+
"""Adversarial sessions (D07/D08) + the expected block/pass scorecard."""
|
|
40
|
+
out = Path(args.out)
|
|
41
|
+
specs = [DEFECTS[d]() for d in ADVERSARIAL for _ in range(args.n)]
|
|
42
|
+
# unique session ids per copy
|
|
43
|
+
seen: dict[str, int] = {}
|
|
44
|
+
final = []
|
|
45
|
+
for s in specs:
|
|
46
|
+
seen[s.session_id] = seen.get(s.session_id, 0) + 1
|
|
47
|
+
suffix = seen[s.session_id]
|
|
48
|
+
import dataclasses
|
|
49
|
+
final.append(dataclasses.replace(
|
|
50
|
+
s, session_id=f"{s.session_id}-{suffix:03d}"))
|
|
51
|
+
write_sessions_db(out / "sessions.db", final, seed=args.seed)
|
|
52
|
+
scorecard = {
|
|
53
|
+
"mode": "inject",
|
|
54
|
+
"sessions": len(final),
|
|
55
|
+
"expected_blocks": {d: args.n for d in ADVERSARIAL},
|
|
56
|
+
"note": "each session must be denied/quarantined by the target "
|
|
57
|
+
"tool; a pass is a hole in the policy",
|
|
58
|
+
}
|
|
59
|
+
(out / "expected.json").write_text(
|
|
60
|
+
json.dumps(scorecard, indent=2) + "\n", encoding="utf-8")
|
|
61
|
+
for s in final:
|
|
62
|
+
write_expected(out / s.session_id, s)
|
|
63
|
+
print(f"inject: {len(final)} adversarial session(s) -> "
|
|
64
|
+
f"{out / 'sessions.db'}")
|
|
65
|
+
return 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _cmd_fleet(args: argparse.Namespace) -> int:
|
|
69
|
+
if args.sessions < 1:
|
|
70
|
+
print("error: --sessions must be >= 1", file=sys.stderr)
|
|
71
|
+
return 2
|
|
72
|
+
out = Path(args.out)
|
|
73
|
+
plan = build_fleet(args.sessions, seed=args.seed)
|
|
74
|
+
write_sessions_db(out / "sessions.db", plan.specs, seed=args.seed)
|
|
75
|
+
counts: dict[str, int] = {}
|
|
76
|
+
for s in plan.specs:
|
|
77
|
+
arch = next((l.split(":", 1)[1] for l in s.labels
|
|
78
|
+
if l.startswith("archetype:")), "unknown")
|
|
79
|
+
counts[arch] = counts.get(arch, 0) + 1
|
|
80
|
+
mix = ", ".join(f"{a}={c}" for a, c in sorted(counts.items()))
|
|
81
|
+
print(f"fleet: {len(plan.specs)} sessions ({mix}) -> "
|
|
82
|
+
f"{out / 'sessions.db'}")
|
|
83
|
+
if args.vscdb:
|
|
84
|
+
vdb = write_state_vscdb(args.vscdb, plan, seed=args.seed)
|
|
85
|
+
print(f"fleet: {len(plan.gui)} GUI binding(s) -> {vdb}")
|
|
86
|
+
if args.manifest:
|
|
87
|
+
mp = write_fleet_manifest(out, plan, seed=args.seed)
|
|
88
|
+
print(f"fleet: manifest -> {mp}")
|
|
89
|
+
return 0
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
93
|
+
p = argparse.ArgumentParser(
|
|
94
|
+
prog="devin-dream",
|
|
95
|
+
description="Synthetic Devin sessions with known verdicts — "
|
|
96
|
+
"regression and adversarial test data for the catalog.")
|
|
97
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
98
|
+
|
|
99
|
+
u = sub.add_parser("unit", help="one sessions.db + expected.json per defect")
|
|
100
|
+
u.add_argument("--out", required=True)
|
|
101
|
+
u.add_argument("--defect", nargs="+", default=["all"],
|
|
102
|
+
help="defect ids or 'all' (unit covers D01-D06 + D09)")
|
|
103
|
+
u.add_argument("--schema-version", type=int, default=None)
|
|
104
|
+
u.add_argument("--seed", type=int, default=0xDEE4)
|
|
105
|
+
u.set_defaults(func=_cmd_unit)
|
|
106
|
+
|
|
107
|
+
i = sub.add_parser("inject",
|
|
108
|
+
help="adversarial sessions + expected scorecard")
|
|
109
|
+
i.add_argument("--out", required=True)
|
|
110
|
+
i.add_argument("--n", type=int, default=1,
|
|
111
|
+
help="copies per adversarial defect (D07, D08)")
|
|
112
|
+
i.add_argument("--seed", type=int, default=0xDEE4)
|
|
113
|
+
i.set_defaults(func=_cmd_inject)
|
|
114
|
+
|
|
115
|
+
f = sub.add_parser("fleet",
|
|
116
|
+
help="seeded population of archetype sessions")
|
|
117
|
+
f.add_argument("--out", required=True)
|
|
118
|
+
f.add_argument("--sessions", "--n", dest="sessions", type=int,
|
|
119
|
+
default=1000,
|
|
120
|
+
help="population size (--n kept as an alias)")
|
|
121
|
+
f.add_argument("--seed", type=int, default=0)
|
|
122
|
+
f.add_argument("--vscdb", default=None, metavar="PATH",
|
|
123
|
+
help="also emit a synthetic state.vscdb at PATH with "
|
|
124
|
+
"windsurfSpace.sessionWorkspace bindings for a "
|
|
125
|
+
"fraction of sessions")
|
|
126
|
+
f.add_argument("--manifest", action="store_true",
|
|
127
|
+
help="write fleet.json with per-session archetype and "
|
|
128
|
+
"expected properties")
|
|
129
|
+
f.set_defaults(func=_cmd_fleet)
|
|
130
|
+
return p
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def main(argv: list[str] | None = None) -> int:
|
|
134
|
+
args = build_parser().parse_args(argv)
|
|
135
|
+
return args.func(args)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
raise SystemExit(main())
|