tryharness 0.7.31 → 0.7.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,328 @@
1
+ // What the swarm learns from the person, step 2 of its learning. Each post the person writes in a thread is read on
2
+ // Harness's server (src/server/swarm/lessons.ts) with the agents' work since they last wrote and the few lessons kept
3
+ // here closest to it. A post that says something lasting comes back as lessons, each standing on the person's exact
4
+ // words: a new one is kept, the same one again adds the new words to it, and an update or a reversal replaces the
5
+ // kept one (which stays as the record of what was). Lessons are about the person (tastes, standards, ways of working),
6
+ // one agent (what it should or should not do), or a kind of work (how it goes well), and hold everywhere or in the
7
+ // thread's folder. They are kept apart from Harness's own memory (owner, October 16: "learnings seperated is how it
8
+ // should be for sure"). Nothing is read on the privacy setting, with this computer off Harness, or from a Harness that
9
+ // does not read lessons (swarm_lessons). Posts are read one at a time, so two quick ones never both add the same lesson.
10
+
11
+ import path from "node:path";
12
+ import { workSince } from "./reactions.mjs";
13
+ import { fromBlob, toBlob, topK } from "./vectors.mjs";
14
+
15
+ // The kept lessons a post is compared with, closest first. At today's scale that is every lesson: in eval swarm-lessons v1
16
+ // (20261017-040509-e29f1c4d-2204) six chosen by words left out the lessons a repeat repeated.
17
+ const KNOWN = 24;
18
+ const POST_CHARS = 4_000;
19
+
20
+ export function createLessons({
21
+ store,
22
+ askHarness,
23
+ readsLessons,
24
+ isCoordinatorPost,
25
+ // A task's own room (one job given to an agent): what the person writes there settles that job.
26
+ isTaskRoom = () => false,
27
+ // Vectors for texts (`kind` query or document), or null while the local search model is not here.
28
+ embed = async () => null,
29
+ log,
30
+ }) {
31
+ let queue = Promise.resolve();
32
+
33
+ const embedLesson = async (lesson) => {
34
+ const [vector] = (await embed([lesson.text], "document")) ?? [];
35
+ if (vector) store.setLessonVector(lesson.id, toBlob(vector));
36
+ };
37
+
38
+ // The kept lessons closest to a text, among `only` when given: by meaning where lessons have vectors, then by words,
39
+ // then the newest.
40
+ const closestAmong = async (text, only = null, limit = KNOWN) => {
41
+ const allowed = (id) => !only || only.has(id);
42
+ const found = new Map();
43
+ const rows = store.lessonVectors().filter((row) => allowed(row.id));
44
+ if (rows.length) {
45
+ const [query] = (await embed([text], "query")) ?? [];
46
+ if (query)
47
+ for (const hit of topK(
48
+ query,
49
+ rows.map((row) => ({ id: row.id, vector: fromBlob(row.vector) })),
50
+ limit
51
+ ))
52
+ found.set(hit.id, store.getLesson(hit.id));
53
+ }
54
+ for (const lesson of store.lessonsMatching(text, limit * 4))
55
+ if (found.size < limit && allowed(lesson.id))
56
+ found.set(lesson.id, lesson);
57
+ for (const lesson of store.lessons({ limit: 200 }))
58
+ if (found.size < limit && allowed(lesson.id))
59
+ found.set(lesson.id, lesson);
60
+ return [...found.values()].filter(Boolean).slice(0, limit);
61
+ };
62
+ const closest = (text) => closestAmong(text);
63
+
64
+ const subjectOf = (lesson) =>
65
+ lesson.about === "agent"
66
+ ? `agent:${lesson.agent}`
67
+ : "the person and the work";
68
+ // Whether a run in this thread that started before the post was given the lesson in its brief.
69
+ const carriedHere = (thread, post, lessonId) =>
70
+ store.threadRuns(thread.id).some((run) => {
71
+ if (!run.created_at || run.created_at > (post.created_at ?? ""))
72
+ return false;
73
+ try {
74
+ return JSON.parse(
75
+ store.meta(`lessons-carried:${run.id}`) || "[]"
76
+ ).includes(lessonId);
77
+ } catch {
78
+ return false;
79
+ }
80
+ });
81
+
82
+ const read = async (thread, post) => {
83
+ // Learned from the person's threads, not from a single job's room, where their words settle that job (owner,
84
+ // October 17, on "the bankr agent will have tools to be able to call the transaction" kept as a lesson: "needs to
85
+ // be fixed"). No wording separated such answers from wishes when Jev read them; the two in the eval's posts were
86
+ // both in task rooms, and none of the person's lasting wishes was.
87
+ if (!readsLessons() || isTaskRoom(thread)) return [];
88
+ const known = await closest(post.text);
89
+ const { before } = workSince(
90
+ store.roomPosts(thread.id, 0, 500),
91
+ post,
92
+ isCoordinatorPost
93
+ );
94
+ let reply;
95
+ try {
96
+ reply = await askHarness(
97
+ "reflect",
98
+ {
99
+ thread: {
100
+ id: thread.id,
101
+ title: thread.objective.slice(0, 500),
102
+ folder: thread.goal?.workspace ?? null,
103
+ agents: (thread.goal?.team ?? []).slice(0, 16),
104
+ },
105
+ post: { seq: post.seq, text: post.text.slice(0, POST_CHARS) },
106
+ before,
107
+ known: known.map((lesson) => ({
108
+ id: lesson.id,
109
+ about: lesson.about,
110
+ agent: lesson.agent,
111
+ text: lesson.text,
112
+ scope: lesson.scope,
113
+ })),
114
+ },
115
+ null
116
+ );
117
+ } catch (error) {
118
+ log(`lessons from post ${post.seq} in ${thread.id}: ${error.message}`);
119
+ return [];
120
+ }
121
+ // The bridge's answer carries the reflection's fields beside `ok`.
122
+ const reflection = reply?.ok ? (reply.result ?? reply) : null;
123
+ if (!reflection) {
124
+ log(
125
+ `lessons from post ${post.seq} in ${thread.id}: ${reply?.error || "no answer"}`
126
+ );
127
+ return [];
128
+ }
129
+ const knownIds = new Set(known.map((lesson) => lesson.id));
130
+ const kept = [];
131
+ for (const lesson of reflection.lessons ?? []) {
132
+ const evidence = {
133
+ goalId: thread.id,
134
+ seq: post.seq,
135
+ postId: post.id,
136
+ quote: String(lesson.quote ?? "").slice(0, 500),
137
+ at: new Date().toISOString(),
138
+ };
139
+ const fresh = {
140
+ about: lesson.about,
141
+ agent: lesson.agent ?? null,
142
+ text: String(lesson.text ?? "").slice(0, 300),
143
+ scope: lesson.scope === "this_folder" ? "this_folder" : "everywhere",
144
+ folder:
145
+ lesson.scope === "this_folder"
146
+ ? (thread.goal?.workspace ?? null)
147
+ : null,
148
+ evidence,
149
+ };
150
+ if (!fresh.text || !["person", "agent", "work"].includes(fresh.about))
151
+ continue;
152
+ // A lesson about one agent stands only to another about the same agent (in the eval, "Harness should be generating
153
+ // the videos" read as the opposite of Claude Code's role, and replaced it); the person's and the work's stand to each
154
+ // other, as the writer words the same wish either way (the candy track came back once as each).
155
+ const target = knownIds.has(lesson.match?.id)
156
+ ? store.getLesson(lesson.match.id)
157
+ : null;
158
+ const match =
159
+ target && subjectOf(target) === subjectOf(fresh) ? lesson.match : null;
160
+ // A match already replaced by an earlier lesson of the same post keeps this one as new instead of losing it.
161
+ let result = null;
162
+ let changed = "new";
163
+ if (match?.relation === "same") {
164
+ result = store.confirmLesson(match.id, {
165
+ ...evidence,
166
+ // Said again after an agent here was given it: the lesson reached the work and did not take.
167
+ ...(carriedHere(thread, post, match.id) ? { carried: true } : {}),
168
+ });
169
+ if (result) changed = "same";
170
+ } else if (match) {
171
+ result = store.replaceLesson(match.id, fresh);
172
+ if (result) changed = match.relation;
173
+ }
174
+ result ??= store.addLesson(fresh);
175
+ kept.push({ ...result, change: changed });
176
+ if (changed !== "same")
177
+ await embedLesson(result).catch((error) =>
178
+ log(`lesson ${result.id} not embedded: ${error.message}`)
179
+ );
180
+ }
181
+ // What the post took back with nothing in its place (a lesson of the post that replaced it already did).
182
+ for (const id of reflection.takenBack ?? []) {
183
+ if (!knownIds.has(id)) continue;
184
+ const taken = store.takeBackLesson(id, {
185
+ goalId: thread.id,
186
+ seq: post.seq,
187
+ postId: post.id,
188
+ // The person's words that took it back: the start of their post, as they wrote it.
189
+ quote: post.text.slice(0, 200),
190
+ at: new Date().toISOString(),
191
+ });
192
+ if (taken) kept.push({ ...taken, change: "taken back" });
193
+ }
194
+ return kept;
195
+ };
196
+
197
+ /** Read the person's post in a thread, after any post still being read; resolves with the lessons it changed. */
198
+ const heard = (thread, post) => {
199
+ const next = queue.then(() => read(thread, post));
200
+ queue = next.catch(() => {});
201
+ return next;
202
+ };
203
+
204
+ // A lesson holds for work in a folder when it holds everywhere or was learned in that folder.
205
+ const holdsIn = (lesson, folder) =>
206
+ lesson.scope === "everywhere" ||
207
+ (folder &&
208
+ lesson.folder &&
209
+ path.resolve(lesson.folder) === path.resolve(folder));
210
+ // Lessons about one agent go to that agent alone; about the person and the work, to everyone.
211
+ const forAgent = (lesson, agent) =>
212
+ lesson.about !== "agent" || lesson.agent === agent;
213
+ const ORDER = { agent: 0, person: 1, work: 2 };
214
+ // What the person had to say again after agents were given it comes first, then what they said most often, then by
215
+ // kind: the agent's own, the person's, the work's.
216
+ const byWeight = (a, b) =>
217
+ b.missed - a.missed ||
218
+ b.evidence.length - a.evidence.length ||
219
+ ORDER[a.about] - ORDER[b.about];
220
+
221
+ /**
222
+ * The lessons for a piece of work: the ones that hold in its folder and are for this agent, the closest to the work's
223
+ * words first when there are more than `limit`.
224
+ */
225
+ const forWork = async ({
226
+ agent = null,
227
+ folder = null,
228
+ text = "",
229
+ limit = 8,
230
+ }) => {
231
+ const fitting = store
232
+ .lessons({ limit: 1000 })
233
+ .filter((lesson) => holdsIn(lesson, folder) && forAgent(lesson, agent));
234
+ if (fitting.length > limit && text.trim()) {
235
+ // The lessons the person had to repeat after agents were told stay in, before any ranking by the work's words.
236
+ const pressing = fitting
237
+ .filter((lesson) => lesson.missed > 0)
238
+ .sort(byWeight);
239
+ const fits = new Set(fitting.map((lesson) => lesson.id));
240
+ const ranked = (await closestAmong(text, fits, limit)).map(
241
+ (lesson) => lesson.id
242
+ );
243
+ const order = [
244
+ ...pressing.map((lesson) => lesson.id),
245
+ ...ranked,
246
+ ...fitting.sort(byWeight).map((lesson) => lesson.id),
247
+ ];
248
+ return [...new Set(order)]
249
+ .slice(0, limit)
250
+ .map((id) => fitting.find((lesson) => lesson.id === id));
251
+ }
252
+ return fitting.sort(byWeight).slice(0, limit);
253
+ };
254
+
255
+ // The lessons the person's words hold for every piece of work, newest first, for an agent's session start.
256
+ const told = (limit = 6) =>
257
+ store
258
+ .lessons({ limit: 200 })
259
+ .filter(
260
+ (lesson) => lesson.scope === "everywhere" && lesson.about !== "agent"
261
+ )
262
+ .slice(0, limit);
263
+
264
+ // The person's lesson by its id or the start of it, among the active ones; null when none or more than one fits.
265
+ const find = (id) => {
266
+ const wanted = String(id || "").trim();
267
+ if (wanted.length < 4) return null;
268
+ const fits = store
269
+ .lessons({ limit: 1000 })
270
+ .filter((lesson) => lesson.id.startsWith(wanted));
271
+ return fits.length === 1 ? fits[0] : null;
272
+ };
273
+ /** The person's edit: their words replace the lesson's. */
274
+ const edit = async (id, text) => {
275
+ const lesson = find(id);
276
+ const words = String(text || "")
277
+ .replace(/\s+/g, " ")
278
+ .trim()
279
+ .slice(0, 300);
280
+ if (!lesson || !words) return null;
281
+ const edited = store.editLesson(lesson.id, words);
282
+ await embedLesson(edited).catch((error) =>
283
+ log(`lesson ${edited.id} not embedded: ${error.message}`)
284
+ );
285
+ return edited;
286
+ };
287
+ const remove = (id) => {
288
+ const lesson = find(id);
289
+ return lesson ? store.removeLesson(lesson.id) : null;
290
+ };
291
+
292
+ /**
293
+ * The person's Undo on the line under their post: a lesson the post kept is taken out, one it changed gives way to the
294
+ * lesson it replaced, and one it took back is in the work again.
295
+ */
296
+ const undo = async (lessonId, change) => {
297
+ const lesson = store.getLesson(lessonId);
298
+ if (!lesson) return null;
299
+ if (change === "kept") return store.removeLesson(lessonId);
300
+ if (change === "changed") {
301
+ const before = store
302
+ .lessons({ status: "replaced", limit: 1000 })
303
+ .find((entry) => entry.replaced_by === lessonId);
304
+ if (!before) return null;
305
+ store.removeLesson(lessonId);
306
+ const restored = store.restoreLesson(before.id);
307
+ await embedLesson(restored).catch(() => {});
308
+ return restored;
309
+ }
310
+ if (change === "taken back") {
311
+ const restored = store.restoreLesson(lessonId, { dropTookBack: true });
312
+ await embedLesson(restored).catch(() => {});
313
+ return restored;
314
+ }
315
+ return null;
316
+ };
317
+
318
+ return {
319
+ heard,
320
+ closest,
321
+ forWork,
322
+ told,
323
+ edit,
324
+ remove,
325
+ undo,
326
+ list: (options) => store.lessons(options),
327
+ };
328
+ }
package/lib/models.mjs CHANGED
@@ -413,11 +413,20 @@ export function createModelRouter({
413
413
  : null);
414
414
  if (!pick) return null;
415
415
  const by = named ? (named.by === "user" ? "you" : named.by) : pick.by;
416
+ // A model picked without an effort runs at the default its list gives it, the one the buddy shows, named on the run
417
+ // so the agent's own settings (Codex's model_reasoning_effort) do not put a different one in its place.
418
+ const withDefaultEffort = (picked) => ({
419
+ ...picked,
420
+ effort:
421
+ picked.effort ??
422
+ findModel(store, agent, picked.model).entry?.defaultEffort ??
423
+ null,
424
+ });
416
425
  if (!pick.auto) {
417
426
  // The list may have changed since the pick: a model the agent no longer lists runs on its own settings, said so.
418
427
  const checked = resolvePick(store, agent, pick.model, pick.effort);
419
428
  return checked.ok
420
- ? { ...checked.pick, by }
429
+ ? { ...withDefaultEffort(checked.pick), by }
421
430
  : {
422
431
  model: null,
423
432
  effort: null,
@@ -480,7 +489,7 @@ export function createModelRouter({
480
489
  );
481
490
  // Jev's confidence in the pick stays with the run, to see later how sure the picks that worked were.
482
491
  return {
483
- ...checked.pick,
492
+ ...withDefaultEffort(checked.pick),
484
493
  by: "auto",
485
494
  judgments: picked.judgments ?? null,
486
495
  estimatedMicroUsd: picked.estimatedMicroUsd ?? null,
package/lib/phones.mjs CHANGED
@@ -127,6 +127,8 @@ const RELAY_PERSON_COMMANDS = [
127
127
  const BROWSER_PERSON_COMMANDS = new Set([
128
128
  ...PHONE_COMMANDS,
129
129
  "board.goal.resume",
130
+ // What the swarm learned from the person (lessons.mjs), read in the PiP; the phone app has no view of it.
131
+ "board.swarm.lessons",
130
132
  "board.standing",
131
133
  "board.standing.check",
132
134
  "board.standing.off",
@@ -0,0 +1,108 @@
1
+ // The person's reaction to their agents' work, step 1b of the swarm's learning. When the person posts in a thread after
2
+ // agents delivered something since they last wrote, Harness's server reads the post against that work
3
+ // (src/server/swarm/reaction.ts): approves, approves with changes, rejects, or not a reaction, and whose work it is
4
+ // about (one agent's post, or what they made together). The verdict is kept on this board for each agent it is about,
5
+ // and each agent's card counts the verdicts on its work (cards.mjs). Nothing is read on the privacy setting, with this
6
+ // computer off Harness, or from a Harness that does not read reactions (swarm_reactions): such a post has no verdict.
7
+
8
+ // What an agent delivers in a thread: its result, a post or a file. Questions, notes and held posts are not work.
9
+ const WORK_KINDS = new Set(["result", "post", "file"]);
10
+ // Never read for a reaction, as the coordinator's round leaves them out: notes, the coordinator's own records, and posts
11
+ // held for the person (one may carry a secret).
12
+ const QUIET_KINDS = new Set(["note", "held", "round", "goal_update"]);
13
+ const RESULTS = 6;
14
+ const BEFORE = 8;
15
+ const POST_CHARS = 4_000;
16
+
17
+ const clip = (post) => ({
18
+ seq: post.seq,
19
+ author: post.author,
20
+ kind: post.kind,
21
+ text: post.text.slice(0, POST_CHARS),
22
+ });
23
+
24
+ /**
25
+ * What the person's post in a thread may react to, from the thread's posts: the posts since they last wrote (`before`)
26
+ * and, among them, the agents' work (`results`), newest last. No results: nothing was delivered to react to.
27
+ */
28
+ export function workSince(posts, post, isCoordinatorPost) {
29
+ const earlier = posts.filter(
30
+ (other) => other.seq < post.seq && !QUIET_KINDS.has(other.kind)
31
+ );
32
+ const lastOwn = earlier.findLast(
33
+ (other) => other.author === "user" && other.kind === "post"
34
+ );
35
+ const since = earlier.filter((other) => other.seq > (lastOwn?.seq ?? 0));
36
+ return {
37
+ before: since.slice(-BEFORE).map(clip),
38
+ results: since
39
+ .filter(
40
+ (other) =>
41
+ WORK_KINDS.has(other.kind) &&
42
+ other.author !== "user" &&
43
+ !isCoordinatorPost(other)
44
+ )
45
+ .slice(-RESULTS)
46
+ .map(clip),
47
+ };
48
+ }
49
+
50
+ export function createReactions({
51
+ store,
52
+ askHarness,
53
+ readsReactions,
54
+ isCoordinatorPost,
55
+ log,
56
+ }) {
57
+ /** Read the person's post in a thread; resolves once its verdicts are kept, or there is none to keep. */
58
+ const heard = async (thread, post) => {
59
+ if (!readsReactions()) return null;
60
+ const { before, results } = workSince(
61
+ store.roomPosts(thread.id, 0, 500),
62
+ post,
63
+ isCoordinatorPost
64
+ );
65
+ if (!results.length) return null;
66
+ let reply;
67
+ try {
68
+ reply = await askHarness(
69
+ "react",
70
+ {
71
+ thread: { id: thread.id, title: thread.objective.slice(0, 500) },
72
+ post: { seq: post.seq, text: post.text.slice(0, POST_CHARS) },
73
+ before,
74
+ results,
75
+ },
76
+ null
77
+ );
78
+ } catch (error) {
79
+ log(`reaction to post ${post.seq} in ${thread.id}: ${error.message}`);
80
+ return null;
81
+ }
82
+ // The bridge's answer carries the reaction's fields beside `ok`.
83
+ const reaction = reply?.ok ? (reply.result ?? reply) : null;
84
+ const delivered = new Set(results.map((result) => result.author));
85
+ const agents = (reaction?.agents ?? []).filter((agent) =>
86
+ delivered.has(agent)
87
+ );
88
+ if (!reaction?.verdict || !agents.length) {
89
+ log(
90
+ `reaction to post ${post.seq} in ${thread.id}: ${reply?.error || "no verdict"}`
91
+ );
92
+ return null;
93
+ }
94
+ for (const agent of agents)
95
+ store.recordReaction({
96
+ postId: post.id,
97
+ goalId: thread.id,
98
+ postSeq: post.seq,
99
+ aboutSeq: reaction.about ?? null,
100
+ agent,
101
+ verdict: reaction.verdict,
102
+ judgments: reaction.judgments ?? null,
103
+ });
104
+ return { agents, verdict: reaction.verdict };
105
+ };
106
+
107
+ return { heard };
108
+ }
package/lib/runner.mjs CHANGED
@@ -417,7 +417,8 @@ export function runAgentTask(task, emit, observe = () => {}) {
417
417
  if (running.has(slot)) {
418
418
  emit(
419
419
  "failed",
420
- `Another run is already working in ${task.cwd || "that folder"}.`
420
+ `Another run is already working in ${task.cwd || "that folder"}.`,
421
+ { failure: "start" }
421
422
  );
422
423
  return;
423
424
  }
@@ -431,7 +432,7 @@ export function runAgentTask(task, emit, observe = () => {}) {
431
432
  environment = agentEnvironment(invocation.command);
432
433
  if (environment.error) throw new Error(environment.error);
433
434
  } catch (error) {
434
- emit("failed", error.message);
435
+ emit("failed", error.message, { failure: "start" });
435
436
  return;
436
437
  }
437
438
  emit("accepted");
@@ -446,7 +447,9 @@ export function runAgentTask(task, emit, observe = () => {}) {
446
447
  child.stdin.end(invocation.stdin);
447
448
  } catch (error) {
448
449
  running.delete(slot);
449
- emit("failed", `Could not start ${task.agent}: ${error.message}`);
450
+ emit("failed", `Could not start ${task.agent}: ${error.message}`, {
451
+ failure: "start",
452
+ });
450
453
  return;
451
454
  }
452
455
  running.add(slot);
@@ -532,7 +535,7 @@ export function runAgentTask(task, emit, observe = () => {}) {
532
535
  0,
533
536
  4_000
534
537
  ),
535
- { usage: usage.result() }
538
+ { usage: usage.result(), failure: "exit" }
536
539
  );
537
540
  else if (code === 0)
538
541
  emit(
@@ -547,12 +550,14 @@ export function runAgentTask(task, emit, observe = () => {}) {
547
550
  emit(
548
551
  "failed",
549
552
  `${task.agent} exited with code ${code}.\n${tail}`.slice(0, 4_000),
550
- { usage: usage.result() }
553
+ { usage: usage.result(), failure: "exit" }
551
554
  );
552
555
  });
553
556
  child.on("error", (error) => {
554
557
  running.delete(slot);
555
558
  children.delete(task.runId);
556
- emit("failed", `Could not start ${task.agent}: ${error.message}`);
559
+ emit("failed", `Could not start ${task.agent}: ${error.message}`, {
560
+ failure: "start",
561
+ });
557
562
  });
558
563
  }