ticket-runner 0.6.5 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/verdict.ts CHANGED
@@ -1,25 +1,88 @@
1
+ import type { ListedCriterion } from "./acceptance-criteria.js";
1
2
  import { NOTES_LIST_SCHEMA } from "./note-schema.js";
2
3
  import { z } from "zod";
3
4
 
4
5
  /**
5
6
  * The structured result of the verify Stage: one status per Acceptance
6
- * Criterion, with the evidence the session gathered for it.
7
+ * Criterion, answered by the number the pipeline listed it under, with the
8
+ * evidence the session gathered for it, and the defects of the branch it found
9
+ * beside them.
7
10
  */
8
- const criterionSchema = z.object({
9
- text: z.string(),
11
+ const answerSchema = z.object({
12
+ number: z.number().int(),
10
13
  status: z.enum(["met", "unmet", "unverifiable"]),
11
14
  evidence: z.string(),
12
15
  });
13
16
 
14
- const verdictSchema = z.object({
15
- criteria: z.array(criterionSchema).min(1),
17
+ const defectSchema = z.object({
18
+ summary: z.string(),
19
+ evidence: z.string(),
20
+ });
21
+
22
+ const answeredSchema = z.object({
23
+ criteria: z.array(answerSchema).min(1),
24
+ defects: z.array(defectSchema).default([]),
16
25
  /** The agent's own opinion. The pipeline recomputes it; see {@link passes}. */
17
26
  pass: z.boolean(),
18
27
  });
19
28
 
20
- export type Criterion = z.infer<typeof criterionSchema>;
21
- export type Verdict = z.infer<typeof verdictSchema>;
22
- export type CriterionStatus = Criterion["status"];
29
+ /** The verdict as the Stage answered it, before its numbers are checked against the list. */
30
+ export type AnsweredVerdict = z.infer<typeof answeredSchema>;
31
+ export type CriterionStatus = z.infer<typeof answerSchema>["status"];
32
+
33
+ /**
34
+ * A defect this branch introduced, or one in code it added, which the verify
35
+ * Stage reports beside the criteria rather than as a Note. It fails the
36
+ * verdict as an `unmet` criterion does.
37
+ */
38
+ export type Defect = z.infer<typeof defectSchema>;
39
+
40
+ /** One listed criterion and what the verdict made of it. */
41
+ export interface GradedCriterion extends ListedCriterion {
42
+ status: CriterionStatus;
43
+ evidence: string;
44
+ }
45
+
46
+ /**
47
+ * A verdict whose numbers are exactly the listed ones: every criterion carries
48
+ * the line the pipeline listed it as, so whatever shows a criterion shows the
49
+ * Ticket's words and never the Stage's copy of them.
50
+ */
51
+ export interface Verdict {
52
+ criteria: GradedCriterion[];
53
+ defects: Defect[];
54
+ }
55
+
56
+ /**
57
+ * How a verdict's numbers missed the list, which means it has not graded the
58
+ * Ticket: a number listed and not answered, one answered and not listed, one
59
+ * answered more than once.
60
+ */
61
+ export interface Misnumbered {
62
+ missing: number[];
63
+ unlisted: number[];
64
+ repeated: number[];
65
+ }
66
+
67
+ /** The `defects` list as the verify Stage reads it. */
68
+ const DEFECTS_LIST_SCHEMA = {
69
+ type: "array",
70
+ description:
71
+ "Defects this branch introduced, or in code it added, beside the criteria. Empty when you found none.",
72
+ items: {
73
+ type: "object",
74
+ properties: {
75
+ summary: { type: "string", description: "One short sentence naming the defect." },
76
+ evidence: {
77
+ type: "string",
78
+ description:
79
+ "Where it sits, what you ran or read, what you expected, what came back, and the counterexample. It proposes no fix",
80
+ },
81
+ },
82
+ required: ["summary", "evidence"],
83
+ additionalProperties: false,
84
+ },
85
+ } as const;
23
86
 
24
87
  /**
25
88
  * The `--json-schema` the verify Stage is invoked with: the verdict, and the
@@ -27,15 +90,13 @@ export type CriterionStatus = Criterion["status"];
27
90
  *
28
91
  * The `notes` list has to be declared here even though {@link parseVerdict}
29
92
  * strips it and the Notes parser reads it off any object at all, because this
30
- * schema closes its top level to additional properties. That closure, not the
31
- * parsing, is what would otherwise leave the most adversarial reader in the
32
- * pipeline with nowhere to put a finding outside the criteria.
93
+ * schema closes its top level to additional properties.
33
94
  *
34
- * It is not required, where the criteria and the flag are. The verdict is the
35
- * Stage's product and a session that graded nothing has done none of its job,
36
- * so every required key here is a way for a Ticket to lose its verdict — and a
37
- * side channel may never cost one. A verdict that leaves the list out is a
38
- * verdict from a session that found nothing beside the criteria.
95
+ * Neither `notes` nor `defects` is required, where the criteria and the flag
96
+ * are. The verdict is the Stage's product and a session that graded nothing has
97
+ * done none of its job, so every required key here is a way for a Ticket to
98
+ * lose its verdict. A verdict that leaves a list out is one from a session that
99
+ * found nothing to put in it.
39
100
  */
40
101
  export const VERDICT_JSON_SCHEMA = {
41
102
  type: "object",
@@ -45,36 +106,80 @@ export const VERDICT_JSON_SCHEMA = {
45
106
  items: {
46
107
  type: "object",
47
108
  properties: {
48
- text: { type: "string", description: "The Acceptance Criterion verbatim" },
109
+ number: {
110
+ type: "integer",
111
+ description: "The criterion's number in the list the prompt gives",
112
+ },
49
113
  status: { type: "string", enum: ["met", "unmet", "unverifiable"] },
50
114
  evidence: {
51
115
  type: "string",
52
116
  description: "What you ran or read, and what it showed. It proposes no fix",
53
117
  },
54
118
  },
55
- required: ["text", "status", "evidence"],
119
+ required: ["number", "status", "evidence"],
56
120
  additionalProperties: false,
57
121
  },
58
122
  },
123
+ defects: DEFECTS_LIST_SCHEMA,
59
124
  pass: { type: "boolean" },
60
- notes: NOTES_LIST_SCHEMA,
125
+ notes: {
126
+ ...NOTES_LIST_SCHEMA,
127
+ description:
128
+ "Defects that belong to another Ticket, or to no Ticket yet. A defect this branch made goes in `defects`, not here. Empty when you found none, which is the ordinary case.",
129
+ },
61
130
  },
62
131
  required: ["criteria", "pass"],
63
132
  additionalProperties: false,
64
133
  } as const;
65
134
 
66
135
  /** Parse the verify Stage's structured output. Throws when it is unusable. */
67
- export function parseVerdict(result: unknown): Verdict {
68
- return verdictSchema.parse(result);
136
+ export function parseVerdict(result: unknown): AnsweredVerdict {
137
+ return answeredSchema.parse(result);
138
+ }
139
+
140
+ /**
141
+ * The answered verdict with each answer joined to the criterion its number
142
+ * names, in the list's order; or how its numbers missed the list, when they do.
143
+ */
144
+ export function gradeVerdict(
145
+ answered: AnsweredVerdict,
146
+ listed: readonly ListedCriterion[],
147
+ ): Verdict | Misnumbered {
148
+ const byNumber = new Map(listed.map((criterion) => [criterion.number, criterion]));
149
+ const answers = new Map<number, AnsweredVerdict["criteria"][number]>();
150
+ const repeated = new Set<number>();
151
+ const unlisted = new Set<number>();
152
+ for (const answer of answered.criteria) {
153
+ if (!byNumber.has(answer.number)) unlisted.add(answer.number);
154
+ else if (answers.has(answer.number)) repeated.add(answer.number);
155
+ else answers.set(answer.number, answer);
156
+ }
157
+ const missing = listed.map(({ number }) => number).filter((number) => !answers.has(number));
158
+ if (missing.length > 0 || unlisted.size > 0 || repeated.size > 0) {
159
+ return { missing, unlisted: [...unlisted], repeated: [...repeated] };
160
+ }
161
+ return {
162
+ criteria: listed.map((criterion) => {
163
+ const { status, evidence } = answers.get(criterion.number) as AnsweredVerdict["criteria"][number];
164
+ return { ...criterion, status, evidence };
165
+ }),
166
+ defects: answered.defects,
167
+ };
168
+ }
169
+
170
+ /** Whether {@link gradeVerdict} found the numbers off. */
171
+ export function isMisnumbered(graded: Verdict | Misnumbered): graded is Misnumbered {
172
+ return "missing" in graded;
69
173
  }
70
174
 
71
175
  /**
72
- * The pipeline's own pass decision: no criterion is unmet and at least one is
73
- * met. A verdict of nothing but `unverifiable` is zero evidence, so it fails.
176
+ * The pipeline's own pass decision: no criterion is unmet, the branch made no
177
+ * defect, and at least one criterion is met. A verdict of nothing but
178
+ * `unverifiable` is zero evidence, so it fails.
74
179
  */
75
180
  export function passes(verdict: Verdict): boolean {
76
181
  const counts = countStatuses(verdict);
77
- return counts.unmet === 0 && counts.met > 0;
182
+ return counts.unmet === 0 && verdict.defects.length === 0 && counts.met > 0;
78
183
  }
79
184
 
80
185
  export function countStatuses(verdict: Verdict): Record<CriterionStatus, number> {
@@ -83,6 +188,6 @@ export function countStatuses(verdict: Verdict): Record<CriterionStatus, number>
83
188
  return counts;
84
189
  }
85
190
 
86
- export function unmetCriteria(verdict: Verdict): Criterion[] {
191
+ export function unmetCriteria(verdict: Verdict): GradedCriterion[] {
87
192
  return verdict.criteria.filter((criterion) => criterion.status === "unmet");
88
193
  }