skilldrop-cli 0.16.8 → 0.16.9

package/packs/dev-team/skills/devils-advocate/evals/evals.json

@@ -43,6 +43,35 @@
         "It gives a concrete fix, such as reading with the csv module and normalising the email with strip and lower",
         "It does not state as fact what the other columns hold or what the real file contains beyond what was given"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "Finished the late fee feature for the tenant billing service (lateFees.ts, tests in lateFees.test.ts). Rules from finance: nothing if the invoice is 5 days or fewer past due. After that it's 1.5% of the balance for each calendar month the invoice has been overdue in, so one due 20 March and still open on 2 May has March, April and May. Capped at $75 per invoice in total. Never on paid or void invoices. A nightly job calls applyLateFees with every invoice. Tests are green. Can you go over it before I merge? Skip the style nits, I want the things that would actually bite us.",
+      "assertions": [
+        "It flags that monthsOverdue subtracts month numbers without the year, so an invoice due in November or December and still open in January gets a zero or negative count and no fee",
+        "It flags that the $75 cap is applied to one month's amount before multiplying by the months, so a large balance overdue for two or more months is charged more than $75 (for example $10,000 over two months gives $150)",
+        "It flags that applyLateFees adds the whole fee to the balance on every nightly run with nothing recording what was already charged, so the same invoice is charged again each night",
+        "It points out that the cap test uses a single month overdue, where a per-month cap and a total cap give the same $75, so it cannot catch the cap bug",
+        "It points out that the applyLateFees test only checks the balance went up after one run, and that no test runs it twice or checks the amount",
+        "It names a missing test for an invoice that is overdue across a year boundary",
+        "It does not report the grace-period check as a bug: charging nothing at 5 days late or fewer matches the stated rule",
+        "It says plainly that the change is not ready to merge as it stands"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "agent wrote this migration for me. adds region onto orders so the regional dashboards don't need the join to customers. ran clean on my laptop db. postgres. the orders table in prod is big-ish, don't know the exact size. top 3 things i should worry about before i run it friday?\n\n```sql\nALTER TABLE orders ADD COLUMN region_code VARCHAR(2) NOT NULL;\n\nUPDATE orders\nSET region_code = (SELECT region FROM customers WHERE customers.id = orders.customer_id);\n\nCREATE INDEX idx_orders_region ON orders(region_code);\n```",
+      "assertions": [
+        "It flags that adding a NOT NULL column with no default fails on a table that already has rows, so the first statement will error on prod",
+        "It says the clean run on the laptop does not show the migration works on prod, for example because the local orders table may have been empty",
+        "It flags locking or long-running work on a large table, from the single full-table UPDATE or the non-concurrent index build, and gives a concrete mitigation such as batching the backfill or CREATE INDEX CONCURRENTLY",
+        "It flags that a customer with no region, or a region longer than 2 characters, would break the backfill, and says what customers.region holds is not known from what was given",
+        "It gives a corrected order of steps: add the column as nullable, backfill it, then add the NOT NULL constraint",
+        "It does not state the row count of orders, how long the migration will take, or what customers.region contains as fact",
+        "It leads with three main concerns as asked, and anything beyond the three is kept to a brief mention"
+      ]
     }
   ]
 }

package/packs/dev-team/skills/devils-advocate/evals/files/4/lateFees.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect } from "vitest";
+import { applyLateFees, computeLateFee, Invoice } from "./lateFees";
+
+const day = (s: string) => new Date(s + "T12:00:00Z");
+
+const invoice = (over: Partial<Invoice> = {}): Invoice => ({
+  id: "INV-1001",
+  tenantId: "T-204",
+  dueDate: "2026-03-01",
+  balance: 1000,
+  status: "open",
+  ...over,
+});
+
+describe("computeLateFee", () => {
+  it("charges nothing inside the grace period", () => {
+    expect(computeLateFee(invoice(), day("2026-03-06"))).toBe(0);
+  });
+
+  it("charges 1.5% once the grace period is over", () => {
+    expect(computeLateFee(invoice(), day("2026-03-07"))).toBe(15);
+  });
+
+  it("charges for each calendar month overdue", () => {
+    const inv = invoice({ dueDate: "2026-03-20" });
+    expect(computeLateFee(inv, day("2026-05-02"))).toBe(45);
+  });
+
+  it("caps the fee at $75", () => {
+    const inv = invoice({ balance: 10000 });
+    expect(computeLateFee(inv, day("2026-03-10"))).toBe(75);
+  });
+
+  it("never charges paid or void invoices", () => {
+    expect(computeLateFee(invoice({ status: "paid" }), day("2026-06-01"))).toBe(0);
+    expect(computeLateFee(invoice({ status: "void" }), day("2026-06-01"))).toBe(0);
+  });
+});
+
+describe("applyLateFees", () => {
+  it("adds the fee to open invoices", () => {
+    const inv = invoice();
+    applyLateFees([inv], day("2026-03-10"));
+    expect(inv.balance).toBeGreaterThan(1000);
+  });
+});

package/packs/dev-team/skills/devils-advocate/evals/files/4/lateFees.ts

@@ -0,0 +1,46 @@
+export interface Invoice {
+  id: string;
+  tenantId: string;
+  dueDate: string; // ISO date, e.g. "2026-03-01"
+  balance: number; // dollars outstanding
+  status: "open" | "paid" | "void";
+}
+
+const GRACE_DAYS = 5;
+const MONTHLY_RATE = 0.015;
+const FEE_CAP = 75;
+const MS_PER_DAY = 86_400_000;
+
+function parseDue(inv: Invoice): Date {
+  return new Date(inv.dueDate + "T00:00:00Z");
+}
+
+export function daysLate(inv: Invoice, today: Date): number {
+  return Math.floor((today.getTime() - parseDue(inv).getTime()) / MS_PER_DAY);
+}
+
+// Number of calendar months the invoice has been overdue in, counting the
+// month it fell due. Due 20 March, today 2 May -> 3.
+export function monthsOverdue(inv: Invoice, today: Date): number {
+  return today.getUTCMonth() - parseDue(inv).getUTCMonth() + 1;
+}
+
+export function computeLateFee(inv: Invoice, today: Date): number {
+  if (inv.status !== "open") return 0;
+  if (daysLate(inv, today) <= GRACE_DAYS) return 0;
+
+  const monthly = Math.min(inv.balance * MONTHLY_RATE, FEE_CAP);
+  const fee = monthly * monthsOverdue(inv, today);
+  return Math.round(fee * 100) / 100;
+}
+
+// Called by the nightly job with every invoice for the portfolio.
+export function applyLateFees(invoices: Invoice[], today: Date): Invoice[] {
+  for (const inv of invoices) {
+    const fee = computeLateFee(inv, today);
+    if (fee > 0) {
+      inv.balance += fee;
+    }
+  }
+  return invoices;
+}

package/packs/dev-team/skills/release-notes/evals/evals.json

@@ -43,6 +43,35 @@
         "It does not invent dates, PR numbers, a commit count for commits it was not shown, or features not in the pasted list",
         "It delivers usable release notes in the reply rather than only asking questions, and lists the items that need confirming before sending"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "Portside WMS 5.0.0 goes out to customers on Thursday. The log since 4.9.2 is in git-log.txt. Two things please: (1) upgrade notes for admins. Our customers run Portside on their own servers, so their IT people need to know what to do before they upgrade. (2) A short paragraph for the customer newsletter, three or four sentences. Sales want the newsletter to lead with batch scanning and to say the duplicate pick list problem is finally fixed. Also remind people that 4.x support ends on 31 December.",
+      "assertions": [
+        "The PostgreSQL 15+ requirement is called out as something admins must deal with before upgrading (the installer aborts on PostgreSQL 13 and 14), even though its commit is labelled chore",
+        "The removal of /api/v1/stock is called out as a breaking change with /api/v2/stock-levels as the replacement, and the breaking items come before the new features in the admin notes",
+        "Batch scanning is described as off by default (flag scan.batch.enabled, missed reads on older handhelds), not as a feature that works out of the box after upgrading",
+        "The duplicate pick list problem is not described as fully fixed; the output says split shipments can still produce a duplicate line (#1893) or flags that the 'finally fixed' claim is not supported by the log",
+        "The end of 4.x maintenance is given as 31 March 2027, or the conflict between the requested 31 December date and the log is flagged; 31 December is not repeated as fact",
+        "The wms.mail.host to wms.smtp.host rename is described as a deprecation where the old key still works, not as a change that breaks existing configuration on upgrade",
+        "The admin notes include the security fix for session tokens not being invalidated on password change (LSA-2026-04)",
+        "The newsletter paragraph does not mention the spring-boot bump, the StockLedger refactor or the CI test shards"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "need the what's new text for the app stores for Tallybird 1.6 (habit tracker, it's just me and Sam). short, people don't read these. stuff that's in: dark mode, home screen widgets (iOS only i think? Sam would know), fixed the bug where streaks reset if you logged right around midnight, a few crash fixes, Sam did something with sync, and we dropped iOS 15. iOS goes out tomorrow, android build is still stuck in review",
+      "assertions": [
+        "It delivers store text that could be pasted in as it stands, not only a list of questions",
+        "Dark mode and the fix for streaks resetting around midnight both appear in the store text",
+        "The end of iOS 15 support is mentioned in the iOS text or raised explicitly as something users on iOS 15 need to be told",
+        "Widgets are not promised to Android users; the output flags that whether widgets are iOS-only needs confirming with Sam, or gives separate iOS and Android text with widgets only in the iOS one",
+        "The sync change is not described as a specific improvement (such as faster or more reliable sync); it is left out or flagged as needing detail from Sam",
+        "It does not invent crash counts, percentages, a release date for Android, or features that are not in the list",
+        "The store text itself is short: about 100 words or fewer per store"
+      ]
     }
   ]
 }

package/packs/dev-team/skills/release-notes/evals/files/4/git-log.txt

@@ -0,0 +1,26 @@
+# git log v4.9.2..v5.0.0-rc2 (oldest first) -- Portside WMS
+9c1e4a7 2026-07-14 feat(picking): wave planning screen groups orders by carrier cutoff (#1802)
+b3d0f52 2026-07-16 fix(receiving): ASN import rejects files that start with a BOM (#1809)
+4e7a918 2026-07-21 chore(deps): bump spring-boot 3.2.5 -> 3.3.2
+a80c3de 2026-07-23 feat(scan): batch barcode scan mode for handhelds (#1777)
+17f6b0c 2026-07-28 chore(db): migrations now use MERGE; requires PostgreSQL 15+ (#1815)
+    Installer aborts on PostgreSQL 13 and 14. 4.x supported 13+.
+d52e9a1 2026-07-30 fix(labels): ZPL label truncates SKUs longer than 20 characters (#1821)
+6fa23c8 2026-08-04 refactor(inventory): split StockLedger into reader and writer
+e19b7d4 2026-08-06 feat(api): /api/v2/stock-levels endpoint with pagination (#1790)
+0b4c8f1 2026-08-11 fix(picking): dedupe pick lists on wave release (#1833)
+    Partial. Split shipments can still produce a duplicate line, tracked in #1893.
+c7719ae 2026-08-13 perf(reports): stock valuation report at 500k SKUs: 212s -> 58s (#1840)
+83ad5e0 2026-08-18 feat(scan): put batch scan behind flag scan.batch.enabled, default false (#1848)
+    Field test at the pilot site showed missed reads on older handhelds.
+    Stays off by default until #1851 is fixed.
+f2061bb 2026-08-20 security: session tokens were not invalidated on password change (LSA-2026-04) (#1852)
+5d9e3c0 2026-08-25 docs: 4.x maintenance ends 2027-03-31 (was 2026-12-31) (#1856)
+a1f47e2 2026-08-27 feat(api)!: remove /api/v1/stock (deprecated since 4.6); use /api/v2/stock-levels (#1858)
+39b8d6f 2026-09-01 ci: split integration tests into 4 shards
+7c40e95 2026-09-03 fix: tweaks
+e6d1a07 2026-09-08 feat(config): rename wms.mail.host to wms.smtp.host (#1866)
+    Old key is still read and logs a deprecation warning. Removal planned for 6.0.
+2ab95f3 2026-09-10 fix(receiving): putaway suggestion ignored the bin weight limit (#1870)
+b90c7e8 2026-09-15 feat(ui): Polish translation of the handheld screens (#1874)
+d07a3b6 2026-09-22 fix(installer): upgrade from 4.9.x failed when the backup directory was a symlink (#1881)

package/packs/research/skills/source-synthesis/evals/evals.json

@@ -44,6 +44,36 @@
         "It says five conversations are too few to set a price or to conclude that cafe owners want it, and gives any price suggestion as tentative",
         "It does not add market figures, competitor prices or cafe-industry facts that are not in the notes"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "I'm the governance officer at Marlowe Vale Academy Trust (9 secondary schools). Next month the board decides whether to roll the phone-pouch scheme out from the two pilot schools to all nine. Everything we've got is in this folder: pilot-report.md, behaviour-logs.md, teacher-focus-group.md, parent-survey.md and vendor-brochure.md. Can you pull together what the evidence actually shows? Two pages max, it's going in the board pack. They'll particularly want to know what it did to behaviour and to exam results.",
+      "assertions": [
+        "It states that logged incidents at Fenwick Park rose from 280 to 291, which contradicts the pilot report's statement that incidents fell at both schools",
+        "It says Ashcombe's fall from 412 to 318 cannot be credited to the pouches because the logging system changed on 5 January 2026 and stopped counting low-level disruption, which was 96 of the 412 the year before",
+        "It traces the 40% reduction figure to the vendor brochure's footnote (a 2019 study of three New Zealand schools commissioned by the vendor) and does not count the pilot report's repetition of it as separate evidence",
+        "It corrects the parent survey response rate: 1,140 out of 3,050 is about 37%, not 47%",
+        "On exam results it says the sources contain no attainment data (the pilot report says results were not analysed and the brochure's 'improves grades' has no source), and it gives no estimated effect on results",
+        "It reports that phone confiscations fell at both schools (87 to 9 at Ashcombe, 64 to 22 at Fenwick Park) as the clearest measured change",
+        "It notes the teacher focus group was six teachers all from Ashcombe, with none from Fenwick Park, and reports the drawbacks they raised (for example ten minutes lost at form time or pupils pouching a dummy phone), not only the positive view in the pilot report",
+        "It does not bring in research, statistics or claims about phones in schools that are not in the five files"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "I'm on the committee for our village hall and the boiler is on its last legs. We're arguing about whether to get a heat pump instead. This is everything I've collected, sorry it's a bit scrappy:\n\n- Quote from Fernbrook Renewables: £14,200 to fit an air source heat pump. They only do heat pumps. The chap reckoned a new gas boiler would be 'about £4,100' but that's not a quote, he doesn't fit them.\n- Spoke to the treasurer at Little Aldwick hall, they got a heat pump in 2024. She said their bills 'went down about a third'. They also had the roof insulated the same year.\n- Cut a paragraph out of a magazine a while ago, no idea which one or when: 'heat pumps are three to four times as efficient as gas boilers'.\n- I think there's a grant, up to £7,500? Read it somewhere. Not sure village halls can get it.\n- We spent about £2,300 on gas last year I think. Hall's in use maybe 4 days a week.\n\nCan you pull together what we actually know? Mainly: would a heat pump be cheaper to run?",
+      "assertions": [
+        "It delivers the write-up from the notes as given and does not stop to ask for more information first",
+        "It says the notes cannot answer whether a heat pump would be cheaper to run, because they contain no electricity or gas prices and no running-cost estimate for this hall, and it states no annual saving as fact",
+        "It says Little Aldwick's bills falling by about a third cannot be put down to the heat pump alone because the roof was insulated in the same year",
+        "It says the 'three to four times as efficient' line is about efficiency, not running cost, and comes from an undated cutting from an unknown magazine",
+        "The grant is treated as unconfirmed on both the amount and whether a village hall qualifies, and the £7,500 is not subtracted from the cost as if it were secured",
+        "It notes that the £4,100 boiler figure is an offhand estimate from an installer who only fits heat pumps, not a quote",
+        "It gives the up-front gap as about £10,100 (£14,200 against about £4,100)",
+        "It does not add energy prices, grant scheme rules or typical heat pump costs from outside the notes, unless clearly labelled as not coming from the notes"
+      ]
     }
   ]
 }

package/packs/research/skills/source-synthesis/evals/files/4/behaviour-logs.md

@@ -0,0 +1,31 @@
+# Behaviour logs: pilot schools
+
+Extract by the trust data team, 9 July 2026. Counts of logged behaviour incidents per term.
+
+## Ashcombe School (1,620 pupils)
+
+| Year | Autumn | Spring | Summer | Total |
+|---|---|---|---|---|
+| 2024-25 (before pilot) | 150 | 138 | 124 | 412 |
+| 2025-26 (pilot year) | 141 | 92 | 85 | 318 |
+
+Note: on 5 January 2026 Ashcombe moved from the old paper-and-spreadsheet log to ClassLedger. Under ClassLedger, "low-level disruption" is recorded as a classroom note and is no longer logged as an incident. In 2024-25, low-level disruption accounted for 96 of Ashcombe's 412 incidents.
+
+## Fenwick Park School (1,430 pupils)
+
+| Year | Autumn | Spring | Summer | Total |
+|---|---|---|---|---|
+| 2024-25 (before pilot) | 98 | 94 | 88 | 280 |
+| 2025-26 (pilot year) | 101 | 97 | 93 | 291 |
+
+Note: no change to the logging system at Fenwick Park in either year.
+
+## Phone confiscations (both years logged the same way at both schools)
+
+| School | 2024-25 | 2025-26 |
+|---|---|---|
+| Ashcombe | 87 | 9 |
+| Fenwick Park | 64 | 22 |
+
+## Not available
+No behaviour extract was requested for the seven schools outside the pilot.

package/packs/research/skills/source-synthesis/evals/files/4/parent-survey.md

@@ -0,0 +1,25 @@
+# Parent survey: summary
+
+Online survey, open 1–19 June 2026. Sent to all 3,050 households with a child at Ashcombe or Fenwick Park. One response per household.
+
+Responses received: 1,140 (Ashcombe 790, Fenwick Park 350)
+
+## Q1. Should the pouch scheme continue next year?
+| Answer | Share of respondents |
+|---|---|
+| Yes | 58% |
+| No | 24% |
+| Not sure | 18% |
+
+Results were not broken down by school.
+
+## Q2. What, if anything, worries you about the scheme? (tick all that apply)
+| Concern | Share of respondents |
+|---|---|
+| Not being able to reach my child in an emergency | 41% |
+| Cost of replacing a lost pouch | 19% |
+| My child needs their phone for a medical reason | 6% |
+| Nothing worries me | 37% |
+
+## Q3. Has your child's behaviour at home around their phone changed?
+Free text, 412 comments. Not yet coded.

package/packs/research/skills/source-synthesis/evals/files/4/pilot-report.md

@@ -0,0 +1,27 @@
+# Phone-pouch pilot: evaluation report
+
+Prepared by Hollis Tran, Office of the Deputy CEO, Marlowe Vale Academy Trust
+17 July 2026
+
+## 1. Background
+From September 2025 pupils at two of our nine secondary schools, Ashcombe School and Fenwick Park School, have locked their phones in a magnetic pouch at morning registration and unlocked them at the gate at the end of the day. The other seven schools kept the existing "off and in your bag" rule.
+
+## 2. Behaviour
+Behaviour incidents fell at both pilot schools over the pilot year. This is in line with evidence from other schools, which shows a 40% reduction in behaviour incidents after pouches are introduced.
+
+Phone confiscations dropped sharply at both schools.
+
+## 3. Staff view
+A focus group of teachers was positive overall. Staff reported fewer confrontations in lessons.
+
+## 4. Parents
+Parents are strongly supportive. 58% of respondents support continuing the scheme. The response rate was 47% (1,140 responses from 3,050 households).
+
+## 5. Attainment
+Summer 2026 exam results had not been analysed at the time of writing.
+
+## 6. Cost
+Year-one cost was £25.40 per pupil, including replacement of the 11% of pouches lost or damaged during the year.
+
+## 7. Recommendation
+Roll out to all nine schools from January 2027.

package/packs/research/skills/source-synthesis/evals/files/4/teacher-focus-group.md

@@ -0,0 +1,19 @@
+# Teacher focus group: notes and quotes
+
+Held 12 May 2026, Ashcombe School library, 50 minutes. Six teachers, all from Ashcombe. Facilitated by Hollis Tran. A session at Fenwick Park was planned for June but did not take place.
+
+[04:10] Maths teacher: "I used to have one argument a week about a phone under the desk. I've had maybe two all year."
+
+[09:30] Head of Year 9: "Form time is a mess now. Pouching takes ten minutes every morning. That's ten minutes of registration and notices gone."
+
+[14:45] English teacher: "Some of them bring an old dead phone to put in the pouch and keep the real one. I don't know how many. More in Year 10 and 11."
+
+[21:00] Science teacher: "I miss them for practicals. We used phone timers and the slow-motion camera. I've had to book the tablet trolley and there's one trolley for the department."
+
+[27:20] PE teacher: "Lunchtime is louder. They actually talk to each other. I'd call that a good thing but the duty staff might not."
+
+[33:05] Head of Year 9: "The gate at ten past three is the flashpoint. One unlocking station per year group, two hundred and seventy kids. We've had pushing."
+
+[41:15] Maths teacher: "Would I go back? No. But I wouldn't say behaviour is transformed. Lessons are calmer. Corridors are the same."
+
+[46:00] English teacher: "Nobody has asked the pupils."

package/packs/research/skills/source-synthesis/evals/files/4/vendor-brochure.md

@@ -0,0 +1,18 @@
+# Stowaway Pouches: Schools Brochure 2026
+
+## Give pupils their attention back
+
+**Schools see 40% fewer behaviour incidents\***
+
+**Improves grades.** Pupils who are not distracted learn more.
+
+**Loved by teachers and parents.**
+
+### Pricing
+£18 per pouch. Unlocking stations supplied free with orders over 500 pouches. Replacement pouches charged at list price.
+
+### What schools say
+"It changed the feel of the building in a week." — Deputy Head, secondary school, West Midlands
+
+---
+\* Based on a 2019 study of three schools in New Zealand, commissioned by Stowaway Pouches Ltd. Individual results vary.

package/packs/solution-architect/skills/db-schema-design/evals/evals.json

@@ -40,6 +40,36 @@
         "It stores the allowed range somewhere (per sensor, per greenhouse or per sensor type) and says the actual range values were not given; any numbers shown are labelled as examples",
         "Reading times are stored with a time zone or in UTC"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "We're rebuilding the loyalty points system at Greenlane Markets (220 stores). My notes are in requirements.md, finance's email is in finance-email.md, and there's a small export of what we have today in members_sample.csv and points_txn_sample.csv. Can you design the new tables? I need the SQL and a short note on the decisions and anything in there that looks off — not an essay.",
+      "assertions": [
+        "It includes CREATE TABLE statements in which points movements (earn, redeem, expire, adjust) are rows in a ledger that is only ever added to, and a member's balance is either computed from those rows or kept as a stored total that it says how to keep in step with them",
+        "It points out that member M-1042's entries in the sample add up to 1,340 points while the members table says 1,410, a difference of 70",
+        "A duplicate till submission is blocked by a unique constraint or key built on the receipt reference (with store and/or till), and it notes that receipt R-88213 appears twice in the sample",
+        "It flags that the notes say points expire after 24 months and finance says 18, does not silently pick one, and stores an expiry date (or the earn date plus a configurable period) per earned lot so either rule can be applied",
+        "It says points by household cannot be produced because nothing in the data identifies a household, and it does not treat a shared postcode as a household",
+        "It points out that 400k transactions a day is about 146 million a year, not the 50 million in the notes",
+        "Points are stored as integers and money amounts as an exact type (numeric/decimal or integer pence), not FLOAT, and the value of a point is not hard-coded as 0.5p in the schema",
+        "It shows how the liability report can be run as at a past month end by store where earned, and gives an index or key on member and time for the newest-first history screen"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "we're a building firm, about 30 of us. right now who-has-what kit is tracked on a whiteboard in the yard and it's useless. I want a little internal tool: where is each thing right now, and where has it been / who had it. stuff isn't always on a site, sometimes it's in someone's van or at the repair shop. also need to know when things are due their safety test. few hundred items I'd guess? the IT guy set up a Postgres for the timesheets I think, could use that. what tables do I need",
+      "assertions": [
+        "It gives concrete table definitions (SQL or a column-by-column layout) in the reply, and does not stop to ask questions before delivering them",
+        "There is a movements or assignments table with one row per hand-over recording the item, where it went, who has it and when, so an item's full history can be listed",
+        "It says how 'where is it right now' is answered (the latest movement row, or a current-location column) and, if both are stored, how they are kept consistent",
+        "The design can record an item being somewhere that is not a site, such as a van or the repair shop, without leaving the location empty",
+        "It stores a test due date or last-tested date per item and says the test interval was not given; any interval shown is labelled as an example or assumption",
+        "It says it has assumed each item is tracked individually, or flags that it is not known whether some kit is counted in bulk (for example a box of identical items)",
+        "It says Postgres is assumed from 'I think' and should be confirmed, and does not recommend partitioning, sharding or a non-relational database for a few hundred items",
+        "It does not state a number of sites, vans or items as fact beyond the 'few hundred' guess given"
+      ]
     }
   ]
 }

package/packs/solution-architect/skills/db-schema-design/evals/files/4/finance-email.md

@@ -0,0 +1,18 @@
+From: Ines Okonjo-Bright (Financial Controller)
+To: Tomasz Wierzba
+Date: 29 September 2026
+Subject: Loyalty rebuild - what finance needs
+
+Tomasz,
+
+Three things before you design anything.
+
+1. The balances do not tie to the log. In the audit sample, member M-1042 shows 1,410 points on the members table. I added up the log myself and I do not get that. The auditors want every balance to be provable from the entries behind it.
+
+2. Since the January change, points expire after 18 months, not 24. Please make sure the new system does that.
+
+3. The liability report (unexpired points by the store where they were earned) has to be re-runnable as at any past month end. The auditors asked us to re-run March in July and we could not, because the old numbers had been overwritten.
+
+Also, the value of a point is a pricing decision and it may change next year. Do not bake 0.5p in.
+
+Ines

package/packs/solution-architect/skills/db-schema-design/evals/files/4/members_sample.csv

@@ -0,0 +1,5 @@
+member_id,card_number,joined_on,postcode,points_balance
+M-0456,6340010000456,2019-03-11,LS6 2AB,0
+M-1042,6340010001042,2021-07-02,LS8 4QT,1410
+M-2077,6340010002077,2023-11-19,HG1 5PL,210
+M-3310,6340010003310,2022-01-30,LS8 4QT,23

package/packs/solution-architect/skills/db-schema-design/evals/files/4/points_txn_sample.csv

@@ -0,0 +1,17 @@
+txn_id,member_id,store_id,till_id,receipt_ref,type,points,basket_value,created_at
+90001,M-1042,S014,T03,R-51120,earn,84,84.19999999999999,2026-05-02 10:14:51
+90002,M-1042,S014,T01,R-51877,earn,126,126.45,2026-05-16 18:02:33
+90003,M-1042,S014,,,bonus,500,,2026-05-20 00:05:00
+90004,M-1042,S031,T07,R-60233,earn,97,97.30000000000001,2026-06-06 12:40:09
+90005,M-1042,S014,T02,R-52990,redeem,-400,61.75,2026-06-20 16:21:47
+90006,M-1042,S014,T02,R-53641,earn,143,143.08,2026-07-11 09:55:12
+90007,M-1042,,,,adjust,250,,2026-07-15 14:30:00
+90008,M-1042,S014,,,bonus,540,,2026-08-01 00:05:00
+90009,M-2077,S102,T04,R-88213,earn,85,85.6,2026-08-09 19:03:21
+90010,M-2077,S102,T04,R-88213,earn,85,85.6,2026-08-09 19:03:25
+90011,M-2077,S102,T01,R-88590,earn,40,40.15,2026-08-23 11:12:02
+90012,M-3310,S014,T03,R-54002,earn,52,52.0,2026-07-04 13:08:40
+90013,M-3310,S057,T02,R-71455,earn,71,71.89999999999999,2026-07-18 17:45:16
+90014,M-3310,S057,T02,R-71901,redeem,-100,33.2,2026-08-01 12:02:58
+90015,M-0456,S031,T05,R-40118,earn,66,66.4,2024-08-30 15:27:03
+90016,M-0456,,,,expire,-66,,2026-08-31 02:00:00

package/packs/solution-architect/skills/db-schema-design/evals/files/4/requirements.md

@@ -0,0 +1,28 @@
+# Greenlane Rewards rebuild — my notes (Tomasz, platform team)
+
+Greenlane Markets: 220 stores, about 1.8M loyalty members.
+
+## What we have today
+- `members` table with a `points_balance` column. The tills update it directly.
+- `points_txn` log table, bolted on in 2023. Nothing enforces that it agrees with the balance.
+- `basket_value` is a FLOAT. Timestamps are whatever the till's clock said, no zone.
+
+## What the new one has to do
+1. Till scans a loyalty card -> look up the member by card number and show the balance. Needs to be fast (under 100 ms), it happens on every loyalty basket.
+2. Till earns points on a basket, or redeems points against a basket.
+3. App: member sees their points history, newest first, 20 rows at a time.
+4. Points expire 24 months after they were earned. A nightly job expires them. Oldest points get spent first.
+5. Customer services can make a manual adjustment. We need the reason and who did it.
+6. Finance: month-end liability = unexpired points outstanding, split by the store where they were earned.
+7. Marketing (Dalia) wants points by household.
+
+## Numbers
+- About 400k loyalty transactions a day, so roughly 50 million a year.
+- 1 point per whole pound spent. A point is worth 0.5p when redeemed.
+
+## Known problems
+- Tills retry on timeout because store networks are flaky. We have definitely had the same receipt sent twice.
+- Balances drift. Finance are not happy, see their email.
+
+## Decided
+- Postgres 16 (managed). Not changing.

package/packs/solution-architect/skills/design-doc/evals/evals.json

@@ -41,6 +41,35 @@
         "It says how a migrated job is checked before its old cron entry is removed, and how to go back if it misbehaves",
         "It is about one page: roughly 700 words or fewer"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "Can you write the design doc for automating customer erasure requests? Ottilie's ask is in request.md, and my notes on the systems and what we discussed are in systems.md. It's going to architecture review next Wednesday. Include the support-hours saving she asked for.",
+      "assertions": [
+        "It flags that the inventory lists 7 systems holding customer data while the request says 6, and the design covers all seven (or names the fraud scoring store as the one the request missed)",
+        "The fraud scoring store is shown as having no owner since the June reorg and an unknown deletion capability, raised as a blocker or open question needing an owner",
+        "It surfaces that the immutable database backups are kept 35 days and single rows cannot be removed, so 'including backups within 30 days' cannot be met as written, and proposes a way to handle it or raises it as a decision for the privacy lead",
+        "It does not promise to delete invoices: it states the 10-year tax retention for invoices (with name and billing address) conflicts with 'all data removed', and limits erasure in the Orders DB to non-invoice fields or raises it as a decision",
+        "The support-hours saving is stated as not calculable because time spent per request has never been measured; no hours figure is invented",
+        "At least two of the three approaches from the notes (central orchestrator, event-based with an acknowledgement ledger, manual with SLA timers) are described with the reason for choosing or rejecting each, and one is recommended",
+        "Seller (business) accounts are stated as out of scope",
+        "The size of the problem is given from the request's figures (412 requests in Q3 2026, 38 late, longest 71 days), and no other volumes, costs or team sizes are presented as facts"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "need a short design doc for rate limiting our public API, a page or so, for Friday's eng sync. we're a 12 person startup, weather data API. one customer hammered us last week and it took everyone down for like 20 min. right now there are no limits at all. we have free and paid tiers. the API sits behind nginx, then a python app, and redis is already there for caching. Dana thinks we should just do it in nginx, I think in the app so we can have different limits per plan. we don't know what the limits should be yet",
+      "assertions": [
+        "It delivers a design doc in the reply rather than only asking questions",
+        "The actual limit values are recorded as not yet decided, with a way to set them (for example from observed traffic), or any numbers shown are clearly labelled as placeholders; none is presented as decided",
+        "Both doing it in nginx and doing it in the app are described, with the trade-off tied to per-plan limits, and it either picks one or says what would decide it",
+        "The incident is described only as given (one customer's traffic took the API down for everyone for about 20 minutes last week); no request rates, customer name or revenue impact are invented",
+        "It says what a client sees when it is limited (for example an HTTP 429 with when to retry)",
+        "The rollout includes a step that observes or logs would-be limiting before requests are actually blocked, or a stated way to switch enforcement off quickly",
+        "It is about one page: roughly 700 words or fewer"
+      ]
     }
   ]
 }

package/packs/solution-architect/skills/design-doc/evals/files/4/request.md

@@ -0,0 +1,31 @@
+# Request: automate customer erasure requests
+
+From: Ottilie Brandt, Privacy Lead, Marrowfield Market
+To: Engineering
+Date: 28 Sep 2026
+
+## Problem
+When a customer asks us to erase their data, a support agent logs it in a spreadsheet and
+raises a ticket with each system team. Each team deletes by hand and replies. Requests get
+stuck when one team is slow or a ticket is missed.
+
+Q3 2026 (July to September):
+- 412 erasure requests received
+- 38 completed after the deadline
+- longest took 71 days
+- two complaints to the regulator so far in 2026, both about late erasure
+
+## What I need
+- Once support has verified the customer's identity and logged the request, erasure runs
+  automatically.
+- All of the customer's data is removed from all 6 systems, including backups, within
+  30 days of the verified request.
+- An audit record per request showing what was erased, where and when.
+- Support can see the status of a request without chasing teams.
+- In production by 15 Jan 2027, which is the date of the internal audit follow-up.
+
+## Not part of this
+Seller (business) accounts. Those are handled by the contracts team under the seller agreement.
+
+## Also
+Please include the expected saving in support hours. I need it for the budget case.

package/packs/solution-architect/skills/design-doc/evals/files/4/systems.md

@@ -0,0 +1,40 @@
+# Erasure automation: systems and discussion notes
+
+Notes by Kasimir (eng lead, Customer Platform), from the thread and the 1 Oct call.
+
+## Where customer personal data lives
+
+| # | System | Personal data | Owner | Deletion today |
+|---|--------|---------------|-------|----------------|
+| 1 | Accounts DB (Postgres) | name, email, phone, addresses | Identity team (Rafael) | soft-delete flag; hard-delete script exists, run by hand |
+| 2 | Orders DB (Postgres) | delivery address, phone, delivery notes, gift messages, invoices | Commerce team (Mei) | none |
+| 3 | Search index | display name on reviews | Discovery team (Sunniva) | rebuilt nightly from Accounts and Reviews, so follows Accounts within 24h |
+| 4 | Postwing (email/CRM, SaaS) | email, name, campaign history | Marketing ops (Jorge) | delete API exists, used by hand |
+| 5 | Helpstack (support desk, SaaS) | tickets, free text | Support (Anneke) | delete API per ticket, used by hand |
+| 6 | Data warehouse | copies of 1, 2, 4, 5 plus event logs keyed by customer_id | Data platform (Tunde) | none, partitions are append-only |
+| 7 | Fraud scoring store | device fingerprints, email hash, address hash | none. Was the Risk team, which was dissolved in the June reorg. Nobody has picked it up | unknown, nobody on the call knew |
+
+## Things people said
+
+- Petra (Finance): invoices, including the customer name and billing address on them, have to
+  be kept for 10 years for tax. "We cannot delete those, full stop."
+- Mei: we could strip everything on an order that is not on the invoice (phone, delivery notes,
+  gift messages) and keep the invoice itself.
+- Backups: Accounts DB and Orders DB backups are immutable snapshots kept for 35 days. You
+  cannot remove single rows from them. Ottilie has not been told this yet.
+- Tunde: deleting from the warehouse means rewriting partitions. He suggested a tombstone table
+  plus a weekly compaction job. "A few weeks of work", not scoped.
+- Identity verification stays with support, as now. Not changing.
+- Nobody has ever timed how long support spends on an erasure request.
+
+## Approaches discussed
+
+a. Central orchestrator: one new service calls each system's delete API in turn and tracks
+   state per request. Rafael likes the single place to look. Mei does not want another team's
+   service holding delete rights on the Orders DB.
+b. Event-based: publish `customer.erasure_requested` on the existing event bus. Each owning team
+   writes a consumer that deletes its own data and publishes an acknowledgement. A small ledger
+   records acks per system per request, and flags any system that has not acked in N days.
+   Most people leaned this way. Concern: a system with no owner never acks.
+c. Keep it manual but move it from the spreadsheet into the ticketing tool with SLA timers and
+   escalation. Cheapest. Does not remove the manual deletes.

package/packs/solution-architect/skills/tech-comparison-matrix/evals/evals.json

@@ -42,6 +42,35 @@
         "It does not invent benchmark figures, throughput numbers or costs",
         "Large-scale performance, horizontal scaling or community popularity is not among the highest-weighted criteria"
       ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-06",
+      "prompt": "We need to decide what to do with our driver app: keep the two native apps, rewrite in React Native, or rewrite in Flutter. Everything I've collected is in mobile-options.md. Leadership wants a scored table and a recommendation on one page. Use the cold start times from the spikes for the performance score.",
+      "assertions": [
+        "It states that a full rewrite cannot deliver the proof-of-delivery photo feature by 1 Feb 2027: 26 weeks (React Native) or 30 weeks (Flutter) from 2 Nov 2026 ends in May 2027, so the feature has to ship in the native apps first or the rewrite options miss the contract date",
+        "It says Orrin does not publish or support React Native or Flutter wrappers (the 9 Sept 2026 email) and that a bridge has to be written in-house; the March 2024 wiki claim of official plugins is not relied on",
+        "It points out that the Flutter cold start of 2.1 s was measured on a Pixel 8 and not on the H4, so it is not comparable with the 2.9 s (React Native) and 1.8 s (native) H4 figures; no H4 figure is invented for Flutter",
+        "It notes that the Flutter prototype's scan screen crashed on the H4 and was not investigated, as an open risk against the scanner must-have",
+        "The share of drivers on the handheld is given as about 70% (2,650 of 3,800), or the '80%' in the notes is flagged as not matching the counts",
+        "All three options are scored against the same named criteria, with weights shown and a total per option",
+        "It gives one clear recommendation and says what would change it",
+        "It does not introduce costs, benchmark figures or effort estimates that are not in the file"
+      ]
+    },
+    {
+      "id": 5,
+      "held_out": "2026-10-06",
+      "prompt": "need to pick how we run inference for our model: the GPU box in the office, cloud GPU instances, or just pay for a hosted inference API. we're 6 people, it's a document classifier, the documents are customers' contracts and a couple of them are touchy about where their data goes. volume is low right now but a big customer might sign. Kofi says the office box is basically free since we already own it. just the table and your pick, I'm pasting it into slack",
+      "assertions": [
+        "All three options (office GPU box, cloud GPU instances, hosted inference API) are compared in one table against the same named criteria, each with a score or rating",
+        "Where customer data goes, or data confidentiality, is one of the criteria and is among the most heavily weighted, tied to the remark that the documents are customers' contracts",
+        "The office box is not treated as free: the output names at least one ongoing cost or risk of it, such as the team's time to run it, no redundancy if it fails, or power and hardware upkeep",
+        "It gives one clear pick, not 'it depends'",
+        "It states that current and future volume are unknown and says how the pick would change if the big customer signs or volume grows",
+        "It does not invent prices, per-hour or per-request costs, latency or throughput figures",
+        "It is short enough to paste into Slack: the table plus no more than about 250 words of other text"
+      ]
     }
   ]
 }