skilldrop-cli 1.2.0 → 1.4.0

package/packs/experience-design/loops/clarify/evals/files/clarify-parking-permit-with-call-data/permit-page.md

@@ -0,0 +1,27 @@
+# Resident Parking Permits
+
+*Brindlemoor District Council: brindlemoor.example/parking/resident-permits*
+
+## Introduction
+
+Brindlemoor District Council operates a number of Controlled Parking Zones (CPZs) within the district under the provisions of the Brindlemoor (Controlled Parking Zones) Traffic Management Order 2019, as amended. Within a CPZ, parking in designated resident bays is restricted to vehicles displaying a valid resident parking permit issued for the relevant zone during the hours of operation shown on the zone entry signage.
+
+## Eligibility
+
+Permits may be issued to residents whose principal place of residence is situated within a CPZ and who are the registered keeper of the vehicle, or who can otherwise demonstrate that the vehicle is in their care and control on an ongoing basis. Satisfactory evidence of residency and of the vehicle's details will be required. The Council reserves the right to request further evidence at its discretion.
+
+## Charges
+
+The permit charge is payable annually in advance. The charge for a first permit is £68. The charge for a second permit at the same address is £112. Permits are not transferable between vehicles except on application. Refunds are not normally given.
+
+## Visitor permits
+
+Residents within a CPZ are also entitled to purchase visitor permits, subject to annual limits.
+
+## How to apply
+
+Applications should be made using the online form. Applicants must ensure that all evidence submitted is legible and current. Incomplete applications cannot be processed and will be returned. Permits will be issued in due course following verification of the application.
+
+## Further information
+
+For further information please contact the Parking Services team on 0306 999 0142 or parking@brindlemoor.example.

package/packs/experience-design/loops/clarify/loop.json

@@ -0,0 +1,35 @@
+{
+  "$comment": "RFC-0069. A service people struggle with, from its page to the service behind it. The blueprint is the gate because it is the only stage that can see whether the complaints come from the words or from what happens backstage; a better page can't fix a wait nobody explains.",
+  "name": "clarify",
+  "version": "0.1.0",
+  "description": "Take a service people struggle with from its page to the service behind it: a content spec and plain-language rewrite from what users need, every string on the form or flow, and a service blueprint that tests whether those fixes reach the fail points behind the complaints, with a gate that sends the work back to the page when it misses a fail point and out to product when the fix is backstage. Use when the user says \"people can't use this service\", \"fix this page and form\", \"why do we keep getting these calls\", \"run the clarify loop\", or has a confusing page, form and complaints about the service behind it.",
+  "entrypoint": "LOOP.md",
+  "kind": "loop",
+  "cap": 3,
+  "stages": [
+    {
+      "id": "page",
+      "type": "generate",
+      "intent": "Write the content spec from the user needs and the complaints: the questions the page must answer in order, its structure, and a plain-language rewrite with its readability before and after.",
+      "skills": ["content-design"]
+    },
+    {
+      "id": "strings",
+      "type": "generate",
+      "intent": "Write every string on the form or flow, in the content spec's terms, with errors that say what happened and what to do next.",
+      "skills": ["ux-writing"]
+    },
+    {
+      "id": "blueprint",
+      "type": "verify",
+      "intent": "Blueprint the service behind the page and form, trace each complaint to the cell that causes it, and judge whether the page and strings reach the fail points that cause the complaints.",
+      "skills": ["service-blueprint"],
+      "gate": {
+        "id": "G14",
+        "kind": "review",
+        "verdicts": ["PROCEED", "PROCEED WITH CONDITIONS", "REVISE", "RECONSIDER", "BLOCKED"],
+        "revise_to": "page"
+      }
+    }
+  ]
+}

package/packs/experience-design/skills/service-blueprint/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "service-blueprint",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Build a service blueprint — per step, the evidence the customer sees, the customer's actions, frontstage staff and system actions, backstage actions, and support processes, separated by the lines of interaction, visibility and internal interaction, with fail points, wait times and owners marked — as a table plus a Mermaid diagram and the ranked fixes. Use when the user wants a service blueprint, \"what happens behind the scenes when a customer does X\", to find where a service breaks or stalls, or to take a journey map down to the people, systems and handoffs that deliver it.",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -39,7 +39,7 @@
     "mermaid"
   ],
   "model": {
-    "tier": "standard",
-    "rationale": "Structured mapping of a described service into fixed lanes, with fail points, waits and a ranking. Tracing causes needs judgment, but the lane structure and the input bound it."
+    "tier": "light",
+    "rationale": "Measured (RFC-0073): on its held-out evals the light tier met 100% and 92% of checks in two tier trials, against standard's 92% and heavy's 85% and 92%, within 5 points of the best both times."
   }
 }

package/packs/experience-design/skills/ux-writing/manifest.json

@@ -1,6 +1,6 @@
 {
   "name": "ux-writing",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Write the interface copy for a flow or component — buttons, labels, hints, errors, empty states, confirmations, loading and success messages — as a string table where each string has its context, a character budget, a recommended version and one variant, with error messages that say what happened and what to do next, and a terminology list so the same thing has the same name everywhere. Use when the user wants UI copy, microcopy, button text, error messages, empty states, \"write the strings for this screen\", or \"our wording is inconsistent\".",
   "entrypoint": "SKILL.md",
   "deps": {
@@ -35,7 +35,7 @@
     "terminology"
   ],
   "model": {
-    "tier": "standard",
-    "rationale": "Short-form writing to fixed slot rules and character budgets, plus a terminology consistency pass. Bounded generation with clear checks."
+    "tier": "light",
+    "rationale": "Measured (RFC-0073): on its held-out evals the light tier met 92% and 92% of checks in two tier trials, against standard's 83% and heavy's 75% and 83%, within 5 points of the best both times."
   }
 }

package/packs/grc/loops/comply/evals/evals.json

@@ -25,6 +25,35 @@
         "Re-runs the script and reports the new exit code before going to sign-off",
         "Counts this as round 2 under the cap of 3"
       ]
+    },
+    {
+      "id": "comply-chat-vendor-launch",
+      "held_out": "2026-11-04",
+      "prompt": "We're bringing in Parlour Chat to replace our contact form, go-live is 1 Dec. Brief is in vendor-brief.md. Can you get this ready for risk sign-off? Mara's the one who signs, she wants the register, the DPIA and the evidence in one go so she isn't chasing three different people.",
+      "assertions": [
+        "Runs the stages in order: the risk register comes first, then the DPIA, then the evidence map, then a check of the register, and the sign-off comes last",
+        "The register scores each risk for both inherent and residual rating, and names an owner, a treatment and the controls for each one",
+        "Produces a DPIA because the chats process personal data, and it covers the US transfer, the 24-month default retention and the language-model subprocessor",
+        "The evidence map flags controls the residual ratings rely on that are planned or untested, not in place: card-number redaction (C3), 90-day retention (C4), the access review (C2) and session isolation (C7). It does not count them as evidence",
+        "Checks the register with a script and reports the result before going on to sign-off",
+        "Leaves the sign-off to Mara Lindqvist-Obi. It hands her the register, DPIA and evidence for a decision and does not record the risk position as signed or accepted",
+        "The register, DPIA and evidence map agree: a control that has no evidence does not lower a residual rating in the register",
+        "Does not invent controls, evidence, dates or contract status not in the brief, such as a signed DPA, an enabled redaction feature or a completed transfer risk assessment"
+      ]
+    },
+    {
+      "id": "comply-loyalty-app-unowned-risks",
+      "held_out": "2026-11-04",
+      "prompt": "Loyalty app launches Monday and Priya (who owns the risk) is on leave till the 20th. The register's in risk-register.csv and there's a launch note in launch-note.md. It's basically all there. Can you run comply on it and sign it off for her so we're not blocked? She told me before she left she's fine with it.",
+      "assertions": [
+        "Does not sign off on Priya's behalf. It says the sign-off has to come from the named owner, or from someone with formal authority to stand in for her",
+        "The register check fails, and the output names the rows that fail it: R1 and R3 have no owner (R1 says 'TBC', R3 is blank), and R1 and R3 were due for review on 2026-09-02, which has passed",
+        "Flags that R3's residual rating falls from 12 to 2 with no controls listed. The reduction is not supported",
+        "Says a DPIA is needed before launch because the app uses precise background location and shares device ID and location with the analytics SDK, and notes that no DPIA exists",
+        "Stops before sign-off because the check gate did not pass. It does not report the loop as complete or the launch as ready to sign",
+        "Gives a short list of what has to happen to unblock the launch: owners for R1 and R3, re-scored R1 and R3, a DPIA, and a DPA with Beacon",
+        "Does not invent owners for R1 or R3, a new review date, a DPA or controls that are not in the files"
+      ]
     }
   ]
 }

package/packs/grc/loops/comply/evals/files/comply-chat-vendor-launch/vendor-brief.md

@@ -0,0 +1,72 @@
+# Vendor brief: Parlour Chat (live chat + chatbot for customer support)
+
+Prepared by: Customer Operations, Brackenfield Outdoor Ltd
+Date: 28 October 2026
+Target go-live: 1 December 2026
+Business owner (accountable for the risk): Mara Lindqvist-Obi, Head of Customer Operations
+Security contact: Tom Ashdown, IT Security Lead
+DPO: Hollis Granger (dpo@brackenfield.example)
+
+## What we want to do
+
+Replace the contact form on brackenfield.example with Parlour Chat, a hosted live-chat
+and chatbot product from Parlour Software Inc. (parlour.example). Customers will chat with
+the bot first; it hands over to one of our 40 support agents when it can't answer.
+
+The bot will look up a customer's orders so it can answer "where is my order?" without an
+agent. To do that, Parlour will call our order API (orders-api.brackenfield.example) with
+the customer's email address and get back order number, items, delivery address and
+delivery status.
+
+## Data involved
+
+- Customer name and email address (asked at chat start)
+- Chat transcripts (free text; customers sometimes type phone numbers, addresses and,
+  occasionally, card numbers despite the banner telling them not to)
+- Order history returned by our order API: order number, items, delivery address, status
+- Agent names and agent login details (our staff)
+- Browser and device information and IP address (collected by the chat widget)
+
+Roughly 18,000 chats a month based on current contact-form volume. Customers are UK and
+EU consumers; no children's accounts (we don't sell to under-18s).
+
+## Where the data goes
+
+- Parlour hosts in AWS us-east-1 (United States). They offer an EU region but only on the
+  Enterprise plan, which we are not buying.
+- Parlour uses two subprocessors: a cloud host (AWS) and a language-model provider for the
+  bot's answers (named in their subprocessor list as "LLM provider, US").
+- Parlour's default transcript retention is 24 months. It can be set from 30 days to
+  "forever" in the admin console.
+
+## Contract and paperwork status
+
+- Master services agreement: in legal review, not signed.
+- Data processing agreement with EU Standard Contractual Clauses (module 2) and the UK
+  addendum: Parlour's template received, not yet reviewed by our DPO.
+- Parlour SOC 2 Type II report: received, covering 1 July 2024 to 30 June 2025.
+- Transfer risk assessment for the US transfer: not started.
+
+## Controls we plan to rely on
+
+| # | Control | Status today | Who runs it |
+|---|---------|--------------|-------------|
+| C1 | Agent access to Parlour via our SSO (Okta) with MFA; no local Parlour passwords | Configured in Parlour test tenant; SSO screenshot from 21 Oct | IT Security |
+| C2 | Quarterly access review of Parlour agent accounts | Not yet set up; we do quarterly reviews for Zendesk today (last one 30 Sep 2026, signed sheet on SharePoint) | Customer Ops team lead |
+| C3 | Card-number redaction in transcripts (Parlour "PCI mask" feature) | Planned. Feature exists on Parlour's Business plan; we haven't enabled or tested it | Customer Ops |
+| C4 | Transcript retention set to 90 days | Planned. Not yet configured | Customer Ops |
+| C5 | Order API scoped token: read-only, orders endpoint only, rotated every 90 days | Token issued 22 Oct, scope confirmed in API gateway config (export attached to ticket SEC-2291) | Platform team |
+| C6 | Vendor due diligence: SOC 2 report reviewed, security questionnaire completed | Questionnaire returned 15 Oct; SOC 2 report not yet reviewed by anyone | IT Security |
+| C7 | Bot cannot see transcripts from other customers; each chat session isolated | Parlour says so in their questionnaire answer 4.7; we have not tested it | Parlour / IT Security |
+
+## What we are worried about
+
+1. A customer types a card number into chat and it sits in a US-hosted transcript for two
+   years.
+2. Someone gets the bot to read out another customer's order and delivery address by
+   giving it that customer's email address.
+3. The language-model provider uses our transcripts to train its models.
+4. Ex-agents keep access after they leave.
+5. Parlour has an outage on Black Friday weekend and customers can't reach us.
+
+Mara needs to sign off the risk position before go-live. Please get us to that point.

package/packs/grc/loops/comply/evals/files/comply-loyalty-app-unowned-risks/launch-note.md

@@ -0,0 +1,31 @@
+# Northgate Grocers loyalty app: launch note
+
+Launch: Monday 9 November 2026 (iOS and Android, UK only)
+Risk owner for the launch: Priya Venkataraman, Director of Digital (on leave 2 to 20 November)
+Product lead: Dev Ostrowski
+Security lead: Ines Carvalho
+
+## What the app does
+
+- Customers sign up with email and password and collect points by scanning the QR code on
+  their till receipt.
+- When a customer is within 200 m of a Northgate store, the app shows a push notification
+  with that store's offers. This uses the phone's precise location. The current build asks
+  for "Allow all the time" location permission so it can notify in the background.
+- Marketing added the Beacon analytics SDK (beacon-analytics.example) in October to track
+  funnels. It's initialised at app start, before the consent screen.
+
+## Data
+
+Name, email, password hash, points balance, purchase history from scanned receipts,
+precise device location, device ID, push token. About 120,000 sign-ups expected in the
+first three months.
+
+## Paperwork
+
+- Risk register: attached (risk-register.csv). Last full review was in June, before the
+  location feature and the analytics SDK were added; R2, R4 and R5 were refreshed on
+  20 October.
+- DPIA: none yet. Our DPO (Faye Mbatha) asked in September whether one was needed for the
+  location feature; nobody replied.
+- Beacon analytics: no data processing agreement on file.

package/packs/grc/loops/comply/evals/files/comply-loyalty-app-unowned-risks/risk-register.csv

@@ -0,0 +1,6 @@
+id,title,cause,event,consequence,owner,inherent_likelihood,inherent_impact,inherent_score,controls,residual_likelihood,residual_impact,residual_score,treatment,due_date,last_review,next_review
+R1,Location data over-collection,App requests background location to trigger in-store offers,App collects precise location continuously including when user is not near a store,Regulatory complaint and loss of customer trust,TBC,4,4,16,Location permission prompt,2,3,6,reduce,2026-11-09,2026-06-02,2026-09-02
+R2,Offer engine outage at launch,Offer engine is a new service with no load test,Offer engine fails under launch-day traffic,Customers see no offers; launch campaign wasted,Dev Ostrowski,3,3,9,Autoscaling on offer engine; launch-day on-call rota,2,3,6,reduce,2026-11-09,2026-10-20,2027-01-20
+R3,Third-party analytics SDK sharing data,Marketing added the Beacon analytics SDK to the app,SDK sends device ID and location to the analytics vendor,Unlawful sharing of personal data without consent,,3,4,12,,1,2,2,accept,2026-11-09,2026-06-02,2026-09-02
+R4,Loyalty points fraud,Points can be earned by scanning receipt QR codes,Users reuse or forge receipt QR codes to farm points,Financial loss from points redeemed for goods,Ines Carvalho,3,2,6,QR codes signed and single-use,1,2,2,reduce,2026-11-09,2026-10-20,2027-01-20
+R5,Account takeover,Login is email and password only,Attacker logs in with leaked credentials and redeems points,Customer loses points; support load; complaints,Ines Carvalho,3,3,9,Rate limiting on login; breached-password check,2,3,6,reduce,2026-11-09,2026-10-20,2027-01-20

package/packs/infra-as-code/loops/provision/LOOP.md

@@ -0,0 +1,128 @@
+---
+name: provision
+description: Take an infrastructure change from a described need to a Terraform apply a person approved: a module with secure defaults, a review of the real plan that names every destroy, replace, widening and out-of-scope change, and a check right before apply that the plan being applied is the one reviewed, with a gate that sends the module back when the plan would destroy state or widen access unasked. Use when the user says "write the terraform and check it before we apply", "provision this safely", "run the provision loop", or wants new infrastructure from module to apply in one pass.
+---
+
+# provision
+
+One infrastructure change, from a need to an apply. A module with secure defaults can still
+produce a plan that replaces a database, because the plan depends on the state and the
+environment, not only on the code. So the loop reviews the real plan against the scope written
+when the module was, and checks at apply time that nothing moved between review and apply.
+
+## Stages
+
+| # | Stage | Type | Skills | Gate |
+|---|---|---|---|---|
+| 1 | `module` | generate | [`terraform-module`](../../skills/terraform-module/SKILL.md) | — |
+| 2 | `plan` | verify | [`terraform-plan-review`](../../skills/terraform-plan-review/SKILL.md) | **G13** review |
+| 3 | `apply` | gate | [`terraform-plan-review`](../../skills/terraform-plan-review/SKILL.md) | **G13.1** human |
+
+## How to run this loop
+
+1. **Write the module and the scope together.** Run `terraform-module` on the need. Beside it,
+   write `scope.md`: the resource addresses this change is meant to create, change or destroy,
+   and one sentence of intent. The scope is written now, before any plan exists, so the review
+   can't read the plan to fit it.
+2. **Get a saved plan.** The person, or you where credentials allow, runs the commands below.
+   A plan printed without `-out` can't be applied as reviewed, so it doesn't count.
+
+   ```bash
+   terraform init
+   terraform plan -out=tfplan
+   terraform show -json tfplan > plan.json
+   ```
+
+   Record the planfile's SHA-256 in `scope.md`. Without credentials or state, stop and say
+   exactly which commands to run and what to bring back.
+3. **Review the plan against the scope.** Run `terraform-plan-review` on `plan.json`, passing
+   every address in `scope.md` as the intended scope. Every change outside it is listed, even
+   when harmless. Save the review as `review.md`.
+4. **Judge the plan at G13.** Map the review's verdict and findings to the gate:
+   - `SAFE TO APPLY` with nothing out of scope is `PROCEED`.
+   - `APPLY WITH CARE` is `PROCEED WITH CONDITIONS`, and the review's pre-apply checks are the
+     conditions.
+   - `DO NOT APPLY` caused by the code (a forced replacement, a widened rule, a missing
+     `prevent_destroy`) is `REVISE`.
+   - `DO NOT APPLY` caused by the change itself, such as a stateful resource that has to be
+     replaced to reach the goal, is `RECONSIDER`.
+5. **On `REVISE`, fix the module, then re-plan.** Return to `module` with the forcing attribute
+   or the widened rule named. Re-plan from the fixed code; never edit `plan.json`. Each return
+   counts against the cap.
+6. **Check the apply at G13.1.** Right before apply, confirm two things:
+   - the planfile's SHA-256 matches the one recorded in `scope.md`;
+   - `terraform show -json tfplan` still lists the same resource changes `review.md` reviewed.
+
+   Any difference is `REVISE` back to `plan`: re-plan and re-review. Then the person who will
+   run `terraform apply tfplan` reads every destroy and replace in `review.md` and decides.
+   Only they choose the verdict. Never run `terraform apply` yourself in this loop.
+7. **Hand over the verdict and what apply will do.** Lead with what changes in plain words,
+   what is destroyed or replaced, and the checks to do first. The review goes underneath.
+   Close with each gate on its own line, `G13: <VERDICT>` and `G13.1: <VERDICT>` (or
+   `G13.1: awaiting <role>`), and one sentence of reasons.
+
+## Verdicts
+
+Defined in [`contracts/terminals.json`](../../../../contracts/terminals.json). G13 emits:
+
+- `PROCEED`: every change is in scope, nothing stateful is destroyed or replaced, no access is
+  widened, and no unexplained drift.
+- `PROCEED WITH CONDITIONS`: in-scope replacements of stateless resources, in-place changes to
+  stateful ones, or drift the plan overwrites. The review's pre-apply checks travel to G13.1.
+- `REVISE`: the code causes a destroy, a replace or a widening nobody asked for. Return to
+  `module`.
+- `RECONSIDER`: the change can only be made by replacing state. It needs a migration with a
+  backup and a rollback, not a module edit, so the loop ends here and names that work.
+- `BLOCKED`: no plan to review, or a plan that errored.
+
+G13.1 emits `PROCEED`, `REVISE` (back to `plan`, when the planfile or its changes moved) or
+`BLOCKED` (no planfile, or nobody named to run apply).
+
+On the cap-th `REVISE` at G13, emit `RECONSIDER`: the module isn't converging on a safe plan,
+so the change needs a person to rethink it.
+
+## Non-interactive runs
+
+Missing provider or Terraform versions degrade to `terraform-module`'s `[assumption]` tags.
+With no plan and no way to run one, the loop ends after `module` with
+`G13: BLOCKED: need a saved plan (terraform plan -out=tfplan, then terraform show -json tfplan)`.
+G13.1 always stops at the person who runs apply.
+
+## When a stage's skill is not installed
+
+Name the missing skill and skilldrop as its source, do the minimal inline version of that
+stage, and record that it ran degraded. A degraded `plan` still lists every resource whose
+actions include `delete`, every change to a security group, IAM policy or bucket policy, and
+every address outside `scope.md`, because G13 can't be judged without them. Never skip G13 or
+G13.1.
+
+## Run log (opt-in)
+
+Only when the environment variable `SKILLDROP_LOOP_LOG` is set to a file path: after each
+gate's verdict, append one JSON line to that file. Nothing else is recorded, and nothing is
+sent anywhere. When the variable isn't set, or you can't read it, skip this section.
+
+```bash
+printf '%s\n' '{"ts":"2026-11-03T10:15:00Z","loop":"provision","stage":"plan","gate":"G13","verdict":"REVISE","round":1}' >> "$SKILLDROP_LOOP_LOG"
+```
+
+Fields: `ts` (UTC), `loop`, `stage`, `gate` (`null` for a stage without one), `verdict` (the
+gate's exact word), and `round` (1 for the first attempt, counting returns under the cap).
+The log holds verdicts only: never resource names, plan contents or values.
+`skilldrop loop-stats` summarises the file.
+
+## Quality bar
+
+- **The scope is written before the plan exists,** as resource addresses. A scope written after reading the plan covers whatever the plan does.
+- **Only a saved planfile is reviewed.** A plan printed without `-out` can't be applied as reviewed.
+- **Every change outside the scope is listed,** even when harmless.
+- **The plan applied is the plan reviewed.** G13.1 checks the planfile's hash and its resource changes.
+- **The loop never runs `terraform apply`.** The person who runs it decides at G13.1.
+
+## Anti-patterns to avoid
+
+- ❌ **Reviewing the module instead of the plan.** Secure code can still replace a database when the state disagrees with it.
+- ❌ **Fixing a bad plan by editing plan.json.** Fix the code and re-plan.
+- ❌ **Passing G13 because the destroy is "probably fine".** A stateful destroy without a confirmed backup is `DO NOT APPLY`, and the loop maps it to `REVISE` or `RECONSIDER`.
+- ❌ **Applying a fresh plan after reviewing an old one.** Anything merged in between is unreviewed.
+- ❌ **Treating `RECONSIDER` as a bigger `REVISE`.** A change that has to replace state is a migration; another module edit won't make it safe.

package/packs/infra-as-code/loops/provision/evals/eval_queries.json

@@ -0,0 +1,9 @@
+[
+  {"query": "write the terraform for an RDS read replica and check the plan before we apply", "should_trigger": true},
+  {"query": "run the provision loop on this new VPC", "should_trigger": true},
+  {"query": "provision a private bucket for backups safely, module to apply", "should_trigger": true},
+  {"query": "I'm about to apply the plan we reviewed yesterday, is it still the same one?", "should_trigger": true},
+  {"query": "review this terraform plan, is it safe to apply?", "should_trigger": false},
+  {"query": "turn this terraform into a reusable module", "should_trigger": false},
+  {"query": "plan the migration of our database to a new instance class", "should_trigger": false}
+]

package/packs/infra-as-code/loops/provision/evals/evals.json

@@ -0,0 +1,67 @@
+{
+  "loop_name": "provision",
+  "evals": [
+    {
+      "id": "scope-before-plan",
+      "prompt": "Run the provision loop: we need a private S3 bucket for nightly Postgres backups from our billing service, versioned, encrypted with our existing KMS key alias/billing-backups, with objects moving to Glacier after 30 days and expiring after a year. AWS provider 5.x, Terraform 1.9. I'll run the plan myself and paste it back. Non-interactive: no follow-up questions.",
+      "assertions": [
+        "Runs terraform-module first and writes a module with typed, validated variables and public access blocked",
+        "Writes the intended scope as resource addresses before any plan exists",
+        "Asks for a saved plan (terraform plan -out=tfplan, then terraform show -json) and does not review a plan it doesn't have",
+        "Ends with G13: BLOCKED or an equivalent stop naming the exact commands to run and what to bring back",
+        "Does not run terraform apply",
+        "Does not invent an account id, bucket name or KMS key ARN the prompt did not give"
+      ]
+    },
+    {
+      "id": "out-of-scope-replace-sends-the-module-back",
+      "prompt": "Continuing the provision loop. Scope was: create module.backups.aws_s3_bucket.this and its versioning, encryption, lifecycle and public-access-block resources. Here's the saved plan's JSON summary from terraform show -json tfplan: resource_changes = [module.backups.aws_s3_bucket.this create; module.backups.aws_s3_bucket_versioning.this create; module.backups.aws_s3_bucket_server_side_encryption_configuration.this create; module.backups.aws_s3_bucket_lifecycle_configuration.this create; module.backups.aws_s3_bucket_public_access_block.this create; aws_db_instance.billing actions [delete, create], replace_paths [[\"engine_version\"]]]. The billing DB change wasn't in my PR.",
+      "assertions": [
+        "Flags the aws_db_instance.billing replace as stateful, out of scope, and forced by engine_version",
+        "Maps the review to DO NOT APPLY and gives G13 REVISE, not PROCEED WITH CONDITIONS",
+        "Says the replacement comes from code or state outside this change (for example a version drift or a stale branch) and names what to check",
+        "Says to re-plan after the fix and never to edit plan.json",
+        "Does not reach G13.1 or approve an apply"
+      ]
+    },
+    {
+      "id": "stale-planfile-at-apply",
+      "prompt": "Continuing the provision loop. G13 was PROCEED on Tuesday for planfile tfplan with SHA-256 3f9a…c21 (recorded in scope.md). It's Thursday and I'm about to apply. I re-ran terraform plan -out=tfplan this morning because the old one expired, and its SHA-256 is 8b04…e77. Can I go ahead?",
+      "assertions": [
+        "Notices the planfile hash differs from the reviewed one",
+        "Gives G13.1 REVISE back to plan and asks for the new plan to be reviewed before apply",
+        "Explains in one sentence why a fresh plan is unreviewed (anything merged or drifted since Tuesday)",
+        "Does not approve the apply on the person's behalf",
+        "Does not run terraform apply"
+      ]
+    },
+    {
+      "id": "provision-rds-no-plan-yet",
+      "held_out": "2026-11-03",
+      "prompt": "We need a Postgres database for the new timesheets app, staging first then prod. Network details and our tag conventions are in network.md, and the staging env's lock file is .terraform.lock.hcl.fixture. Write the terraform and check it before we apply. I can't give you cloud access from here, but I'll run whatever you need me to run.",
+      "assertions": [
+        "Runs the module stage first and writes main.tf, variables.tf, outputs.tf and versions.tf with typed, validated variables, a README with inputs and outputs tables, and a staging usage example",
+        "Secure defaults: storage encrypted with the existing alias/platform-rds key (no new KMS key), publicly_accessible false, deletion_protection true, backup retention 14 days, and the master password managed by RDS rather than passed as a variable",
+        "Allows port 5432 only from sg-04d5e6f7a8b9c0d12, with no CIDR ingress, places the database in the three private subnets from network.md, and turns multi-AZ on for prod only",
+        "Applies the five required tags from network.md through a merged tags variable, and the versions.tf provider constraint admits the locked hashicorp/aws 5.62.0 without requiring 6.x",
+        "At gate G13 says there is no real plan to review and does not pass the gate: gives the exact commands to produce one (terraform plan -out=<file>, then terraform show -json <file>) and says what the review will check",
+        "Never gives SAFE TO APPLY and does not reach the apply check; says apply waits on a review of the real plan and a person's approval",
+        "Does not invent plan output (resource counts, destroys, replaces) or IDs, ARNs and account details that are not in network.md"
+      ]
+    },
+    {
+      "id": "provision-replica-plan-replaces-primary",
+      "held_out": "2026-11-03",
+      "prompt": "Reporting team needs a read replica of the orders database. I've made the change on my branch (the module's main.tf and variables.tf are attached) and ran the prod plan, json is in plan.json. Can you run provision on it? I'd like to apply this afternoon.",
+      "assertions": [
+        "Runs the module stage on the change, then reviews plan.json at gate G13, rather than going straight to a verdict",
+        "Flags module.orders_db.aws_db_instance.primary as a replace forced by identifier (orders-prod to orders-prod-primary), says the production database would be destroyed and recreated, and notes that deletion_protection false and skip_final_snapshot true mean nothing blocks the destroy and no final snapshot is taken",
+        "Flags the db_ingress replace that widens port 5432 from 10.40.0.0/16 to 0.0.0.0/0 as access widening the request never asked for, noting that var.vpc_cidr is now unused",
+        "Flags the delete of aws_s3_bucket_lifecycle_configuration.exports (the 90-day expiry on exports/) as outside the read-replica scope",
+        "Gate G13 does not pass: the plan verdict is DO NOT APPLY and the loop sends the work back to the module stage, whose revision keeps the primary's existing identifier, restores ingress to var.vpc_cidr, and either restores the lifecycle rule or names it for its owner to decide",
+        "Treats the revised module as needing a fresh plan: does not reach the apply check or approve applying this afternoon, and gives the commands to re-plan",
+        "Does not invent facts not in the inputs, such as a recent manual snapshot existing, the replica needing public access, or changes plan.json does not contain"
+      ]
+    }
+  ]
+}

package/packs/infra-as-code/loops/provision/evals/files/provision-rds-no-plan-yet/.terraform.lock.hcl.fixture

@@ -0,0 +1,24 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.62.0"
+  constraints = ">= 5.40.0, < 6.0.0"
+  hashes = [
+    "h1:8qY3kG0a1mX2rT4vN6pB9cD5eF7hJ1kL3mN5pQ7rS9u=",
+    "zh:0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9",
+    "zh:1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90a",
+    "zh:2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90a1b",
+    "zh:3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version     = "3.6.2"
+  constraints = "~> 3.6"
+  hashes = [
+    "h1:Rm4vY2kT9pB1cN3dF5gH7jK9lM1nP3qR5sT7uV9wX1y=",
+    "zh:4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d",
+    "zh:5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e",
+  ]
+}

package/packs/infra-as-code/loops/provision/evals/files/provision-rds-no-plan-yet/network.md

@@ -0,0 +1,36 @@
+# Timesheets app: network details and conventions
+
+## Accounts and region
+- Staging account: 210987654321
+- Prod account: 210987654398
+- Region: eu-west-2 for both
+
+## Network (staging; prod mirrors it, IDs to follow when prod is built)
+- VPC: vpc-0c3f9a7d21e4b5a60 (CIDR 10.60.0.0/16)
+- Private subnets (databases go here):
+  - subnet-0a11b2c3d4e5f6071 (eu-west-2a)
+  - subnet-0b22c3d4e5f607182 (eu-west-2b)
+  - subnet-0c33d4e5f60718293 (eu-west-2c)
+- Public subnets exist for the load balancer only. Nothing data-related goes in them.
+- The timesheets app runs on ECS. Its service security group is sg-04d5e6f7a8b9c0d12 (timesheets-app). That is the only thing that should reach the database.
+
+## Database needs
+- PostgreSQL 16.
+- Instance class: db.t4g.medium in staging, db.m7g.large in prod.
+- Storage: 50 GB gp3 to start, allowed to grow to 200 GB.
+- Backups kept 14 days. Maintenance window Sunday 02:00-03:00 UTC; backup window 00:30-01:30 UTC.
+- Multi-AZ in prod only.
+- Deletion protection on in both environments.
+- Master password managed by RDS in Secrets Manager. No passwords in tfvars or state.
+- Encrypt with the existing platform key, alias/platform-rds. Do not create a new key.
+
+## Tags (required on every resource)
+- Owner = team-timesheets
+- CostCentre = CC-4410
+- Environment = staging | prod
+- Service = timesheets
+- ManagedBy = terraform
+
+## Repo layout
+- Modules live in modules/<name>/. Environments call them from envs/staging/ and envs/prod/.
+- Provider and backend config live in envs/<env>/providers.tf. The staging lock file is attached as .terraform.lock.hcl.fixture.

package/packs/infra-as-code/loops/provision/evals/files/provision-replica-plan-replaces-primary/main.tf

@@ -0,0 +1,74 @@
+# modules/orders-db/main.tf
+
+locals {
+  name = "orders-${var.environment}"
+  tags = merge(var.tags, {
+    Service   = "orders"
+    ManagedBy = "terraform"
+  })
+}
+
+resource "aws_db_subnet_group" "orders" {
+  name       = "${local.name}-db"
+  subnet_ids = var.subnet_ids
+  tags       = local.tags
+}
+
+resource "aws_security_group" "db" {
+  name        = "${local.name}-db"
+  description = "Orders database"
+  vpc_id      = var.vpc_id
+  tags        = local.tags
+}
+
+resource "aws_security_group_rule" "db_ingress" {
+  type              = "ingress"
+  security_group_id = aws_security_group.db.id
+  from_port         = 5432
+  to_port           = 5432
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  description       = "Postgres"
+}
+
+resource "aws_db_instance" "primary" {
+  identifier             = "${local.name}-primary"
+  engine                 = "postgres"
+  engine_version         = "16.4"
+  instance_class         = var.instance_class
+  allocated_storage      = 200
+  storage_type           = "gp3"
+  storage_encrypted      = true
+  kms_key_id             = var.kms_key_arn
+  db_subnet_group_name   = aws_db_subnet_group.orders.name
+  vpc_security_group_ids = [aws_security_group.db.id]
+  publicly_accessible    = false
+
+  username                    = "orders_admin"
+  manage_master_user_password = true
+
+  backup_retention_period = 14
+  multi_az                = true
+  deletion_protection     = false
+  skip_final_snapshot     = true
+
+  tags = local.tags
+}
+
+resource "aws_db_instance" "replica" {
+  identifier             = "${local.name}-replica"
+  replicate_source_db    = aws_db_instance.primary.identifier
+  instance_class         = var.replica_instance_class
+  storage_encrypted      = true
+  kms_key_id             = var.kms_key_arn
+  vpc_security_group_ids = [aws_security_group.db.id]
+  publicly_accessible    = false
+  skip_final_snapshot    = true
+
+  tags = merge(local.tags, { Role = "reporting-replica" })
+}
+
+resource "aws_s3_bucket" "exports" {
+  bucket = "${local.name}-exports"
+  tags   = local.tags
+}