rcf-lite 0.26.0 → 0.28.0

package/blueprints/jobs-background/contributions/probes/retry-and-fail-real-account.mjs

@@ -0,0 +1,355 @@
+/**
+ * Retry-and-fail real-account probe (live-engine coverage).
+ *
+ * Exercises REAL Cloudflare Queues end-to-end through the vendor's
+ * pull-consumer + dead-letter-queue mechanism (per
+ * https://developers.cloudflare.com/queues/configuration/pull-consumers/
+ * and https://developers.cloudflare.com/queues/configuration/dead-letter-queues/).
+ *
+ * Flow:
+ *   1. Mint a scratch queue (name minted at run time from a probe
+ *      constant plus a short random suffix; see the code below and
+ *      the run-notes.md in the evidence tree for the resolved name)
+ *      and a paired DLQ via the CF REST Queues API.
+ *   2. Attach an `http_pull` consumer to the main queue with
+ *      max_retries=3, dead_letter_queue=<dlq name>, visibility timeout
+ *      short (2 s) so the observation window stays tight. Attach an
+ *      `http_pull` consumer to the DLQ so a subsequent pull can
+ *      positively record DLQ landing.
+ *   3. Publish one message.
+ *   4. Pull, retry, repeat until the message no longer appears on the
+ *      primary queue (Cloudflare moved it to the DLQ after
+ *      max_retries+1 deliveries). Record the observed attempts on each
+ *      pull.
+ *   5. Pull the DLQ and assert the message id landed there.
+ *   6. Teardown: delete both queues; confirm each is absent from the
+ *      account queue listing.
+ *
+ * Skip on missing account creds: exactly one variable per skip row.
+ *
+ * Row anchoring:
+ *  - The account-bound skip row anchors AC-jobs-retryOnHandlerFailure
+ *    (the shipped retry AC) with `accountBoundSkipped: true` and a
+ *    `reason` naming the one unset gate variable.
+ *  - Every non-skip row is CONFORMANCE-ONLY with `anchorAcId: null`
+ *    and a `limitation` naming the nearest shipped AC id and the
+ *    property it states that this probe does NOT positively observe:
+ *      - Provisioning row -> AC-jobs-requiresQueue (states init-time
+ *        queue requirement; this row records vendor queue creation).
+ *      - Attempts-counter row -> AC-jobs-retryOnHandlerFailure (states
+ *        the applied in-memory queue redelivery and three jobStarted
+ *        events on the sink; this row records Cloudflare Queues'
+ *        vendor attempts counter, not the sink events).
+ *      - DLQ landing row -> AC-30109-1 (states DLQ-producer invocation
+ *        on the in-memory driver; this row records real Cloudflare
+ *        Queues DLQ landing).
+ *      - Teardown row -> AC-jobs-requiresQueue (no AC states scratch
+ *        teardown; this row records DELETE + post-run absence).
+ */
+
+export const accountBound = true;
+export const DECLARED_ENV = Object.freeze([
+  'CI_HAS_CLOUDFLARE_ACCOUNT',
+  'CF_ACCOUNT_ID',
+  'CF_API_TOKEN',
+  'CF_API_BASE',
+]);
+
+// CF_API_BASE is a required declared variable per the endpoint-hosts rule;
+// no literal endpoint host in probe source. The runProbe() gate checks
+// it before use and returns an exact one-variable skip when unset.
+
+const AC_RETRY_FIRST8 = 'With SIMULATE_HANDLER_THROW=true set on the shared sample-app fixture,';
+const AC_REQUIRES_FIRST8 = 'On a fresh init scratch project with NO';
+const AC_PROVISION_LIMITATION = 'AC-jobs-requiresQueue: On a fresh init scratch project with NO messaging-queue provider applied, rcf-lite refuses to apply jobs-background with the stable message id jobs-background-no-queue. Not observed on this row: this row records vendor queue and DLQ provisioning on a real Cloudflare account, which the AC does not state (its property is CLI refusal at init time, not vendor-side queue creation).';
+const AC_ATTEMPTS_LIMITATION = 'AC-jobs-retryOnHandlerFailure: With SIMULATE_HANDLER_THROW=true set on the shared sample-app fixture, the retry-and-fail probe schedules a job whose handler throws a retryable error; the applied in-memory queue redelivers the message per retryPolicy.maxAttempts. Three jobStarted events fire on the sink with the same jobId and attempts counter 1, 2, 3; after the third failure jobFailed fires with a terminal error code and no further re-delivery follows. Not observed on this row: the AC states applied in-memory queue redelivery and jobStarted-on-sink events; this row records the vendor Cloudflare Queues attempts counter across pulls [0,1,2,3], not the jobStarted sink events the AC states.';
+const AC_DLQ_LIMITATION = 'AC-30109-1: The retry-and-fail probe report, after the three failing attempts, records the DLQ-producer invocation on the in-memory driver via a jobDeadLettered event carrying the terminal job id and the original job body. Not observed on this row: the AC pins the in-memory driver DLQ-producer invocation; this row records real Cloudflare Queues DLQ landing (backlog + payload-id match on a real DLQ), which the AC does not state.';
+const AC_TEARDOWN_LIMITATION = 'AC-jobs-requiresQueue: On a fresh init scratch project with NO messaging-queue provider applied, rcf-lite refuses to apply jobs-background with the stable message id jobs-background-no-queue. Not observed on this row: this row records scratch Cloudflare Queues DELETE + post-run absence via the account queue listing, which no jobs-background AC or REQ states; the AC used as a signpost is the nearest shipped AC on queue existence.';
+
+function skipRow(reason) {
+  return {
+    results: [{
+      anchorAcId: 'AC-jobs-retryOnHandlerFailure',
+      verdict: 'pass',
+      detail: `${AC_RETRY_FIRST8} - accountBound: skipped (${reason})`,
+      accountBoundSkipped: true,
+      reason,
+      evidence: { skip: true, reason, envDeclared: [...DECLARED_ENV] },
+    }],
+    extra: { accountBoundSkipped: true, reason, envDeclared: [...DECLARED_ENV] },
+  };
+}
+
+async function cf(apiBase, method, path, token, body) {
+  const res = await fetch(`${apiBase}${path}`, {
+    method,
+    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+    body: body === undefined ? undefined : JSON.stringify(body),
+  });
+  const text = await res.text();
+  let json = null;
+  try { json = JSON.parse(text); } catch { /* keep raw */ }
+  return { httpStatus: res.status, json, raw: text };
+}
+
+function shortId() {
+  return Math.random().toString(36).slice(2, 8);
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+export default async function runProbe() {
+  const gate = process.env.CI_HAS_CLOUDFLARE_ACCOUNT;
+  if (gate == null || gate === '') return skipRow('CI_HAS_CLOUDFLARE_ACCOUNT unset');
+  if (gate !== 'true') return skipRow('CI_HAS_CLOUDFLARE_ACCOUNT (not "true")');
+  if (!process.env.CF_ACCOUNT_ID) return skipRow('CF_ACCOUNT_ID unset');
+  if (!process.env.CF_API_TOKEN) return skipRow('CF_API_TOKEN unset');
+  if (!process.env.CF_API_BASE) return skipRow('CF_API_BASE unset');
+
+  const API_BASE = process.env.CF_API_BASE;
+  const accountId = process.env.CF_ACCOUNT_ID;
+  const token = process.env.CF_API_TOKEN;
+  const short = shortId();
+  const queueName = `qa-e-jobs-q-${short}`;
+  const dlqName = `qa-e-jobs-dlq-${short}`;
+  const results = [];
+  const teardown = { deletePrimary: null, deleteDlq: null, primaryAbsent: null, dlqAbsent: null };
+  let qid = null;
+  let dlqId = null;
+  try {
+    // 1. mint DLQ first, then the main queue (main queue's consumer
+    // config references the DLQ by name).
+    const dlqCreate = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues`, token, { queue_name: dlqName });
+    dlqId = dlqCreate.json && dlqCreate.json.result && dlqCreate.json.result.queue_id;
+    if (!dlqId) throw new Error(`DLQ create failed: httpStatus=${dlqCreate.httpStatus} body=${dlqCreate.raw}`);
+    const dlqConsumer = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${dlqId}/consumers`, token, {
+      type: 'http_pull',
+      settings: { batch_size: 10, visibility_timeout_ms: 10000, max_retries: 3, retry_delay: 0 },
+    });
+    if (!(dlqConsumer.json && dlqConsumer.json.success)) throw new Error(`DLQ consumer attach failed: ${dlqConsumer.raw}`);
+
+    const primaryCreate = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues`, token, { queue_name: queueName });
+    qid = primaryCreate.json && primaryCreate.json.result && primaryCreate.json.result.queue_id;
+    if (!qid) throw new Error(`primary create failed: httpStatus=${primaryCreate.httpStatus} body=${primaryCreate.raw}`);
+    const primaryConsumer = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${qid}/consumers`, token, {
+      type: 'http_pull',
+      dead_letter_queue: dlqName,
+      settings: { batch_size: 10, visibility_timeout_ms: 2000, max_retries: 3, retry_delay: 0 },
+    });
+    if (!(primaryConsumer.json && primaryConsumer.json.success)) throw new Error(`primary consumer attach failed: ${primaryConsumer.raw}`);
+
+    results.push({
+      anchorAcId: null,
+      conformanceOnly: true,
+      limitation: AC_PROVISION_LIMITATION,
+      verdict: 'pass',
+      detail: `conformanceOnly (${AC_PROVISION_LIMITATION}) - provisioned scratch queue ${queueName} (id=${qid}) with DLQ ${dlqName} (id=${dlqId}), both with http_pull consumers, max_retries=3`,
+      evidence: {
+        queueName, dlqName,
+        primaryConsumerResponseMetadata: primaryConsumer.json && primaryConsumer.json.result,
+        dlqConsumerResponseMetadata: dlqConsumer.json && dlqConsumer.json.result,
+      },
+    });
+
+    // 3. publish one message
+    const publishJobId = `job-${short}`;
+    const pubBody = { jobId: publishJobId, jobName: 'send-welcome-email', jobInput: { userId: 7, email: 'ok@example.com' } };
+    const pub = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${qid}/messages`, token, { body: pubBody, content_type: 'json' });
+    if (!(pub.json && pub.json.success)) throw new Error(`publish failed: ${pub.raw}`);
+
+    // 4. pull + retry loop; observe attempts increasing to 3, then absent
+    const attemptsObserved = [];
+    let messageIdObserved = null;
+    for (let i = 0; i < 20; i += 1) {
+      const pull = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${qid}/messages/pull`, token, {
+        batch_size: 10,
+        visibility_timeout_ms: 2000,
+      });
+      const msgs = (pull.json && pull.json.result && pull.json.result.messages) || [];
+      if (msgs.length === 0) {
+        // Wait a bit; message may have been consumed by the retry cycle
+        // and not yet visible again, OR moved to DLQ.
+        attemptsObserved.push({ pulled: 0 });
+        await sleep(2500);
+        continue;
+      }
+      const m = msgs[0];
+      messageIdObserved = m.id;
+      attemptsObserved.push({ pulled: msgs.length, attempts: m.attempts, id: m.id });
+      // Retry: tell CF to reschedule with retry_delay=0
+      const ack = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${qid}/messages/ack`, token, {
+        retries: [{ lease_id: m.lease_id, delay_seconds: 0 }],
+      });
+      if (!(ack.json && ack.json.success)) throw new Error(`ack retry failed: ${ack.raw}`);
+      await sleep(500);
+    }
+    // After the loop, message should be either absent (moved to DLQ) or
+    // in the DLQ. Poll the DLQ a few times to allow for the vendor's
+    // internal retry-to-DLQ transfer window.
+    let dlqMsgs = [];
+    let dlqBacklog = 0;
+    let dlqPullAttempts = 0;
+    for (let dp = 0; dp < 10; dp += 1) {
+      await sleep(2000);
+      dlqPullAttempts += 1;
+      const dlqPull = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${dlqId}/messages/pull`, token, {
+        batch_size: 10,
+        visibility_timeout_ms: 30000,
+      });
+      dlqMsgs = (dlqPull.json && dlqPull.json.result && dlqPull.json.result.messages) || [];
+      dlqBacklog = (dlqPull.json && dlqPull.json.result && dlqPull.json.result.message_backlog_count) || dlqMsgs.length;
+      if (dlqMsgs.length > 0 || dlqBacklog >= 1) break;
+    }
+    // Cloudflare Queues assigns fresh transport message ids when a
+    // message lands in the DLQ, so a strict transport id-match is not
+    // achievable at the vendor surface. The probe carries its own
+    // application-level correlation id in the message body payload
+    // (`publishJobId`) and matches on that: the row is only a pass
+    // when the same payload jobId that was published to the primary
+    // is observed on the DLQ landing.
+    const dlqPayloadIds = dlqMsgs.map((mm) => (mm.body && (typeof mm.body === 'string'
+      ? (() => { try { return JSON.parse(mm.body).jobId; } catch { return null; } })()
+      : mm.body.jobId))).filter(Boolean);
+    const idMatch = dlqPayloadIds.includes(publishJobId);
+    const seenInDlq = idMatch; // strict: payload correlation-id match required
+    // Ack the DLQ message so it doesn't loiter (deletion also happens
+    // in teardown). Ack failures are NOT swallowed - they land on
+    // teardown.dlqAcks so a failure surfaces on the teardown row.
+    teardown.dlqAcks = [];
+    for (const mm of dlqMsgs) {
+      try {
+        const ack = await cf(API_BASE, 'POST', `/accounts/${accountId}/queues/${dlqId}/messages/ack`, token, {
+          acks: [{ lease_id: mm.lease_id }],
+        });
+        teardown.dlqAcks.push({
+          lease_id: mm.lease_id,
+          httpStatus: ack.httpStatus,
+          success: !!(ack.json && ack.json.success === true),
+        });
+      } catch (err) {
+        teardown.dlqAcks.push({ lease_id: mm.lease_id, ok: false, error: err && err.message });
+      }
+    }
+
+    // Assertion 1: observed attempts sequence includes non-zero values
+    // (real engine incremented the counter).
+    const uniqueAttempts = [...new Set(attemptsObserved.filter((a) => a.attempts != null).map((a) => a.attempts))].sort((a, b) => a - b);
+    // REQ-004 states the applied queue redelivers per its own retry
+    // contract (Cloudflare Queues ceiling 100 per ADR-3003) and the
+    // jobs runtime records the attempt counter on each jobStarted
+    // event. With max_retries=3 the vendor's attempts counter must
+    // reach exactly the [0..3] consecutive sequence (initial delivery
+    // 0 plus three retries 1, 2, 3). A gapped sequence such as [0,2,3]
+    // fails the row.
+    const expectedAttempts = [0, 1, 2, 3];
+    const sawRetries = uniqueAttempts.length === expectedAttempts.length
+      && uniqueAttempts.every((v, i) => v === expectedAttempts[i]);
+    results.push({
+      anchorAcId: null,
+      conformanceOnly: true,
+      limitation: AC_ATTEMPTS_LIMITATION,
+      verdict: sawRetries ? 'pass' : 'fail',
+      detail: sawRetries
+        ? `conformanceOnly (AC-jobs-retryOnHandlerFailure: With SIMULATE_HANDLER_THROW=true set on the shared sample-app fixture) - Cloudflare Queues incremented the vendor attempts counter across pulls: observed attempts values ${JSON.stringify(uniqueAttempts)}`
+        : `conformanceOnly (AC-jobs-retryOnHandlerFailure: With SIMULATE_HANDLER_THROW=true set on the shared sample-app fixture) - did not observe the vendor attempts counter increment across pulls; attemptsObserved=${JSON.stringify(attemptsObserved)}`,
+      evidence: { attemptsObserved, uniqueAttempts, messageId: messageIdObserved, queueName },
+    });
+
+    // Assertion 2: message ended up on DLQ (positive dead-letter
+    // landing in a real Cloudflare Queue).
+    results.push({
+      anchorAcId: null,
+      conformanceOnly: true,
+      limitation: AC_DLQ_LIMITATION,
+      verdict: seenInDlq ? 'pass' : 'fail',
+      detail: seenInDlq
+        ? `conformanceOnly (${AC_DLQ_LIMITATION}) - after ${attemptsObserved.length} primary pulls and ${dlqPullAttempts} DLQ polls the message landed on the DLQ ${dlqName} (backlog=${dlqBacklog}, payload-id-match=true for ${publishJobId})`
+        : `conformanceOnly (${AC_DLQ_LIMITATION}) - published payload jobId ${publishJobId} did not appear on the DLQ ${dlqName} within ${dlqPullAttempts} polls; dlqBacklog=${dlqBacklog} observedDlqPayloadIds=${JSON.stringify(dlqPayloadIds)} primary pulls=${attemptsObserved.length}`,
+      evidence: { dlqName, dlqId, dlqBacklog, dlqTransportMessageIds: dlqMsgs.map((mm) => mm.id), dlqPayloadJobIds: dlqPayloadIds, expectedPayloadJobId: publishJobId, primaryTransportMessageId: messageIdObserved, payloadIdMatch: idMatch, dlqPollAttempts: dlqPullAttempts },
+    });
+  } finally {
+    // 6. Teardown: delete both queues; confirm absent from listing.
+    // A teardown failure FAILS the verdict (authoring-standard rule 5).
+    if (qid) {
+      try {
+        const d = await cf(API_BASE, 'DELETE', `/accounts/${accountId}/queues/${qid}`, token);
+        teardown.deletePrimary = { queueId: qid, ok: d.json && d.json.success === true, httpStatus: d.httpStatus };
+      } catch (err) {
+        teardown.deletePrimary = { queueId: qid, ok: false, error: err && err.message };
+      }
+    }
+    if (dlqId) {
+      try {
+        const d = await cf(API_BASE, 'DELETE', `/accounts/${accountId}/queues/${dlqId}`, token);
+        teardown.deleteDlq = { queueId: dlqId, ok: d.json && d.json.success === true, httpStatus: d.httpStatus };
+      } catch (err) {
+        teardown.deleteDlq = { queueId: dlqId, ok: false, error: err && err.message };
+      }
+    }
+    // Confirm absence via a queue list. Assert HTTP status 200 AND
+    // json.success === true before treating the listing as authoritative;
+    // record both on the teardown so an unsuccessful response never
+    // normalises to an empty list (absence must be positive evidence).
+    // Cloudflare Queues DELETE is asynchronous - the listing can still
+    // carry the queue for a few seconds after DELETE 200. Poll the
+    // listing up to ~30 s (six 5-second sleeps) before accepting the
+    // result as authoritative. The final observed listing (with the
+    // number of polls it took) lands on teardown.listing.
+    let listingPolls = 0;
+    let list = null;
+    try {
+      for (let i = 0; i < 6; i += 1) {
+        listingPolls += 1;
+        list = await cf(API_BASE, 'GET', `/accounts/${accountId}/queues`, token);
+        const httpOk = list.httpStatus === 200;
+        const successFlag = !!(list.json && list.json.success === true);
+        if (!httpOk || !successFlag) break;
+        const nms = (list.json.result || []).map((q) => q.queue_name);
+        if (!nms.includes(queueName) && !nms.includes(dlqName)) break;
+        await sleep(5000);
+      }
+      const httpOk = list && list.httpStatus === 200;
+      const successFlag = !!(list && list.json && list.json.success === true);
+      const listingOk = httpOk && successFlag;
+      const names = listingOk
+        ? ((list.json && list.json.result) || []).map((q) => q.queue_name)
+        : [];
+      teardown.listing = {
+        httpStatus: list ? list.httpStatus : null,
+        success: successFlag,
+        listingOk,
+        listedCount: names.length,
+        pollsTaken: listingPolls,
+        errors: list && list.json && list.json.errors ? list.json.errors : null,
+      };
+      teardown.primaryAbsent = listingOk
+        ? { ok: !names.includes(queueName), listedCount: names.length, httpStatus: list.httpStatus, success: successFlag }
+        : { ok: false, listedCount: names.length, httpStatus: list.httpStatus, success: successFlag, note: 'listing unsuccessful; absence cannot be asserted' };
+      teardown.dlqAbsent = listingOk
+        ? { ok: !names.includes(dlqName), listedCount: names.length, httpStatus: list.httpStatus, success: successFlag }
+        : { ok: false, listedCount: names.length, httpStatus: list.httpStatus, success: successFlag, note: 'listing unsuccessful; absence cannot be asserted' };
+    } catch (err) {
+      teardown.listing = { httpStatus: null, success: false, listingOk: false, error: err && err.message };
+      teardown.primaryAbsent = { ok: false, error: err && err.message };
+      teardown.dlqAbsent = { ok: false, error: err && err.message };
+    }
+    const dlqAcksOk = Array.isArray(teardown.dlqAcks)
+      && teardown.dlqAcks.every((a) => a.success === true);
+    const teardownOk = teardown.deletePrimary && teardown.deletePrimary.ok
+      && teardown.deleteDlq && teardown.deleteDlq.ok
+      && teardown.primaryAbsent && teardown.primaryAbsent.ok
+      && teardown.dlqAbsent && teardown.dlqAbsent.ok
+      && dlqAcksOk;
+    results.push({
+      anchorAcId: null,
+      conformanceOnly: true,
+      limitation: AC_TEARDOWN_LIMITATION,
+      verdict: teardownOk ? 'pass' : 'fail',
+      detail: teardownOk
+        ? `conformanceOnly (${AC_TEARDOWN_LIMITATION}) - scratch queues ${queueName} and ${dlqName} deleted and confirmed absent from post-run account queue listing`
+        : `conformanceOnly (${AC_TEARDOWN_LIMITATION}) - queue teardown FAILED: ${JSON.stringify(teardown)}`,
+      evidence: { teardown },
+    });
+  }
+  return { results, extra: { envDeclared: [...DECLARED_ENV], queueName, dlqName, teardown } };
+}

package/blueprints/jobs-background/contributions/probes/retry-and-fail.mjs

@@ -19,6 +19,8 @@ import { createScheduler, createFakeClock } from '../../../../packages/rcf-lite/
 import { PROJECT_ROOT } from './probe-utils.mjs';
 import { resolve } from 'node:path';
 
+const AC_FIRST8 = 'With SIMULATE_HANDLER_THROW=true set on the shared sample-app fixture,';
+
 export default async function runProbe() {
   const cfg = queueConfigFromEnv();
   // maxRetries on the driver matches send-welcome-email.maxAttempts=3.
@@ -54,12 +56,41 @@ export default async function runProbe() {
   const oneJobId = jobIds.size === 1;
   const failedOk = !!jobFailed && typeof jobFailed.terminalErrorCode === 'string';
   const pass = attemptsOk && oneJobId && failedOk;
+  const scalarJobId = jobIds.size > 0 ? [...jobIds][0] : null;
   results.push({
     anchorAcId: 'AC-jobs-retryOnHandlerFailure',
     verdict: pass ? 'pass' : 'fail',
     detail: pass
-      ? `three jobStarted events attempts=[1,2,3] same jobId; jobFailed carries terminalErrorCode=${jobFailed.terminalErrorCode}`
-      : `attemptsSeq=${JSON.stringify(attemptsSeq)}; jobIds.size=${jobIds.size}; jobFailed=${JSON.stringify(jobFailed)}`,
+      ? `${AC_FIRST8} - three jobStarted events attempts=[1,2,3] same jobId; jobFailed carries terminalErrorCode=${jobFailed.terminalErrorCode}`
+      : `${AC_FIRST8} - attemptsSeq=${JSON.stringify(attemptsSeq)}; jobIds.size=${jobIds.size}; jobFailed=${JSON.stringify(jobFailed)}`,
+    evidence: {
+      jobId: scalarJobId,
+      attemptsSequence: attemptsSeq,
+      distinctJobIdList: [...jobIds],
+      jobStartedCount: jobStartedEvents.length,
+      terminalJobFailed: jobFailed || null,
+      dlqInvoked,
+    },
+  });
+  // AC-30109-1: after the three failing jobStarted events and the
+  // terminal jobFailed event, the underlying in-memory queue driver
+  // invoked the DLQ producer path. Observed as `dlqInvoked:true`
+  // (state.dlq carries the failing message id after the drain).
+  const dlqEntry = pair.state.dlq[0] || null;
+  const dlqPass = dlqInvoked === true && failedOk;
+  results.push({
+    anchorAcId: 'AC-30109-1',
+    verdict: dlqPass ? 'pass' : 'fail',
+    detail: dlqPass
+      ? `The retry-and-fail probe's report, after the three failing jobStarted events and the terminal jobFailed event, records that the in-memory queue driver invoked the DLQ producer path for the failing message id: AC-30109-1 observed with dlqInvoked:true; terminal jobFailed event carries terminalErrorCode=${jobFailed.terminalErrorCode}; the failing message id was routed to state.dlq after maxAttempts exhausted`
+      : `The retry-and-fail probe's report, after the three failing jobStarted events and the terminal jobFailed event, did not record dlqInvoked true: AC-30109-1 not observed - dlqInvoked=${dlqInvoked} jobFailed=${JSON.stringify(jobFailed)}`,
+    evidence: {
+      jobId: scalarJobId,
+      dlqInvoked,
+      dlqLength: pair.state.dlq.length,
+      attemptsSequence: attemptsSeq,
+      terminalJobFailed: jobFailed || null,
+    },
   });
   return { results, extra: { events, dlqInvoked, attemptsSeq } };
 }

package/blueprints/jobs-background/contributions/probes/run-retry-and-fail-real-account.mjs

@@ -0,0 +1,12 @@
+/**
+ * Shim: run retry-and-fail-real-account against the the real-account credentials
+ * Cloudflare account. Records real HTTP status + per-pull observations
+ * as engine descriptor.
+ */
+import runProbe from './retry-and-fail-real-account.mjs';
+import { runShim } from './probe-utils.mjs';
+
+const engine = process.env.CI_HAS_CLOUDFLARE_ACCOUNT === 'true'
+  ? { kind: 'queue', vendor: 'cloudflare-queues', observed: 'per-request on results' }
+  : { kind: 'queue', vendor: 'cloudflare-queues', observed: false, reason: 'CI_HAS_CLOUDFLARE_ACCOUNT unset' };
+runShim('retry-and-fail-real-account', engine, runProbe);

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-001.json

@@ -2,7 +2,7 @@
   "reqId": "jobs-background-REQ-001",
   "prdId": "PRD-001",
   "title": "Requires an applied queue capability; refuses apply on a bare project with exit 3 and the stable message id jobs-background-no-queue",
-  "description": "The jobs-background blueprint composes on an applied queue capability via the T-5 mechanism from the visual round. The apply verb refuses on a bare project (no applied blueprint declares capabilities: [queue]) with exit code 3 and a stable, dashboard-friendly message id jobs-background-no-queue on the first stderr line. The refusal message also names the shipped provider messaging-queue-cloudflare and the override flag --allow-no-queue-yet explicitly, per infra round 5 spec section 10 Q3 default asserting BOTH exit code AND stable message id. The override flag records a note on the sidecar rcf/blueprints/jobs-background.applied.json under the queue family so rcf define validate later flags surfaces gated on queue that never activated.",
+  "description": "The jobs-background blueprint composes on an applied queue capability via the applied-capability mechanism. The apply verb refuses on a bare project (no applied blueprint declares capabilities: [queue]) with exit code 3 and a stable, dashboard-friendly message id jobs-background-no-queue on the first stderr line. The refusal message also names the shipped provider messaging-queue-cloudflare and the override flag --allow-no-queue-yet explicitly, asserting both the exit code and the stable message id. The override flag records a note on the sidecar rcf/blueprints/jobs-background.applied.json under the queue family so rcf define validate later flags surfaces gated on queue that never activated.",
   "category": "functional",
   "domain": "jobs",
   "priority": "must",
@@ -11,8 +11,12 @@
     "blueprint:jobs-background",
     "category:00-composition"
   ],
-  "version": "1.0.0",
+  "version": "1.0.1",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-10T00:00:00.000Z",
+  "deliveredBy": {
+    "adrId": "ADR-3103-jobs-background-requires-applied-capabilities",
+    "field": "decision.refusalMessageId"
+  }
 }

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-002.json

@@ -2,7 +2,7 @@
   "reqId": "jobs-background-REQ-002",
   "prdId": "PRD-001",
   "title": "Job-definition contract: name, handler, input schema (opaque per project), retry policy, timeout",
-  "description": "A job definition is a module in the applying project's jobs directory (default ./jobs/, elicited at apply time) exporting a shape { name, handler, inputSchema, retryPolicy, timeoutMs }. Name is a stable string; handler is an async function receiving the message body and returning a value or throwing; inputSchema is opaque per project (the applying project's own validation library); retryPolicy is { maxAttempts, backoff }; timeoutMs is a positive integer (default 60000, floor 1000, ceiling bounded by the applied queue consumer wall-clock ceiling per ADR-3104). The jobs runtime registers every job at boot on the applied queue consumer and dispatches by name.",
+  "description": "A job definition is a module in the applying project's jobs directory (default ./jobs/, elicited at apply time) exporting the shape owned on TAC-3101-jobs-background-job-definition.interfaces.jobModule.default. The jobs runtime registers every job at boot on the applied queue consumer and dispatches by name. This blueprint declares the backgroundJobs applied capability at runtime: the registration entry point walks the elicited jobs directory once at boot, and the applied surface exposes the resulting set through the operator-facing surface elicited on the manifest. The handler-timeout-seconds is elicited at apply time per ADR-3104 (default 60; floor 1; ceiling bounded by the applied queue's consumer wall-clock). A handler exceeding its timeout raises a retryable timeout error routed through the queue redelivery mechanism.",
   "category": "functional",
   "domain": "jobs",
   "priority": "must",
@@ -11,8 +11,12 @@
     "blueprint:jobs-background",
     "category:01-definition"
   ],
-  "version": "1.0.0",
+  "version": "1.0.2",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-10T00:00:00Z",
+  "deliveredBy": {
+    "tacId": "TAC-3101-jobs-background-job-definition",
+    "field": "interfaces.jobModule.default"
+  }
 }

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-003.json

@@ -14,5 +14,9 @@
   "version": "1.0.0",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-09T00:00:00Z",
+  "deliveredBy": {
+    "tacId": "TAC-3102-jobs-background-scheduler",
+    "field": "responsibilities.cronAndDelayed"
+  }
 }

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-004.json

@@ -14,5 +14,9 @@
   "version": "1.0.0",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-09T00:00:00Z",
+  "deliveredBy": {
+    "tacId": "TAC-3101-jobs-background-job-definition",
+    "field": "responsibilities.retry"
+  }
 }

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-005.json

@@ -14,5 +14,9 @@
   "version": "1.0.0",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-09T00:00:00Z",
+  "deliveredBy": {
+    "tacId": "TAC-3103-jobs-background-run-log",
+    "field": "responsibilities.fourEvents"
+  }
 }

package/blueprints/jobs-background/contributions/requirements/jobs-background-req-006.json

@@ -14,5 +14,9 @@
   "version": "1.0.0",
   "status": "approved",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-09T00:00:00Z",
+  "deliveredBy": {
+    "adrId": "ADR-3102-jobs-background-scheduler-mode",
+    "field": "decision.operatorSurface"
+  }
 }

package/blueprints/jobs-background/contributions/tacs/tac-3101-jobs-background-job-definition.json

@@ -2,12 +2,12 @@
   "tacId": "TAC-3101-jobs-background-job-definition",
   "prdId": "PRD-001",
   "tadId": "TAD-001",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "status": "approved",
   "name": "Jobs-background job-definition module contract",
-  "purpose": "The shape every job-definition module in the applying project's jobs directory exports. One shape carries name, handler, inputSchema (opaque per project), retryPolicy, and timeoutMs. The jobs runtime registers every job at boot on the applied queue consumer and dispatches by name.",
+  "purpose": "The shape every job-definition module in the applying project's jobs directory exports, owned on interfaces.jobModule.default below. The jobs runtime registers every job at boot on the applied queue consumer and dispatches by name.",
   "responsibilities": [
-    "Declare a job by exporting { name, handler, inputSchema, retryPolicy, timeoutMs } (AC-30102-1, AC-jobs-retryOnHandlerFailure).",
+    "Declare a job by exporting the shape owned on interfaces.jobModule.default below (AC-30102-1, AC-jobs-retryOnHandlerFailure).",
     "Bound handler wall-clock via timeoutMs; jobs runtime cancels a handler that exceeds timeoutMs and treats it as a retryable error (AC-jobs-retryOnHandlerFailure).",
     "Respect the applied queue's max-attempts ceiling; retryPolicy.maxAttempts must not exceed the ceiling (AC-jobs-retryOnHandlerFailure).",
     "Refuse composition on a bare project via the requiresAppliedCapabilities gate (AC-jobs-requiresQueue)."
@@ -41,10 +41,10 @@
     {
       "name": "Applied queue capability",
       "kind": "external",
-      "description": "Composed on the applied blueprint declaring capabilities: [queue] via the T-5 mechanism; refuses apply on a bare project per REQ-001."
+      "description": "Composed on the applied blueprint declaring capabilities: [queue] via the applied-capability mechanism; refuses apply on a bare project per REQ-001."
     }
   ],
   "tradeoffs": "The v1.0.0 job-definition shape does not prescribe an inputSchema validator; the applying project owns validation. A later minor could add an optional validate hook if a shipped validation library becomes the shelf default. The dispatch table is by job name; two modules exporting the same name collide at boot and refuse.",
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-10T00:00:00.000Z"
 }

package/blueprints/jobs-background/contributions/user-stories/jobs-background-us-30101.json

@@ -16,10 +16,27 @@
       "when": "rcf define blueprint add ./blueprints/jobs-background runs",
       "then": "exit code is 3; stderr first-line grep matches [jobs-background-no-queue]; stderr grep matches messaging-queue-cloudflare; stderr grep matches --allow-no-queue-yet; the manifest has no jobs-background entry",
       "testable": true,
-      "scope": "runtime"
+      "scope": "runtime",
+      "disposition": "template"
+    },
+    {
+      "id": "AC-30101-2",
+      "description": "--allow-no-queue-yet override lands sidecar note. Given a bare project without an applied queue capability, when apply runs with --allow-no-queue-yet, then apply succeeds with exit 0, writes rcf/blueprints/jobs-background.applied.json with allowNoAuthYet: true, appliedCapabilities: [], notes matching /no queue yet/ and /--allow-no-queue-yet/, and the operator can then apply a queue provider to lift the sidecar note on the next apply cycle.",
+      "given": "a bare project without an applied queue capability",
+      "when": "apply runs with --allow-no-queue-yet",
+      "then": "apply exits 0; sidecar written with allowNoAuthYet: true, empty appliedCapabilities, notes naming \"no queue yet\" and the flag",
+      "testable": true,
+      "scope": "runtime",
+      "ownerRef": {
+        "adrId": "ADR-3103-jobs-background-requires-applied-capabilities",
+        "field": "decision.allowSkipFlag"
+      },
+      "disposition": "fixed"
     }
   ],
-  "tacIds": ["TAC-3101-jobs-background-job-definition"],
+  "tacIds": [
+    "TAC-3101-jobs-background-job-definition"
+  ],
   "createdAt": "2026-09-06T20:30:00.000Z",
-  "updatedAt": "2026-09-06T20:30:00.000Z"
+  "updatedAt": "2026-09-09T00:00:00Z"
 }