tokens-for-good 0.4.21 → 0.4.22

package/README.md

@@ -29,7 +29,7 @@ Each org takes about 5 minutes:
 2. **Verify** — every citation URL checked, hallucinations flagged and corrected
 3. **Humanize** — voice pass (no em dashes, no AI-tells, analyst voice)
 
-Under v3 dual-research, every org is researched by two contributors independently; a third contributor consolidates both reports' EVIDENCE TABLEs into one merged table that the server scores deterministically (out of 120). A human reviewer finalizes it for the directory.
+Under v3 dual-research, every org is researched by two contributors independently. An independent validator then prunes any unsupported or fabricated evidence from both reports — reading the server's cached page text, so it does zero web fetches — and a consolidator merges the validated EVIDENCE TABLEs into one table the server scores deterministically (out of 120). A human reviewer finalizes it for the directory.
 
 ## Contributor tiers
 
@@ -46,10 +46,12 @@ Once installed, these are available to your AI via the MCP server:
 
 | Tool | Purpose |
 |---|---|
-| `next_action` | Tells you whether to research or consolidate next |
+| `next_action` | Tells you whether to validate, consolidate, or research next |
 | `claim_org` | Reserves the next available nonprofit (up to 2 researchers per org under v3) |
-| `get_methodology` | Fetches research / verify / humanize / consolidate instructions |
+| `get_methodology` | Fetches research / verify / humanize / validate / consolidate instructions |
 | `submit_report` | Submits a finished report (or a merged consolidation report with `disagreement_rows`) |
+| `get_next_validation` | v3 validator: fetch both reports + cached page text to prune unsupported evidence |
+| `submit_validation` | v3 validator: submit corrected reports (subtract/correct only, never add) |
 | `get_next_consolidation` | v3 consolidator: fetch your assignment + both source reports to merge |
 | `setup_automation` | Emits `/schedule` prompt (normally called by `/tfg-schedule` skill) |
 | `my_impact` / `research_status` / `get_badge` | Stats, leaderboard, GitHub README badge |

package/package.json

@@ -1,45 +1,45 @@
-{
-  "name": "tokens-for-good",
-  "version": "0.4.21",
-  "type": "module",
-  "description": "Donate your spare AI tokens to research nonprofits for Fierce Philanthropy",
-  "homepage": "https://tokensforgood.ai",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/Tokens-for-Good/tokens-for-good.git"
-  },
-  "bugs": {
-    "url": "https://github.com/Tokens-for-Good/tokens-for-good/issues"
-  },
-  "bin": {
-    "tokens-for-good": "src/cli.js"
-  },
-  "main": "./src/mcp-server.js",
-  "files": [
-    "src/",
-    "pipeline/",
-    "skills/",
-    "LICENSE"
-  ],
-  "scripts": {
-    "start": "node src/mcp-server.js",
-    "test": "node --test src/**/*.test.js"
-  },
-  "keywords": [
-    "mcp",
-    "philanthropy",
-    "nonprofit",
-    "research",
-    "tokens",
-    "social-impact"
-  ],
-  "license": "MIT",
-  "engines": {
-    "node": ">=18"
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.0",
-    "prompts": "^2.4.2",
-    "zod": "^4.3.6"
-  }
-}
+{
+  "name": "tokens-for-good",
+  "version": "0.4.22",
+  "type": "module",
+  "description": "Donate your spare AI tokens to research nonprofits for Fierce Philanthropy",
+  "homepage": "https://tokensforgood.ai",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Tokens-for-Good/tokens-for-good.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Tokens-for-Good/tokens-for-good/issues"
+  },
+  "bin": {
+    "tokens-for-good": "src/cli.js"
+  },
+  "main": "./src/mcp-server.js",
+  "files": [
+    "src/",
+    "pipeline/",
+    "skills/",
+    "LICENSE"
+  ],
+  "scripts": {
+    "start": "node src/mcp-server.js",
+    "test": "node --test src/**/*.test.js"
+  },
+  "keywords": [
+    "mcp",
+    "philanthropy",
+    "nonprofit",
+    "research",
+    "tokens",
+    "social-impact"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "prompts": "^2.4.2",
+    "zod": "^4.3.6"
+  }
+}

package/pipeline/04-validate/PROMPT.md

@@ -0,0 +1,58 @@
+# Validate Two Independent Research Reports (v3)
+
+## Your Role
+
+You are a validator — an independent fourth contributor who sits between the two researchers and the consolidator. Two researchers independently researched the same organization; before their reports are merged and scored, your job is to catch **unsupported or fabricated evidence** and hand back cleaned-up ("validated") versions of both reports.
+
+You do **no web research and no fetching**. The server already fetched every page the researchers cited and gives you the cached text. You validate using ONLY that cached text. This is deliberate: it makes validation cheap and possible even on a local model with no scraping budget.
+
+## Why this stage exists
+
+A report's score comes entirely from its EVIDENCE TABLE — eight rows, each a quote + a source URL. The most dangerous failure isn't a thin report; it's a confident one citing a quote that **isn't actually on the page**, or a page that **doesn't exist**. That evidence would score real points it didn't earn. The server already runs a deterministic check (does the quote literally appear on the cached page?), and you'll see its verdicts. Your job is the judgment the deterministic check can't do:
+
+- Does the quote, even if it appears on the page, actually **support the row's criterion**? (e.g. an "ultimate outcome" row quoting an *output* count, or a counterfactual row quoting a result instead of a study design.)
+- Is the quote **cherry-picked** — technically present but contradicted or heavily qualified elsewhere on the same page?
+- Do the two reports **contradict each other** on the same row in a way that means at least one is wrong?
+
+## Your Inputs
+
+The `/tfg-validate` skill (or the `get_next_validation` MCP tool) returns:
+- Your validation **claim_id** (a UUID)
+- The **organization** (name, slug, url)
+- Two **source_reports** (markdown), each with its EVIDENCE TABLE and each with the server's deterministic **citation_verdicts** (per row: `verified` / `fabricated` / `unverifiable`)
+- **cached_pages**: `{url, fetch_status, http_status, text}` for every URL cited across both reports. `fetch_status` is `ok` (text present), `unreachable` (couldn't load — includes 404s), `non_text` (a PDF or similar the server can't read), or `missing` (wasn't cached).
+
+## Your Task — per row, per report
+
+Start from the server's verdicts, then apply judgment:
+
+1. **`fabricated`** (quote not on the page, or the URL 404s) → **remove the row** (blank its quote/URL/name). The evidence isn't real.
+2. **`verified`** (quote is on the page) → check it actually **supports the criterion**. If the quote is on-page but doesn't match what the row is asking for, **correct it** to a better quote *from the same cached page* if one exists, otherwise **remove the row**. You may shorten or fix a quote to match the page verbatim. You may NOT swap in a different page or invent a quote.
+3. **`unverifiable`** (`non_text` PDF / `unreachable` non-404 / `missing`) → you can't machine-check it. Keep it **only if** the surrounding report makes it credible; if it looks invented, remove it and say so in your notes. Don't punish a real PDF citation just because it can't be auto-read.
+4. **Counterfactual rows e/f** → the quote must name the study **DESIGN** (randomized/RCT, control or comparison group, quasi-experimental), not the result. If the cached page only supports a result, remove the row.
+
+**The one hard rule: you may only SUBTRACT rows or CORRECT a quote to match its cited page. You may NEVER ADD new evidence, new rows, or new URLs.** Adding evidence is the researcher's job, not yours. A validated report is always a subset (or verbatim-corrected version) of the original.
+
+This is **enforced by the server**: your submission is rejected (422) if it introduces a URL the original didn't cite, OR if any EVIDENCE TABLE row's quote is not actually found on its cited page. So when you correct a quote, copy it **verbatim from the cached page text** — don't paraphrase. If you can't find supporting text on the page, remove the row instead of rewriting it. (Rows whose page is a PDF/blocked/un-readable are `unverifiable` and pass through — you don't need to fix those.)
+
+If a report was already clean, return it unchanged (or simply omit it from your submission — omitted reports keep their original).
+
+## Submit
+
+Call `submit_validation` with:
+- `claim_id`: your validation claim ID
+- `validated_reports`: an array of `{claim_id, report_markdown}` — one entry per report you changed, where `claim_id` is that **source report's** claim_id and `report_markdown` is its full corrected markdown. Omit reports you didn't change.
+- `validation_notes`: a short summary of what you cut and why (e.g. "rc1: dropped row f — the RCT quote was a result, not a design; dropped row c — quote not found on the cited page").
+- `estimated_tokens`: honest estimate (validation is read-heavy but no web work).
+
+If **both reports are already clean** (nothing to remove or correct), submit `validated_reports` as an **empty array** with a note saying so — the round then proceeds to consolidation unchanged. Don't resubmit a report you didn't change.
+
+Each validated report must still be a structurally valid v3 report (all PROMPT/Section headings + an EVIDENCE TABLE + SOURCES). You're pruning evidence, not dismantling the document — leave the prose and structure intact; just remove or correct the offending EVIDENCE TABLE rows (and you may drop a now-unused entry from SOURCES). **Pruning is allowed to drop a report below the usual fresh-submit floors** (≥5 citations, ≥3 domains, ≥1500 words) — those aren't enforced on a validation, because removing fake evidence legitimately shrinks the report.
+
+## What not to do
+
+- **Don't add evidence.** Ever. Subtract or correct-to-source only.
+- **Don't fetch anything.** Use the cached_pages text. If a page is `missing`/`unreachable`, treat the citation as unverifiable — don't go find it yourself.
+- **Don't punish honest blanks or real PDFs.** A blank row is fine. An `unverifiable` PDF from a credible source can stay.
+- **Don't rewrite prose.** Your edits are confined to the EVIDENCE TABLE (and removing orphaned SOURCES lines).
+- **Don't second-guess the deterministic `fabricated` verdict.** If the quote isn't on the page, it goes — the page text is right there for you to confirm.

package/skills/tfg-validate.md

@@ -0,0 +1,51 @@
+---
+name: tfg-validate
+description: Validate two independent v3 research reports by pruning unsupported or fabricated EVIDENCE TABLE rows, using ONLY the cached page text the server provides (no web fetches). Use this when the user has been assigned a validation, OR to check whether they have one pending. Ideal for local models — it does zero scraping.
+---
+
+The user wants to validate a pair of independent research reports before they're merged and scored. Under v3 dual-research, every org is researched by two contributors; you (an independent fourth contributor) catch unsupported or fabricated evidence and hand back cleaned-up versions of both reports. The consolidator then merges already-validated tables.
+
+You do **no web research and no fetching**. The server already fetched every cited page and gives you the cached text. This is deliberate — validation is cheap and works on a local model with no scraping budget.
+
+## What to do
+
+1. **Call `get_next_validation` on the TFG MCP.** It returns:
+   - the org being validated,
+   - your validation `claim_id` (a UUID — submit against this one),
+   - two `source_reports` (each a full v3 report with its EVIDENCE TABLE) plus the server's deterministic `citation_verdicts` per row (`verified` / `fabricated` / `unverifiable`),
+   - `cached_pages`: the text of every URL cited across both reports, with a `fetch_status` (`ok` / `unreachable` / `non_text` / `missing`),
+   - the full validation methodology inline.
+
+   If nothing is assigned, the tool says so — tell the user and stop here.
+
+2. **For each report, go row by row** (a1, a2, a3, b, c, d, e, f). Start from the server's verdict, then apply judgment using the cached page text:
+   - **`fabricated`** (quote not on the page, or URL 404s) → **remove the row** (blank quote/URL/name).
+   - **`verified`** (quote is on the page) → check it actually **supports the row's criterion**. If on-page but off-target, correct it to a better quote *from the same cached page*, or remove the row. Counterfactual rows e/f must quote the study DESIGN (RCT/control/comparison), not a result.
+   - **`unverifiable`** (PDF / unreachable non-404 / missing) → keep only if the report makes it credible; otherwise remove and note why. Don't punish a real PDF just because it can't be auto-read.
+
+3. **The one hard rule:** you may only **SUBTRACT** rows or **CORRECT** a quote to match its cited page. **Never ADD** new evidence, rows, or URLs. A validated report is always a subset (or verbatim-corrected version) of the original. If a report was already clean, leave it unchanged (omit it from your submission).
+
+4. **Call `submit_validation`** with:
+   - `claim_id`: your validation claim ID from step 1
+   - `validated_reports`: an array of `{ claim_id, report_markdown }` — one entry per report you changed, where `claim_id` is that source report's claim_id and `report_markdown` is its full corrected markdown. Omit reports you didn't change. **If both reports are clean, pass an empty array `[]`** — the round proceeds to consolidation unchanged. Pruning is allowed to drop a report below the usual ≥5-citation / ≥3-domain / ≥1500-word floors; those aren't enforced on a validation.
+   - `validation_notes`: a short summary of what you cut/corrected and why
+   - `estimated_tokens`: an honest estimate (validation is read-heavy, no web work)
+
+5. **Tell the user what you changed** (e.g. "Validated 2 reports for X: dropped report-1 row f (RCT quote was a result, not a design) and report-2 row c (quote not on the cited page). Consolidation will follow.").
+
+## What not to do
+
+- **Don't add evidence.** Ever. Subtract or correct-to-source only.
+- **Don't fetch anything.** Use the `cached_pages` text. A `missing`/`unreachable` page means the citation is unverifiable — don't go find it yourself.
+- **Don't punish honest blanks or real PDFs.** A blank row is fine; an `unverifiable` PDF from a credible source can stay.
+- **Don't rewrite prose.** Your edits are confined to the EVIDENCE TABLE (and removing now-orphaned SOURCES lines).
+- **Don't second-guess a `fabricated` verdict** — the page text is right there; if the quote isn't in it, the row goes.
+
+## If something goes wrong
+
+- **`get_next_validation` returns "no validations assigned"** → there isn't one queued. The user can run `/tfg` to do a fresh research run instead.
+- **`submit_validation` says a report is "no longer structurally valid"** → you removed a heading/section, not just evidence. It still needs all PROMPT/Section headings + an EVIDENCE TABLE + SOURCES. Restore the structure and resubmit with the same `claim_id`.
+- **`submit_validation` says it "introduces new URLs"** → you added or changed a citation URL. Validation can't add evidence; revert to the original URLs (only remove rows or fix quotes) and resubmit.
+- **`submit_validation` says a row's "quote is not on the cited page"** → you rewrote a quote to text the page doesn't contain. Copy the quote verbatim from the cached page text, or remove the row entirely. Resubmit with the same `claim_id`.
+- **You referenced a claim_id you weren't given** → only the source `claim_id`s from `get_next_validation` are valid in `validated_reports`. Re-read its output.
+- **No cached page text for a citation** → treat that citation as unverifiable; do not fetch it yourself.

package/src/api-client.js

@@ -90,6 +90,24 @@ export class ApiClient {
     return this.request('GET', '/research/consolidate/next');
   }
 
+  async getNextValidation() {
+    // 204 -> null when nothing is assigned, same as consolidation.
+    return this.request('GET', '/research/validate/next');
+  }
+
+  async submitValidation(claimId, validatedReports, validationNotes = null, tokenUsage = null) {
+    const normalizedTokenUsage = typeof tokenUsage === 'number'
+      ? { total_tokens: tokenUsage }
+      : tokenUsage;
+
+    return this.request('POST', '/research/validate/submit', {
+      claim_id: claimId,
+      validated_reports: validatedReports,
+      validation_notes: validationNotes,
+      token_usage: normalizedTokenUsage,
+    });
+  }
+
   async getStatus() {
     const response = await fetch(`${BASE_URL}/research/status`, {
       headers: this.headers(),

package/src/api-client.test.js

@@ -98,3 +98,30 @@ test('request() returns null on 204 No Content (consolidation queue empty)', asy
     globalThis.fetch = original;
   }
 });
+
+test('getNextValidation returns null on 204 (validation queue empty)', async () => {
+  const original = globalThis.fetch;
+  globalThis.fetch = async () => new Response(null, { status: 204 });
+  try {
+    const client = new ApiClient('tfg_test_key');
+    const result = await client.getNextValidation();
+    assert.equal(result, null);
+  } finally {
+    globalThis.fetch = original;
+  }
+});
+
+test('submitValidation posts validated_reports and wraps numeric token usage', async () => {
+  await withMockFetch(async (calls) => {
+    const client = new ApiClient('tfg_test_key');
+    const validated = [{ claim_id: 'src-uuid', report_markdown: 'cleaned report' }];
+    await client.submitValidation('val-uuid', validated, 'dropped row f', 5000);
+
+    assert.match(calls[0].url, /\/research\/validate\/submit$/);
+    const body = JSON.parse(calls[0].opts.body);
+    assert.equal(body.claim_id, 'val-uuid');
+    assert.deepEqual(body.validated_reports, validated);
+    assert.equal(body.validation_notes, 'dropped row f');
+    assert.deepEqual(body.token_usage, { total_tokens: 5000 });
+  });
+});

package/src/init.js

@@ -127,6 +127,8 @@ export async function runInit() {
     console.log(`✓ ${plans[3].label}`);
     writeSkillFile('tfg-consolidate');
     console.log(`✓ ${plans[4].label}`);
+    writeSkillFile('tfg-validate');
+    console.log(`✓ ${plans[5].label}`);
   } else if (platform === 'qwen-code') {
     writeQwenCommand('tfg');
     console.log(`✓ ${plans[1].label}`);
@@ -163,6 +165,7 @@ function planWrites(platform) {
     plans.push({ label: `${skillPath('tfg-schedule')}  (/tfg-schedule skill)` });
     plans.push({ label: `${skillPath('tfg')}  (/tfg skill)` });
     plans.push({ label: `${skillPath('tfg-consolidate')}  (/tfg-consolidate skill)` });
+    plans.push({ label: `${skillPath('tfg-validate')}  (/tfg-validate skill)` });
   } else if (platform === 'qwen-code') {
     plans.push({ label: `${qwenCommandPath('tfg')}  (/tfg slash command)` });
   }

package/src/mcp-server.js

@@ -129,13 +129,14 @@ server.tool('claim_org', 'Claim the next available nonprofit org to research.',
   }
 });
 
-server.tool('get_methodology', 'Get the full instructions for a pipeline step: research (the v3 EVIDENCE TABLE flow), verify, humanize, or consolidate (the v3 dual-research merge step).', {
-  step: z.enum(['research', 'verify', 'humanize', 'consolidate']).describe('Which pipeline step to get instructions for'),
+server.tool('get_methodology', 'Get the full instructions for a pipeline step: research (the v3 EVIDENCE TABLE flow), verify, humanize, validate (prune unsupported evidence from two reports using cached page text), or consolidate (the v3 dual-research merge step).', {
+  step: z.enum(['research', 'verify', 'humanize', 'validate', 'consolidate']).describe('Which pipeline step to get instructions for'),
 }, async ({ step }) => {
   const stepMap = {
     'research': '01-research/PROMPT.md',
     'verify': '02-verify/PROMPT.md',
     'humanize': '03-humanize/PROMPT.md',
+    'validate': '04-validate/PROMPT.md',
     'consolidate': '05-consolidate/PROMPT.md',
   };
 
@@ -204,6 +205,66 @@ server.tool('get_next_consolidation', 'Get your assigned v3 consolidation: the o
   }
 });
 
+server.tool('get_next_validation', 'Get your assigned v3 validation: both researchers\' reports, the server\'s deterministic citation verdicts, and the cached text of every page they cite. You validate using ONLY that cached text (no web fetches), pruning unsupported/fabricated EVIDENCE TABLE rows. Returns "no assignments" when nothing is queued for you.', {}, async () => {
+  if (!client) return { content: [{ type: 'text', text: 'Error: TFG_API_KEY not set.' }] };
+
+  try {
+    const result = await client.getNextValidation();
+    if (!result || !result.claim_id) {
+      return { content: [{ type: 'text', text: 'No validations assigned to you right now.' }] };
+    }
+    let validateMethodology = '';
+    try {
+      validateMethodology = readFileSync(join(PIPELINE_DIR, '04-validate/PROMPT.md'), 'utf-8');
+    } catch {
+      validateMethodology = 'Using ONLY the cached page text provided, remove EVIDENCE TABLE rows whose quote is not on its cited page (verdict "fabricated"), and correct quotes that do not match. You may only SUBTRACT or CORRECT-to-source, never ADD. Submit the corrected reports with submit_validation.';
+    }
+    const reports = (result.source_reports || []).map((r, i) =>
+      `### Source report ${i + 1} — claim_id ${r.claim_id} (submitted ${r.submitted_at || 'unknown'})\n\nServer citation verdicts: ${JSON.stringify(r.citation_verdicts || {})}\n\n${r.report_markdown}`
+    ).join('\n\n---\n\n');
+    const pages = (result.cached_pages || []).map((p) =>
+      `#### ${p.url}  [${p.fetch_status}${p.http_status ? ' ' + p.http_status : ''}]\n${p.text ? p.text : '(no text — not machine-checkable)'}`
+    ).join('\n\n');
+    return {
+      content: [{ type: 'text', text: `Validation assigned:\nOrg: ${result.org?.name}\nRound: ${result.round_id}\nYour validation claim ID (submit against this one): ${result.claim_id}\n\n---\n\n${validateMethodology}\n\n--- SOURCE REPORTS ---\n\n${reports}\n\n--- CACHED PAGE TEXT (use ONLY this; do not fetch) ---\n\n${pages}` }],
+    };
+  } catch (err) {
+    if (err.status === 404 || err.status === 204) {
+      return { content: [{ type: 'text', text: 'No validations assigned to you right now.' }] };
+    }
+    return { content: [{ type: 'text', text: `Error: ${err.message}` }] };
+  }
+});
+
+server.tool('submit_validation', 'Submit a validation: corrected ("validated") versions of the source reports. You may only SUBTRACT unsupported/fabricated EVIDENCE TABLE rows or CORRECT a quote to match its cited page — never ADD new evidence. Provide full corrected markdown for each report you changed; omit reports you left unchanged.', {
+  claim_id: z.string().describe('Your validation claim_id from get_next_validation'),
+  validated_reports: z.array(z.object({
+    claim_id: z.string().describe('A source report\'s claim_id'),
+    report_markdown: z.string().describe('That report\'s full corrected markdown'),
+  })).describe('One entry per report you changed. Omit reports you did not change.'),
+  validation_notes: z.string().optional().describe('Short summary of what you cut/corrected and why'),
+  estimated_tokens: z.number().optional().describe('Honest total token estimate for this validation run'),
+}, async ({ claim_id, validated_reports, validation_notes, estimated_tokens }) => {
+  if (!client) return { content: [{ type: 'text', text: 'Error: TFG_API_KEY not set.' }] };
+
+  try {
+    const result = await client.submitValidation(claim_id, validated_reports, validation_notes, estimated_tokens ?? null);
+    markContributed();
+    return {
+      content: [{ type: 'text', text: `Validation submitted for ${result.org_name}: ${result.reports_validated} report(s) validated. Consolidation will follow.` }],
+    };
+  } catch (err) {
+    // Surface the structured detail the server returns so the agent knows
+    // exactly what to fix: which rows' quotes weren't on the page, which URLs
+    // were newly introduced, or which structure checks failed.
+    const detail = [];
+    if (err.data?.validation_errors) detail.push(err.data.validation_errors.join('\n'));
+    if (err.data?.fabricated_rows) detail.push(`Rows whose quote is not on the cited page: ${err.data.fabricated_rows.join(', ')}. Copy the quote verbatim from the cached page text, or remove the row.`);
+    if (err.data?.new_urls) detail.push(`New URLs not in the original (not allowed): ${err.data.new_urls.join(', ')}`);
+    return { content: [{ type: 'text', text: `Validation error: ${err.message}${detail.length ? '\n' + detail.join('\n') : ''}` }] };
+  }
+});
+
 server.tool('research_status', 'See the overall Tokens for Good project progress and leaderboard.', {}, async () => {
   try {
     const clientForStatus = client || new ApiClient('dummy', { version: PKG_VERSION, platform, installId }); // Status is public