seo 0.13.3 → 0.14.0

package/site-kit/starter/.github/workflows/refresh-data.yml

@@ -0,0 +1,88 @@
+name: Refresh data
+
+# Checks the data sources once a month and opens a pull request when one
+# changed. Nothing reaches the live site until a person reviews and merges
+# it, so a bad source can never rewrite a price, a rate or a health number
+# on its own. Set the schedule to how often your sources really change.
+on:
+  schedule:
+    - cron: "23 4 1 * *"
+  workflow_dispatch:
+
+concurrency:
+  group: refresh-data
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  refresh:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      BRANCH: automation/refresh-data-${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Stop when a refresh pull request is already open
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          open=$(gh pr list --state open --json headRefName --jq '[.[] | select(.headRefName | startswith("automation/refresh-data-"))] | length')
+          if [ "$open" -gt 0 ]; then
+            echo "A data refresh pull request is already open. Review it first."
+            exit 0
+          fi
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Fetch changed sources
+        run: pnpm data:refresh
+
+      - name: Validate the data
+        run: pnpm data:validate
+
+      - name: Check what changed
+        id: changes
+        run: |
+          changed=$( { git diff --name-only; git ls-files --others --exclude-standard; } | sort -u)
+          if [ -z "$changed" ]; then
+            echo "No source changed."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          unexpected=$(echo "$changed" | grep -Ev '^(src/data/|data/raw/)' || true)
+          if [ -n "$unexpected" ]; then
+            echo "The refresh changed files outside src/data and data/raw:"
+            echo "$unexpected"
+            exit 1
+          fi
+          echo "changed=true" >> "$GITHUB_OUTPUT"
+
+      - name: Build and check the site
+        if: steps.changes.outputs.changed == 'true'
+        run: pnpm build
+
+      - name: Open a pull request
+        if: steps.changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git switch -c "$BRANCH"
+          git add -A -- src/data data/raw
+          git commit -m "chore(data): refresh data sources"
+          git push origin "$BRANCH"
+          gh pr create --base "${{ github.event.repository.default_branch }}" --head "$BRANCH" \
+            --title "Refresh data sources" \
+            --body "A data source changed. Check the changed numbers against the source before you merge. \`seo site check\` passed on this build."

package/site-kit/starter/.node-version

@@ -0,0 +1 @@
+24

package/site-kit/starter/AGENTS.md

@@ -0,0 +1,65 @@
+# Agent notes
+
+This site was made with `seo site new`. It is a static Astro site on
+Cloudflare Workers static assets, built from the data in `src/data`.
+
+Work one step at a time. Print the instructions for a step with
+`seo site guide <step>`: setup, market, data, plan, build, golden-page,
+writing, scale, launch, grow. Follow the step, then show the person the
+result before the next one.
+
+`pnpm build` runs `seo site check`, and Cloudflare will not deploy a build
+that fails it. Fix the page, the template or the data, never the check.
+When a word the copy rules ban is part of the site's subject, add it to
+`allowWords` in `site.config.json`.
+
+## Components
+
+The design matches a clean blank theme: Tailwind 4, Inter and JetBrains
+Mono, soft `bg-muted` panels with no borders or shadows, and automatic
+light and dark mode. Reuse these before writing new markup:
+
+- `Header.astro`, `Footer.astro`: links come from `nav` and `footerLinks`
+  in `src/site.ts`.
+- `PageHeader.astro`: the H1 and the one-paragraph answer under it.
+- `Breadcrumbs.astro`: drawn by the layout from its `breadcrumbs` prop.
+- `SectionHeading.astro`: an H2 with a short line of text.
+- `Card.astro`: a muted panel with an icon, title and text. A linked card
+  takes `link`, a label that says where it goes. Put cards in
+  `grid grid-cols-1 gap-2 sm:grid-cols-2`.
+- `FactTable.astro`: facts with their source and checked date.
+- `FaqAccordion.astro`: questions and answers. Pass the same list to the
+  layout's `faqs` prop.
+- `Button.astro` (and `ui/button.ts`), `FormInput.astro`,
+  `FormSelect.astro`, `CodeBlock.astro`, `HugeIcon.astro`, `Logo.astro`.
+- `site-prose`: put it on an `<article>` for plain headings, paragraphs,
+  lists and tables.
+
+Every colour comes from `src/styles/globals.css`. Use token classes such
+as `bg-muted`, `text-foreground`, `text-foreground-muted` and
+`text-primary`. `seo site check` fails on raw colours (`#fff`, `rgb()`,
+`bg-blue-600`) and on `<style>` blocks in pages, components and layouts,
+so the theme changes in one place. To restyle the site, change the tokens
+there, the mark in `Logo.astro` and `public/favicon.svg`, and the card
+colours in `src/lib/og/card.ts`.
+
+## Where things live
+
+- `site.config.json`: the site name, domain, description and choices.
+- `src/data`: the dataset and its sources, each with a checked date.
+- `src/lib/narrative.ts`: page prose built from facts in the data.
+- `src/layouts/Base.astro`: the title, canonical, sharing tags and JSON-LD.
+- `src/components`: the components listed above.
+- `src/styles/globals.css`: the theme, the only place for colours.
+- `src/integrations`: sharing images and sitemaps, drawn from the build.
+- `scripts`: data refresh and validation, deploy with IndexNow, and the
+  Markdown rewrite rules.
+
+## Never
+
+- Never add noindex to a page people should find. Protect previews with
+  Cloudflare Access instead.
+- Never guess a fact. Keep an unknown value unknown and say so on the page.
+- Never use `run_worker_first` for Markdown copies.
+- Never change a live URL without a 301 to the closest page.
+- Never write about the build, the tools or search terms in page copy.

package/site-kit/starter/README.md

@@ -0,0 +1,11 @@
+# Example Parks
+
+A data-driven site made with `seo site new`.
+
+```sh
+pnpm dev       # local preview
+pnpm build     # build, then seo site check
+pnpm run deploy
+```
+
+Read `AGENTS.md`, then run `seo site guide setup` for the first step.

package/site-kit/starter/astro.config.mjs

@@ -0,0 +1,31 @@
+// @ts-check
+import { agentMarkdown } from '@iannuttall/seo-graph-astro'
+import tailwindcss from '@tailwindcss/vite'
+import { defineConfig } from 'astro/config'
+import ogImages from './src/integrations/og-images.ts'
+import sitemaps from './src/integrations/sitemaps.ts'
+import { llmsTxt } from './llms.config.mjs'
+import config from './site.config.json' with { type: 'json' }
+
+// Static HTML on Cloudflare Workers static assets. Every URL has one form:
+// no trailing slash, served from `page.html`.
+export default defineConfig({
+  site: `https://${config.domain}`,
+  output: 'static',
+  trailingSlash: 'never',
+  build: { format: 'file' },
+  devToolbar: { enabled: false },
+  integrations: [
+    // A Markdown copy of every page, llms.txt and agent-routes.json.
+    agentMarkdown({ llmsTxt, routeMigrationWarnings: false }),
+    // These two read the finished HTML, so they run last.
+    ogImages(),
+    sitemaps(),
+  ],
+  vite: {
+    // Tailwind 4. The theme is src/styles/globals.css.
+    plugins: [tailwindcss()],
+    // Orb and tunnel previews use generated hostnames.
+    server: { allowedHosts: true },
+  },
+})

package/site-kit/starter/gitignore

@@ -0,0 +1,7 @@
+node_modules
+dist
+.astro
+.wrangler
+.env
+.env.*
+.dev.vars*

package/site-kit/starter/llms.config.mjs

@@ -0,0 +1,38 @@
+import parks from './src/data/parks.json' with { type: 'json' }
+import config from './site.config.json' with { type: 'json' }
+
+const regions = [...new Set(parks.map((park) => park.region))].sort()
+const slug = (text) =>
+  text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+
+/** @satisfies {import('@iannuttall/seo-graph-core').LlmsTxtConfig} */
+export const llmsTxt = {
+  title: config.name,
+  summary: config.description,
+  sections: [
+    {
+      heading: 'Start here',
+      items: [
+        { path: '/', label: config.name },
+        { path: '/about', label: 'Where the data comes from' },
+      ],
+    },
+    {
+      heading: 'Regions',
+      items: regions.map((region) => ({
+        path: `/regions/${slug(region)}`,
+        label: `Parks in ${region}`,
+      })),
+    },
+    {
+      heading: 'Parks',
+      items: parks.map((park) => ({
+        path: `/parks/${park.slug}`,
+        label: park.name,
+      })),
+    },
+  ],
+}

package/site-kit/starter/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "example-site",
+  "private": true,
+  "type": "module",
+  "engines": {
+    "node": ">=22.12"
+  },
+  "scripts": {
+    "dev": "astro dev",
+    "build": "astro build && seo site check",
+    "preview": "astro preview",
+    "check": "seo site check",
+    "deploy": "node scripts/deploy.mjs",
+    "data:refresh": "node scripts/refresh-data.mjs",
+    "data:validate": "node scripts/validate-data.mjs",
+    "rewrites": "node scripts/markdown-rewrites.mjs"
+  },
+  "dependencies": {
+    "@hugeicons/core-free-icons": "^4.4.0",
+    "@iannuttall/seo-graph-astro": "0.0.12",
+    "@iannuttall/seo-graph-core": "0.0.5",
+    "@tailwindcss/vite": "^4.3.4",
+    "astro": "^7.4.5",
+    "tailwindcss": "^4.3.4"
+  },
+  "devDependencies": {
+    "seo": "latest",
+    "takumi-js": "2.19.0",
+    "wrangler": "^4.156.2"
+  }
+}

package/site-kit/starter/pnpm-workspace.yaml

@@ -0,0 +1,5 @@
+# Packages whose install scripts may run: the bundler Astro uses and the
+# local Cloudflare runtime that wrangler uses.
+allowBuilds:
+  esbuild: true
+  workerd: true

package/site-kit/starter/public/_headers

@@ -0,0 +1,5 @@
+/_astro/*
+  Cache-Control: public, max-age=31536000, immutable
+
+/og/*
+  Cache-Control: public, max-age=86400

package/site-kit/starter/public/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><title>Site icon</title><style>path{fill:#111}@media (prefers-color-scheme:dark){path{fill:#fff}}</style><path d="M16 3c-4 5-7 9-7 13a7 7 0 0 0 6 6.9V29h2v-6.1A7 7 0 0 0 23 16c0-4-3-8-7-13z"/></svg>

package/site-kit/starter/public/fonts/LICENSE-inter.txt

@@ -0,0 +1,93 @@
+Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

package/site-kit/starter/public/fonts/LICENSE-jetbrains-mono.txt

@@ -0,0 +1,93 @@
+Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono) JetBrainsMono-Italic[wght].ttf: Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

package/site-kit/starter/scripts/deploy.mjs

@@ -0,0 +1,60 @@
+// Deploys the built site, then tells search engines which pages changed.
+// It compares the page hashes in dist/indexnow.json with the copy on the
+// live site and sends only added, updated and removed URLs to IndexNow.
+// Set this as the Cloudflare Workers Builds deploy command: pnpm run deploy
+import { execFileSync } from 'node:child_process'
+import { readFile } from 'node:fs/promises'
+import {
+  changedUrls,
+  diffManifests,
+  parseManifest,
+  submitToIndexNow,
+} from '@iannuttall/seo-graph-core'
+import config from '../site.config.json' with { type: 'json' }
+
+const origin = `https://${config.domain}`
+const current = parseManifest(await readFile('dist/indexnow.json', 'utf8'))
+if (!current) throw new Error('dist/indexnow.json is missing. Run the build.')
+
+let previous = null
+try {
+  const response = await fetch(`${origin}/indexnow.json`, {
+    signal: AbortSignal.timeout(10_000),
+  })
+  if (response.ok) previous = parseManifest(await response.text())
+} catch {
+  // The first deploy has no live copy.
+}
+
+execFileSync('npx', ['wrangler', 'deploy'], { stdio: 'inherit' })
+
+// Only tell search engines once the domain serves this site, which the
+// key file proves. Before launch the site lives on workers.dev.
+let live = false
+try {
+  const response = await fetch(`${origin}/${config.indexNowKey}.txt`, {
+    signal: AbortSignal.timeout(10_000),
+  })
+  live = response.ok && (await response.text()).trim() === config.indexNowKey
+} catch {
+  live = false
+}
+if (!config.indexNowKey || !live) {
+  console.log(`IndexNow skipped: ${origin} does not serve this site yet.`)
+} else {
+  const urls = changedUrls(diffManifests(previous ?? {}, current))
+  if (!urls.length) {
+    console.log('IndexNow: no pages changed.')
+  } else {
+    const results = await submitToIndexNow({
+      host: config.domain,
+      key: config.indexNowKey,
+      urls,
+    })
+    for (const result of results) {
+      console.log(
+        `IndexNow: ${result.submitted} URLs, status ${result.status}${result.ok ? '' : ` (${result.message})`}`,
+      )
+    }
+  }
+}

package/site-kit/starter/scripts/markdown-rewrites.mjs

@@ -0,0 +1,33 @@
+// Prints the two Cloudflare URL rewrite rules that answer
+// `Accept: text/markdown` at a page's normal URL with its Markdown copy.
+// Add them under Rules > Transform Rules > Rewrite URL for the zone.
+// Rewrite rules run before the static assets, so no Worker code runs and
+// bots cost nothing (never use run_worker_first for this).
+import config from '../site.config.json' with { type: 'json' }
+
+const host = `http.host eq "${config.domain}"`
+const method = '(http.request.method eq "GET" or http.request.method eq "HEAD")'
+const accept =
+  '(starts_with(lower(http.request.headers["accept"][0]), "text/markdown"))'
+
+const rules = [
+  {
+    name: 'Markdown copy of the home page',
+    expression: `(${host} and ${method} and http.request.uri.path eq "/" and ${accept})`,
+    rewrite: 'Static: /index.md',
+  },
+  {
+    name: 'Markdown copy of every other page',
+    expression: `(${host} and ${method} and http.request.uri.path ne "/" and not ends_with(http.request.uri.path, "/") and not (http.request.uri.path contains ".") and ${accept})`,
+    rewrite: 'Dynamic: concat(http.request.uri.path, ".md")',
+  },
+]
+
+for (const rule of rules) {
+  console.log(
+    `${rule.name}\n\nWhen incoming requests match (Edit expression):\n${rule.expression}\n\nThen rewrite the path to:\n${rule.rewrite}\n\nQuery: preserve\n`,
+  )
+}
+console.log(
+  `Check it after the next deploy:\ncurl -sI -H "Accept: text/markdown" https://${config.domain}/ | grep -i content-type`,
+)

package/site-kit/starter/scripts/refresh-data.mjs

@@ -0,0 +1,60 @@
+// Checks every source in src/data/sources.json for changes. A source that
+// answers with the same ETag, Last-Modified date or file hash is unchanged.
+// A changed source is saved to data/raw/<id>, its checked date moves to
+// today, and the refresh workflow opens a pull request for a person to
+// review. Turning the raw file into src/data/*.json is site specific: write
+// that step in scripts/build-data.mjs when you add your own data.
+import { createHash } from 'node:crypto'
+import { existsSync } from 'node:fs'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+
+const sourcesFile = 'src/data/sources.json'
+const sources = JSON.parse(await readFile(sourcesFile, 'utf8'))
+const today = new Date().toISOString().slice(0, 10)
+let changed = 0
+
+for (const source of sources) {
+  const headers = {}
+  if (source.etag) headers['If-None-Match'] = source.etag
+  if (source.lastModified) headers['If-Modified-Since'] = source.lastModified
+  let response
+  try {
+    response = await fetch(source.url, {
+      headers,
+      signal: AbortSignal.timeout(60_000),
+    })
+  } catch (error) {
+    console.error(`${source.id}: could not fetch ${source.url}: ${error}`)
+    process.exitCode = 1
+    continue
+  }
+  if (response.status === 304) {
+    console.log(`${source.id}: unchanged`)
+    continue
+  }
+  if (!response.ok) {
+    console.error(`${source.id}: ${source.url} returned ${response.status}`)
+    process.exitCode = 1
+    continue
+  }
+  const body = Buffer.from(await response.arrayBuffer())
+  const sha256 = createHash('sha256').update(body).digest('hex')
+  source.etag = response.headers.get('etag')
+  source.lastModified = response.headers.get('last-modified')
+  if (sha256 === source.sha256) {
+    console.log(`${source.id}: unchanged`)
+    continue
+  }
+  await mkdir('data/raw', { recursive: true })
+  await writeFile(`data/raw/${source.id}`, body)
+  source.sha256 = sha256
+  source.checked = today
+  changed += 1
+  console.log(`${source.id}: changed, saved to data/raw/${source.id}`)
+}
+
+await writeFile(sourcesFile, `${JSON.stringify(sources, null, 2)}\n`)
+if (changed && existsSync('scripts/build-data.mjs')) {
+  await import('./build-data.mjs')
+}
+console.log(`${changed} of ${sources.length} sources changed.`)