poops 1.5.3 → 1.7.0

package/lib/markup/indexer.js

@@ -2,6 +2,9 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { readJsonFile, fileSize } from '../utils/helpers.js'
+import { parseFrontMatter } from './helpers.js'
+import { renderMarkdownCached } from './renderer.js'
+import { humanize } from 'book-of-spells'
 import log from '../utils/log.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -37,6 +40,12 @@ function normalizeConfig(config) {
   return { ...DEFAULTS, ...config }
 }
 
+// A page front-matter `robots: noindex` (or `none`) opts it out of the
+// crawler/discovery artifacts — the sitemap and llms.txt.
+function isNoindex(entry) {
+  return entry.robots ? /\b(noindex|none)\b/i.test(String(entry.robots)) : false
+}
+
 export function extractKeywords(htmlContent, options = {}) {
   const { minWordLength = DEFAULTS.minWordLength, stopWords = new Set() } = options
 
@@ -163,6 +172,7 @@ export function generateSitemap(pageEntries, outputDir, siteUrl, config) {
   xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
 
   for (const entry of pageEntries) {
+    if (isNoindex(entry)) continue
     const loc = baseUrl ? `${baseUrl}/${entry.url}` : entry.url
     xml += '  <url>\n'
     xml += `    <loc>${escapeXml(loc)}</loc>\n`
@@ -182,12 +192,185 @@ export function generateSitemap(pageEntries, outputDir, siteUrl, config) {
   log({ tag: 'indexer', text: 'Generated sitemap:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
 }
 
-function humanizeSegment(seg) {
-  return seg
-    .replace(/[-_]+/g, ' ')
-    .replace(/\s+/g, ' ')
-    .trim()
-    .replace(/\b\w/g, c => c.toUpperCase())
+// Generates an llms.txt (https://llmstxt.org): a markdown index of the site's
+// pages that LLMs / generative engines ingest to understand the site (GEO).
+// Structure: `# title`, a `> description` blockquote, then one `## section` per
+// collection (uncollected pages go under a lead "Pages" section) with
+// `- [title](absolute url): description` links. `title`/`description` are
+// resolved from site data by the caller. Mirrors generateSitemap's page set —
+// isIndex (collection landing/pagination) pages are skipped.
+export function generateLlmsTxt(pageEntries, outputDir, siteUrl, config) {
+  config = normalizeConfig(config)
+  if (!config) return
+
+  const baseUrl = siteUrl ? siteUrl.replace(/\/+$/, '') : ''
+  const absUrl = (url) => baseUrl ? `${baseUrl}/${url}` : url
+
+  let out = `# ${config.title || 'Site'}\n`
+  if (config.description) out += `\n> ${config.description}\n`
+
+  // Optional free-form body (llmstxt.org allows markdown between the blockquote
+  // and the link sections). `intro` is a path to a markdown file authored for
+  // LLM context — inserted verbatim. Avoid H2s in it; they'd read as sections.
+  if (config.intro) {
+    try {
+      const body = fs.readFileSync(path.resolve(process.cwd(), config.intro), 'utf-8').trim()
+      if (body) out += `\n${body}\n`
+    } catch {
+      log({ tag: 'indexer', warn: true, text: 'llms.txt intro file not found:', link: config.intro })
+    }
+  }
+
+  // Group into two levels from the URL path, first-seen order preserved: first
+  // folder = `## section`, second folder = `### subsection` nested under it,
+  // root-level pages under the lead section. A collection's items already live
+  // under `collectionName/…`, so the folder segments double as its grouping.
+  const lead = config.sectionTitle || 'Pages'
+  const sections = new Map() // name -> { direct: [entry], subs: Map<name, [entry]> }
+  const sectionOf = (name) => {
+    if (!sections.has(name)) sections.set(name, { direct: [], subs: new Map() })
+    return sections.get(name)
+  }
+
+  for (const e of pageEntries) {
+    if (e.isIndex || isNoindex(e)) continue
+    const slash = e.url.lastIndexOf('/')
+    const segs = slash === -1 ? [] : e.url.slice(0, slash).split('/').filter(Boolean)
+    if (segs.length === 0) {
+      sectionOf(lead).direct.push(e)
+    } else if (segs.length === 1) {
+      sectionOf(humanize(segs[0])).direct.push(e)
+    } else {
+      const subs = sectionOf(humanize(segs[0])).subs
+      const key = humanize(segs[1])
+      if (!subs.has(key)) subs.set(key, [])
+      subs.get(key).push(e)
+    }
+  }
+
+  const linkLine = (e) => {
+    const link = `[${e.title || e.url}](${absUrl(e.url)})`
+    return e.description ? `- ${link}: ${e.description}\n` : `- ${link}\n`
+  }
+
+  // Collection buckets are chronological (newest first); other sections (docs)
+  // keep their file/order sequence.
+  const ordered = (entries) => {
+    if (!entries.length || !entries.every(e => e.collection != null)) return entries
+    return [...entries].sort((a, b) => new Date(b.date || 0) - new Date(a.date || 0))
+  }
+
+  for (const [name, group] of sections) {
+    out += `\n## ${name}\n\n`
+    for (const e of ordered(group.direct)) out += linkLine(e)
+    for (const [subName, entries] of group.subs) {
+      out += `\n### ${subName}\n\n`
+      for (const e of ordered(entries)) out += linkLine(e)
+    }
+  }
+
+  const outputPath = path.resolve(process.cwd(), outputDir, config.output)
+  fs.writeFileSync(outputPath, out)
+  log({ tag: 'indexer', text: 'Generated llms.txt:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
+}
+
+// Generates llms-full.txt: every page's full content concatenated, the
+// companion to llms.txt's link index (llmstxt.org). Enabled by `llms.full`
+// (string path, or `true` → the index filename with a `-full` suffix). The content comes from each
+// page's markdown SOURCE (entry._src), not entry.content — that's the rendered
+// HTML page (layout + chrome). Only markdown sources qualify: an .njk/.liquid
+// source is template code, not prose.
+// ponytail: raw markdown, so unrendered {% %} tags / shortcodes in a body leak
+// through verbatim. Upgrade path if that bites: render + HTML→markdown the body.
+export function generateLlmsFull(pageEntries, outputDir, siteUrl, config) {
+  config = normalizeConfig(config)
+  if (!config || !config.full) return
+  // Default filename pairs with the index: llms.txt → llms-full.txt, ai.txt →
+  // ai-full.txt (suffix the index's `output`, dir/ext preserved). A string
+  // `full` overrides the path outright.
+  let output = config.full
+  if (typeof output !== 'string') {
+    const p = path.parse(config.output || 'llms.txt')
+    output = path.join(p.dir, `${p.name}-full${p.ext}`)
+  }
+
+  const baseUrl = siteUrl ? siteUrl.replace(/\/+$/, '') : ''
+  const absUrl = (url) => baseUrl ? `${baseUrl}/${url}` : url
+  const isMarkdown = (src) => /\.(md|markdown)$/i.test(src || '')
+
+  const blocks = []
+  for (const e of pageEntries) {
+    if (e.isIndex || isNoindex(e) || !isMarkdown(e._src)) continue
+    const body = parseFrontMatter(e._src).content.trim()
+    if (!body) continue
+    const url = `URL: ${absUrl(e.url)}`
+    // If the body already opens with its own ATX H1, reuse it as the block
+    // heading and slot the URL beneath — otherwise the wrapper `# title` plus
+    // the body's H1 would emit two H1s per block (often exact duplicates).
+    const h1 = body.match(/^# .*(?:\r?\n|$)/)
+    blocks.push(h1
+      ? `${h1[0].trim()}\n${url}\n\n${body.slice(h1[0].length).trimStart()}`
+      : `# ${e.title || e.url}\n${url}\n\n${body}`)
+  }
+  if (!blocks.length) return
+
+  // Corpus header: names the file and its purpose so a whole-file ingest opens
+  // with context instead of the first page's H1. Reuses the index title/desc.
+  const name = config.title || 'this site'
+  let head = `# Full Documentation Archive for ${name}\n`
+  head += `\nThis file contains the complete Markdown documentation for ${name}.`
+  if (config.description) head += `\n\n> ${config.description}`
+  // Optional author-written preamble after the header — the full-file counterpart
+  // to `intro` for the index. Inserted verbatim; a missing file warns and skips.
+  if (config.fullIntro) {
+    try {
+      const body = fs.readFileSync(path.resolve(process.cwd(), config.fullIntro), 'utf-8').trim()
+      if (body) head += `\n\n${body}`
+    } catch {
+      log({ tag: 'indexer', warn: true, text: 'llms-full intro file not found:', link: config.fullIntro })
+    }
+  }
+  blocks.unshift(head)
+
+  const outputPath = path.resolve(process.cwd(), outputDir, output)
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true })
+  fs.writeFileSync(outputPath, blocks.join('\n\n---\n\n') + '\n')
+  log({ tag: 'indexer', text: 'Generated llms-full.txt:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
+}
+
+// Generates a robots.txt. Defaults to allow-all with a `Sitemap:` line pointing
+// at the generated sitemap (absolute, needs `site.url`). The object form takes
+// `userAgent`, `allow`/`disallow` (string or array of paths) and `sitemap`
+// (an explicit URL, or `false` to omit the line). `sitemapOutput` is the
+// sitemap's filename, threaded in by generateIndexFiles.
+// ponytail: single user-agent group; add per-agent groups if a real need shows up.
+export function generateRobotsTxt(outputDir, siteUrl, config, sitemapOutput) {
+  config = normalizeConfig(config)
+  if (!config) return
+
+  const arr = (v) => v == null ? [] : (Array.isArray(v) ? v : [v])
+  const allow = arr(config.allow)
+  const disallow = arr(config.disallow)
+
+  const lines = [`User-agent: ${config.userAgent || '*'}`]
+  for (const p of allow) lines.push(`Allow: ${p}`)
+  for (const p of disallow) lines.push(`Disallow: ${p}`)
+  // Empty `Disallow:` is the canonical "allow everything" when nothing is set.
+  if (allow.length === 0 && disallow.length === 0) lines.push('Disallow:')
+
+  let sitemapUrl = null
+  if (config.sitemap === false) {
+    sitemapUrl = null
+  } else if (typeof config.sitemap === 'string') {
+    sitemapUrl = config.sitemap
+  } else if (sitemapOutput && siteUrl) {
+    sitemapUrl = `${siteUrl.replace(/\/+$/, '')}/${sitemapOutput}`
+  }
+  if (sitemapUrl) lines.push('', `Sitemap: ${sitemapUrl}`)
+
+  const outputPath = path.resolve(process.cwd(), outputDir, config.output)
+  fs.writeFileSync(outputPath, lines.join('\n') + '\n')
+  log({ tag: 'indexer', text: 'Generated robots.txt:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
 }
 
 function navNodeTitle(entry) {
@@ -211,7 +394,7 @@ function navFilterEntries(pageEntries, collectionsOpt) {
       // pagination pages (url has a slash) always dropped. Landing titles are
       // the raw collection name ("blog"), so humanize for display.
       if (mode === 'index' && !e.url.includes('/')) {
-        result.push({ ...e, title: humanizeSegment(e.title) })
+        result.push({ ...e, title: humanize(e.title) })
       }
       continue
     }
@@ -264,7 +447,7 @@ function serializeNavNode(node) {
   const children = [...node.children.values()].map(serializeNavNode)
   sortNavSiblings(children)
 
-  const out = { title: node.title != null ? node.title : humanizeSegment(node.segment) }
+  const out = { title: node.title != null ? node.title : humanize(node.segment) }
   if (node.url != null) out.url = node.url
 
   let order = node.order
@@ -333,8 +516,188 @@ export function generateNav(pageEntries, outputDir, config) {
   log({ tag: 'indexer', text: 'Generated nav:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
 }
 
+// RFC-822 date for RSS pubDate/lastBuildDate (toUTCString is RFC-822 compliant).
+function rssDate(d) {
+  const date = new Date(d)
+  return isNaN(date) ? '' : date.toUTCString()
+}
+
+// ISO-8601 (RFC-3339) date for Atom updated timestamps.
+function isoDate(d) {
+  const date = new Date(d)
+  return isNaN(date) ? new Date().toISOString() : date.toISOString()
+}
+
+function feedAuthorName(author) {
+  if (!author) return ''
+  return String(author.name || author)
+}
+
+// Normalizes the `feed` option into concrete per-collection specs. Accepts:
+//   true | "feed.xml"           → RSS for every collection
+//   { output, type, limit, … }  → those options for every collection
+//   { collection, … }           → one named collection
+//   [ {…}, {…} ]                → an explicit list
+// `output` containing a slash is a full path under the output dir; a bare
+// filename is placed in the collection's own folder (default "feed.xml").
+function resolveFeedSpecs(config, collectionNames, site) {
+  if (!config) return []
+  const raw = Array.isArray(config) ? config : [config]
+  const specs = []
+  for (let item of raw) {
+    if (item === true) item = {}
+    else if (typeof item === 'string') item = { output: item }
+    if (!item || typeof item !== 'object') continue
+
+    const targets = item.collection ? [item.collection] : collectionNames
+    for (const name of targets) {
+      const outName = item.output || 'feed.xml'
+      specs.push({
+        collection: name,
+        output: outName.includes('/') ? outName : `${name}/${outName}`,
+        type: item.type === 'atom' ? 'atom' : 'rss',
+        limit: Number(item.limit) > 0 ? Number(item.limit) : 20,
+        title: item.title || (site.title ? `${humanize(name)} | ${site.title}` : humanize(name)),
+        description: item.description || site.description || '',
+        author: feedAuthorName(item.author || site.author),
+        lang: item.lang || site.lang || '',
+        content: item.content === true // full article HTML in <content:encoded>
+      })
+    }
+  }
+  return specs
+}
+
+// Article-body HTML for a feed item's <content:encoded> / Atom <content>. Renders
+// the page's markdown SOURCE (entry._src) with the shared marked instance — the
+// same extensions the site uses — so it's clean article HTML, not entry.content
+// (the whole rendered page, layout + chrome). Markdown sources only; the render
+// cache is warm from the compile, so this is a lookup, not a re-render.
+// ponytail: raw markdown render, so unrendered {% %} tags / shortcodes in a body
+// leak; same ceiling as llms-full. Returns '' when there's no clean body to emit.
+function feedContentHtml(e) {
+  if (!e._src || !/\.(md|markdown)$/i.test(e._src)) return ''
+  return renderMarkdownCached(e._src, parseFrontMatter(e._src).content).trim()
+}
+
+// `]]>` would close the CDATA section early; split it so the payload survives.
+function cdata(html) {
+  return `<![CDATA[${html.replace(/]]>/g, ']]]]><![CDATA[>')}]]>`
+}
+
+function buildRssFeed(spec, items, { selfUrl, channelLink, abs }) {
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
+  xml += '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"'
+  xml += spec.content ? ' xmlns:content="http://purl.org/rss/1.0/modules/content/">\n' : '>\n'
+  xml += '<channel>\n'
+  xml += `  <title>${escapeXml(spec.title)}</title>\n`
+  xml += `  <link>${escapeXml(channelLink)}</link>\n`
+  if (spec.description) xml += `  <description>${escapeXml(spec.description)}</description>\n`
+  if (selfUrl) xml += `  <atom:link href="${escapeXml(selfUrl)}" rel="self" type="application/rss+xml"/>\n`
+  if (spec.lang) xml += `  <language>${escapeXml(spec.lang)}</language>\n`
+  if (items[0] && items[0].date) xml += `  <lastBuildDate>${rssDate(items[0].date)}</lastBuildDate>\n`
+  for (const e of items) {
+    const link = abs(e.url)
+    const desc = e.description || e.excerpt
+    xml += '  <item>\n'
+    xml += `    <title>${escapeXml(e.title || e.url)}</title>\n`
+    xml += `    <link>${escapeXml(link)}</link>\n`
+    xml += `    <guid isPermaLink="true">${escapeXml(link)}</guid>\n`
+    if (desc) xml += `    <description>${escapeXml(desc)}</description>\n`
+    if (e.date) xml += `    <pubDate>${rssDate(e.date)}</pubDate>\n`
+    if (spec.content) {
+      const html = feedContentHtml(e)
+      if (html) xml += `    <content:encoded>${cdata(html)}</content:encoded>\n`
+    }
+    xml += '  </item>\n'
+  }
+  xml += '</channel>\n</rss>\n'
+  return xml
+}
+
+function buildAtomFeed(spec, items, { selfUrl, channelLink, abs }) {
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
+  xml += '<feed xmlns="http://www.w3.org/2005/Atom">\n'
+  xml += `  <title>${escapeXml(spec.title)}</title>\n`
+  if (spec.description) xml += `  <subtitle>${escapeXml(spec.description)}</subtitle>\n`
+  xml += `  <id>${escapeXml(selfUrl || channelLink)}</id>\n`
+  xml += `  <link href="${escapeXml(channelLink)}"/>\n`
+  if (selfUrl) xml += `  <link href="${escapeXml(selfUrl)}" rel="self" type="application/atom+xml"/>\n`
+  xml += `  <updated>${isoDate(items[0] && items[0].date)}</updated>\n`
+  if (spec.author) xml += `  <author><name>${escapeXml(spec.author)}</name></author>\n`
+  for (const e of items) {
+    const link = abs(e.url)
+    const desc = e.description || e.excerpt
+    const author = feedAuthorName(e.author)
+    xml += '  <entry>\n'
+    xml += `    <title>${escapeXml(e.title || e.url)}</title>\n`
+    xml += `    <link href="${escapeXml(link)}"/>\n`
+    xml += `    <id>${escapeXml(link)}</id>\n`
+    xml += `    <updated>${isoDate(e.date)}</updated>\n`
+    if (desc) xml += `    <summary>${escapeXml(desc)}</summary>\n`
+    if (author) xml += `    <author><name>${escapeXml(author)}</name></author>\n`
+    if (spec.content) {
+      const html = feedContentHtml(e)
+      if (html) xml += `    <content type="html">${cdata(html)}</content>\n`
+    }
+    xml += '  </entry>\n'
+  }
+  xml += '</feed>\n'
+  return xml
+}
+
+// Generates RSS/Atom feeds from collections — one file per resolved spec, its
+// items the collection's posts newest-first (by date), capped at `limit`.
+// Items carry description/excerpt by default; `content: true` on a spec adds
+// full article HTML (<content:encoded> / Atom <content>) via feedContentHtml,
+// which renders the markdown source rather than reusing the whole-page HTML.
+export function generateFeeds(pageEntries, outputDir, siteUrl, config, site = {}) {
+  if (!config) return
+
+  // Distinct collections present, first-seen order.
+  const collectionNames = []
+  const byCollection = new Map()
+  for (const e of pageEntries) {
+    if (e.isIndex || e.collection == null) continue
+    if (!byCollection.has(e.collection)) {
+      byCollection.set(e.collection, [])
+      collectionNames.push(e.collection)
+    }
+    byCollection.get(e.collection).push(e)
+  }
+
+  const baseUrl = siteUrl ? siteUrl.replace(/\/+$/, '') : ''
+  const abs = (u) => {
+    const rel = String(u == null ? '' : u).replace(/^\/+/, '')
+    if (/^https?:\/\//i.test(u)) return u
+    return baseUrl ? (rel ? `${baseUrl}/${rel}` : baseUrl) : `/${rel}`
+  }
+
+  for (const spec of resolveFeedSpecs(config, collectionNames, site)) {
+    const items = (byCollection.get(spec.collection) || [])
+      .filter(e => !isNoindex(e))
+      .sort((a, b) => new Date(b.date || 0) - new Date(a.date || 0))
+      .slice(0, spec.limit)
+
+    const ctx = { selfUrl: abs(spec.output), channelLink: abs(spec.collection), abs }
+    const xml = spec.type === 'atom' ? buildAtomFeed(spec, items, ctx) : buildRssFeed(spec, items, ctx)
+
+    const outputPath = path.resolve(process.cwd(), outputDir, spec.output)
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true })
+    fs.writeFileSync(outputPath, xml)
+    log({ tag: 'indexer', text: 'Generated feed:', link: path.relative(process.cwd(), outputPath), size: fileSize(outputPath) })
+  }
+}
+
 export function generateIndexFiles(pageEntries, outputDir, siteUrl, config) {
   generateSearchIndex(pageEntries, outputDir, config.searchIndex)
   generateSitemap(pageEntries, outputDir, siteUrl, config.sitemap)
+  generateLlmsTxt(pageEntries, outputDir, siteUrl, config.llms)
+  generateLlmsFull(pageEntries, outputDir, siteUrl, config.llms)
+  const sitemapOutput = config.sitemap
+    ? (typeof config.sitemap === 'string' ? config.sitemap : config.sitemap.output)
+    : null
+  generateRobotsTxt(outputDir, siteUrl, config.robots, sitemapOutput)
   generateNav(pageEntries, outputDir, config.nav)
+  generateFeeds(pageEntries, outputDir, siteUrl, config.feed, config.site)
 }

package/lib/markups.js

@@ -1,6 +1,6 @@
 import { pathExists, pathIsDirectory, readDataFile, mkDir, buildTime, toPosix } from './utils/helpers.js'
-import { replaceOutExtensions, getRelativePathPrefix, getPageUrlRelativeToOutput, parseFrontMatter, wordcount } from './markup/helpers.js'
-import { collectionAutoDiscovery, getCollectionDataBasedOnConfig, buildCollectionPaginationData, generateCollectionPaginationPages } from './markup/collections.js'
+import { replaceOutExtensions, getRelativePathPrefix, getPageUrlRelativeToOutput, parseFrontMatter, wordcount, excerpt } from './markup/helpers.js'
+import { collectionAutoDiscovery, getCollectionDataBasedOnConfig, buildCollectionPaginationData, generateCollectionPaginationPages, buildTaxonomyData, generateTaxonomyPages } from './markup/collections.js'
 import { generateIndexFiles, buildNavTree } from './markup/indexer.js'
 import NunjucksEngine from './markup/engines/nunjucks.js'
 import LiquidEngine from './markup/engines/liquid.js'
@@ -37,6 +37,9 @@ export default class Markups {
     this.includePaths = moduleConfig.includePaths || moduleConfig.options.includePaths || []
     this.searchIndexConfig = moduleConfig.options.searchIndex || moduleConfig.searchIndex
     this.sitemapConfig = moduleConfig.options.sitemap || moduleConfig.sitemap
+    this.llmsConfig = moduleConfig.options.llms || moduleConfig.llms
+    this.robotsConfig = moduleConfig.options.robots || moduleConfig.robots
+    this.feedConfig = moduleConfig.options.feed || moduleConfig.feed
     this.navConfig = moduleConfig.options.nav || moduleConfig.nav
     this.baseURL = moduleConfig.baseURL || moduleConfig.options.baseURL || null
 
@@ -107,6 +110,16 @@ export default class Markups {
     this.loadDataFiles(this.dataConfig)
   }
 
+  // Normalizes the llms config (string shorthand or object) and fills in the
+  // title/description from site data so generateLlmsTxt gets ready values.
+  resolveLlmsConfig() {
+    if (!this.llmsConfig) return undefined
+    const base = typeof this.llmsConfig === 'string' ? { output: this.llmsConfig } : { ...this.llmsConfig }
+    base.title = base.title || this.siteData.title
+    base.description = base.description || this.siteData.description
+    return base
+  }
+
   loadDataFiles(files) {
     if (!files) return
 
@@ -212,11 +225,18 @@ export default class Markups {
     const context = { page: {} }
     let pageUrl
 
+    let pageExtra
     if (additionalContext) {
       if (additionalContext._url) {
         pageUrl = additionalContext._url
         delete additionalContext._url
       }
+      // extra props to stamp on page.* after front-matter parse (e.g. taxonomy
+      // pages inject the breadcrumb hint the index template's front matter lacks)
+      if (additionalContext._page) {
+        pageExtra = additionalContext._page
+        delete additionalContext._page
+      }
       Object.assign(context, additionalContext)
     }
 
@@ -225,12 +245,14 @@ export default class Markups {
       context.page = frontMatterResult.frontMatter
       context.page.content = frontMatterResult.content
       context.page.wordcount = wordcount(frontMatterResult.content)
+      context.page.excerpt = excerpt(frontMatterResult.content)
     } catch (err) {
       log({ tag: 'error', text: 'Failed parsing front matter:', link: templateName })
       console.error(err)
     }
 
     if (pageUrl) context.page.url = pageUrl
+    if (pageExtra) Object.assign(context.page, pageExtra)
 
     const frontMatter = context.page
 
@@ -502,16 +524,22 @@ export default class Markups {
       ...getCollectionDataBasedOnConfig(this.markupIn, this.collectionsConfig)
     }
 
-    const shouldIndex = this.searchIndexConfig || this.sitemapConfig || this.navConfig
+    const shouldIndex = this.searchIndexConfig || this.sitemapConfig || this.navConfig || this.feedConfig
     const pageEntries = shouldIndex ? [] : null
 
     buildCollectionPaginationData(collectionData)
+    // attach collection.taxonomies before any render so index/listing templates
+    // can link tag/category pages on page 1
+    buildTaxonomyData(collectionData)
 
     // must precede any rendering (incl. pagination pages) so every template
     // sees the current build's nav
     this.buildNavGlobal(markupIn, collectionData)
 
-    const collectionPromises = generateCollectionPaginationPages(collectionData, this.markupIn, this.markupOut, this.compileEntry.bind(this), this.baseURL)
+    const collectionPromises = [
+      ...generateCollectionPaginationPages(collectionData, this.markupIn, this.markupOut, this.compileEntry.bind(this), this.baseURL, this.siteData),
+      ...generateTaxonomyPages(collectionData, this.markupIn, this.markupOut, this.compileEntry.bind(this), this.baseURL, this.siteData)
+    ]
 
     await Promise.all(collectionPromises)
 
@@ -528,6 +556,20 @@ export default class Markups {
             isIndex: true
           })
         }
+
+        // taxonomy term pages: crawlable (sitemap) but isIndex, so out of the
+        // search index, llms.txt and nav — same as pagination pages
+        for (const tax of collection.taxonomies || []) {
+          for (const term of tax.terms) {
+            for (let p = 0; p < term.totalPages; p++) {
+              pageEntries.push({
+                url: p === 0 ? term.url : `${term.url}/${p + 1}`,
+                title: term.term,
+                isIndex: true
+              })
+            }
+          }
+        }
       }
     }
 
@@ -541,7 +583,11 @@ export default class Markups {
       generateIndexFiles(pageEntries, this.markupOut, this.siteData.url, {
         searchIndex: this.searchIndexConfig,
         sitemap: this.sitemapConfig,
-        nav: this.navConfig
+        llms: this.resolveLlmsConfig(),
+        robots: this.robotsConfig,
+        nav: this.navConfig,
+        feed: this.feedConfig,
+        site: this.siteData
       })
     }
 
@@ -650,7 +696,11 @@ export default class Markups {
       generateIndexFiles(this.lastPageEntries, this.markupOut, this.siteData.url, {
         searchIndex: this.searchIndexConfig,
         sitemap: this.sitemapConfig,
-        nav: this.navConfig
+        llms: this.resolveLlmsConfig(),
+        robots: this.robotsConfig,
+        nav: this.navConfig,
+        feed: this.feedConfig,
+        site: this.siteData
       })
     }
 
@@ -662,6 +712,6 @@ export default class Markups {
 // Everything except the fields that change on every content edit — used to
 // detect front matter drift that would invalidate the shared nav global
 function entrySignature(entry) {
-  const { content, wordcount, ...rest } = entry
+  const { content, wordcount, excerpt, ...rest } = entry
   return JSON.stringify(rest)
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "poops",
   "description": "Straightforward, no-bullshit bundler for the web.",
-  "version": "1.5.3",
+  "version": "1.7.0",
   "license": "MIT",
   "type": "module",
   "main": "poops.js",
@@ -52,7 +52,7 @@
   },
   "dependencies": {
     "argoyle": "^1.0.0",
-    "book-of-spells": "^1.0.32",
+    "book-of-spells": "^1.3.0",
     "chokidar": "^3.5.3",
     "connect": "^3.7.0",
     "dayjs": "^1.11.19",