poops 1.5.1 → 1.5.3

package/lib/markup/collections.js

@@ -262,7 +262,8 @@ export function generateCollectionPaginationPages(collectionData, markupInDir, m
       const compilePromise = compileEntryFn(file, context).then(({ result, skipped }) => {
         if (skipped) return
         mkDir(markupOutDirFull)
-        fs.writeFileSync(markupOut, result)
+        // async write so I/O overlaps rendering of the other pages
+        return fs.promises.writeFile(markupOut, result)
       })
       compilePromises.push(compilePromise)
     }

package/lib/markup/engines/liquid.js

@@ -2,12 +2,22 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { Liquid } from 'liquidjs'
 import { highlightCode } from '../highlight.js'
-import { marked, renderMarkdown } from '../renderer.js'
+import { marked, renderMarkdownCached } from '../renderer.js'
 import { discoverImageVariants, parseFrontMatter, groupby, buildImageTag, buildPaginationTag, renderToc } from '../helpers.js'
 import { getImageExif, listImages } from '../image-cache.js'
 import { slugify } from 'book-of-spells'
 import dayjs from 'dayjs'
 
+// liquidjs parse-cache keys: "<lookupType>:<name>" (root/layouts/partials)
+// for named partials, "<includerPath>,<name>" for relative references — see
+// liquidjs Parser._parseFileCached. The name part is what the template wrote.
+function depNameFromKey(key) {
+  const m = /^(root|layouts|partials):/.exec(key)
+  if (m) return key.slice(m[0].length)
+  const comma = key.indexOf(',')
+  return comma === -1 ? key : key.slice(comma + 1)
+}
+
 export default class LiquidEngine {
   constructor(templatesDir, includePaths) {
     const roots = [templatesDir]
@@ -24,14 +34,32 @@ export default class LiquidEngine {
       }
     } catch { /* ignore */ }
 
+    // Dependency index for incremental rebuilds, fed by the recording parse
+    // cache below. Persists across compiles; each page render overwrites its
+    // own entry.
+    this.deps = new Map()
+    this._currentDeps = null
+
+    // Parsed-template cache: without it every {% render %}/{% include %}
+    // re-resolves (realpath per root) and re-parses the partial on every
+    // page — dominates build time on partial-heavy sites. Cleared per
+    // compile (clearCache) so watch-mode edits are picked up.
+    // Recording: liquidjs consults this cache for every partial use, hits
+    // included, so read+write touches are a complete per-page partial trace
+    // (the loader/fs only fire on misses).
+    const store = new Map()
+    const touch = (key) => { if (this._currentDeps) this._currentDeps.add(depNameFromKey(key)) }
+    const parseCache = {
+      read: (key) => { const value = store.get(key); if (value !== undefined) touch(key); return value },
+      write: (key, value) => { store.set(key, value); touch(key) },
+      remove: (key) => store.delete(key),
+      clear: () => store.clear()
+    }
+
     this.engine = new Liquid({
       root: roots,
       extname: '.liquid',
-      // Parsed-template cache: without it every {% render %}/{% include %}
-      // re-resolves (realpath per root) and re-parses the partial on every
-      // page — dominates build time on partial-heavy sites. Cleared per
-      // compile (clearCache) so watch-mode edits are picked up.
-      cache: true,
+      cache: parseCache,
       dynamicPartials: true,
       strictFilters: false,
       strictVariables: false,
@@ -114,13 +142,32 @@ export default class LiquidEngine {
     if (this.engine.options.cache) this.engine.options.cache.clear()
   }
 
+  // Incremental rebuilds: pages whose last render touched this file. Dep
+  // names are partial names as written in templates ('sections/foo', no
+  // extension), so match by basename — a same-named file elsewhere
+  // over-rebuilds (safe), never under-rebuilds.
+  pagesDependingOn(file) {
+    const full = path.resolve(process.cwd(), file)
+    const base = path.basename(full, path.extname(full))
+    const pages = []
+    for (const [page, deps] of this.deps) {
+      for (const dep of deps) {
+        if (dep === full || path.posix.basename(dep, path.posix.extname(dep)) === base) {
+          pages.push(page)
+          break
+        }
+      }
+    }
+    return pages
+  }
+
   async render(templateName, context) {
     let source
     const frontMatterResult = parseFrontMatter(templateName)
     source = frontMatterResult.content
 
     if (path.extname(templateName) === '.md') {
-      source = renderMarkdown(source)
+      source = renderMarkdownCached(templateName, source)
     }
 
     const frontMatter = context.page || {}
@@ -128,13 +175,32 @@ export default class LiquidEngine {
       source = `{% layout '${frontMatter.layout}${this.fileExtension}' %}{% block content %}${source}{% endblock %}`
     }
 
-    return this.engine.parseAndRender(source, { ...this.globals, ...context }, {
-      globals: this.globals
-    })
+    // Dep tracking: parseAndRenderSync is fully synchronous, so every parse
+    // cache touch in this window belongs to this page. The page's own file
+    // seeds the set (the page source never goes through parseFile).
+    const pageKey = path.resolve(templateName)
+    const deps = new Set([pageKey])
+    this._currentDeps = deps
+
+    // Sync render path: ~20% faster than parseAndRender — thousands of small
+    // renders pay for the promise scaffolding (toPromise/isPromise), not the
+    // template work. Constraint: filters/tags must be sync or generator-based
+    // (all of ours are); an async filter won't throw in sync mode, it silently
+    // stringifies to "[object Promise]". Tags eval args via liquid._evalValue
+    // (generator, works in both modes) — liquid.evalValue returns a promise
+    // and would corrupt sync renders the same way.
+    try {
+      return this.engine.parseAndRenderSync(source, { ...this.globals, ...context }, {
+        globals: this.globals
+      })
+    } finally {
+      this._currentDeps = null
+      this.deps.set(pageKey, deps)
+    }
   }
 
   async renderString(source, context) {
-    return this.engine.parseAndRender(source, { ...this.globals, ...context }, {
+    return this.engine.parseAndRenderSync(source, { ...this.globals, ...context }, {
       globals: this.globals
     })
   }
@@ -148,7 +214,7 @@ function registerGoogleFontsTag(engine) {
       this.value = tagToken.args.trim()
     },
     * render(ctx) {
-      const fonts = yield this.liquid.evalValue(this.value, ctx)
+      const fonts = yield this.liquid._evalValue(this.value, ctx)
       if (!fonts || (Array.isArray(fonts) && fonts.length === 0)) return ''
 
       const fontList = typeof fonts === 'string' ? [fonts] : fonts
@@ -221,7 +287,7 @@ function registerPaginationTag(engine) {
       this.value = tagToken.args.trim()
     },
     * render(ctx) {
-      const pagination = yield this.liquid.evalValue(this.value, ctx)
+      const pagination = yield this.liquid._evalValue(this.value, ctx)
       const prefix = (yield ctx.get(['relativePathPrefix'])) || ''
       return buildPaginationTag(pagination, prefix)
     }

package/lib/markup/engines/nunjucks.js

@@ -6,29 +6,49 @@ import { discoverImageVariants, parseFrontMatter, groupby, buildImageTag, buildP
 import { getImageExif, listImages } from '../image-cache.js'
 import { toPosix } from '../../utils/helpers.js'
 import { highlightCode } from '../highlight.js'
-import { marked, renderMarkdown } from '../renderer.js'
+import { marked, renderMarkdownCached } from '../renderer.js'
 import { slugify } from 'book-of-spells'
 import dayjs from 'dayjs'
 import log from '../../utils/log.js'
 
 class RelativeLoader extends nunjucks.Loader {
-  constructor(templatesDir, includePaths) {
+  constructor(templatesDir, includePaths, onTouch) {
     super()
     this.templatesDir = templatesDir
     this.includePaths = includePaths || []
     this.includePaths.push('_*')
+    // Reports every compiled-template cache touch (hit or write) with the
+    // backing file path — the Environment reads loader.cache directly on
+    // hits, so getSource alone never sees reuse of already-compiled templates.
+    this.onTouch = onTouch || null
+  }
+
+  // nunjucks assigns `loader.cache = {}` and then reads/writes entries on it
+  // directly (Environment.getTemplate); the accessor wraps whatever it
+  // assigns in a proxy that feeds the dependency index.
+  get cache() { return this._cache }
+  set cache(store) {
+    const touch = (tmpl) => { if (tmpl && tmpl.path && this.onTouch) this.onTouch(tmpl.path) }
+    this._cache = new Proxy(store, {
+      get(target, key) { const tmpl = target[key]; touch(tmpl); return tmpl },
+      set(target, key, tmpl) { touch(tmpl); target[key] = tmpl; return true }
+    })
   }
 
   getSource(name) {
     let fullPath = name
     if (!fs.existsSync(name)) {
       let pattern = `**/${name}`
-      if (this.includePaths) {
+      if (this.includePaths.length > 1) {
         pattern = `{${this.includePaths.join(',')}}/${pattern}`
+      } else if (this.includePaths.length === 1) {
+        // no braces for a single path: glob treats one-element braces
+        // (`{_*}`) as literal characters and the pattern never matches
+        pattern = `${this.includePaths[0]}/${pattern}`
       }
       fullPath = globSync(toPosix(path.join(this.templatesDir, pattern)))[0]
     }
-    if (!fs.existsSync(fullPath)) {
+    if (!fullPath || !fs.existsSync(fullPath)) {
       log({ tag: 'markup', error: true, text: 'Template not found:', link: name })
       // noCache so a template created mid-watch is found on the next compile
       return { src: '', path: fullPath, noCache: true }
@@ -47,7 +67,7 @@ class RelativeLoader extends nunjucks.Loader {
     }
 
     if (path.extname(fullPath) === '.md') {
-      source = renderMarkdown(source)
+      source = renderMarkdownCached(fullPath, source)
     }
 
     if (frontMatter.layout) {
@@ -55,8 +75,9 @@ class RelativeLoader extends nunjucks.Loader {
     }
 
     // Cached: getSource does glob resolution + front matter + markdown per
-    // call, so re-running it per include per page dominates builds. Cleared
-    // per compile (clearCache) so watch-mode edits are picked up.
+    // call, so re-running it per include per page dominates builds. The cache
+    // survives across compiles; the watcher drops a changed file's entries
+    // via invalidate(), so watch-mode edits are still picked up.
     return { src: source, path: fullPath, noCache: false }
   }
 
@@ -69,8 +90,17 @@ export default class NunjucksEngine {
   constructor(templatesDir, includePaths, options) {
     const autoescape = (options && options.autoescape) || false
 
+    // Dependency index for incremental rebuilds: while a page renders, every
+    // template the Environment touches (cache hits included, via the loader's
+    // cache proxy) lands in that page's dep set. Persists across compiles —
+    // each page render overwrites its own entry.
+    this.deps = new Map()
+    this._currentDeps = null
+
     this.env = new nunjucks.Environment(
-      new RelativeLoader(templatesDir, includePaths),
+      new RelativeLoader(templatesDir, includePaths, (file) => {
+        if (this._currentDeps) this._currentDeps.add(file)
+      }),
       { autoescape, watch: false }
     )
   }
@@ -80,6 +110,22 @@ export default class NunjucksEngine {
     for (const loader of this.env.loaders) loader.cache = {}
   }
 
+  // Watch-mode: drop only the compiled template(s) backed by a changed or
+  // deleted file, instead of the whole cache being wiped per compile. Cache
+  // keys vary (absolute page paths, resolved include paths, bare include
+  // names), but every cached Template records the source path getSource
+  // returned — match on that. Prefix match covers a deleted directory.
+  invalidate(file) {
+    const full = path.resolve(process.cwd(), file)
+    for (const loader of this.env.loaders) {
+      for (const [name, tmpl] of Object.entries(loader.cache)) {
+        if (tmpl && tmpl.path && (tmpl.path === full || tmpl.path.startsWith(full + path.sep))) {
+          delete loader.cache[name]
+        }
+      }
+    }
+  }
+
   get fileExtension() { return '.njk' }
   get indexableExtensions() { return new Set(['.html', '.md', '.njk']) }
   get markupExtensions() { return 'html|xml|rss|atom|json|njk|md' }
@@ -154,16 +200,33 @@ export default class NunjucksEngine {
     delete this.env.globals[key]
   }
 
-  render(templateName, context) {
-    return new Promise((resolve, reject) => {
-      this.env.getTemplate(templateName).render(context, (error, result) => {
-        if (!error) {
-          resolve(result)
-        } else {
-          reject(error)
-        }
-      })
-    })
+  // Incremental rebuilds: pages whose last render loaded this file.
+  pagesDependingOn(file) {
+    const full = path.resolve(process.cwd(), file)
+    const pages = []
+    for (const [page, deps] of this.deps) {
+      if (deps.has(full)) pages.push(page)
+    }
+    return pages
+  }
+
+  async render(templateName, context) {
+    // Sync render (no callback): the callback API defers the result a tick
+    // per page, and dep attribution needs the render to finish inside this
+    // window — every template the loader cache serves in it belongs to this
+    // page. Constraint: filters/extensions must be sync (all of ours are;
+    // nunjucks throws loudly if an async one sneaks into a sync render).
+    // Deps kept on errors too: everything loaded up to the failure is
+    // recorded, so fixing the failing partial re-renders this page.
+    const pageKey = path.resolve(templateName)
+    const deps = new Set([pageKey])
+    this._currentDeps = deps
+    try {
+      return this.env.getTemplate(templateName).render(context)
+    } finally {
+      this._currentDeps = null
+      this.deps.set(pageKey, deps)
+    }
   }
 
   renderString(source, context) {

package/lib/markup/indexer.js

@@ -29,7 +29,7 @@ const DEFAULTS = {
   globalFrequencyCeiling: 0.8
 }
 
-const INTERNAL_FIELDS = new Set(['content', 'isIndex', 'layout', 'published'])
+const INTERNAL_FIELDS = new Set(['content', 'isIndex', 'layout', 'published', '_src'])
 
 function normalizeConfig(config) {
   if (!config) return null
@@ -94,11 +94,33 @@ function escapeXml(str) {
     .replace(/'/g, ''')
 }
 
+// Keyword extraction survives compiles: keyed by url, validated by comparing
+// rendered content (V8 string equality is a memcmp — far cheaper than the
+// regex passes in extractKeywords). Rebuilt per call, so deleted pages drop
+// out. ponytail: retains each page's HTML across compiles; switch to content
+// hashes if memory matters on very large sites.
+let keywordCache = new Map()
+let keywordCacheSig = ''
+
+// Test hook: extraction always builds a fresh array, so reference identity
+// across calls is proof of a memo hit
+export function _getKeywordCache() {
+  return keywordCache
+}
+
 export function generateSearchIndex(pageEntries, outputDir, config) {
   config = normalizeConfig(config)
   if (!config) return
 
+  // Any option that changes extraction output invalidates the whole cache
+  const sig = JSON.stringify([config.minWordLength, config.maxKeywords, config.stopWords])
+  if (sig !== keywordCacheSig) {
+    keywordCache = new Map()
+    keywordCacheSig = sig
+  }
+
   const stopWords = loadStopWords(config.stopWords)
+  const nextCache = new Map()
 
   let entries = pageEntries
     .filter(e => !e.isIndex)
@@ -108,12 +130,20 @@ export function generateSearchIndex(pageEntries, outputDir, config) {
         if (!INTERNAL_FIELDS.has(key)) entry[key] = value
       }
       if (!entry.keywords) {
-        entry.keywords = extractKeywords(e.content || '', { ...config, stopWords })
-          .slice(0, config.maxKeywords)
+        const content = e.content || ''
+        const cached = keywordCache.get(e.url)
+        entry.keywords = cached && cached.content === content
+          ? cached.keywords
+          : extractKeywords(content, { ...config, stopWords }).slice(0, config.maxKeywords)
+        // Cached pre-ceiling: the frequency ceiling reassigns entry.keywords
+        // to a new filtered array, so this reference stays unfiltered
+        nextCache.set(e.url, { content, keywords: entry.keywords })
       }
       return entry
     })
 
+  keywordCache = nextCache
+
   entries = applyGlobalFrequencyCeiling(entries, config.globalFrequencyCeiling)
 
   // resolve, not join: outputDir may be absolute (join would mangle it,

package/lib/markup/renderer.js

@@ -69,3 +69,16 @@ marked.use(markedGithubFootnote())
 export function renderMarkdown(source) {
   return mapRawSegments(marked.parse(source), decodeTemplateEntities, (s) => s)
 }
+
+// marked is a measurable slice of a build and its output only changes when
+// the file does. Keyed by path, validated by content identity: parseFrontMatter
+// hands back the same cached string object until the file's mtime changes, so
+// a === check invalidates exactly when the source was actually re-read.
+const markdownCache = new Map()
+export function renderMarkdownCached(filePath, source) {
+  const cached = markdownCache.get(filePath)
+  if (cached && cached.source === source) return cached.html
+  const html = renderMarkdown(source)
+  markdownCache.set(filePath, { source, html })
+  return html
+}

package/lib/markups.js

@@ -1,5 +1,5 @@
 import { pathExists, pathIsDirectory, readDataFile, mkDir, buildTime, toPosix } from './utils/helpers.js'
-import { replaceOutExtensions, getRelativePathPrefix, getPageUrlRelativeToOutput, parseFrontMatter, clearFrontMatterCache, wordcount } from './markup/helpers.js'
+import { replaceOutExtensions, getRelativePathPrefix, getPageUrlRelativeToOutput, parseFrontMatter, wordcount } from './markup/helpers.js'
 import { collectionAutoDiscovery, getCollectionDataBasedOnConfig, buildCollectionPaginationData, generateCollectionPaginationPages } from './markup/collections.js'
 import { generateIndexFiles, buildNavTree } from './markup/indexer.js'
 import NunjucksEngine from './markup/engines/nunjucks.js'
@@ -305,64 +305,79 @@ export default class Markups {
     this.engine.setGlobal('nav', buildNavTree(entries, navOptions))
   }
 
+  // One page: render, write, index. Shared by full directory compiles and
+  // incremental rebuilds. Resolves to { skipped, entry } so incremental can
+  // patch the previous build's entries.
+  compilePage(file, markupIn, collectionData, pageEntries) {
+    const relativePath = path.relative(markupIn, file)
+    const relativePathParts = relativePath.split(path.sep)
+
+    // Map before deriving dir/prefix: engines may relocate output (e.g.
+    // Shopify's templates/ flattening), and links must match the final path.
+    const markupOut = this.mapOutputPath(path.resolve(process.cwd(), this.markupOut, relativePath))
+    const fromPath = path.resolve(process.cwd(), this.markupOut)
+    const markupOutDir = path.dirname(markupOut)
+
+    mkDir(markupOutDir)
+
+    const fileContext = {
+      ...collectionData,
+      relativePathPrefix: getRelativePathPrefix(markupOutDir, fromPath, this.baseURL),
+      // output-relative so page.url matches nav.json urls (getPageUrlRelativeToOutput),
+      // otherwise `item.url == page.url` sidebar/active checks never fire
+      _url: getPageUrlRelativeToOutput(markupOut, this.markupOut)
+    }
+
+    const shouldIndex = pageEntries && this.engine.indexableExtensions.has(path.extname(file))
+    const fileCollection = relativePathParts.length > 1 && collectionData[relativePathParts[0]]
+      ? relativePathParts[0]
+      : null
+
+    return this.compileEntry(file, fileContext).then(({ result, frontMatter, skipped }) => {
+      if (skipped) {
+        if (fs.existsSync(markupOut)) {
+          fs.unlinkSync(markupOut)
+          log({ tag: this.logTag, text: 'Removed unpublished:', link: path.relative(process.cwd(), markupOut) })
+        }
+        return { skipped: true, entry: null }
+      }
+      // async write so I/O overlaps rendering of the other pages; the
+      // returned promise keeps Promise.all waiting for it
+      const write = fs.promises.writeFile(markupOut, result)
+
+      let entry = null
+      if (shouldIndex && frontMatter.published !== false) {
+        if (!frontMatter.title) frontMatter.title = path.basename(file, path.extname(file))
+        if (fileCollection && !frontMatter.collection) frontMatter.collection = fileCollection
+
+        entry = {
+          ...frontMatter,
+          url: getPageUrlRelativeToOutput(markupOut, this.markupOut),
+          content: result,
+          isIndex: false,
+          // source path (normalized) so incremental rebuilds can replace this
+          // entry; stripped from emitted artifacts (indexer INTERNAL_FIELDS)
+          _src: path.resolve(file)
+        }
+        pageEntries.push(entry)
+      }
+      return write.then(() => ({ skipped: false, entry }))
+    })
+  }
+
   async compileDirectory(markupIn, collectionData, pageEntries) {
     const markupStart = performance.now()
     const markupFiles = this.getMarkupFiles(markupIn)
     const compilePromises = []
-    const indexableExtensions = this.engine.indexableExtensions
 
     for (const file of markupFiles) {
-      const relativePath = path.relative(markupIn, file)
-      const relativePathParts = relativePath.split(path.sep)
+      const relativePathParts = path.relative(markupIn, file).split(path.sep)
 
       if (this.isCollectionIndexOverride(relativePathParts, collectionData)) {
         continue
       }
 
-      // Map before deriving dir/prefix: engines may relocate output (e.g.
-      // Shopify's templates/ flattening), and links must match the final path.
-      const markupOut = this.mapOutputPath(path.resolve(process.cwd(), this.markupOut, relativePath))
-      const fromPath = path.resolve(process.cwd(), this.markupOut)
-      const markupOutDir = path.dirname(markupOut)
-
-      mkDir(markupOutDir)
-
-      const fileContext = {
-        ...collectionData,
-        relativePathPrefix: getRelativePathPrefix(markupOutDir, fromPath, this.baseURL),
-        // output-relative so page.url matches nav.json urls (getPageUrlRelativeToOutput),
-        // otherwise `item.url == page.url` sidebar/active checks never fire
-        _url: getPageUrlRelativeToOutput(markupOut, this.markupOut)
-      }
-
-      const shouldIndex = pageEntries && indexableExtensions.has(path.extname(file))
-      const fileCollection = relativePathParts.length > 1 && collectionData[relativePathParts[0]]
-        ? relativePathParts[0]
-        : null
-
-      const compilePromise = this.compileEntry(file, fileContext).then(({ result, frontMatter, skipped }) => {
-        if (skipped) {
-          if (fs.existsSync(markupOut)) {
-            fs.unlinkSync(markupOut)
-            log({ tag: this.logTag, text: 'Removed unpublished:', link: path.relative(process.cwd(), markupOut) })
-          }
-          return
-        }
-        fs.writeFileSync(markupOut, result)
-
-        if (shouldIndex && frontMatter.published !== false) {
-          if (!frontMatter.title) frontMatter.title = path.basename(file, path.extname(file))
-          if (fileCollection && !frontMatter.collection) frontMatter.collection = fileCollection
-
-          pageEntries.push({
-            ...frontMatter,
-            url: getPageUrlRelativeToOutput(markupOut, this.markupOut),
-            content: result,
-            isIndex: false
-          })
-        }
-      })
-      compilePromises.push(compilePromise)
+      compilePromises.push(this.compilePage(file, markupIn, collectionData, pageEntries))
     }
 
     try {
@@ -374,13 +389,28 @@ export default class Markups {
       console.error(err)
       throw err
     } finally {
-      clearFrontMatterCache()
-      // Engine template caches live for one compile: fast within a build,
-      // watch-mode edits always picked up on the next one
-      if (typeof this.engine.clearCache === 'function') this.engine.clearCache()
+      this.clearEngineCaches()
     }
   }
 
+  // Front matter/markdown/data caches are mtime- or identity-validated and
+  // survive across compiles — a watch rebuild only re-reads what changed.
+  // Engines with per-file invalidation (nunjucks) keep their compiled
+  // templates too — the watcher routes changed paths to invalidate(). Engines
+  // without it (liquid) still clear their parse cache every compile so
+  // watch-mode edits are always picked up.
+  clearEngineCaches() {
+    if (typeof this.engine.invalidate === 'function') return
+    if (typeof this.engine.clearCache === 'function') this.engine.clearCache()
+  }
+
+  // Watch-mode hook: drop engine state backed by one changed/deleted file.
+  // The mtime-validated caches need no help — this only reaches engines that
+  // cache compiled templates across compiles.
+  invalidate(file) {
+    if (this.engine && typeof this.engine.invalidate === 'function') this.engine.invalidate(file)
+  }
+
   async compileSingleFile(markupIn, collectionData, pageEntries) {
     const markupStart = performance.now()
     let markupOut = path.resolve(process.cwd(), this.markupOut)
@@ -428,8 +458,7 @@ export default class Markups {
       console.error(err)
       throw err
     } finally {
-      clearFrontMatterCache()
-      if (typeof this.engine.clearCache === 'function') this.engine.clearCache()
+      this.clearEngineCaches()
     }
   }
 
@@ -440,6 +469,11 @@ export default class Markups {
     await this.init()
     if (!this.engine) return
 
+    // Reset up front so a failed build can't leave a stale snapshot —
+    // compileIncremental treats a missing snapshot as "must full-compile"
+    this.lastCollectionData = null
+    this.lastPageEntries = null
+
     if (this.config.reactorData) {
       // A removed reactor component must also drop its injected global,
       // same staleness rule as data files in loadDataFiles()
@@ -510,5 +544,124 @@ export default class Markups {
         nav: this.navConfig
       })
     }
+
+    // Snapshot for incremental rebuilds: they render with the same
+    // collection/pagination context and patch these entries in place
+    this.lastCollectionData = collectionData
+    this.lastPageEntries = pageEntries
+  }
+
+  // Watch hook for .json/.yaml changes. Data extensions are ambiguous: a
+  // .json may be a data file feeding the globals, or engine-owned markup
+  // (Shopify templates/*.json, section-group JSON). Engines that can tell
+  // claim theirs via isMarkupSource() and get the incremental path;
+  // everything else reloads data globals and full-compiles, as before.
+  async compileDataChange(file) {
+    await this.init()
+    if (this.engine && typeof this.engine.isMarkupSource === 'function' &&
+      this.engine.isMarkupSource(path.resolve(process.cwd(), file))) {
+      return this.compileIncremental(file)
+    }
+    return this.reloadDataFiles().then(() => this.compile())
   }
+
+  // Watch-mode entry point: rebuild only the pages whose last render touched
+  // the changed file. Falls back to a full compile() whenever the result
+  // isn't provably identical to a full build — unsupported engine, no prior
+  // successful full build, deletions/renames, collection membership (listing
+  // and pagination pages read item content outside the dep index), an
+  // affected pagination-rendered template, a published flip, or front matter
+  // drift while a nav global is baked into every page.
+  async compileIncremental(changedFile) {
+    const moduleConfig = this.config.markup
+    if (!moduleConfig || !moduleConfig.in) return
+
+    await this.init()
+    if (!this.engine) return
+
+    if (typeof this.engine.pagesDependingOn !== 'function') return this.compile()
+    if (!this.lastCollectionData) return this.compile()
+
+    const markupIn = path.resolve(process.cwd(), this.markupIn)
+    if (!pathIsDirectory(markupIn)) return this.compile()
+
+    const abs = path.resolve(process.cwd(), changedFile)
+    if (!fs.existsSync(abs)) return this.compile()
+
+    // Drop the changed file's compiled template (idempotent — the watcher
+    // already does this, but incremental must not depend on that)
+    this.invalidate(abs)
+
+    const collectionData = this.lastCollectionData
+    const relParts = path.relative(markupIn, abs).split(path.sep)
+    if (relParts[0] !== '..' && collectionData[relParts[0]]) return this.compile()
+
+    let affected = this.engine.pagesDependingOn(abs)
+    if (affected === null) return this.compile() // dep index unreliable
+    affected = affected.filter((page) => fs.existsSync(page))
+    if (affected.length === 0) return this.compile() // unknown file — don't guess
+
+    for (const page of affected) {
+      const rel = path.relative(markupIn, page)
+      if (rel.startsWith('..')) return this.compile()
+      if (this.isCollectionIndexOverride(rel.split(path.sep), collectionData)) return this.compile()
+    }
+
+    // A page's own dep set always contains itself, so an edited page is in
+    // `affected` (exact match, no glob needed). If it isn't, but the file IS a
+    // page source, this is a new page whose basename collides with a recorded
+    // dep — rendering only the collided pages would leave it unbuilt.
+    if (!affected.includes(abs) && this.getMarkupFiles(markupIn).some((f) => path.resolve(f) === abs)) {
+      return this.compile()
+    }
+
+    const markupStart = performance.now()
+    const newEntries = this.lastPageEntries ? [] : null
+
+    let results
+    try {
+      results = await Promise.all(affected.map((page) => this.compilePage(page, markupIn, collectionData, newEntries)))
+    } catch (err) {
+      log({ tag: this.logTag, error: true, text: 'Failed compiling' })
+      console.error(err)
+      throw err
+    } finally {
+      this.clearEngineCaches()
+    }
+
+    // A skipped (published: false) page changes the page set — nav, index and
+    // collection listings all shift, only a full build gets them right
+    if (results.some((r) => r.skipped)) return this.compile()
+
+    if (this.lastPageEntries) {
+      for (const entry of newEntries) {
+        const idx = this.lastPageEntries.findIndex((e) => e._src === entry._src)
+        if (idx === -1) return this.compile() // page wasn't part of the last full build
+        // The nav global is baked into every page's HTML — front matter
+        // drift (title, order, …) can change it for pages we're not
+        // rebuilding. Content/wordcount change on every edit and never feed
+        // the nav, so they're excluded from the comparison.
+        if (this.navConfig && entrySignature(this.lastPageEntries[idx]) !== entrySignature(entry)) {
+          return this.compile()
+        }
+        this.lastPageEntries[idx] = entry
+      }
+
+      generateIndexFiles(this.lastPageEntries, this.markupOut, this.siteData.url, {
+        searchIndex: this.searchIndexConfig,
+        sitemap: this.sitemapConfig,
+        nav: this.navConfig
+      })
+    }
+
+    const markupEnd = performance.now()
+    log({ tag: this.logTag, text: `Recompiled: ${affected.length} page${affected.length > 1 ? 's' : ''} into`, link: this.markupOut, time: buildTime(markupStart, markupEnd) })
+  }
+}
+
+// Everything except the fields that change on every content edit — used to
+// detect front matter drift that would invalidate the shared nav global
+function entrySignature(entry) {
+  const { content, wordcount, ...rest } = entry
+  return JSON.stringify(rest)
 }

package/lib/utils/helpers.js

@@ -96,10 +96,24 @@ export function readYamlFile(filePath) {
   }
 }
 
+// mtime+size-keyed memo (same pattern as the front matter cache): a watch
+// change to one data file re-reads only that file instead of re-parsing every
+// data file on each reload. Entries self-invalidate; callers share the parsed
+// object across reloads, so treat it as read-only.
+const dataFileCache = new Map()
+
 export function readDataFile(filePath) {
-  if (/(\.json)$/i.test(filePath)) return readJsonFile(filePath)
-  if (/(\.ya?ml)$/i.test(filePath)) return readYamlFile(filePath)
-  return fs.readFileSync(filePath, 'utf8')
+  const stat = fs.statSync(filePath)
+  const cached = dataFileCache.get(filePath)
+  if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached.value
+
+  let value
+  if (/(\.json)$/i.test(filePath)) value = readJsonFile(filePath)
+  else if (/(\.ya?ml)$/i.test(filePath)) value = readYamlFile(filePath)
+  else value = fs.readFileSync(filePath, 'utf8')
+
+  dataFileCache.set(filePath, { mtimeMs: stat.mtimeMs, size: stat.size, value })
+  return value
 }
 
 export function deleteDirectoryContents(directory) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "poops",
   "description": "Straightforward, no-bullshit bundler for the web.",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "license": "MIT",
   "type": "module",
   "main": "poops.js",

package/poops.js

@@ -55,31 +55,54 @@ async function resolveLiveReloadPort(config) {
   config.livereload_port = liveReloadPort
 }
 
-function setupLiveReloadServer(config) {
-  if (!config.livereload) return null
-  const liveReloadExcludes = ['.git', '.svn', '.hg']
-
-  if (config.watch) {
-    liveReloadExcludes.push(...config.watch)
-  }
-
-  if (config.includePaths) {
-    liveReloadExcludes.push(...config.includePaths)
-  }
+// The livereload server never fs-watches: watching the project meant every
+// output file written during a build fired its own reload (dozens of browser
+// flickers per build, some mid-write). Instead the rebuild chains in
+// setupWatchers call reload() once their compile settles; the debounce folds
+// the several module compiles one save triggers into a single refresh.
+// 500ms: long enough to fold cascaded chains — a style/script compile writes
+// into a watched copy source (Shopify theme assets/), whose chokidar event
+// (awaitWriteFinish 150ms) triggers the copy-to-dist chain ~300-400ms after
+// the first chain's reload. One save = one refresh, after dist is current.
+//
+// reload(file) collects paths over the debounce window. If everything in the
+// window is .css, each path is sent so the livereload client hot-swaps
+// stylesheets in place (no page reload, styles update without flicker);
+// anything else escalates to one full '/' refresh.
+let liveReloadServer = null
+let reloadTimer = null
+const reloadPaths = new Set()
+function reload(file) {
+  if (!liveReloadServer) return
+  reloadPaths.add(file || '/')
+  clearTimeout(reloadTimer)
+  reloadTimer = setTimeout(() => {
+    const paths = [...reloadPaths]
+    reloadPaths.clear()
+    if (paths.every((p) => p.endsWith('.css'))) {
+      paths.forEach((p) => liveReloadServer.refresh(p))
+    } else {
+      liveReloadServer.refresh('/')
+    }
+  }, 500)
+}
 
-  if (config.livereload.exclude) {
-    liveReloadExcludes.push(...config.livereload.exclude)
-  }
+// The css output paths of the styles entries — what the styles chain reports
+// to reload() so style edits hot-swap. A directory `out` maps to the entry
+// point's basename, mirroring how the styles compiler names its output file.
+function styleOutputs(config) {
+  return [config.styles].flat()
+    .filter((entry) => entry && entry.in && entry.out)
+    .map((entry) => (path.extname(entry.out)
+      ? entry.out
+      : path.join(entry.out, path.basename(entry.in).replace(/\.(sass|scss)$/i, '.css'))))
+}
 
-  const liveReloadServer = livereload.createServer({
-    exclusions: [...new Set(liveReloadExcludes)],
-    port: config.livereload_port,
-    exts: config.livereload.exts, // replaces the default watched extensions
-    extraExts: config.livereload.extraExts // adds to the defaults
-  })
+function setupLiveReloadServer(config) {
+  if (!config.livereload) return
+  liveReloadServer = livereload.createServer({ port: config.livereload_port })
   styledLog(`🔃 {dim}LiveReload  :{/} ${liveReloadServer.config.port}`)
   console.log()
-  liveReloadServer.watch(cwd)
 }
 
 function setupWatchers(config, modules) {
@@ -105,27 +128,37 @@ function setupWatchers(config, modules) {
   const isBuildOutput = (file) => outputZones.some((zone) => pathContainsPathSegment(file, zone))
 
   const rebuild = (file) => {
+    // Engines that keep compiled templates across compiles (nunjucks) drop
+    // exactly this file's entries; shared by change/add/unlink via rebuild.
+    modules.markups.invalidate(file)
     if (/(\.m?jsx?|\.tsx?)$/i.test(file) && !isBuildOutput(file)) {
-      modules.scripts.compile().catch(err => console.error(err))
+      modules.scripts.compile().then(() => reload()).catch(err => console.error(err))
 
       if (modules.reactor.belongsToReactor(file)) {
         modules.reactor.compile().then(() => {
           if (modules.reactor.renderedChanged) {
             config.reactorData = modules.reactor.getRendered()
-            modules.markups.compile().then(() => modules.postcss.compile()).catch(err => console.error(err))
+            modules.markups.compile().then(() => modules.postcss.compile()).then(() => reload()).catch(err => console.error(err))
           }
         }).catch(err => console.error(err))
       }
     }
     if (/(\.sass|\.scss|\.css)$/i.test(file) && !isBuildOutput(file)) {
-      modules.styles.compile().then(() => modules.postcss.compile()).catch(err => console.error(err))
+      modules.styles.compile().then(() => modules.postcss.compile())
+        .then(() => styleOutputs(config).forEach((out) => reload(out)))
+        .catch(err => console.error(err))
     }
     if (/(\.html|\.xml|\.rss|\.atom|\.njk|\.liquid|\.md)$/i.test(file)) {
-      modules.markups.compile().then(() => modules.postcss.compile()).catch(err => console.error(err))
+      // Incremental: re-render only the pages whose last render touched this
+      // file; falls back to a full compile for anything it can't prove safe
+      // (deletions, new files, collection members, engines without dep info).
+      modules.markups.compileIncremental(file).then(() => modules.postcss.compile()).then(() => reload()).catch(err => console.error(err))
     }
 
     if (/(\.json|\.ya?ml)$/i.test(file)) {
-      modules.markups.reloadDataFiles().then(() => modules.markups.compile()).catch(err => console.error(err))
+      // Engine-owned markup with a data extension (Shopify templates/*.json)
+      // goes incremental; real data files reload globals + full compile.
+      modules.markups.compileDataChange(file).then(() => reload()).catch(err => console.error(err))
     }
   }
 
@@ -141,9 +174,14 @@ function setupWatchers(config, modules) {
       modules.images.compile()
         .then(() => modules.markups.compile())
         .then(() => modules.postcss.compile())
+        .then(() => reload())
         .catch(err => console.error(err))
     }
-    doesFileBelongToPath(file, config.copy) && modules.copy.execute().catch(err => console.error(err))
+    // A copied .css (e.g. the styles compiler's own output landing in a copy
+    // source) stays a hot-swap; any other copied file needs a full reload.
+    doesFileBelongToPath(file, config.copy) && modules.copy.execute()
+      .then(() => reload(/\.css$/i.test(file) ? file : undefined))
+      .catch(err => console.error(err))
   }
 
   // Atomic-save editors (rename-write) fire unlink+add for every save, so an
@@ -169,6 +207,7 @@ function setupWatchers(config, modules) {
       modules.images.remove(file)
         .then(() => modules.markups.compile())
         .then(() => modules.postcss.compile())
+        .then(() => reload())
         .catch(err => console.error(err))
     }
     rebuild(file)
@@ -176,9 +215,10 @@ function setupWatchers(config, modules) {
   }
 
   const handleDeletedDir = (dirPath) => {
+    modules.markups.invalidate(dirPath) // prefix match drops every template under it
     modules.markups.removeOutput(dirPath)
     if (doesFileBelongToPath(dirPath, config.markup)) {
-      modules.markups.compile().then(() => modules.postcss.compile()).catch(err => console.error(err))
+      modules.markups.compile().then(() => modules.postcss.compile()).then(() => reload()).catch(err => console.error(err))
     }
     modules.copy.unlink(dirPath, doesFileBelongToPath(dirPath, config.copy))
   }