votive 0.2.1 → 0.3.1

package/README.md

@@ -1,18 +1,11 @@
 # Votive
 
-*File processor*
-
-- Powers [Voot](https://github.com/samlfair/voot)
-- Bundles [Vowel](https://github.com/samlfair/vowel)
-
 ## Roadmap
 
+- [ ] ReadPaths
 - [ ] Flesh out job runners
-- [ ] Rename jobs and paths
-- [ ] Better dependency tracking
-- [ ] Better query filters
-- [x] File deletion handling
+- [ ] Destination query filters
 
-## Project: Jobs
+## Helpers
 
-Jobs was originally a generic concept, but after working it's clear that there are two main categories of jobs: async writes and data fetching. We can probably handle async writes with inbuilt logic. So, instead of a "job", we should have "read uri"? That way we can cache all the uris.
+- Read

package/lib/bundle.js

@@ -245,6 +245,8 @@ async function bundle(config, cache) {
     // Write destination files
     await writeDestinations(config, database)
 
+    const stale = database.target.getStale()
+
     writeTime()
 
     // database.commit()
@@ -280,7 +282,6 @@ async function bundler(config) {
         queue.push(bundle(config))
         cache = await queue[0]
       } else {
-        "cache"
         queue.push(bundle(config, cache))
         await queue[0]
       }

package/lib/cleanDeletions.js

@@ -0,0 +1,32 @@
+import { readdir, rm } from "node:fs/promises"
+import path from "node:path"
+
+/** @import {Abstract, Abstracts, VotiveConfig} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+
+
+/**
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ */
+async function cleanDeletions(config, database) {
+  // Must remove from sources first??
+  const dirents = new Set((await readdir(config.destinationFolder, {
+    recursive: true,
+    withFileTypes: true
+  })).map(d => d.isFile() && path.relative(config.destinationFolder, path.join(d.parentPath, d.name)))
+    .filter(a => a)
+  )
+
+  const targets = new Set(database.target.getAll().map(t => t.path))
+
+  const deletions = dirents.difference(targets)
+
+  deletions.forEach(async deletion => {
+    const deletionPath = path.join(config.destinationFolder, deletion)
+    await rm(deletionPath)
+  })
+
+}
+
+export default cleanDeletions

package/lib/createDatabase.js

@@ -230,7 +230,18 @@ function prepareStatements(database) {
         WITH matches AS (
           SELECT m.destination, COUNT(*) AS match_count
           FROM metadata m
-          INNER JOIN json_each(:filter) j ON j.key = m.label AND j.value = m.value
+          INNER JOIN json_each(:filter) j ON j.key = m.label AND (
+            j.value = m.value
+            OR (
+              json_valid(m.value)
+              AND json_type(m.value) = 'array'
+              AND EXISTS (
+                SELECT 1
+                FROM json_each(m.value)
+                WHERE value = j.value
+              )
+            )
+          )
           GROUP BY m.destination
         ),
         filter_count AS (
@@ -250,9 +261,46 @@ function prepareStatements(database) {
           OR d.dir LIKE :recursivePath
         )
         GROUP BY d.path
-        ORDER BY d.path ASC
+        ORDER BY d.path ASC;
       `))),
 
+      /*
+          WITH matches AS (
+            SELECT m.destination, COUNT(*) AS match_count
+            FROM metadata m
+            INNER JOIN json_each('{ "breadcrumb": "Hello World" }') j ON j.key = m.label AND (
+              j.value = m.value
+              OR (
+                json_valid(m.value)
+                AND json_type(m.value) = 'array'
+                AND EXISTS (
+                  SELECT 1
+                  FROM json_each(m.value)
+                  WHERE value = j.value
+                )
+              )
+            )
+            GROUP BY m.destination
+          ),
+          filter_count AS (
+            SELECT COUNT(*) AS total FROM json_each('{ "breadcrumb": "Hello World" }')
+          )
+          SELECT d.*, json_group_object(i.label, i.value) AS metadata
+          FROM destinations d
+          INNER JOIN metadata i ON d.path = i.destination
+          LEFT JOIN matches ON matches.destination = d.path
+          CROSS JOIN filter_count f
+          WHERE (
+            f.total = 0
+            OR matches.match_count = f.total
+          )
+          AND (
+            d.dir = '/'
+            OR d.dir LIKE '%'
+          )
+          GROUP BY d.path
+          ORDER BY d.path ASC;
+          */
 
       /**
        * @callback SQLiteTargetGetAll
@@ -694,6 +742,19 @@ function createDatabase(databasePath = ".votive.db") {
 
     target: {
 
+      /**
+       * @param {string} filePath
+       */
+      delete(filePath) {
+        // Note: Need to run clean-up after
+        prepared.target.delete.get(filePath)
+        prepared.metadata.deleteByTarget.all(filePath)
+        const deps = queries.dependency.getAllByTarget(filePath)
+        deps.forEach(dep => {
+          queries.target.markStale(dep.dependent)
+        })
+      },
+
       /**
        * @param {string} filePath
        * @returns {TargetOutput | undefined}
@@ -755,7 +816,7 @@ function createDatabase(databasePath = ".votive.db") {
       getWithTrackers(filePath, dependent) {
         const target = queries.target.get(filePath)
 
-        if(!target) return
+        if (!target) return
 
         const trackedTarget = { metadata: {} }
 
@@ -790,7 +851,7 @@ function createDatabase(databasePath = ".votive.db") {
 
       /** @param {TargetGetManyWithTrackersParams | undefined} params */
       getManyWithTrackers(params) {
-        const { folder = "%", recursive, dependent, query = {} } = params
+        const { folder = "", recursive, dependent, query = {} } = params
 
         const recursivePath = [folder, "%"].filter(a => a).join("/")
 

package/lib/readAbstracts.js

@@ -1,6 +1,6 @@
 /** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
-/** @import {ReadSourceFilesResult} from "./readSources.js"
+/** @import {ReadSourceFileResult} from "./readSources.js"
 
 /**
  * @typedef {object} ReadAbstractsResult
@@ -13,17 +13,15 @@
  */
 
 /**
- * @param {ReadSourceFilesResult} files
+ * @param {ReadSourceFileResult} abstracts
  * @param {VotiveConfig} config
  * @param {Database} database
  * @param {FlatProcessors} processors
  * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
  */
-function readAbstracts(files, config, database, processors) {
+function readAbstracts(abstracts, config, database, processors) {
 
-  const processed = files.flatMap(file => {
-
-    const { abstract: unprocessedAbstract } = file
+  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
 
     /**
      * @param {FlatProcessors} processors
@@ -34,14 +32,13 @@ function readAbstracts(files, config, database, processors) {
       const [flatProcessor, ...rest] = processors
       if (!flatProcessor) return { abstract, jobs }
       const { processor, plugin } = flatProcessor
-      if (!processor.extensions.includes(file.syntax)
-        || !processor.transformFile
+      if (processor.syntax !== syntax
+        || !processor.read
+        || !processor.read.abstract
       ) return recursiveProcess(rest, abstract, jobs)
 
-      const settings = database.setting.getByFolder(file.dir)
-      
-      const processed = processor.transformFile(abstract, database, config)
-      processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
+      const processed = processor.read.abstract(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.plugin = plugin.name)
       return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
     }
 

package/lib/readFolders.js

@@ -1,9 +1,8 @@
-import path from "node:path"
-
 /** @import {VotiveConfig, FlatProcessors} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
 /** @import {Dirent} from "node:fs" */
 
+import path from "node:path"
 
 /**
  * @param {Dirent[]} folders
@@ -13,35 +12,33 @@ import path from "node:path"
  */
 function readFolders(folders = [], config, database, processors) {
 
-  const folderProcessors = processors.filter(({ processor }) => processor.readFolder)
+  const folderProcessors = processors.filter(({ processor }) => processor.read && processor.read.folder)
 
   const processed = folderProcessors.flatMap(({ processor, plugin }) => {
 
     const jobs = folders.flatMap(folder => {
-      let folderPath = path.join(folder.parentPath, folder.name)
-      if(folderPath) folderPath += path.sep
-
-      const { targets, jobs } = processor.readFolder(folderPath, database, config)
-      jobs.forEach(job => job.syntax = processor.syntax)
-      if (targets) {
-        targets.forEach(target => {
-          database.target.create(target)
+      const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
+
+      /** @ts-ignore `.read` is throwing a warning, but it's guarded above */
+      const { destinations, jobs } = processor.read.folder(folderPath, database, config)
+      jobs.forEach(job => job.plugin = plugin.name)
+      if (destinations) {
+        destinations.forEach(destination => {
+          database.createOrUpdateDestination(destination)
         })
         return jobs
       }
     })
 
-    
-    const rootPath = path.relative(config.sourceFolder, "")
-    const rootFolder = processor.readFolder(rootPath, database, config, true)
+    const rootFolder = processor.read.folder("", database, config)
 
-    if (rootFolder.targets) {
-      rootFolder.targets.forEach(target => {
-        database.target.create(target)
+    if (rootFolder.destinations) {
+      rootFolder.destinations.forEach(destination => {
+        database.createOrUpdateDestination(destination)
       })
     }
 
-    rootFolder.jobs.forEach(job => job.syntax = processor.syntax)
+    rootFolder.jobs.forEach(job => job.plugin = plugin.name)
 
     if(rootFolder.jobs) jobs.push(... rootFolder.jobs)
     return jobs

package/lib/readSources.js

@@ -1,16 +1,16 @@
 import { decodeBuffer } from "encoding-sniffer"
 import fs from "node:fs/promises"
 import path from "node:path"
-import pLimit from "p-limit"
+import { visit } from "unist-util-visit"
 
-/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax, Router} from "./bundle.js" */
+/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax} from "./bundle.js" */
 /** @import {Dirent} from "node:fs" */
 /** @import {Database} from "./createDatabase.js" */
 
 /**
  * @typedef {object} ReadSourcesResult
  * @property {Dirent[]} folders
- * @property {ReadSourceFileResult[]} sources
+ * @property {ReadSourceFilePlugin[]} sources
  */
 
 /**
@@ -25,56 +25,24 @@ async function readSources(config, database, processors) {
     recursive: true
   })
 
-  const filteredDirents = (dirents || []).filter(fileFilter(config))
-
+  const filteredDirents = (dirents || []).filter(fileFilter(config, database))
   const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
-  const loadingFiles = files.map(a => path.normalize(path.format({ name: a.name, dir: a.parentPath })))
   if (!files) return { folders, sources: [] }
-  const limit = pLimit(5)
   const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config))
-
-  const reading = [
-    (await Promise.all(readingSourceFiles)).filter(a => a),
-    pruneDeletions(config, database, filteredDirents)
-  ]
-
-  const [sources, deletedSources] = await Promise.all(reading)
-
+  const sources = readingSourceFiles && await Promise.all(readingSourceFiles)
   return {
     folders,
-    sources: [...sources, ...deletedSources]
+    sources
   }
 }
 
 /**
  * @param {VotiveConfig} config
  * @param {Database} database
- * @param {Dirent[]} dirents
- */
-async function pruneDeletions(config, database, dirents) {
-  const sourceFilePaths = new Set(dirents.map(d => d.isFile() && path.join(d.parentPath, d.name)))
-  const sourceRecords = database.source.getAll()
-  const sourceRecordPaths = new Set(sourceRecords.map(r => r.path))
-
-  const deletions = sourceRecordPaths.difference(sourceFilePaths)
-
-  let deletedSources = []
-
-  deletions.forEach(deletion => {
-    deletedSources.push(database.source.delete(deletion))
-  })
-
-  return deletedSources.filter(a => a)
-}
-
-/**
- * @param {VotiveConfig} config
  */
-function fileFilter(config) {
+function fileFilter(config, database) {
   /** @param {Dirent} dirent */
   return (dirent) => {
-    const isDestinationFolder = !path.relative(config.destinationFolder, path.join(dirent.parentPath, dirent.name))
-
     if (dirent.parentPath === config.destinationFolder) {
       return false
     } else if (dirent.parentPath.startsWith(config.destinationFolder + path.sep)) {
@@ -83,28 +51,21 @@ function fileFilter(config) {
       return false // Ignore hidden files
     } else if (dirent.parentPath.includes(path.sep + ".")) {
       return false // Ignore hidden folders
-    } else if (dirent.parentPath.match(/^\.\w/)) {
-      return false // Ignore hidden folders
-    } if (isDestinationFolder) {
-      return false
     }
     return true
   }
 }
 
 /**
- * @typedef {ReadSourceFileResult[]} ReadSourceFilesResult
+ * @typedef {ReadSourceFilePlugin[]} ReadSourceFileResult
  */
 
 /**
- * @typedef {object} ReadSourceFileResult
+ * @typedef {object} ReadSourceFilePlugin
  * @property {Abstract} abstract
  * @property {ProcessorSyntax} syntax
- * @property {object} [metadata]
- * @property {string} [targetFilePath]
- * @property {string} [dir]
  * @property {Jobs} jobs
- * @property {string} sourceFilePath
+ * @property {string} destinationPath
  */
 
 /**
@@ -115,12 +76,12 @@ function fileFilter(config) {
 function readSourceFile(processors, database, config) {
   /**
    * @param {import("node:fs").Dirent} dirent
-   * @returns {Promise<ReadSourceFileResult>[]}
+   * @returns {Promise<ReadSourceFilePlugin>[]}
    */
   return (dirent) => {
     const { name, parentPath } = dirent
-    const sourceFilePath = path.join(parentPath, name)
-    const sourceFileInfo = path.parse(sourceFilePath)
+    const filePath = path.join(parentPath, name)
+    const fileInfo = path.parse(filePath)
 
     const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config))
 
@@ -130,98 +91,43 @@ function readSourceFile(processors, database, config) {
      * @param {VotiveConfig} config
      */
     async function process(plugin, processor, config) {
-      const { readFile: read, extensions: filter, format } = processor
-      if (filter.includes(sourceFileInfo.ext) && read) {
+      const { read, filter, syntax } = processor
+      if (filter.extensions.includes(fileInfo.ext) && read && read.text) {
 
         // Check modified time
-        const stat = await fs.stat(sourceFilePath)
-        const source = database.source.get(sourceFilePath)
-        const diff = source && source.lastModified - Number(Math.floor(stat.mtimeMs))
-
-        if (source && diff > -1) {
-          return null
-        }
-
-        database.setting.deleteBySource(sourceFilePath)
-
-        const targetFilePath = route(sourceFilePath, plugin, config)
-        const targetFileExtension = path.extname(targetFilePath)
-
-        // Set URL if exists
-        const allJobs = []
-
-        if (format === "buffer") {
-
-          // FIXME update this type
-          const data = read(sourceFilePath, database, config)
-          const { metadata, abstract, jobs } = data
+        const stat = await fs.stat(filePath)
+        const source = database.getSource(filePath)
 
-          const target = database.target.create({
-            metadata,
-            abstract,
-            path: targetFilePath,
-          })
-
-
-          if (Array.isArray(jobs)) {
-            allJobs.push(...jobs)
-          }
-        }
 
-        if (format === "text") {
-          const stats = await fs.stat(sourceFilePath)
-          const data = await fs.readFile(sourceFilePath, { encoding: "utf-8" })
+        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
 
-          const { jobs, abstract, metadata } = read(data, sourceFilePath, targetFilePath, database, config)
+        database.deleteSettings(filePath)
 
+        // Get destination route
+        const destinationPath = route(filePath, plugin)
 
-          if (Array.isArray(jobs)) {
-            allJobs.push(...jobs)
-          }
-
-          const target = database.target.create({
-            abstract,
-            path: targetFilePath,
-            metadata,
-          })
-
-          // FIXME this won't work with the refactor
-          allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
-          updateSource()
-          return {
-            abstract,
-            metadata,
-            syntax: targetFileExtension,
-            targetFilePath: target.path,
-            dir: target.dir,
-            sourceFilePath,
-            jobs: allJobs
-          }
-        }
+        // Set URL if exists
+        const buffer = await fs.readFile(path.format(fileInfo))
+        const data = decodeBuffer(buffer)
+        const { jobs, abstract, metadata } = read.text(data, filePath, database, config)
+        jobs && jobs.forEach(job => job.plugin = plugin.name)
 
-        // FIXME Job syntax
-        allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
-        updateSource()
+        const timeStamp = stat.mtimeMs.toFixed()
 
-        return {
-          jobs: allJobs,
-          abstract: null,
-          targetFilePath: null,
-          sourceFilePath: null,
-          metadata: null,
-          syntax: null
+        if(source) {
+          database.updateSource(filePath, Number(timeStamp))
+        } else {
+          database.createSource(filePath, destinationPath, Number(timeStamp))
         }
 
-        function updateSource() {
-          const timeStamp = stat.mtimeMs.toFixed()
-
-          if (source) {
-            database.source.updateTimestamp(sourceFilePath, Number(timeStamp))
-          } else {
-            database.source.create(sourceFilePath, targetFilePath, Number(timeStamp))
-          }
-        }
+        database.createOrUpdateDestination({
+          metadata,
+          path: destinationPath,
+          abstract: abstract,
+          syntax: syntax
+        })
 
+        return { abstract, destinationPath, syntax, jobs }
       }
     }
 
@@ -232,43 +138,10 @@ function readSourceFile(processors, database, config) {
 /**
  * @param {string} filePath
  * @param {VotivePlugin} plugin
- * @param {VotiveConfig} config
  */
-function route(filePath, plugin, config) {
-  const { dir, ...parsedPath } = path.parse(filePath)
-  const rooty = !path.relative(config.sourceFolder, dir)
-  const segments = ["", ...dir.split(path.sep).filter(a => a)]
-  const pathInfo = {
-    inRootDir: rooty,
-    dir: segments,
-    ...parsedPath
-  }
-
-  if (!plugin.router) return "0"
-
-  const routedPath = plugin.router(pathInfo)
-
-  if (!routedPath) return "0"
-
-  if (routedPath.hasOwnProperty("dir") && Array.isArray(routedPath.dir)) {
-    return path.normalize(path.format({
-      dir: path.join(...routedPath.dir),
-      root: routedPath.root || "",
-      base: routedPath.base,
-      name: routedPath.name,
-      ext: routedPath.ext
-    }))
-  }
-
-  const routedInfo = {
-    dir: routedPath.dir || "",
-    root: routedPath.root || "",
-    base: routedPath.base,
-    name: routedPath.name,
-    ext: routedPath.ext
-  }
-
-  return path.normalize(path.format(routedInfo))
+function route(filePath, plugin) {
+  if (!plugin.router) return ""
+  return plugin.router(filePath) || ""
 }
 
 export default readSources

package/lib/runJobs.js

@@ -1,5 +1,8 @@
+import workerpool from "workerpool"
+
 /** @import {Job, Jobs, Database, VotiveConfig} from "./bundle.js" */
 
+const pool = workerpool.pool()
 
 /**
  * @param {Jobs} jobs
@@ -7,35 +10,14 @@
  * @param {Database} database
  */
 async function runJobs(jobs, config, database) {
-  const processors = config.plugins.flatMap(plugin => plugin.processors.flatMap(processor => processor.read?.url && ({ fetcher: processor.read.url, syntax: processor.syntax }))).filter(a => a)
-  const running = jobs.flatMap(async job => {
+  const running = jobs.map(async job => {
     if (!job) return
-    const cachedURL = database.getURL(job.data)
-    if (cachedURL) return
-    const processing = processors.map(async processor => {
-      if (processor.syntax === job.syntax) {
-        try {
-          const response = await fetch(job.data)
-          if (response.status >= 200 && response.status < 300) {
-            const data = await response[job.runner]()
-            const processed = processor.fetcher(data)
-            database.createURL(job.data, processed, job.destination)
-            return
-          } else {
-            console.warn(`Error fetching URL: ${job.data}`)
-          }
-        } catch (e) {
-          console.error(e)
-          return
-        }
-      }
-    })
-
-    return Promise.allSettled(processing)
+    const plugin = config.plugins.find(plugin => plugin.name === job.plugin)
+    return pool.exec(plugin.runners[job.runner], [job.data])
   })
 
-  const settled = await Promise.allSettled(running)
-  return settled
+  const ran = await Promise.allSettled(running)
+  pool.terminate()
 }
 
 export default runJobs

package/lib/utils/index.js

@@ -1,19 +1,14 @@
 import { styleText } from "node:util"
 import { statSync } from "node:fs"
-import path from "path"
 
-/**
- * @param {string} label
- * @param {string} message
- * @param {boolean} verbose
- */
-export function stopwatch(label, message, verbose) {
+/** @param {string} label */
+export function stopwatch(label) {
   const start = performance.now()
 
   function stop() {
     const end = performance.now()
     const duration = end - start
-    if(verbose) console.info(`${styleText("dim", label + ":")} ${styleText("magenta", message)} ${styleText("magenta", duration.toFixed(2) + "ms")}`)
+    console.info(`${label}: ${styleText("red", duration.toFixed(2) + "ms")}`)
   }
 
   return stop
@@ -23,11 +18,8 @@ export function stopwatch(label, message, verbose) {
  * @param {string} urlPath
  */
 export function splitURL(urlPath) {
-  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").filter(a => a).reverse()
-  const segments = urlDirSegmentsReversed.reverse()
-  segments.push('')
-  const folder = path.relative("", segments.join(path.sep))
-  return folder
+  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
+  return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
 }
 
 /** @param {string} dbPath */

package/lib/writeDestinations.js

@@ -1,4 +1,4 @@
-import { mkdir, writeFile } from "node:fs/promises"
+import { mkdir, writeFile, rm } from "node:fs/promises"
 import path from "node:path"
 import { checkFile } from "./utils/index.js"
 
@@ -31,7 +31,15 @@ async function writeDestinations(config, database) {
     return writeProcessors.map(async processor => {
       if (processor.extensions.includes(destination.syntax)) {
         const writeInfo = await processor.writeFile(destination, database, config)
-        if (!writeInfo) return
+        if (!writeInfo) {
+          try {
+            database.target.delete(destination.path)
+            return await rm(destinationPath)
+          } catch (e) {
+            console.error(e)
+          }
+        }
+
         const { data, encoding = 'utf-8' } = writeInfo
 
         async function write() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {
@@ -24,7 +24,6 @@
   },
   "dependencies": {
     "encoding-sniffer": "^0.2.1",
-    "p-limit": "^7.3.0",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "workerpool": "^10.0.1"

package/tests/index.js

@@ -13,84 +13,21 @@ import { checkFile } from "./../lib/utils/index.js"
 
 process.chdir("./tests")
 
-test("database creation in memory", (t) => {
-  const database = votive.createDatabase(":memory:")
-
-  t.test("database functions", () => {
-    const databaseKeys = new Set(Object.keys(database))
-    const expectedKeys = new Set(["saveDB", "source", "setting", "dependency", "target", "url"])
-    const difference = databaseKeys.difference(expectedKeys)
-    assert(difference.size === 0)
-  })
-
-  t.test("create sources", () => {
-    database.source.create("abc.in", "abc.out", 1)
-    database.source.create("def.in", "def.out", 2)
-    const sources = database.source.getAll()
-    assert(sources.length === 2)
-  })
-
-  t.test("create targets", () => {
-    database.target.create({
-      path: "abc.out",
-      abstract: ["abc"],
-      metadata: {
-        color: "blue",
-      }
-    })
-
-    database.target.create({
-      path: "def.out",
-      abstract: ["def"],
-      metadata: {
-        color: "red",
-      }
-    })
-
-    const targets = database.target.getAll()
-
-    assert(targets.length === 2)
-  })
-})
-
-test("bundle", async () => {
-  const tempDest = fs.mkdtempDisposableSync("target-")
+test("empty directory", async () => {
+  const tempDest = fs.mkdtempDisposableSync("destination-")
   const tempSource = fs.mkdtempDisposableSync("source-")
 
-  fs.writeFileSync(
-    path.join(tempSource.path, "abc.in"),
-    JSON.stringify({ abstract: ["abc"], metadata: { color: "blue" } }),
-    "utf-8"
-  )
-
   try {
-    const stop = stopwatch("Bundle folder")
-    const database = await votive.bundle({
+    const stop = stopwatch("Bundle empty folder")
+    await votive.bundle({
       sourceFolder: tempSource.path,
       destinationFolder: tempDest.path,
-      plugins: [{
-        name: "test-plugin",
-        processors: [{
-          format: "text",
-          extensions: [".in"],
-          // readResource: null,
-          writeFile: (destination) => ({ data: destination.abstract[0] }),
-          readFile: (data) => ({ abstract: [data], metadata: { color: "green" } }),
-          // readFolder: null,
-          // transformFile: null
-        }],
-        router: ({ base, dir, root }) => ({ base, root, ext: ".out" }),
-      }]
-
+      plugins: []
     })
 
-    const sources = database.source.getAll()
-
-    const targets = database.target.getAll()
-
     stop()
 
-    const dir = fs.readdirSync(tempDest.path, { recursive: true })
+    const dir = fs.readdirSync(tempDest.path)
     assert(dir.length === 0, "Destination directory is not empty.")
   } catch (error) {
     tempDest.remove()
@@ -103,275 +40,281 @@ test("bundle", async () => {
   tempSource.remove()
 })
 
-// test("internals", async (t) => {
-//   const dbExists = checkFile(".votive.db")
-//   if (dbExists) await rm(".votive.db")
-
-//   const temp = fs.mkdtempDisposableSync("destination-")
-
-//   /** @returns {Job} */
-//   function createExampleJob() {
-//     return {
-//       data: Math.floor(Math.random() * 1000),
-//       runner: "exampleRunner"
-//     }
-//   }
-
-
-//   /** @type {ReadText} */
-//   function exampleTextReader(text, filePath, destinationPath, database, config) {
-//     const matches = text.match(/\b\w+\b/)
-//     const title = matches ? matches[0] : "Untitled"
-
-//     /** @type {ReadTextResult} */
-//     return {
-//       abstract: {
-//         content: text,
-//       },
-//       metadata: {
-//         title
-//       },
-//       jobs: [
-//         createExampleJob()
-//       ]
-//     }
-//   }
-
-//   /** @type {ReadAbstract} */
-//   function exampleAbstractReader(abstract, database, config) {
-//     abstract.exampleAppend = true
-//     return { abstract, jobs: [createExampleJob()] }
-//   }
-
-//   /** @type {ReadFolder} */
-//   function exampleFolderReader(folder, database, config) {
-//     return {
-//       jobs: [createExampleJob()],
-//       destinations: [
-//         {
-//           path: "index.html",
-//           abstract: { content: "" },
-//           metadata: {
-//             title: "home"
-//           },
-//           syntax: "md"
-//         }
-//       ]
-//     }
-//   }
-
-//   /** @type {VotiveProcessor} */
-//   const exampleProcessor = {
-//     syntax: "md",
-//     filter: {
-//       extensions: [".md"]
-//     },
-//     read: {
-//       text: exampleTextReader,
-//       abstract: exampleAbstractReader,
-//       folder: exampleFolderReader
-//     },
-//     write: (destination, database, config) => {
-//       return {
-//         data: "lorem ipsum",
-//       }
-//     }
-//   }
-
-//   /** @param {string} sourcePath */
-//   function router({ base, ...parsed }) {
-//     return { ...parsed, ext: ".html" }
-//   }
-
-//   /** @type {VotivePlugin} */
-//   const examplePlugin = {
-//     name: "example plugin",
-//     runners: {
-//       exampleRunner: exampleRunner
-//     },
-//     router,
-//     processors: [exampleProcessor]
-//   }
-
-//   /** @type {Runner} */
-//   async function exampleRunner(data, database) {
-//     return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
-//   }
-
-//   /** @type {VotiveConfig} */
-//   const config = {
-//     sourceFolder: "./markdown",
-//     destinationFolder: temp.path,
-//     plugins: [examplePlugin]
-//   }
-
-//   /** @type {FlatProcessors} */
-//   const processors = [
-//     {
-//       plugin: examplePlugin,
-//       processor: exampleProcessor
-//     }
-//   ]
-
-//   try {
-//     fs.writeFileSync("markdown/prunee.md", "A little content")
-
-//     const database = votive.createDatabase()
-//     const sourcesOne = database.getAllSources()
-
-//     t.test("no sources on startup", () => {
-//       const isArray = Array.isArray(sourcesOne)
-//       const isEmpty = !sourcesOne.length
-//       assert(isArray && isEmpty)
-//     })
-
-
-
-//     const stop = stopwatch("Read sources")
-
-//     const { folders, sources } = await votive.readSources(config, database, processors)
-
-//     t.test('one folder exists', () => {
-//       assert(folders.length === 1)
-//     })
-
-//     t.test('three sources exist', () => {
-//       assert(sources.length === 4)
-//     })
-
-//     stop()
-
-//     const sourcesJobs = sources.flatMap(source => source.jobs)
+test("internals", async (t) => {
+  const dbExists = checkFile(".votive.db")
+  if (dbExists) await rm(".votive.db")
+
+  const temp = fs.mkdtempDisposableSync("destination-")
+
+  /** @returns {Job} */
+  function createExampleJob() {
+    return {
+      data: Math.floor(Math.random() * 1000),
+      runner: "exampleRunner"
+    }
+  }
+
+
+  /** @type {ReadText} */
+  function exampleTextReader(text, filePath, database, config) {
+    const matches = text.match(/\b\w+\b/)
+    const title = matches ? matches[0] : "Untitled"
+
+    /** @type {ReadTextResult} */
+    return {
+      abstract: {
+        content: text,
+      },
+      metadata: {
+        title
+      },
+      jobs: [
+        createExampleJob()
+      ]
+    }
+  }
+
+  /** @type {ReadAbstract} */
+  function exampleAbstractReader(abstract, database, config) {
+    abstract.exampleAppend = true
+    return { abstract, jobs: [createExampleJob()] }
+  }
+
+  /** @type {ReadFolder} */
+  function exampleFolderReader(folder, database, config) {
+    return {
+      jobs: [createExampleJob()],
+      destinations: [
+        {
+          path: "/index.html",
+          abstract: { content: "" },
+          metadata: {
+            title: "home"
+          },
+          syntax: "md"
+        }
+      ]
+    }
+  }
+
+  /** @type {VotiveProcessor} */
+  const exampleProcessor = {
+    syntax: "md",
+    filter: {
+      extensions: [".md"]
+    },
+    read: {
+      text: exampleTextReader,
+      abstract: exampleAbstractReader,
+      folder: exampleFolderReader
+    },
+    write: (destination, database, config) => {
+      return {
+        data: "lorem ipsum",
+      }
+    }
+  }
+
+  /** @param {string} sourcePath */
+  function router(sourcePath) {
+    const { base, ...parsed }= path.parse(sourcePath)
+    return path.format({ ...parsed, ext: ".html" })
+  }
+
+  /** @type {VotivePlugin} */
+  const examplePlugin = {
+    name: "example plugin",
+    runners: {
+      exampleRunner: exampleRunner
+    },
+    router,
+    processors: [exampleProcessor]
+  }
+
+  /** @type {Runner} */
+  async function exampleRunner(data, database) {
+    return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
+  }
+
+  /** @type {VotiveConfig} */
+  const config = {
+    sourceFolder: "./markdown",
+    destinationFolder: temp.path,
+    plugins: [examplePlugin]
+  }
 
-//     const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
-
-//     t.test('transformation succeeded', () => {
-//       assert(processedAbstracts.find(abstract => abstract.exampleAppend))
-//     })
+  /** @type {FlatProcessors} */
+  const processors = [
+    {
+      plugin: examplePlugin,
+      processor: exampleProcessor
+    }
+  ]
 
-//     t.test('abstract jobs exist', () => {
-//       assert(abstractsJobs.length > 0)
-//       assert(abstractsJobs.every(job => job.data && job.runner))
-//     })
+  try {
+    fs.writeFileSync("markdown/prunee.md", "A little content")
 
-//     const foldersJobs = readFolders(folders, config, database, processors)
+    const database = votive.createDatabase()
+    const sourcesOne = database.getAllSources()
 
-//     t.test("folders jobs exist", () => {
-//       assert(foldersJobs.length > 0)
-//       assert(foldersJobs.every(job => job.data && job.runner))
-//     })
+    t.test("no sources on startup", () => {
+      const isArray = Array.isArray(sourcesOne)
+      const isEmpty = !sourcesOne.length
+      assert(isArray && isEmpty)
+    })
 
-//     database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
-//     const firstDestination = database.getDestinationIndependently("abc.html", [])
 
-//     t.test('first destination created', () => {
-//       assert(firstDestination.path === 'abc.html')
-//     })
 
-//     database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
+    const stop = stopwatch("Read sources")
 
-//     /* TODO: Test that dependencies are working */
+    const { folders, sources } = await votive.readSources(config, database, processors)
 
-//     const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
-//     const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-//     const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-//     const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
+    t.test('one folder exists', () => {
+      assert(folders.length === 1)
+    })
 
-//     t.test('settings created', () => {
-//       assert(setting.value === 'blue')
-//       assert(newSetting.value === 'Dog')
-//       assert(!sameSetting)
-//     })
+    t.test('three sources exist', () => {
+      assert(sources.length === 4)
+    })
 
-//     const abcSettings = database.getSettings("abc.html")
+    stop()
 
-//     t.test('settings retrieved', () => {
-//       assert(abcSettings.theme[0] === "blue")
-//     })
+    const sourcesJobs = sources.flatMap(source => source.jobs)
 
-//     const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
-//     const defSettings = database.getSettings("abc/def.html")
+    const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
 
-//     t.test('settings retrieved', () => {
-//       assert(abcSettings.theme[0] === "blue")
-//       assert(defSettings.theme[1] === "green")
-//     })
+    t.test('transformation succeeded', () => {
+      assert(processedAbstracts.find(abstract => abstract.exampleAppend))
+    })
 
-//     const staleDestinations = database.getStaleDestinations()
+    t.test('abstract jobs exist', () => {
+      assert(abstractsJobs.length > 0)
+      assert(abstractsJobs.every(job => job.data && job.runner))
+    })
 
-//     t.test('all destinations are stale', () => {
-//       assert(staleDestinations.length === 7)
-//     })
+    const foldersJobs = readFolders(folders, config, database, processors)
 
-//     const written = await votive.writeDestinations(config, database)
+    t.test("folders jobs exist", () => {
+      assert(foldersJobs.length > 0)
+      assert(foldersJobs.every(job => job.data && job.runner))
+    })
 
-//     const staleDestinationsAfterWriting = database.getStaleDestinations()
+    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
+    const firstDestination = database.getDestinationIndependently("abc.html", [])
 
-//     t.test('all destinations are fresh', () => {
-//       assert(staleDestinationsAfterWriting.length === 0)
-//     })
+    t.test('first destination created', () => {
+      assert(firstDestination.path === 'abc.html')
+    })
 
-//     const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-//     const staleAfterAbstractUpdate = database.getStaleDestinations()
+    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
+    const destination = database.getDestinationDependently("abc.html", ["a"], "abc/def.html")
 
-//     t.test('updated document with no side effects is stale', () => {
-//       assert(staleAfterAbstractUpdate.length === 1)
-//     })
+    const dependencies = database.getDependencies()
 
-//     await votive.writeDestinations(config, database)
-//     const staleAfterSecondWrite = database.getStaleDestinations()
+    t.test('dependency created', () => {
+      assert(dependencies[0].dependent === 'abc/def.html')
+    })
 
-//     t.test('everything fresh again', () => {
-//       assert(staleAfterSecondWrite.length === 0)
-//     })
 
-//     const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-//     const staleAfterSideEffects = database.getStaleDestinations()
+    const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
+    const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+    const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+    const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
 
-//     t.test('side effects work', () => {
-//       assert(staleAfterSideEffects.length === 2)
-//     })
+    t.test('settings created', () => {
+      assert(setting.value === 'blue')
+      assert(newSetting.value === 'Dog')
+      assert(!sameSetting)
+    })
 
+    const abcSettings = database.getSettings("abc.html")
 
-//     await votive.writeDestinations(config, database)
+    t.test('settings retrieved', () => {
+      assert(abcSettings.theme[0] === "blue")
+    })
 
-//     const oldSetting = database.setSetting("abc", "theme", "green")
+    const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
+    const defSettings = database.getSettings("abc/def.html")
 
-//     const staleAfterSettingsChange = database.getStaleDestinations()
+    t.test('settings retrieved', () => {
+      assert(abcSettings.theme[0] === "blue")
+      assert(defSettings.theme[1] === "green")
+    })
 
-//     t.test('setting affects descendents', () => {
-//       assert(staleAfterSettingsChange.length === 1)
-//     })
+    const staleDestinations = database.getStaleDestinations()
 
-//     // TODO: Write tests for jobs
-//     const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
-//     runJobs(jobs, config, database)
+    t.test('all destinations are stale', () => {
+      assert(staleDestinations.length === 7)
+    })
 
-//     fs.rmSync("markdown/prunee.md")
+    const written = await votive.writeDestinations(config, database)
 
-//     await votive.pruneSources(config, database)
+    const staleDestinationsAfterWriting = database.getStaleDestinations()
 
-//     const result = database.getDestinationIndependently("markdown/prunee.html")
+    t.test('all destinations are fresh', () => {
+      assert(staleDestinationsAfterWriting.length === 0)
+    })
+
+    const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+    const staleAfterAbstractUpdate = database.getStaleDestinations()
 
-//     /* TODO: Check pruning metadata works (I removed it) */
+    t.test('updated document with no side effects is stale', () => {
+      assert(staleAfterAbstractUpdate.length === 1)
+    })
 
-//     const finalDestinations = database.getDestinations({
-//       filter: [
-//         {
-//           property: "a",
-//           operator: "gt",
-//           value: 3
-//         }
-//       ]
-//     }, "markdown/prunee.html")
+    await votive.writeDestinations(config, database)
+    const staleAfterSecondWrite = database.getStaleDestinations()
 
-//     // TODO: Write a test for get destinations
+    t.test('everything fresh again', () => {
+      assert(staleAfterSecondWrite.length === 0)
+    })
 
-//     await database.saveDB()
+    const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+    const staleAfterSideEffects = database.getStaleDestinations()
 
-//     temp.remove()
-//   } catch (error) {
-//     temp.remove()
-//     throw error
-//   }
-// })
+    t.test('side effects work', () => {
+      assert(staleAfterSideEffects.length === 2)
+    })
+
+    const freshDestinations = database.getAllDestinations()
+
+    await votive.writeDestinations(config, database)
+
+    const oldSetting = database.setSetting("abc", "theme", "green")
+
+    const staleAfterSettingsChange = database.getStaleDestinations()
+
+    t.test('setting affects descendents', () => {
+      assert(staleAfterSettingsChange.length === 1)
+    })
+
+    // TODO: Write tests for jobs
+    const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
+    runJobs(jobs, config, database)
+
+    const destinations = database.getAllDestinations()
+    database.getDestinationDependently("markdown/prunee.html", ["title"], "abc/def.html")
+    const newDependencies = database.getDependencies()
+
+    fs.rmSync("markdown/prunee.md")
+
+    await votive.pruneSources(config, database)
+
+    const result = database.getDestinationIndependently("markdown/prunee.html")
+    const prunedDependencies = database.getDependencies()
+    const prunedMetadata = database.getAllMetadataByPath("markdown/prunee.html")
+
+    t.test('source pruned', () => {
+      assert(!result)
+      assert(newDependencies.length - prunedDependencies.length === 1)
+      assert(!prunedMetadata)
+    })
+
+    await database.saveDB()
+
+    temp.remove()
+  } catch (error) {
+    temp.remove()
+    throw error
+  }
+})