votive 0.3.2 → 0.4.0

package/README.md

@@ -1,11 +1,18 @@
 # Votive
 
+*File processor*
+
+- Powers [Voot](https://github.com/samlfair/voot)
+- Bundles [Vowel](https://github.com/samlfair/vowel)
+
 ## Roadmap
 
-- [ ] ReadPaths
 - [ ] Flesh out job runners
-- [ ] Destination query filters
+- [ ] Rename jobs and paths
+- [ ] Better dependency tracking
+- [ ] Better query filters
+- [x] File deletion handling
 
-## Helpers
+## Project: Jobs
 
-- Read
+Jobs was originally a generic concept, but after working it's clear that there are two main categories of jobs: async writes and data fetching. We can probably handle async writes with inbuilt logic. So, instead of a "job", we should have "read uri"? That way we can cache all the uris.

package/lib/bundle.js

@@ -245,8 +245,6 @@ async function bundle(config, cache) {
     // Write destination files
     await writeDestinations(config, database)
 
-    const stale = database.target.getStale()
-
     writeTime()
 
     // database.commit()
@@ -282,6 +280,7 @@ async function bundler(config) {
         queue.push(bundle(config))
         cache = await queue[0]
       } else {
+        "cache"
         queue.push(bundle(config, cache))
         await queue[0]
       }

package/lib/createDatabase.js

@@ -264,43 +264,43 @@ function prepareStatements(database) {
         ORDER BY d.path ASC;
       `))),
 
-      /*
-          WITH matches AS (
-            SELECT m.destination, COUNT(*) AS match_count
-            FROM metadata m
-            INNER JOIN json_each('{ "breadcrumb": "Hello World" }') j ON j.key = m.label AND (
-              j.value = m.value
-              OR (
-                json_valid(m.value)
-                AND json_type(m.value) = 'array'
-                AND EXISTS (
-                  SELECT 1
-                  FROM json_each(m.value)
-                  WHERE value = j.value
-                )
+    /*
+        WITH matches AS (
+          SELECT m.destination, COUNT(*) AS match_count
+          FROM metadata m
+          INNER JOIN json_each('{ "breadcrumb": "Hello World" }') j ON j.key = m.label AND (
+            j.value = m.value
+            OR (
+              json_valid(m.value)
+              AND json_type(m.value) = 'array'
+              AND EXISTS (
+                SELECT 1
+                FROM json_each(m.value)
+                WHERE value = j.value
               )
             )
-            GROUP BY m.destination
-          ),
-          filter_count AS (
-            SELECT COUNT(*) AS total FROM json_each('{ "breadcrumb": "Hello World" }')
-          )
-          SELECT d.*, json_group_object(i.label, i.value) AS metadata
-          FROM destinations d
-          INNER JOIN metadata i ON d.path = i.destination
-          LEFT JOIN matches ON matches.destination = d.path
-          CROSS JOIN filter_count f
-          WHERE (
-            f.total = 0
-            OR matches.match_count = f.total
           )
-          AND (
-            d.dir = '/'
-            OR d.dir LIKE '%'
-          )
-          GROUP BY d.path
-          ORDER BY d.path ASC;
-          */
+          GROUP BY m.destination
+        ),
+        filter_count AS (
+          SELECT COUNT(*) AS total FROM json_each('{ "breadcrumb": "Hello World" }')
+        )
+        SELECT d.*, json_group_object(i.label, i.value) AS metadata
+        FROM destinations d
+        INNER JOIN metadata i ON d.path = i.destination
+        LEFT JOIN matches ON matches.destination = d.path
+        CROSS JOIN filter_count f
+        WHERE (
+          f.total = 0
+          OR matches.match_count = f.total
+        )
+        AND (
+          d.dir = '/'
+          OR d.dir LIKE '%'
+        )
+        GROUP BY d.path
+        ORDER BY d.path ASC;
+        */
 
       /**
        * @callback SQLiteTargetGetAll
@@ -746,13 +746,14 @@ function createDatabase(databasePath = ".votive.db") {
        * @param {string} filePath
        */
       delete(filePath) {
-        // Note: Need to run clean-up after
-        prepared.target.delete.get(filePath)
+        console.log("deleting")
+        const deleted = prepared.target.delete.get(filePath)
         prepared.metadata.deleteByTarget.all(filePath)
         const deps = queries.dependency.getAllByTarget(filePath)
         deps.forEach(dep => {
           queries.target.markStale(dep.dependent)
         })
+        return deleted
       },
 
       /**
@@ -816,7 +817,7 @@ function createDatabase(databasePath = ".votive.db") {
       getWithTrackers(filePath, dependent) {
         const target = queries.target.get(filePath)
 
-        if (!target) return
+        if(!target) return
 
         const trackedTarget = { metadata: {} }
 

package/lib/readAbstracts.js

@@ -1,6 +1,6 @@
 /** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
-/** @import {ReadSourceFileResult} from "./readSources.js"
+/** @import {ReadSourceFilesResult} from "./readSources.js"
 
 /**
  * @typedef {object} ReadAbstractsResult
@@ -13,15 +13,17 @@
  */
 
 /**
- * @param {ReadSourceFileResult} abstracts
+ * @param {ReadSourceFilesResult} files
  * @param {VotiveConfig} config
  * @param {Database} database
  * @param {FlatProcessors} processors
  * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
  */
-function readAbstracts(abstracts, config, database, processors) {
+function readAbstracts(files, config, database, processors) {
 
-  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
+  const processed = files.flatMap(file => {
+
+    const { abstract: unprocessedAbstract } = file
 
     /**
      * @param {FlatProcessors} processors
@@ -32,13 +34,14 @@ function readAbstracts(abstracts, config, database, processors) {
       const [flatProcessor, ...rest] = processors
       if (!flatProcessor) return { abstract, jobs }
       const { processor, plugin } = flatProcessor
-      if (processor.syntax !== syntax
-        || !processor.read
-        || !processor.read.abstract
+      if (!processor.extensions.includes(file.syntax)
+        || !processor.transformFile
       ) return recursiveProcess(rest, abstract, jobs)
 
-      const processed = processor.read.abstract(abstract, database, config)
-      processed.jobs && processed.jobs.forEach(job => job.plugin = plugin.name)
+      const settings = database.setting.getByFolder(file.dir)
+      
+      const processed = processor.transformFile(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
       return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
     }
 

package/lib/readFolders.js

@@ -1,8 +1,9 @@
+import path from "node:path"
+
 /** @import {VotiveConfig, FlatProcessors} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
 /** @import {Dirent} from "node:fs" */
 
-import path from "node:path"
 
 /**
  * @param {Dirent[]} folders
@@ -12,33 +13,35 @@ import path from "node:path"
  */
 function readFolders(folders = [], config, database, processors) {
 
-  const folderProcessors = processors.filter(({ processor }) => processor.read && processor.read.folder)
+  const folderProcessors = processors.filter(({ processor }) => processor.readFolder)
 
   const processed = folderProcessors.flatMap(({ processor, plugin }) => {
 
     const jobs = folders.flatMap(folder => {
-      const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
-
-      /** @ts-ignore `.read` is throwing a warning, but it's guarded above */
-      const { destinations, jobs } = processor.read.folder(folderPath, database, config)
-      jobs.forEach(job => job.plugin = plugin.name)
-      if (destinations) {
-        destinations.forEach(destination => {
-          database.createOrUpdateDestination(destination)
+      let folderPath = path.join(folder.parentPath, folder.name)
+      if(folderPath) folderPath += path.sep
+
+      const { targets, jobs } = processor.readFolder(folderPath, database, config)
+      jobs.forEach(job => job.syntax = processor.syntax)
+      if (targets) {
+        targets.forEach(target => {
+          database.target.create(target)
         })
         return jobs
       }
     })
 
-    const rootFolder = processor.read.folder("", database, config)
+    
+    const rootPath = path.relative(config.sourceFolder, "")
+    const rootFolder = processor.readFolder(rootPath, database, config, true)
 
-    if (rootFolder.destinations) {
-      rootFolder.destinations.forEach(destination => {
-        database.createOrUpdateDestination(destination)
+    if (rootFolder.targets) {
+      rootFolder.targets.forEach(target => {
+        database.target.create(target)
       })
     }
 
-    rootFolder.jobs.forEach(job => job.plugin = plugin.name)
+    rootFolder.jobs.forEach(job => job.syntax = processor.syntax)
 
     if(rootFolder.jobs) jobs.push(... rootFolder.jobs)
     return jobs

package/lib/readSources.js

@@ -1,16 +1,16 @@
 import { decodeBuffer } from "encoding-sniffer"
 import fs from "node:fs/promises"
 import path from "node:path"
-import { visit } from "unist-util-visit"
+import pLimit from "p-limit"
 
-/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax} from "./bundle.js" */
+/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax, Router} from "./bundle.js" */
 /** @import {Dirent} from "node:fs" */
 /** @import {Database} from "./createDatabase.js" */
 
 /**
  * @typedef {object} ReadSourcesResult
  * @property {Dirent[]} folders
- * @property {ReadSourceFilePlugin[]} sources
+ * @property {ReadSourceFileResult[]} sources
  */
 
 /**
@@ -25,24 +25,56 @@ async function readSources(config, database, processors) {
     recursive: true
   })
 
-  const filteredDirents = (dirents || []).filter(fileFilter(config, database))
+  const filteredDirents = (dirents || []).filter(fileFilter(config))
+
   const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
+  const loadingFiles = files.map(a => path.normalize(path.format({ name: a.name, dir: a.parentPath })))
   if (!files) return { folders, sources: [] }
+  const limit = pLimit(5)
   const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config))
-  const sources = readingSourceFiles && await Promise.all(readingSourceFiles)
+
+  const reading = [
+    (await Promise.all(readingSourceFiles)).filter(a => a),
+    pruneDeletions(config, database, filteredDirents)
+  ]
+
+  const [sources, deletedSources] = await Promise.all(reading)
+
   return {
     folders,
-    sources
+    sources: [...sources, ...deletedSources]
   }
 }
 
 /**
  * @param {VotiveConfig} config
  * @param {Database} database
+ * @param {Dirent[]} dirents
+ */
+async function pruneDeletions(config, database, dirents) {
+  const sourceFilePaths = new Set(dirents.map(d => d.isFile() && path.join(d.parentPath, d.name)))
+  const sourceRecords = database.source.getAll()
+  const sourceRecordPaths = new Set(sourceRecords.map(r => r.path))
+
+  const deletions = sourceRecordPaths.difference(sourceFilePaths)
+
+  let deletedSources = []
+
+  deletions.forEach(deletion => {
+    deletedSources.push(database.source.delete(deletion))
+  })
+
+  return deletedSources.filter(a => a)
+}
+
+/**
+ * @param {VotiveConfig} config
  */
-function fileFilter(config, database) {
+function fileFilter(config) {
   /** @param {Dirent} dirent */
   return (dirent) => {
+    const isDestinationFolder = !path.relative(config.destinationFolder, path.join(dirent.parentPath, dirent.name))
+
     if (dirent.parentPath === config.destinationFolder) {
       return false
     } else if (dirent.parentPath.startsWith(config.destinationFolder + path.sep)) {
@@ -51,21 +83,28 @@ function fileFilter(config, database) {
       return false // Ignore hidden files
     } else if (dirent.parentPath.includes(path.sep + ".")) {
       return false // Ignore hidden folders
+    } else if (dirent.parentPath.match(/^\.\w/)) {
+      return false // Ignore hidden folders
+    } if (isDestinationFolder) {
+      return false
     }
     return true
   }
 }
 
 /**
- * @typedef {ReadSourceFilePlugin[]} ReadSourceFileResult
+ * @typedef {ReadSourceFileResult[]} ReadSourceFilesResult
  */
 
 /**
- * @typedef {object} ReadSourceFilePlugin
+ * @typedef {object} ReadSourceFileResult
  * @property {Abstract} abstract
  * @property {ProcessorSyntax} syntax
+ * @property {object} [metadata]
+ * @property {string} [targetFilePath]
+ * @property {string} [dir]
  * @property {Jobs} jobs
- * @property {string} destinationPath
+ * @property {string} sourceFilePath
  */
 
 /**
@@ -76,12 +115,12 @@ function fileFilter(config, database) {
 function readSourceFile(processors, database, config) {
   /**
    * @param {import("node:fs").Dirent} dirent
-   * @returns {Promise<ReadSourceFilePlugin>[]}
+   * @returns {Promise<ReadSourceFileResult>[]}
    */
   return (dirent) => {
     const { name, parentPath } = dirent
-    const filePath = path.join(parentPath, name)
-    const fileInfo = path.parse(filePath)
+    const sourceFilePath = path.join(parentPath, name)
+    const sourceFileInfo = path.parse(sourceFilePath)
 
     const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config))
 
@@ -91,43 +130,98 @@ function readSourceFile(processors, database, config) {
      * @param {VotiveConfig} config
      */
     async function process(plugin, processor, config) {
-      const { read, filter, syntax } = processor
-      if (filter.extensions.includes(fileInfo.ext) && read && read.text) {
+      const { readFile: read, extensions: filter, format } = processor
+      if (filter.includes(sourceFileInfo.ext) && read) {
 
         // Check modified time
-        const stat = await fs.stat(filePath)
-        const source = database.getSource(filePath)
+        const stat = await fs.stat(sourceFilePath)
+        const source = database.source.get(sourceFilePath)
+        const diff = source && source.lastModified - Number(Math.floor(stat.mtimeMs))
 
+        if (source && diff > -1) {
+          return null
+        }
 
-        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
-
-        database.deleteSettings(filePath)
+        database.setting.deleteBySource(sourceFilePath)
 
-        // Get destination route
-        const destinationPath = route(filePath, plugin)
+        const targetFilePath = route(sourceFilePath, plugin, config)
+        const targetFileExtension = path.extname(targetFilePath)
 
         // Set URL if exists
-        const buffer = await fs.readFile(path.format(fileInfo))
-        const data = decodeBuffer(buffer)
-        const { jobs, abstract, metadata } = read.text(data, filePath, database, config)
-        jobs && jobs.forEach(job => job.plugin = plugin.name)
+        const allJobs = []
+
+        if (format === "buffer") {
+
+          // FIXME update this type
+          const data = read(sourceFilePath, database, config)
+          const { metadata, abstract, jobs } = data
+
+          const target = database.target.create({
+            metadata,
+            abstract,
+            path: targetFilePath,
+          })
 
-        const timeStamp = stat.mtimeMs.toFixed()
 
-        if(source) {
-          database.updateSource(filePath, Number(timeStamp))
-        } else {
-          database.createSource(filePath, destinationPath, Number(timeStamp))
+          if (Array.isArray(jobs)) {
+            allJobs.push(...jobs)
+          }
         }
 
-        database.createOrUpdateDestination({
-          metadata,
-          path: destinationPath,
-          abstract: abstract,
-          syntax: syntax
-        })
+        if (format === "text") {
+          const stats = await fs.stat(sourceFilePath)
+          const data = await fs.readFile(sourceFilePath, { encoding: "utf-8" })
+
+          const { jobs, abstract, metadata } = read(data, sourceFilePath, targetFilePath, database, config)
+
+
+          if (Array.isArray(jobs)) {
+            allJobs.push(...jobs)
+          }
+
+          const target = database.target.create({
+            abstract,
+            path: targetFilePath,
+            metadata,
+          })
+
+          // FIXME this won't work with the refactor
+          allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
+          updateSource()
+          return {
+            abstract,
+            metadata,
+            syntax: targetFileExtension,
+            targetFilePath: target.path,
+            dir: target.dir,
+            sourceFilePath,
+            jobs: allJobs
+          }
+        }
+
+        // FIXME Job syntax
+        allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
+        updateSource()
+
+        return {
+          jobs: allJobs,
+          abstract: null,
+          targetFilePath: null,
+          sourceFilePath: null,
+          metadata: null,
+          syntax: null
+        }
+
+        function updateSource() {
+          const timeStamp = stat.mtimeMs.toFixed()
+
+          if (source) {
+            database.source.updateTimestamp(sourceFilePath, Number(timeStamp))
+          } else {
+            database.source.create(sourceFilePath, targetFilePath, Number(timeStamp))
+          }
+        }
 
-        return { abstract, destinationPath, syntax, jobs }
       }
     }
 
@@ -138,10 +232,43 @@ function readSourceFile(processors, database, config) {
 /**
  * @param {string} filePath
  * @param {VotivePlugin} plugin
+ * @param {VotiveConfig} config
  */
-function route(filePath, plugin) {
-  if (!plugin.router) return ""
-  return plugin.router(filePath) || ""
+function route(filePath, plugin, config) {
+  const { dir, ...parsedPath } = path.parse(filePath)
+  const rooty = !path.relative(config.sourceFolder, dir)
+  const segments = ["", ...dir.split(path.sep).filter(a => a)]
+  const pathInfo = {
+    inRootDir: rooty,
+    dir: segments,
+    ...parsedPath
+  }
+
+  if (!plugin.router) return "0"
+
+  const routedPath = plugin.router(pathInfo)
+
+  if (!routedPath) return "0"
+
+  if (routedPath.hasOwnProperty("dir") && Array.isArray(routedPath.dir)) {
+    return path.normalize(path.format({
+      dir: path.join(...routedPath.dir),
+      root: routedPath.root || "",
+      base: routedPath.base,
+      name: routedPath.name,
+      ext: routedPath.ext
+    }))
+  }
+
+  const routedInfo = {
+    dir: routedPath.dir || "",
+    root: routedPath.root || "",
+    base: routedPath.base,
+    name: routedPath.name,
+    ext: routedPath.ext
+  }
+
+  return path.normalize(path.format(routedInfo))
 }
 
 export default readSources

package/lib/runJobs.js

@@ -1,8 +1,5 @@
-import workerpool from "workerpool"
-
 /** @import {Job, Jobs, Database, VotiveConfig} from "./bundle.js" */
 
-const pool = workerpool.pool()
 
 /**
  * @param {Jobs} jobs
@@ -10,14 +7,35 @@ const pool = workerpool.pool()
  * @param {Database} database
  */
 async function runJobs(jobs, config, database) {
-  const running = jobs.map(async job => {
+  const processors = config.plugins.flatMap(plugin => plugin.processors.flatMap(processor => processor.read?.url && ({ fetcher: processor.read.url, syntax: processor.syntax }))).filter(a => a)
+  const running = jobs.flatMap(async job => {
     if (!job) return
-    const plugin = config.plugins.find(plugin => plugin.name === job.plugin)
-    return pool.exec(plugin.runners[job.runner], [job.data])
+    const cachedURL = database.getURL(job.data)
+    if (cachedURL) return
+    const processing = processors.map(async processor => {
+      if (processor.syntax === job.syntax) {
+        try {
+          const response = await fetch(job.data)
+          if (response.status >= 200 && response.status < 300) {
+            const data = await response[job.runner]()
+            const processed = processor.fetcher(data)
+            database.createURL(job.data, processed, job.destination)
+            return
+          } else {
+            console.warn(`Error fetching URL: ${job.data}`)
+          }
+        } catch (e) {
+          console.error(e)
+          return
+        }
+      }
+    })
+
+    return Promise.allSettled(processing)
   })
 
-  const ran = await Promise.allSettled(running)
-  pool.terminate()
+  const settled = await Promise.allSettled(running)
+  return settled
 }
 
 export default runJobs

package/lib/utils/index.js

@@ -1,14 +1,19 @@
 import { styleText } from "node:util"
 import { statSync } from "node:fs"
+import path from "path"
 
-/** @param {string} label */
-export function stopwatch(label) {
+/**
+ * @param {string} label
+ * @param {string} message
+ * @param {boolean} verbose
+ */
+export function stopwatch(label, message, verbose) {
   const start = performance.now()
 
   function stop() {
     const end = performance.now()
     const duration = end - start
-    console.info(`${label}: ${styleText("red", duration.toFixed(2) + "ms")}`)
+    if(verbose) console.info(`${styleText("dim", label + ":")} ${styleText("magenta", message)} ${styleText("magenta", duration.toFixed(2) + "ms")}`)
   }
 
   return stop
@@ -18,8 +23,11 @@ export function stopwatch(label) {
  * @param {string} urlPath
  */
 export function splitURL(urlPath) {
-  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
-  return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
+  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").filter(a => a).reverse()
+  const segments = urlDirSegmentsReversed.reverse()
+  segments.push('')
+  const folder = path.relative("", segments.join(path.sep))
+  return folder
 }
 
 /** @param {string} dbPath */

package/lib/writeDestinations.js

@@ -39,7 +39,6 @@ async function writeDestinations(config, database) {
             console.error(e)
           }
         }
-
         const { data, encoding = 'utf-8' } = writeInfo
 
         async function write() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "encoding-sniffer": "^0.2.1",
+    "p-limit": "^7.3.0",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "workerpool": "^10.0.1"

package/tests/index.js

@@ -13,21 +13,84 @@ import { checkFile } from "./../lib/utils/index.js"
 
 process.chdir("./tests")
 
-test("empty directory", async () => {
-  const tempDest = fs.mkdtempDisposableSync("destination-")
+test("database creation in memory", (t) => {
+  const database = votive.createDatabase(":memory:")
+
+  t.test("database functions", () => {
+    const databaseKeys = new Set(Object.keys(database))
+    const expectedKeys = new Set(["saveDB", "source", "setting", "dependency", "target", "url"])
+    const difference = databaseKeys.difference(expectedKeys)
+    assert(difference.size === 0)
+  })
+
+  t.test("create sources", () => {
+    database.source.create("abc.in", "abc.out", 1)
+    database.source.create("def.in", "def.out", 2)
+    const sources = database.source.getAll()
+    assert(sources.length === 2)
+  })
+
+  t.test("create targets", () => {
+    database.target.create({
+      path: "abc.out",
+      abstract: ["abc"],
+      metadata: {
+        color: "blue",
+      }
+    })
+
+    database.target.create({
+      path: "def.out",
+      abstract: ["def"],
+      metadata: {
+        color: "red",
+      }
+    })
+
+    const targets = database.target.getAll()
+
+    assert(targets.length === 2)
+  })
+})
+
+test("bundle", async () => {
+  const tempDest = fs.mkdtempDisposableSync("target-")
   const tempSource = fs.mkdtempDisposableSync("source-")
 
+  fs.writeFileSync(
+    path.join(tempSource.path, "abc.in"),
+    JSON.stringify({ abstract: ["abc"], metadata: { color: "blue" } }),
+    "utf-8"
+  )
+
   try {
-    const stop = stopwatch("Bundle empty folder")
-    await votive.bundle({
+    const stop = stopwatch("Bundle folder")
+    const database = await votive.bundle({
       sourceFolder: tempSource.path,
       destinationFolder: tempDest.path,
-      plugins: []
+      plugins: [{
+        name: "test-plugin",
+        processors: [{
+          format: "text",
+          extensions: [".in"],
+          // readResource: null,
+          writeFile: (destination) => ({ data: destination.abstract[0] }),
+          readFile: (data) => ({ abstract: [data], metadata: { color: "green" } }),
+          // readFolder: null,
+          // transformFile: null
+        }],
+        router: ({ base, dir, root }) => ({ base, root, ext: ".out" }),
+      }]
+
     })
 
+    const sources = database.source.getAll()
+
+    const targets = database.target.getAll()
+
     stop()
 
-    const dir = fs.readdirSync(tempDest.path)
+    const dir = fs.readdirSync(tempDest.path, { recursive: true })
     assert(dir.length === 0, "Destination directory is not empty.")
   } catch (error) {
     tempDest.remove()
@@ -40,281 +103,275 @@ test("empty directory", async () => {
   tempSource.remove()
 })
 
-test("internals", async (t) => {
-  const dbExists = checkFile(".votive.db")
-  if (dbExists) await rm(".votive.db")
-
-  const temp = fs.mkdtempDisposableSync("destination-")
-
-  /** @returns {Job} */
-  function createExampleJob() {
-    return {
-      data: Math.floor(Math.random() * 1000),
-      runner: "exampleRunner"
-    }
-  }
-
-
-  /** @type {ReadText} */
-  function exampleTextReader(text, filePath, database, config) {
-    const matches = text.match(/\b\w+\b/)
-    const title = matches ? matches[0] : "Untitled"
-
-    /** @type {ReadTextResult} */
-    return {
-      abstract: {
-        content: text,
-      },
-      metadata: {
-        title
-      },
-      jobs: [
-        createExampleJob()
-      ]
-    }
-  }
-
-  /** @type {ReadAbstract} */
-  function exampleAbstractReader(abstract, database, config) {
-    abstract.exampleAppend = true
-    return { abstract, jobs: [createExampleJob()] }
-  }
-
-  /** @type {ReadFolder} */
-  function exampleFolderReader(folder, database, config) {
-    return {
-      jobs: [createExampleJob()],
-      destinations: [
-        {
-          path: "/index.html",
-          abstract: { content: "" },
-          metadata: {
-            title: "home"
-          },
-          syntax: "md"
-        }
-      ]
-    }
-  }
-
-  /** @type {VotiveProcessor} */
-  const exampleProcessor = {
-    syntax: "md",
-    filter: {
-      extensions: [".md"]
-    },
-    read: {
-      text: exampleTextReader,
-      abstract: exampleAbstractReader,
-      folder: exampleFolderReader
-    },
-    write: (destination, database, config) => {
-      return {
-        data: "lorem ipsum",
-      }
-    }
-  }
-
-  /** @param {string} sourcePath */
-  function router(sourcePath) {
-    const { base, ...parsed }= path.parse(sourcePath)
-    return path.format({ ...parsed, ext: ".html" })
-  }
-
-  /** @type {VotivePlugin} */
-  const examplePlugin = {
-    name: "example plugin",
-    runners: {
-      exampleRunner: exampleRunner
-    },
-    router,
-    processors: [exampleProcessor]
-  }
-
-  /** @type {Runner} */
-  async function exampleRunner(data, database) {
-    return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
-  }
-
-  /** @type {VotiveConfig} */
-  const config = {
-    sourceFolder: "./markdown",
-    destinationFolder: temp.path,
-    plugins: [examplePlugin]
-  }
+// test("internals", async (t) => {
+//   const dbExists = checkFile(".votive.db")
+//   if (dbExists) await rm(".votive.db")
+
+//   const temp = fs.mkdtempDisposableSync("destination-")
+
+//   /** @returns {Job} */
+//   function createExampleJob() {
+//     return {
+//       data: Math.floor(Math.random() * 1000),
+//       runner: "exampleRunner"
+//     }
+//   }
+
+
+//   /** @type {ReadText} */
+//   function exampleTextReader(text, filePath, destinationPath, database, config) {
+//     const matches = text.match(/\b\w+\b/)
+//     const title = matches ? matches[0] : "Untitled"
+
+//     /** @type {ReadTextResult} */
+//     return {
+//       abstract: {
+//         content: text,
+//       },
+//       metadata: {
+//         title
+//       },
+//       jobs: [
+//         createExampleJob()
+//       ]
+//     }
+//   }
+
+//   /** @type {ReadAbstract} */
+//   function exampleAbstractReader(abstract, database, config) {
+//     abstract.exampleAppend = true
+//     return { abstract, jobs: [createExampleJob()] }
+//   }
+
+//   /** @type {ReadFolder} */
+//   function exampleFolderReader(folder, database, config) {
+//     return {
+//       jobs: [createExampleJob()],
+//       destinations: [
+//         {
+//           path: "index.html",
+//           abstract: { content: "" },
+//           metadata: {
+//             title: "home"
+//           },
+//           syntax: "md"
+//         }
+//       ]
+//     }
+//   }
+
+//   /** @type {VotiveProcessor} */
+//   const exampleProcessor = {
+//     syntax: "md",
+//     filter: {
+//       extensions: [".md"]
+//     },
+//     read: {
+//       text: exampleTextReader,
+//       abstract: exampleAbstractReader,
+//       folder: exampleFolderReader
+//     },
+//     write: (destination, database, config) => {
+//       return {
+//         data: "lorem ipsum",
+//       }
+//     }
+//   }
+
+//   /** @param {string} sourcePath */
+//   function router({ base, ...parsed }) {
+//     return { ...parsed, ext: ".html" }
+//   }
+
+//   /** @type {VotivePlugin} */
+//   const examplePlugin = {
+//     name: "example plugin",
+//     runners: {
+//       exampleRunner: exampleRunner
+//     },
+//     router,
+//     processors: [exampleProcessor]
+//   }
+
+//   /** @type {Runner} */
+//   async function exampleRunner(data, database) {
+//     return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
+//   }
+
+//   /** @type {VotiveConfig} */
+//   const config = {
+//     sourceFolder: "./markdown",
+//     destinationFolder: temp.path,
+//     plugins: [examplePlugin]
+//   }
+
+//   /** @type {FlatProcessors} */
+//   const processors = [
+//     {
+//       plugin: examplePlugin,
+//       processor: exampleProcessor
+//     }
+//   ]
+
+//   try {
+//     fs.writeFileSync("markdown/prunee.md", "A little content")
+
+//     const database = votive.createDatabase()
+//     const sourcesOne = database.getAllSources()
+
+//     t.test("no sources on startup", () => {
+//       const isArray = Array.isArray(sourcesOne)
+//       const isEmpty = !sourcesOne.length
+//       assert(isArray && isEmpty)
+//     })
+
+
+
+//     const stop = stopwatch("Read sources")
+
+//     const { folders, sources } = await votive.readSources(config, database, processors)
+
+//     t.test('one folder exists', () => {
+//       assert(folders.length === 1)
+//     })
+
+//     t.test('three sources exist', () => {
+//       assert(sources.length === 4)
+//     })
+
+//     stop()
+
+//     const sourcesJobs = sources.flatMap(source => source.jobs)
 
-  /** @type {FlatProcessors} */
-  const processors = [
-    {
-      plugin: examplePlugin,
-      processor: exampleProcessor
-    }
-  ]
+//     const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
+
+//     t.test('transformation succeeded', () => {
+//       assert(processedAbstracts.find(abstract => abstract.exampleAppend))
+//     })
 
-  try {
-    fs.writeFileSync("markdown/prunee.md", "A little content")
+//     t.test('abstract jobs exist', () => {
+//       assert(abstractsJobs.length > 0)
+//       assert(abstractsJobs.every(job => job.data && job.runner))
+//     })
 
-    const database = votive.createDatabase()
-    const sourcesOne = database.getAllSources()
+//     const foldersJobs = readFolders(folders, config, database, processors)
 
-    t.test("no sources on startup", () => {
-      const isArray = Array.isArray(sourcesOne)
-      const isEmpty = !sourcesOne.length
-      assert(isArray && isEmpty)
-    })
+//     t.test("folders jobs exist", () => {
+//       assert(foldersJobs.length > 0)
+//       assert(foldersJobs.every(job => job.data && job.runner))
+//     })
 
+//     database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
+//     const firstDestination = database.getDestinationIndependently("abc.html", [])
 
+//     t.test('first destination created', () => {
+//       assert(firstDestination.path === 'abc.html')
+//     })
 
-    const stop = stopwatch("Read sources")
+//     database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
 
-    const { folders, sources } = await votive.readSources(config, database, processors)
+//     /* TODO: Test that dependencies are working */
 
-    t.test('one folder exists', () => {
-      assert(folders.length === 1)
-    })
+//     const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
+//     const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+//     const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+//     const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
 
-    t.test('three sources exist', () => {
-      assert(sources.length === 4)
-    })
+//     t.test('settings created', () => {
+//       assert(setting.value === 'blue')
+//       assert(newSetting.value === 'Dog')
+//       assert(!sameSetting)
+//     })
 
-    stop()
+//     const abcSettings = database.getSettings("abc.html")
 
-    const sourcesJobs = sources.flatMap(source => source.jobs)
+//     t.test('settings retrieved', () => {
+//       assert(abcSettings.theme[0] === "blue")
+//     })
 
-    const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
+//     const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
+//     const defSettings = database.getSettings("abc/def.html")
 
-    t.test('transformation succeeded', () => {
-      assert(processedAbstracts.find(abstract => abstract.exampleAppend))
-    })
+//     t.test('settings retrieved', () => {
+//       assert(abcSettings.theme[0] === "blue")
+//       assert(defSettings.theme[1] === "green")
+//     })
 
-    t.test('abstract jobs exist', () => {
-      assert(abstractsJobs.length > 0)
-      assert(abstractsJobs.every(job => job.data && job.runner))
-    })
+//     const staleDestinations = database.getStaleDestinations()
 
-    const foldersJobs = readFolders(folders, config, database, processors)
+//     t.test('all destinations are stale', () => {
+//       assert(staleDestinations.length === 7)
+//     })
 
-    t.test("folders jobs exist", () => {
-      assert(foldersJobs.length > 0)
-      assert(foldersJobs.every(job => job.data && job.runner))
-    })
+//     const written = await votive.writeDestinations(config, database)
 
-    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
-    const firstDestination = database.getDestinationIndependently("abc.html", [])
+//     const staleDestinationsAfterWriting = database.getStaleDestinations()
 
-    t.test('first destination created', () => {
-      assert(firstDestination.path === 'abc.html')
-    })
+//     t.test('all destinations are fresh', () => {
+//       assert(staleDestinationsAfterWriting.length === 0)
+//     })
 
-    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
-    const destination = database.getDestinationDependently("abc.html", ["a"], "abc/def.html")
+//     const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+//     const staleAfterAbstractUpdate = database.getStaleDestinations()
 
-    const dependencies = database.getDependencies()
+//     t.test('updated document with no side effects is stale', () => {
+//       assert(staleAfterAbstractUpdate.length === 1)
+//     })
 
-    t.test('dependency created', () => {
-      assert(dependencies[0].dependent === 'abc/def.html')
-    })
+//     await votive.writeDestinations(config, database)
+//     const staleAfterSecondWrite = database.getStaleDestinations()
 
+//     t.test('everything fresh again', () => {
+//       assert(staleAfterSecondWrite.length === 0)
+//     })
 
-    const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
-    const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-    const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-    const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
+//     const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+//     const staleAfterSideEffects = database.getStaleDestinations()
 
-    t.test('settings created', () => {
-      assert(setting.value === 'blue')
-      assert(newSetting.value === 'Dog')
-      assert(!sameSetting)
-    })
+//     t.test('side effects work', () => {
+//       assert(staleAfterSideEffects.length === 2)
+//     })
 
-    const abcSettings = database.getSettings("abc.html")
 
-    t.test('settings retrieved', () => {
-      assert(abcSettings.theme[0] === "blue")
-    })
+//     await votive.writeDestinations(config, database)
 
-    const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
-    const defSettings = database.getSettings("abc/def.html")
+//     const oldSetting = database.setSetting("abc", "theme", "green")
 
-    t.test('settings retrieved', () => {
-      assert(abcSettings.theme[0] === "blue")
-      assert(defSettings.theme[1] === "green")
-    })
+//     const staleAfterSettingsChange = database.getStaleDestinations()
 
-    const staleDestinations = database.getStaleDestinations()
+//     t.test('setting affects descendents', () => {
+//       assert(staleAfterSettingsChange.length === 1)
+//     })
 
-    t.test('all destinations are stale', () => {
-      assert(staleDestinations.length === 7)
-    })
+//     // TODO: Write tests for jobs
+//     const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
+//     runJobs(jobs, config, database)
 
-    const written = await votive.writeDestinations(config, database)
+//     fs.rmSync("markdown/prunee.md")
 
-    const staleDestinationsAfterWriting = database.getStaleDestinations()
+//     await votive.pruneSources(config, database)
 
-    t.test('all destinations are fresh', () => {
-      assert(staleDestinationsAfterWriting.length === 0)
-    })
-
-    const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-    const staleAfterAbstractUpdate = database.getStaleDestinations()
+//     const result = database.getDestinationIndependently("markdown/prunee.html")
 
-    t.test('updated document with no side effects is stale', () => {
-      assert(staleAfterAbstractUpdate.length === 1)
-    })
+//     /* TODO: Check pruning metadata works (I removed it) */
 
-    await votive.writeDestinations(config, database)
-    const staleAfterSecondWrite = database.getStaleDestinations()
+//     const finalDestinations = database.getDestinations({
+//       filter: [
+//         {
+//           property: "a",
+//           operator: "gt",
+//           value: 3
+//         }
+//       ]
+//     }, "markdown/prunee.html")
 
-    t.test('everything fresh again', () => {
-      assert(staleAfterSecondWrite.length === 0)
-    })
+//     // TODO: Write a test for get destinations
 
-    const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-    const staleAfterSideEffects = database.getStaleDestinations()
+//     await database.saveDB()
 
-    t.test('side effects work', () => {
-      assert(staleAfterSideEffects.length === 2)
-    })
-
-    const freshDestinations = database.getAllDestinations()
-
-    await votive.writeDestinations(config, database)
-
-    const oldSetting = database.setSetting("abc", "theme", "green")
-
-    const staleAfterSettingsChange = database.getStaleDestinations()
-
-    t.test('setting affects descendents', () => {
-      assert(staleAfterSettingsChange.length === 1)
-    })
-
-    // TODO: Write tests for jobs
-    const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
-    runJobs(jobs, config, database)
-
-    const destinations = database.getAllDestinations()
-    database.getDestinationDependently("markdown/prunee.html", ["title"], "abc/def.html")
-    const newDependencies = database.getDependencies()
-
-    fs.rmSync("markdown/prunee.md")
-
-    await votive.pruneSources(config, database)
-
-    const result = database.getDestinationIndependently("markdown/prunee.html")
-    const prunedDependencies = database.getDependencies()
-    const prunedMetadata = database.getAllMetadataByPath("markdown/prunee.html")
-
-    t.test('source pruned', () => {
-      assert(!result)
-      assert(newDependencies.length - prunedDependencies.length === 1)
-      assert(!prunedMetadata)
-    })
-
-    await database.saveDB()
-
-    temp.remove()
-  } catch (error) {
-    temp.remove()
-    throw error
-  }
-})
+//     temp.remove()
+//   } catch (error) {
+//     temp.remove()
+//     throw error
+//   }
+// })

package/lib/cleanDeletions.js

@@ -1,32 +0,0 @@
-import { readdir, rm } from "node:fs/promises"
-import path from "node:path"
-
-/** @import {Abstract, Abstracts, VotiveConfig} from "./bundle.js" */
-/** @import {Database} from "./createDatabase.js" */
-
-
-/**
- * @param {VotiveConfig} config
- * @param {Database} database
- */
-async function cleanDeletions(config, database) {
-  // Must remove from sources first??
-  const dirents = new Set((await readdir(config.destinationFolder, {
-    recursive: true,
-    withFileTypes: true
-  })).map(d => d.isFile() && path.relative(config.destinationFolder, path.join(d.parentPath, d.name)))
-    .filter(a => a)
-  )
-
-  const targets = new Set(database.target.getAll().map(t => t.path))
-
-  const deletions = dirents.difference(targets)
-
-  deletions.forEach(async deletion => {
-    const deletionPath = path.join(config.destinationFolder, deletion)
-    await rm(deletionPath)
-  })
-
-}
-
-export default cleanDeletions