votive 0.0.3 → 0.0.5

package/README.md

@@ -1,3 +1,11 @@
 # Votive
 
-TK
+## Roadmap
+
+- [ ] ReadPaths
+- [ ] Flesh out job runners
+- [ ] Destination query filters
+
+## Helpers
+
+- Read

package/lib/bundle.js

@@ -1,9 +1,10 @@
-import { default as createDatabase } from "./createDatabase.js"
-import readSources from "./readSources.js"
+import path from "node:path"
 import readAbstracts from "./readAbstracts.js"
 import readFolders from "./readFolders.js"
-import writeDestinations from "./writeDestinations.js"
+import readSources from "./readSources.js"
 import runJobs from "./runJobs.js"
+import writeDestinations from "./writeDestinations.js"
+import { default as createDatabase } from "./createDatabase.js"
 
 /** @import {Database} from "./createDatabase.js" */
 
@@ -54,6 +55,7 @@ import runJobs from "./runJobs.js"
 /**
  * @callback ReadText
  * @param {string} text
+ * @param {string} filePath
  * @param {Database} database
  * @param {VotiveConfig} config
  * @returns {ReadTextResult | undefined}
@@ -101,6 +103,15 @@ import runJobs from "./runJobs.js"
  * @param {object} Folder
  * @param {Database} database
  * @param {VotiveConfig} config
+ * @returns {{ jobs?: Jobs, destinations?: Destination[] }}
+ */
+
+/**
+ * @typedef {object} Destination
+ * @property {string} path
+ * @property {object} metadata
+ * @property {Abstract} abstract
+ * @property {string} syntax
  */
 
 /**
@@ -158,21 +169,54 @@ import runJobs from "./runJobs.js"
 
 /**
  * @param {VotiveConfig} config
+ * @param {Database | undefined} [cache]
  */
-async function bundle(config) {
+async function bundle(config, cache) {
+
+  // Map out all processors
   const processors = config.plugins
     && config.plugins.flatMap(plugin => plugin.processors && plugin.processors.map(processor => ({ plugin, processor })))
 
-  const database = createDatabase()
+  // Create database
+  const database = cache || createDatabase(path.join(config.sourceFolder, ".votive.db"))
+
+  /*
+    Note: If no cache is provided or located, the database
+    will automatically run in memory, based on the assumption
+    that Votive is running for the first time. The in-
+    memory database will run much faster and then back up
+    to the file system. When Votive next launches from the
+    cached disk database, the read/writes will be a little
+    slower, but startup will be much faster, so it should
+    even out.
+  */
+
+  // Read folders and source files
   const { folders, sources } = await readSources(config, database, processors)
 
+  // Map out jobs from source files
   const sourcesJobs = sources.flatMap(source => source.jobs) || []
-  const { processedAbstracts, abstractsJobs } = readAbstracts(sources, config, database, processors)
+
+  // Process source file abstracts and map jobs
+  const { abstractsJobs } = readAbstracts(sources, config, database, processors)
+
+  // Scan folders and map out jobs
   const foldersJobs = readFolders(folders, config, database, processors) || []
+
+  // Write destination files
   await writeDestinations(config, database)
 
-  const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
-  await runJobs(jobs, config, database)
+  // Run all jobs
+  await runJobs([
+    ...sourcesJobs,
+    ...abstractsJobs,
+    ...foldersJobs
+  ], config, database)
+
+  // Back up database (only if in-memory first run)
+  await database.saveDB()
+
+  return database
 }
 
 export default bundle

package/lib/createDatabase.js

@@ -1,23 +1,23 @@
 import { DatabaseSync, backup } from "node:sqlite"
-import { splitURL } from "./utils/index.js"
+import { splitURL, checkFile } from "./utils/index.js"
 import { statSync } from "node:fs"
+import path from "node:path"
 
 /**
  * @typedef {ReturnType<createDatabase>} Database
  */
 
 
-function checkFile(path) {
-  try {
-    return statSync(path)
-  } catch(e) {
-    return null
-  }
+/** @param {string} dbPath */
+function loadDB(dbPath) {
+  if (checkFile(dbPath)) return new DatabaseSync(dbPath)
+  return new DatabaseSync(":memory:")
 }
 
 
-function createDatabase() {
-  const databaseSync = new DatabaseSync(".votive.db")
+/** @param {string} dbPath */
+function createDatabase(dbPath = ".votive.db") {
+  const databaseSync = loadDB(dbPath)
 
   const store = databaseSync.createTagStore()
 
@@ -25,6 +25,11 @@ function createDatabase() {
 
   const database = {}
 
+  database.saveDB = async () => {
+    if (databaseSync.location()) return // Only save if running in memory
+    await backup(databaseSync, dbPath)
+  }
+
   /**
    * @param {string} folderPath 
    * @param {string} urlPath
@@ -42,9 +47,21 @@ function createDatabase() {
    */
   database.createSource = (source, destination, lastModified) => {
     databaseSync.prepare(``) // SQLite bug. Query fails without this.
-    const inserted = createSource.get(source, destination, lastModified)
+    createSource.get(source, destination, lastModified)
+  }
+
+  const updateSource = databaseSync.prepare(`UPDATE sources SET lastModified = ? WHERE path = ?`)
+
+  /**
+   * @param {string} source
+   * @param {number} lastModified
+   */
+  database.updateSource = (source, lastModified) => {
+    updateSource.get(lastModified, source)
   }
 
+  const getSettingsBySource = databaseSync.prepare(`SELECT * FROM settings WHERE source = ?`)
+
 
   // These statements are cached, no need to refactor
   const createDest = databaseSync.prepare(`INSERT INTO destinations (path, dir, syntax, stale, abstract) VALUES (?, ?, ?, 1, ?)`)
@@ -52,8 +69,25 @@ function createDatabase() {
   const createMeta = databaseSync.prepare(`INSERT INTO metadata (destination, label, value, type) VALUES (?, ?, ?, ?)`)
   const updateMeta = databaseSync.prepare(`UPDATE metadata SET value = ? WHERE destination = ? AND label = ?`)
   const getDepends = databaseSync.prepare(`SELECT * FROM dependencies WHERE destination = ? AND property = ?`)
+  const getDependenciesByDestination = databaseSync.prepare(`SELECT * FROM dependencies WHERE destination = ?`)
+  const getAllDeps = databaseSync.prepare(`SELECT * FROM dependencies`)
   const staleDepen = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path = ?`)
+  const freshDepen = databaseSync.prepare(`UPDATE destinations SET stale = 0 WHERE path = ? RETURNING *`)
   const staleDescendents = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path LIKE ? RETURNING *`)
+  const getAllSettings = databaseSync.prepare(`SELECT * FROM settings`)
+
+  database.getAllSettings = () => {
+    return getAllSettings.all()
+  }
+
+  /** @param {string} path */
+  database.freshenDependency = (path) => {
+    freshDepen.get(path)
+  }
+
+  database.getDependencies = () => {
+    return getAllDeps.all()
+  }
 
   /**
    * @param {object} params
@@ -75,10 +109,18 @@ function createDatabase() {
     }
 
     const changedAbstract = JSON.stringify(dest.abstract) !== JSON.stringify(extant.abstract)
-    const changedMetadata = []
+
+    let changedMetadata = false
+
+
+    // TODO: This is redundant with `getDestinationIndependently`
+    const cachedMetadata = getMetadataByPath.all(dest.path)
 
     for (const key in metadata) {
+      const index = cachedMetadata.findIndex(el => el.label === key)
+      cachedMetadata.splice(index, 1)
       if (JSON.stringify(metadata[key]) !== JSON.stringify(extant.metadata[key])) {
+        changedMetadata = true
         updateMeta.get(metadata[key], dest.path, key)
         const dependencies = getDepends.all(dest.path, key)
         dependencies.forEach(({ dependent }) => {
@@ -87,8 +129,19 @@ function createDatabase() {
       }
     }
 
+    cachedMetadata.forEach(deletedDatum => {
+      changedMetadata = true
+      const dependencies = getDepends.all(dest.path, deletedDatum.label)
+      dependencies.forEach(({ dependent }) => {
+        staleDepen.get(dependent)
+      })
+      deleteMetadata.get(deletedDatum.destination, deletedDatum.label)
+    })
+
+    if (changedAbstract || changedMetadata) staleDepen.get(dest.path)
+
     if (changedAbstract) {
-      const updated = updateDest.get(JSON.stringify(dest.abstract), dest.path)
+      updateDest.get(JSON.stringify(dest.abstract), dest.path)
       const dependencies = getDepends.all(dest.path, "abstract")
       dependencies.forEach(({ dependent }) => {
         staleDepen.get(dependent)
@@ -96,17 +149,43 @@ function createDatabase() {
     }
   }
 
+  const getMetadataByPath = databaseSync.prepare(`
+      SELECT * FROM metadata WHERE destination = ?
+    `)
+
+  /** @param {string} path */
+  database.getAllMetadataByPath = (path) => {
+    return getMetadataByPath.get(path)
+  }
+
+  /** @param {string[]} params */
+  database.getMetadataIndependently = (params) => {
+    return databaseSync.prepare(`
+      SELECT * FROM destinations d
+      LEFT JOIN metadata m ON d.path = m.destination AND m.label IN (${params.map(p => `'${p}'`).join(", ")})
+      WHERE d.path = ?
+    `)
+  }
+
+  database.getDestinationWithoutMetadata = () => {
+    return databaseSync.prepare(`
+        SELECT * FROM destinations
+        WHERE path = ?
+      `)
+  }
+
   /**
    * @param {string} path
-   * @param {string[]} params
+   * @param {string[]} [params]
    */
   database.getDestinationIndependently = (path, params) => {
     // TODO: Could potentially speed up the next two db queries by using the tag store
-    const metadatas = databaseSync.prepare(`
-      SELECT * FROM destinations d
-      LEFT JOIN metadata m ON d.path = m.destination AND m.label IN (${params.map(p => `'${p}'`).join(", ")})
-      WHERE d.path = ?
-    `).all(path)
+
+    const metadatas = (params && params.length)
+      ? database.getMetadataIndependently(params).all(path)
+      : [database.getDestinationWithoutMetadata().get(path)]
+
+    if (!metadatas) return
 
     const [first] = metadatas
 
@@ -123,7 +202,7 @@ function createDatabase() {
               : typeof value !== "string"
                 ? value
                 : JSON.parse(value)
-        return [label, value]
+        return [label, originalValue]
       })
     )
 
@@ -135,7 +214,7 @@ function createDatabase() {
       metadata
     }
 
-    const includeAbstract = params.includes("abstract")
+    const includeAbstract = params && params.includes("abstract")
     if (includeAbstract) result.abstract = JSON.parse(first.abstract)
 
     return result
@@ -157,7 +236,7 @@ function createDatabase() {
 
     const result = database.getDestinationIndependently(path, params)
     if (result.abstract) deps.push(`('${path}, 'abstract', '${dependent}')`)
-    const createDepp = databaseSync.prepare(`INSERT INTO dependencies (destination, property, dependent) VALUES ${deps.join(", ")}`).all()
+    databaseSync.prepare(`INSERT INTO dependencies (destination, property, dependent) VALUES ${deps.join(", ")} RETURNING *`).all()
   }
 
   const selectSource = databaseSync.prepare(`SELECT * FROM sources WHERE path = ?`)
@@ -175,45 +254,138 @@ function createDatabase() {
     return sources
   }
 
+  const deleteSource = databaseSync.prepare(`DELETE FROM sources WHERE path = ? RETURNING *`)
+  const deleteAllDependenciesByDestination = databaseSync.prepare(`DELETE FROM dependencies WHERE destination = ? RETURNING dependent`)
+  const deleteAllMetadataByDestination = databaseSync.prepare(`DELETE FROM metadata WHERE destination = ?`)
+  const deleteMetadata = databaseSync.prepare(`DELETE FROM metadata WHERE destination = ? AND label = ?`)
+  const deleteDestination = databaseSync.prepare(`DELETE FROM destinations WHERE path = ?`)
+
+  /** @param {string} path */
+  database.deleteSource = (path) => {
+    const deleted = deleteSource.all(path)
+    deleted.forEach(source => {
+      database.deleteSettings(String(source.path))
+      getDependenciesByDestination.all(source.destination)
+      deleteAllDependenciesByDestination.all(source.destination)
+        .forEach(({ dependent }) => staleDepen.get(dependent))
+      deleteAllMetadataByDestination.all(source.destination)
+      deleteDestination.get(source.destination)
+    })
+  }
+
+  // TODO: DELETE SETTING
+
   database.getAllDestinations = () => {
     return store.all`SELECT * FROM destinations`
   }
 
   database.getStaleDestinations = () => {
-    return store.all`SELECT * FROM destinations WHERE stale = 1`
+    const metadatas = store.all`
+      SELECT * FROM destinations d
+      LEFT JOIN metadata m on d.path = m.destination
+      WHERE stale = 1`
+
+    console.log({ metadatas })
+
+    const grouped = metadatas.map((value, index, array) => {
+      if (array.slice(0, index)
+        .find(prior => prior.path === value.path)
+      ) {
+        return null
+      } else {
+        return {
+          path: value.path,
+          dir: value.dir,
+          syntax: value.syntax,
+          stale: value.stale,
+          metadata: Object.fromEntries(
+            array.slice(index)
+              .filter(following => following.path === value.path)
+              .map(succeeding => {
+                return reconstituteMetadata(succeeding)
+              })
+          )
+
+        }
+      }
+    }).filter(a => a)
+
+    function reconstituteMetadata({ label, value, type }) {
+      const originalValue = type === "undefined"
+        ? undefined
+        : type === "string"
+          ? String(value)
+          : type === "boolean"
+            ? Boolean(value)
+            : typeof value !== "string"
+              ? value
+              : JSON.parse(value)
+      return [label, originalValue]
+    }
+
+    return grouped
   }
 
-  const createSetting = databaseSync.prepare(`INSERT INTO settings (destination, label, value) VALUES (?, ?, ?) RETURNING *`)
+  const createSetting = databaseSync.prepare(`INSERT INTO settings (destination, label, value, source) VALUES (?, ?, ?, ?) RETURNING *`)
   const selectSetting = databaseSync.prepare(`SELECT * FROM settings WHERE label = ? AND destination = ?`)
-  const updateSetting = databaseSync.prepare(`UPDATE settings SET value = ? WHERE destination = ? AND label = ?`)
+  const updateSetting = databaseSync.prepare(`UPDATE settings SET value = ? WHERE destination = ? AND label = ? RETURNING *`)
+  const deleteSettings = databaseSync.prepare(`DELETE FROM settings WHERE source = ?`)
 
   // const updateDest = databaseSync.prepare(`UPDATE destinations SET abstract = ? WHERE path = ?`)
 
+  // WRITE AN UPDATE SETTINGS FUNCTION FOR WHEN A SOURCE CHANGES
+
+  /** @param {string} source */
+  database.deleteSettings = (source) => {
+    const deleted = deleteSettings.all(source)
+    deleted.forEach(setting => {
+      staleDepen.get(setting.destination)
+      staleDescendents.all(`${setting.destination}/%`)
+    })
+  }
+
   /**
-   * @param {string} destination
+   * @param {string} destinationFolder
    * @param {string} key
    * @param {string} value
+   * @param {string} [source]
+   * @example
+   * setSetting("", "theme", "summer", "settings.md")
+   * setSetting("data", "format", "csv", "/config.yaml")
    */
-  database.setSetting = (destination, key, value) => {
-    // TODO: Check if already exists
-    const extant = selectSetting.get(key, destination)
+  database.setSetting = (destinationFolder, key, value, source = "") => {
+    const extant = selectSetting.get(key, destinationFolder)
 
-    if (!extant) return createSetting.get(destination, key, value)
+    if (!extant) return createSetting.get(destinationFolder, key, value, source)
     if (extant.value === value) return
+    updateSetting.get(value, destinationFolder, key)
 
-    updateSetting.get(value, destination, key)
-    const stale = staleDescendents.all(`${destination}/%`)
+    destinationFolder === ""
+      ? staleDescendents.all("%")
+      : staleDescendents.all(`${destinationFolder}/%`)
   }
 
 
   /**
-   * @param {string} destination
-   * @param {string} dependent
+   * @param {string} destinationFolder
    */
-  database.getSettings = (destination, dependent) => {
-    return Object.fromEntries(databaseSync.prepare(`SELECT * FROM settings WHERE destination = '${destination}' OR destination LIKE '${destination}/%';`)
-      .all()
-      .map(({ label, value }) => [label, value]))
+  database.getSettings = (destinationFolder) => {
+    const segments = destinationFolder.split(path.sep)
+      .map((_, index, array) => `'${array.slice(0, index + 1).join(path.sep)}'`)
+
+    segments.unshift("''")
+    const statement = segments.join(", ")
+
+    const records = databaseSync.prepare(`SELECT * FROM settings WHERE destination IN (${statement})`).all()
+
+    const grouped = {}
+    records.sort((a, b) => a.length - b.length)
+      .forEach(record => {
+        if (!grouped[record.label]) grouped[record.label] = [record.value]
+        else grouped[record.label].push(record.value)
+      })
+
+    return grouped
   }
 
   database.getEverything = () => {
@@ -230,50 +402,7 @@ function createDatabase() {
   return database
 }
 
-/**
- * @param {object} data
- * @param {string} table
- * @param {DatabaseSync} databaseSync
- */
-function updateColumns(data, table, databaseSync) {
-  const keys = new Set(Object.keys(data))
-  const columns = listColumns(table, databaseSync)
-  const newColumns = keys.difference(columns)
-  for (const column of newColumns) addColumn(column, table, databaseSync)
-}
-
-/**
- * @param {string} table
- * @param {DatabaseSync} databaseSync
- * @description list the columns on a table.
- */
-function listColumns(table, databaseSync) {
-  const columns = new Set()
-  const tableInfo = databaseSync.prepare(`PRAGMA table_info(${table})`).all()
-  for (const column of tableInfo) columns.add(column.name)
-  return columns
-}
-
-/**
- * @param {string} column
- * @param {string} table
- * @param {DatabaseSync} databaseSync
- * @description Add a column to a table and update binary.
- */
-function addColumn(column, table, databaseSync) {
-  databaseSync.exec(`ALTER TABLE ${table} ADD COLUMN ${column} STRING;`)
-}
-
 const sqlCreateTables = `
-CREATE TABLE IF NOT EXISTS jobs (
-  id INTEGER PRIMARY KEY,
-  pluginName TEXT NOT NULL,
-  data STRING NOT NULL,
-  done INTEGER NOT NULL DEFAULT 0,
-  runner STRING NOT NULL
-);
-
--- Source/destination relationships
 CREATE TABLE IF NOT EXISTS sources (
   id INTEGER PRIMARY KEY,
   destination STRING,
@@ -297,11 +426,6 @@ CREATE TABLE IF NOT EXISTS destinations (
   abstract
 );
 
-CREATE TABLE IF NOT EXISTS folders (
-  path STRING PRIMARY KEY,
-  urlPath STRING
-);
-
 CREATE TABLE IF NOT EXISTS metadata (
   id INTEGER PRIMARY KEY,
   destination STRING,
@@ -314,6 +438,7 @@ CREATE TABLE IF NOT EXISTS metadata (
 CREATE TABLE IF NOT EXISTS settings (
   id INTEGER PRIMARY KEY,
   destination STRING,
+  source STRING,
   label STRING,
   value STRING
 );

package/lib/index.js

@@ -1,5 +1,6 @@
 export { default as bundle } from "./bundle.js"
 export { default as createDatabase } from "./createDatabase.js"
+export { default as pruneSources } from "./pruneSources.js"
 export { default as readAbstracts } from "./readAbstracts.js"
 export { default as readFolders } from "./readFolders.js"
 export { default as readSources } from "./readSources.js"

package/lib/pruneSources.js

@@ -0,0 +1,28 @@
+import { readdir } from "node:fs/promises"
+import path from "node:path"
+/** @import {VotiveConfig} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+
+
+
+/**
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ */
+async function pruneSources(config, database) {
+  const sources = database.getAllSources()
+  const files = (await readdir(config.sourceFolder, {
+    withFileTypes: true,
+    recursive: true
+  })).map(file => {
+    return file.isFile() ? path.join(file.parentPath, file.name) : null
+  }).filter(a => typeof a === "string")
+
+  sources.forEach(source => {
+    if(files.includes(String(source.path))) return
+    database.deleteSource(String(source.path))
+  })
+
+}
+
+export default pruneSources

package/lib/readAbstracts.js

@@ -48,9 +48,9 @@ function readAbstracts(abstracts, config, database, processors) {
   const processedAbstracts = []
   const abstractsJobs = []
 
-  processed.forEach(({ abstract, job}) => {
+  processed.forEach(({ abstract, jobs }) => {
     processedAbstracts.push(abstract)
-    abstractsJobs.push(job)
+    abstractsJobs.push(...jobs)
   })
 
   return { processedAbstracts, abstractsJobs }

package/lib/readFolders.js

@@ -1,4 +1,4 @@
-/** @import {VotiveConfig, FlatProcessors, Jobs} from "./bundle.js" */
+/** @import {VotiveConfig, FlatProcessors} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
 /** @import {Dirent} from "node:fs" */
 
@@ -9,20 +9,42 @@ import path from "node:path"
  * @param {VotiveConfig} config
  * @param {Database} database
  * @param {FlatProcessors} processors
- * @returns {Jobs}
  */
-function readFolders(folders, config, database, processors) {
-  if(!folders) return
-  return folders.flatMap(folder => {
-    return config.plugins.flatMap(plugin => {
-      return plugin.processors.flatMap(processor => {
-        const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
-        return processor.read
-          && processor.read.folder
-          && processor.read.folder(folderPath, database, config)
-      })
+function readFolders(folders = [], config, database, processors) {
+
+  const folderProcessors = processors.filter(({ processor }) => processor.read && processor.read.folder)
+
+  const processed = folderProcessors.flatMap(({ processor, plugin }) => {
+
+    const jobs = folders.flatMap(folder => {
+      const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
+
+      /** @ts-ignore `.read` is throwing a warning, but it's guarded above */
+      const { destinations, jobs } = processor.read.folder(folderPath, database, config)
+      jobs.forEach(job => job.plugin = plugin.name)
+      if (destinations) {
+        destinations.forEach(destination => {
+          database.createOrUpdateDestination(destination)
+        })
+        return jobs
+      }
     })
+
+    const rootFolder = processor.read.folder("", database, config)
+
+    if (rootFolder.destinations) {
+      rootFolder.destinations.forEach(destination => {
+        database.createOrUpdateDestination(destination)
+      })
+    }
+
+    rootFolder.jobs.forEach(job => job.plugin = plugin.name)
+
+    if(rootFolder.jobs) jobs.push(... rootFolder.jobs)
+    return jobs
   })
+
+  return processed
 }
 
 export default readFolders

package/lib/readSources.js

@@ -25,8 +25,6 @@ async function readSources(config, database, processors) {
     recursive: true
   })
 
-  // TODO: Check file modified time
-
   const filteredDirents = (dirents || []).filter(fileFilter(config, database))
   const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
   if (!files) return { folders, sources: [] }
@@ -101,7 +99,9 @@ function readSourceFile(processors, database, config) {
         const source = database.getSource(filePath)
 
 
-        if (source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
+        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
+
+        database.deleteSettings(filePath)
 
         // Get destination route
         const destinationPath = route(filePath, plugin)
@@ -109,12 +109,17 @@ function readSourceFile(processors, database, config) {
         // Set URL if exists
         const buffer = await fs.readFile(path.format(fileInfo))
         const data = decodeBuffer(buffer)
-        const { jobs, abstract, metadata } = read.text(data, database, config)
+        const { jobs, abstract, metadata } = read.text(data, filePath, database, config)
         jobs && jobs.forEach(job => job.plugin = plugin.name)
-        // TODO: Cache: Check if file already exists
 
         const timeStamp = stat.mtimeMs.toFixed()
-        database.createSource(filePath, destinationPath, Number(timeStamp))
+
+        if(source) {
+          database.updateSource(filePath, Number(timeStamp))
+        } else {
+          database.createSource(filePath, destinationPath, Number(timeStamp))
+        }
+
         database.createOrUpdateDestination({
           metadata,
           path: destinationPath,

package/lib/utils/index.js

@@ -1,4 +1,5 @@
 import { styleText } from "node:util"
+import { statSync } from "node:fs"
 
 /** @param {string} label */
 export function stopwatch(label) {
@@ -20,3 +21,12 @@ export function splitURL(urlPath) {
   const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
   return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
 }
+
+/** @param {string} dbPath */
+export function checkFile(dbPath) {
+  try {
+    return statSync(path.normalize(dbPath))
+  } catch (e) {
+    return null
+  }
+}

package/lib/writeDestinations.js

@@ -29,6 +29,7 @@ async function writeDestinations(config, database) {
         async function write() {
           await mkdir(dir, { recursive: true })
           await writeFile(destinationPath, data, encoding)
+          database.freshenDependency(destination.path)
         }
         
         return write()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {
@@ -19,7 +19,8 @@
     "./internals": "./lib/index.js"
   },
   "scripts": {
-    "test": "node --test --watch tests/*.js"
+    "test": "node --test --watch tests/*.js",
+    "coverage": "node --test --experimental-test-coverage --watch tests/*.js"
   },
   "dependencies": {
     "encoding-sniffer": "^0.2.1",

package/tests/index.js

@@ -5,148 +5,313 @@ import path from "node:path"
 import * as votive from "votive/internals"
 import { stopwatch } from "./../lib/utils/index.js"
 import { readFolders, runJobs } from "../lib/index.js"
+import { readdir, rm } from "node:fs/promises"
+import { checkFile } from "./../lib/utils/index.js"
 
 /** @import {VotiveConfig, VotivePlugin, FlatProcessors, VotiveProcessor, Runner, Job} from "./../lib/bundle.js" */
+/** @import {ProcessorRead, ProcessorWrite, ReadAbstract, ReadText, ReadTextResult, ReadFolder} from "./../lib/bundle.js" */
 
 process.chdir("./tests")
 
 test("empty directory", async () => {
-  const temp = fs.mkdtempDisposableSync("destination-")
+  const tempDest = fs.mkdtempDisposableSync("destination-")
+  const tempSource = fs.mkdtempDisposableSync("source-")
 
   try {
     const stop = stopwatch("Bundle empty folder")
     await votive.bundle({
-      sourceFolder: "./empty",
-      destinationFolder: temp.path
+      sourceFolder: tempSource.path,
+      destinationFolder: tempDest.path,
+      plugins: []
     })
 
     stop()
 
-    const dir = fs.readdirSync(temp.path)
+    const dir = fs.readdirSync(tempDest.path)
     assert(dir.length === 0, "Destination directory is not empty.")
   } catch (error) {
-    temp.remove()
+    tempDest.remove()
+    tempSource.remove()
     console.error(error)
 
   }
 
-  temp.remove()
+  tempDest.remove()
+  tempSource.remove()
 })
 
-test("create empty database", () => {
-  const stop = stopwatch("Create empty database")
-  const database = votive.createDatabase()
-  stop()
+test("internals", async (t) => {
+  const dbExists = checkFile(".votive.db")
+  if (dbExists) await rm(".votive.db")
 
-  database.createSource('exampleSource', 'exampleDestination', 123)
-  const [source] = database.getAllSources()
-  
-  assert(source.destination === 'exampleDestination')
-})
-
-test("read sources", async () => {
   const temp = fs.mkdtempDisposableSync("destination-")
 
-  /** @type {Job} */
-  const testJob = {
-    data: "123",
-    runner: "testRunner"
+  /** @returns {Job} */
+  function createExampleJob() {
+    return {
+      data: Math.floor(Math.random() * 1000),
+      runner: "exampleRunner"
+    }
+  }
+
+
+  /** @type {ReadText} */
+  function exampleTextReader(text, filePath, database, config) {
+    const matches = text.match(/\b\w+\b/)
+    const title = matches ? matches[0] : "Untitled"
+
+    /** @type {ReadTextResult} */
+    return {
+      abstract: {
+        content: text,
+      },
+      metadata: {
+        title
+      },
+      jobs: [
+        createExampleJob()
+      ]
+    }
+  }
+
+  /** @type {ReadAbstract} */
+  function exampleAbstractReader(abstract, database, config) {
+    abstract.exampleAppend = true
+    return { abstract, jobs: [createExampleJob()] }
+  }
+
+  /** @type {ReadFolder} */
+  function exampleFolderReader(folder, database, config) {
+    return {
+      jobs: [createExampleJob()],
+      destinations: [
+        {
+          path: "/index.html",
+          abstract: { content: "" },
+          metadata: {
+            title: "home"
+          },
+          syntax: "md"
+        }
+      ]
+    }
   }
 
   /** @type {VotiveProcessor} */
-  const testProcessor = {
-    syntax: "txt",
+  const exampleProcessor = {
+    syntax: "md",
     filter: {
       extensions: [".md"]
     },
     read: {
-      path: (filePath, database, config) => undefined,
-      text: (text, database, config) => ({ metadata: { foo: "bar", bang: "baz" }, abstract: { baz: "bang" } }),
-      abstract: (abstract, database, config) => ({ abstract: { ...abstract, newAddition: true }, jobs: [testJob] }),
-      folder: (folder, database, config) => [testJob]
+      text: exampleTextReader,
+      abstract: exampleAbstractReader,
+      folder: exampleFolderReader
     },
     write: (destination, database, config) => {
       return {
-        data: "yo there",
+        data: "lorem ipsum",
       }
     }
   }
 
+  /** @param {string} sourcePath */
+  function router(sourcePath) {
+    const { base, ...parsed }= path.parse(sourcePath)
+    return path.format({ ...parsed, ext: ".html" })
+  }
+
   /** @type {VotivePlugin} */
-  const testPlugin = {
-    name: "test plugin",
+  const examplePlugin = {
+    name: "example plugin",
     runners: {
-      testRunner
+      exampleRunner: exampleRunner
     },
-    router: (path) => path,
-    processors: [testProcessor]
+    router,
+    processors: [exampleProcessor]
   }
 
   /** @type {Runner} */
-  async function testRunner(data, database) {
-    const waiting = await new Promise((resolve) => setTimeout(() => resolve(data), 1000))
-
-    return waiting
+  async function exampleRunner(data, database) {
+    return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
   }
 
   /** @type {VotiveConfig} */
   const config = {
     sourceFolder: "./markdown",
     destinationFolder: temp.path,
-    plugins: [testPlugin]
+    plugins: [examplePlugin]
   }
 
   /** @type {FlatProcessors} */
   const processors = [
     {
-      plugin: testPlugin,
-      processor: testProcessor
+      plugin: examplePlugin,
+      processor: exampleProcessor
     }
   ]
 
   try {
+    fs.writeFileSync("markdown/prunee.md", "A little content")
+
     const database = votive.createDatabase()
+    const sourcesOne = database.getAllSources()
+
+    t.test("no sources on startup", () => {
+      const isArray = Array.isArray(sourcesOne)
+      const isEmpty = !sourcesOne.length
+      assert(isArray && isEmpty)
+    })
+
 
-    const moresources = database.getAllSources()
-    console.log({ moresources })
 
     const stop = stopwatch("Read sources")
+
     const { folders, sources } = await votive.readSources(config, database, processors)
+
+    t.test('one folder exists', () => {
+      assert(folders.length === 1)
+    })
+
+    t.test('three sources exist', () => {
+      assert(sources.length === 4)
+    })
+
     stop()
 
     const sourcesJobs = sources.flatMap(source => source.jobs)
+
     const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
 
+    t.test('transformation succeeded', () => {
+      assert(processedAbstracts.find(abstract => abstract.exampleAppend))
+    })
+
+    t.test('abstract jobs exist', () => {
+      assert(abstractsJobs.length > 0)
+      assert(abstractsJobs.every(job => job.data && job.runner))
+    })
+
     const foldersJobs = readFolders(folders, config, database, processors)
 
-    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.txt", abstract: { c: 3 }, syntax: "txt" })
-    const destination = database.getDestinationDependently("abc.txt", ["a"], "def")
-    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "def.txt", abstract: { c: 5 }, syntax: "txt" })
-    database.createOrUpdateDestination({ metadata: { a: 3, b: 9 }, path: "abc.txt", abstract: { c: 4 }, syntax: "txt" })
-    database.createOrUpdateDestination({ metadata: { a: 3, b: 9 }, path: "abc/def.txt", abstract: { c: 4 }, syntax: "txt" })
+    t.test("folders jobs exist", () => {
+      assert(foldersJobs.length > 0)
+      assert(foldersJobs.every(job => job.data && job.runner))
+    })
+
+    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
+    const firstDestination = database.getDestinationIndependently("abc.html", [])
+
+    t.test('first destination created', () => {
+      assert(firstDestination.path === 'abc.html')
+    })
+
+    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
+    const destination = database.getDestinationDependently("abc.html", ["a"], "abc/def.html")
 
+    const dependencies = database.getDependencies()
+
+    t.test('dependency created', () => {
+      assert(dependencies[0].dependent === 'abc/def.html')
+    })
+
+
+    const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
+    const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+    const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+    const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
+
+    t.test('settings created', () => {
+      assert(setting.value === 'blue')
+      assert(newSetting.value === 'Dog')
+      assert(!sameSetting)
+    })
+
+    const abcSettings = database.getSettings("abc.html")
+
+    t.test('settings retrieved', () => {
+      assert(abcSettings.theme[0] === "blue")
+    })
+
+    const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
+    const defSettings = database.getSettings("abc/def.html")
+
+    t.test('settings retrieved', () => {
+      assert(abcSettings.theme[0] === "blue")
+      assert(defSettings.theme[1] === "green")
+    })
+
+    const staleDestinations = database.getStaleDestinations()
+
+    t.test('all destinations are stale', () => {
+      assert(staleDestinations.length === 7)
+    })
+
+    const written = await votive.writeDestinations(config, database)
+
+    const staleDestinationsAfterWriting = database.getStaleDestinations()
+
+    t.test('all destinations are fresh', () => {
+      assert(staleDestinationsAfterWriting.length === 0)
+    })
+
+    const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+    const staleAfterAbstractUpdate = database.getStaleDestinations()
+
+    t.test('updated document with no side effects is stale', () => {
+      assert(staleAfterAbstractUpdate.length === 1)
+    })
+
+    await votive.writeDestinations(config, database)
+    const staleAfterSecondWrite = database.getStaleDestinations()
+
+    t.test('everything fresh again', () => {
+      assert(staleAfterSecondWrite.length === 0)
+    })
+
+    const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+    const staleAfterSideEffects = database.getStaleDestinations()
+
+    t.test('side effects work', () => {
+      assert(staleAfterSideEffects.length === 2)
+    })
+
+    const freshDestinations = database.getAllDestinations()
+
+    await votive.writeDestinations(config, database)
 
-    const setting = database.setSetting("abc", "theme", "blue")
-    const newSetting = database.setSetting("def", "category", "Dog")
-    const sameSetting = database.setSetting("def", "category", "Dog")
     const oldSetting = database.setSetting("abc", "theme", "green")
 
-    const settings = database.getSettings("abc", "def")
+    const staleAfterSettingsChange = database.getStaleDestinations()
+
+    t.test('setting affects descendents', () => {
+      assert(staleAfterSettingsChange.length === 1)
+    })
 
+    // TODO: Write tests for jobs
     const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
     runJobs(jobs, config, database)
 
-    const dbResults = database.getAllSources()
+    const destinations = database.getAllDestinations()
+    database.getDestinationDependently("markdown/prunee.html", ["title"], "abc/def.html")
+    const newDependencies = database.getDependencies()
 
-    const written = await votive.writeDestinations(config, database)
+    fs.rmSync("markdown/prunee.md")
+
+    await votive.pruneSources(config, database)
 
-    // const everything = database.getEverything()
-    // console.log(everything)
+    const result = database.getDestinationIndependently("markdown/prunee.html")
+    const prunedDependencies = database.getDependencies()
+    const prunedMetadata = database.getAllMetadataByPath("markdown/prunee.html")
+
+    t.test('source pruned', () => {
+      assert(!result)
+      assert(newDependencies.length - prunedDependencies.length === 1)
+      assert(!prunedMetadata)
+    })
 
-    const newsources = database.getAllSources()
-    console.log({ newsources })
+    await database.saveDB()
 
-    assert(true, "Read test tk")
     temp.remove()
   } catch (error) {
     temp.remove()

package/tests/markdown/fldr/dog.md

@@ -0,0 +1,3 @@
+# Dog
+
+Here is a page inside a folder.

package/tests/markdown/hey.md

@@ -0,0 +1,3 @@
+# Hey
+
+Here is some content.

package/tests/markdown/index.md

@@ -0,0 +1 @@
+This is an index root.