votive 0.0.1 → 0.0.3

package/README.md

@@ -0,0 +1,3 @@
+# Votive
+
+TK

package/jsconfig.json

@@ -0,0 +1,10 @@
+{
+	"compilerOptions": {
+		"allowJs": true,
+		"checkJs": true,
+		"noImplicitlyAny": true,
+		"module": "esnext",
+		"target": "esnext",
+		"moduleResolution": "nodenext"
+	}
+}

package/lib/bundle.js

@@ -0,0 +1,178 @@
+import { default as createDatabase } from "./createDatabase.js"
+import readSources from "./readSources.js"
+import readAbstracts from "./readAbstracts.js"
+import readFolders from "./readFolders.js"
+import writeDestinations from "./writeDestinations.js"
+import runJobs from "./runJobs.js"
+
+/** @import {Database} from "./createDatabase.js" */
+
+/**
+ * @typedef {object} VotivePlugin
+ * @property {string} name
+ * @property {VotiveProcessor[]} [processors]
+ * @property {Record<string, Runner>} [runners]
+ * @property {Router} [router]
+ */
+
+/**
+ * @typedef {object} VotiveProcessor
+ * @property {ProcessorFilter} [filter]
+ * @property {ProcessorSyntax} syntax
+ * @property {ProcessorRead} [read]
+ * @property {ProcessorWrite} [write]
+ */
+
+/**
+ * Filter for files that the processor will read from.
+ * @typedef {object} ProcessorFilter
+ * @property {string[]} extensions
+ */
+
+/**
+ * A name for the syntax that processors will read-to and write-from (e.g. Unified.js's "hast").
+ * @typedef {string} ProcessorSyntax
+ */
+
+/**
+ * @typedef {object} ProcessorRead
+ * @property {ReadPath} [path]
+ * @property {ReadText} [text]
+ * @property {ReadAbstract} [abstract]
+ * @property {ReadFolder} [folder]
+ */
+
+/**
+ * Read a file path and return any necessary jobs.
+ * @callback ReadPath
+ * @param {string} filePath
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ * @returns {Job[] | undefined}
+ */
+
+/**
+ * @callback ReadText
+ * @param {string} text
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ * @returns {ReadTextResult | undefined}
+ */
+
+/**
+ * @typedef {object} ReadTextResult
+ * @property {Jobs} [jobs]
+ * @property {object} [metadata]
+ * @property {Abstract} [abstract]
+ */
+
+/**
+ * @typedef {Job[]} Jobs
+ */
+
+/**
+ * @callback ReadAbstract
+ * @param {Abstract} abstract
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ * @returns {{abstract: Abstract, jobs: Jobs}}
+ */
+
+/**
+ * @typedef {Record<string, any>} Abstract
+ */
+
+/**
+ * @typedef {Abstract[]} Abstracts
+ */
+
+/**
+ * @typedef {AbstractWithSyntax[]} AbstractsWithSyntax
+ */
+
+/**
+ * @typedef {object} AbstractWithSyntax
+ * @property {Abstract} abstract
+ * @property {ProcessorSyntax} syntax
+ */
+
+/**
+ * @callback ReadFolder
+ * @param {object} Folder
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ */
+
+/**
+ * @typedef {object} Folder
+ * @property {string} path
+ */
+
+/**
+ * Build and write destination files.
+ * @callback ProcessorWrite
+ * @param {object} destination
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ * @returns {{ data: string, encoding?: BufferEncoding = 'utf-8' }}
+ */
+
+/**
+ * A function that runs a job, suggest as fetching data or formatting an image.
+ * @callback Runner
+ * @param {string} file
+ * @param {Database} database
+ * @returns {Promise<any>}
+ */
+
+/**
+ * Run a job.
+ * @typedef {object} Job
+ * @property {object} data
+ * @property {string} runner
+ * @property {string} [plugin]
+ */
+
+/**
+ * @callback Router
+ * @param {string} path
+ * @returns {string | false | undefined}
+ */
+
+/**
+ * @typedef {object} VotiveConfig
+ * @property {string} sourceFolder
+ * @property {string} destinationFolder
+ * @property {VotivePlugin[]} plugins
+ */
+
+/**
+ * @typedef {object} FlatProcessor
+ * @property {VotivePlugin} plugin
+ * @property {VotiveProcessor} processor
+ */
+
+/**
+ * @typedef {FlatProcessor[]} FlatProcessors
+ */
+
+/**
+ * @param {VotiveConfig} config
+ */
+async function bundle(config) {
+  const processors = config.plugins
+    && config.plugins.flatMap(plugin => plugin.processors && plugin.processors.map(processor => ({ plugin, processor })))
+
+  const database = createDatabase()
+  const { folders, sources } = await readSources(config, database, processors)
+
+  const sourcesJobs = sources.flatMap(source => source.jobs) || []
+  const { processedAbstracts, abstractsJobs } = readAbstracts(sources, config, database, processors)
+  const foldersJobs = readFolders(folders, config, database, processors) || []
+  await writeDestinations(config, database)
+
+  const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
+  await runJobs(jobs, config, database)
+}
+
+export default bundle

package/lib/createDatabase.js

@@ -0,0 +1,323 @@
+import { DatabaseSync, backup } from "node:sqlite"
+import { splitURL } from "./utils/index.js"
+import { statSync } from "node:fs"
+
+/**
+ * @typedef {ReturnType<createDatabase>} Database
+ */
+
+
+function checkFile(path) {
+  try {
+    return statSync(path)
+  } catch(e) {
+    return null
+  }
+}
+
+
+function createDatabase() {
+  const databaseSync = new DatabaseSync(".votive.db")
+
+  const store = databaseSync.createTagStore()
+
+  databaseSync.exec(sqlCreateTables)
+
+  const database = {}
+
+  /**
+   * @param {string} folderPath 
+   * @param {string} urlPath
+   */
+  database.createFolder = (folderPath, urlPath) => {
+    return store.get`INSERT INTO folders (folderPath, urlPath) VALUES :${folderPath} :${urlPath} RETURNING *`
+  }
+
+  const createSource = databaseSync.prepare(`INSERT INTO sources (path, destination, lastModified) VALUES (?, ?, ?)`)
+
+  /**
+   * @param {string} source - Source file path.
+   * @param {string} destination - Destination file path.
+   * @param {number} lastModified - Source file date last modified.
+   */
+  database.createSource = (source, destination, lastModified) => {
+    databaseSync.prepare(``) // SQLite bug. Query fails without this.
+    const inserted = createSource.get(source, destination, lastModified)
+  }
+
+
+  // These statements are cached, no need to refactor
+  const createDest = databaseSync.prepare(`INSERT INTO destinations (path, dir, syntax, stale, abstract) VALUES (?, ?, ?, 1, ?)`)
+  const updateDest = databaseSync.prepare(`UPDATE destinations SET abstract = ? WHERE path = ?`)
+  const createMeta = databaseSync.prepare(`INSERT INTO metadata (destination, label, value, type) VALUES (?, ?, ?, ?)`)
+  const updateMeta = databaseSync.prepare(`UPDATE metadata SET value = ? WHERE destination = ? AND label = ?`)
+  const getDepends = databaseSync.prepare(`SELECT * FROM dependencies WHERE destination = ? AND property = ?`)
+  const staleDepen = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path = ?`)
+  const staleDescendents = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path LIKE ? RETURNING *`)
+
+  /**
+   * @param {object} params
+   * @param {string} params.path - Destination file path
+   * @param {object} params.abstract
+   * @param {object} params.metadata
+   * @param {string} params.syntax - Destination abstract syntax
+   */
+  database.createOrUpdateDestination = ({ metadata, ...dest }) => {
+    const [dir] = dest.path ? splitURL(dest.path) : []
+    const params = Object.keys(metadata)
+    if (dest.abstract) params.push("abstract")
+    const extant = database.getDestinationIndependently(dest.path, params)
+
+    if (!extant) {
+      createDest.get(dest.path, dir, dest.syntax, JSON.stringify(dest.abstract))
+      Object.entries(metadata).map(([k, v]) => createMeta.get(dest.path, k, v, typeof v))
+      return
+    }
+
+    const changedAbstract = JSON.stringify(dest.abstract) !== JSON.stringify(extant.abstract)
+    const changedMetadata = []
+
+    for (const key in metadata) {
+      if (JSON.stringify(metadata[key]) !== JSON.stringify(extant.metadata[key])) {
+        updateMeta.get(metadata[key], dest.path, key)
+        const dependencies = getDepends.all(dest.path, key)
+        dependencies.forEach(({ dependent }) => {
+          staleDepen.get(dependent)
+        })
+      }
+    }
+
+    if (changedAbstract) {
+      const updated = updateDest.get(JSON.stringify(dest.abstract), dest.path)
+      const dependencies = getDepends.all(dest.path, "abstract")
+      dependencies.forEach(({ dependent }) => {
+        staleDepen.get(dependent)
+      })
+    }
+  }
+
+  /**
+   * @param {string} path
+   * @param {string[]} params
+   */
+  database.getDestinationIndependently = (path, params) => {
+    // TODO: Could potentially speed up the next two db queries by using the tag store
+    const metadatas = databaseSync.prepare(`
+      SELECT * FROM destinations d
+      LEFT JOIN metadata m ON d.path = m.destination AND m.label IN (${params.map(p => `'${p}'`).join(", ")})
+      WHERE d.path = ?
+    `).all(path)
+
+    const [first] = metadatas
+
+    if (!first) return
+
+    const metadata = Object.fromEntries(
+      metadatas.map(({ label, value, type }) => {
+        const originalValue = type === "undefined"
+          ? undefined
+          : type === "string"
+            ? String(value)
+            : type === "boolean"
+              ? Boolean(value)
+              : typeof value !== "string"
+                ? value
+                : JSON.parse(value)
+        return [label, value]
+      })
+    )
+
+
+    const result = {
+      path: first.path,
+      dir: first.dir,
+      syntax: first.syntax,
+      metadata
+    }
+
+    const includeAbstract = params.includes("abstract")
+    if (includeAbstract) result.abstract = JSON.parse(first.abstract)
+
+    return result
+  }
+
+  /**
+   * @param {string} path
+   * @param {string[]} properties
+   * @param {string} dependent
+   */
+  database.getDestinationDependently = (path, properties, dependent) => {
+    const params = []
+    const deps = []
+
+    properties.forEach(p => {
+      params.push(p)
+      deps.push(`('${path}', '${p}', '${dependent}')`)
+    })
+
+    const result = database.getDestinationIndependently(path, params)
+    if (result.abstract) deps.push(`('${path}, 'abstract', '${dependent}')`)
+    const createDepp = databaseSync.prepare(`INSERT INTO dependencies (destination, property, dependent) VALUES ${deps.join(", ")}`).all()
+  }
+
+  const selectSource = databaseSync.prepare(`SELECT * FROM sources WHERE path = ?`)
+
+  /**
+   * @param {string} path
+   */
+  database.getSource = (path) => {
+    const source = selectSource.get(path)
+    return source
+  }
+
+  database.getAllSources = () => {
+    const sources = store.all`SELECT * FROM sources`
+    return sources
+  }
+
+  database.getAllDestinations = () => {
+    return store.all`SELECT * FROM destinations`
+  }
+
+  database.getStaleDestinations = () => {
+    return store.all`SELECT * FROM destinations WHERE stale = 1`
+  }
+
+  const createSetting = databaseSync.prepare(`INSERT INTO settings (destination, label, value) VALUES (?, ?, ?) RETURNING *`)
+  const selectSetting = databaseSync.prepare(`SELECT * FROM settings WHERE label = ? AND destination = ?`)
+  const updateSetting = databaseSync.prepare(`UPDATE settings SET value = ? WHERE destination = ? AND label = ?`)
+
+  // const updateDest = databaseSync.prepare(`UPDATE destinations SET abstract = ? WHERE path = ?`)
+
+  /**
+   * @param {string} destination
+   * @param {string} key
+   * @param {string} value
+   */
+  database.setSetting = (destination, key, value) => {
+    // TODO: Check if already exists
+    const extant = selectSetting.get(key, destination)
+
+    if (!extant) return createSetting.get(destination, key, value)
+    if (extant.value === value) return
+
+    updateSetting.get(value, destination, key)
+    const stale = staleDescendents.all(`${destination}/%`)
+  }
+
+
+  /**
+   * @param {string} destination
+   * @param {string} dependent
+   */
+  database.getSettings = (destination, dependent) => {
+    return Object.fromEntries(databaseSync.prepare(`SELECT * FROM settings WHERE destination = '${destination}' OR destination LIKE '${destination}/%';`)
+      .all()
+      .map(({ label, value }) => [label, value]))
+  }
+
+  database.getEverything = () => {
+    return Object.fromEntries(
+      databaseSync.prepare(`SELECT name FROM sqlite_master WHERE type = 'table'`)
+        .all()
+        .map(table => [
+          table.name,
+          databaseSync.prepare(`SELECT * FROM ${table.name}`).all()
+        ])
+    )
+  }
+
+  return database
+}
+
+/**
+ * @param {object} data
+ * @param {string} table
+ * @param {DatabaseSync} databaseSync
+ */
+function updateColumns(data, table, databaseSync) {
+  const keys = new Set(Object.keys(data))
+  const columns = listColumns(table, databaseSync)
+  const newColumns = keys.difference(columns)
+  for (const column of newColumns) addColumn(column, table, databaseSync)
+}
+
+/**
+ * @param {string} table
+ * @param {DatabaseSync} databaseSync
+ * @description list the columns on a table.
+ */
+function listColumns(table, databaseSync) {
+  const columns = new Set()
+  const tableInfo = databaseSync.prepare(`PRAGMA table_info(${table})`).all()
+  for (const column of tableInfo) columns.add(column.name)
+  return columns
+}
+
+/**
+ * @param {string} column
+ * @param {string} table
+ * @param {DatabaseSync} databaseSync
+ * @description Add a column to a table and update binary.
+ */
+function addColumn(column, table, databaseSync) {
+  databaseSync.exec(`ALTER TABLE ${table} ADD COLUMN ${column} STRING;`)
+}
+
+const sqlCreateTables = `
+CREATE TABLE IF NOT EXISTS jobs (
+  id INTEGER PRIMARY KEY,
+  pluginName TEXT NOT NULL,
+  data STRING NOT NULL,
+  done INTEGER NOT NULL DEFAULT 0,
+  runner STRING NOT NULL
+);
+
+-- Source/destination relationships
+CREATE TABLE IF NOT EXISTS sources (
+  id INTEGER PRIMARY KEY,
+  destination STRING,
+  path STRING,
+  lastModified INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS dependencies (
+  key INTEGER PRIMARY KEY,
+  destination STRING NOT NULL,
+  property STRING NOT NULL,
+  dependent STRING NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS destinations (
+  key INTEGER PRIMARY KEY,
+  path STRING UNIQUE,
+  dir TEXT,
+  syntax TEXT,
+  stale INTEGER,
+  abstract
+);
+
+CREATE TABLE IF NOT EXISTS folders (
+  path STRING PRIMARY KEY,
+  urlPath STRING
+);
+
+CREATE TABLE IF NOT EXISTS metadata (
+  id INTEGER PRIMARY KEY,
+  destination STRING,
+  label STRING,
+  value STRING,
+  type STRING,
+  UNIQUE(destination, label)
+);
+
+CREATE TABLE IF NOT EXISTS settings (
+  id INTEGER PRIMARY KEY,
+  destination STRING,
+  label STRING,
+  value STRING
+);
+
+`
+
+export default createDatabase

package/lib/index.js

@@ -0,0 +1,7 @@
+export { default as bundle } from "./bundle.js"
+export { default as createDatabase } from "./createDatabase.js"
+export { default as readAbstracts } from "./readAbstracts.js"
+export { default as readFolders } from "./readFolders.js"
+export { default as readSources } from "./readSources.js"
+export { default as runJobs } from "./runJobs.js"
+export { default as writeDestinations } from "./writeDestinations.js"

package/lib/readAbstracts.js

@@ -0,0 +1,59 @@
+/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+/** @import {ReadSourceFileResult} from "./readSources.js"
+
+/**
+ * @typedef {object} ReadAbstractsResult
+ * @property {Jobs} jobs
+ * @property {ReadAbstractResult[]} abstracts
+ */
+
+/**
+ * @typedef {object} ReadAbstractResult
+ */
+
+/**
+ * @param {ReadSourceFileResult} abstracts
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ * @param {FlatProcessors} processors
+ * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
+ */
+function readAbstracts(abstracts, config, database, processors) {
+
+  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
+
+    /**
+     * @param {FlatProcessors} processors
+     * @param {Abstract} abstract
+     * @param {Jobs} jobs
+     */
+    function recursiveProcess(processors, abstract, jobs = []) {
+      const [flatProcessor, ...rest] = processors
+      if (!flatProcessor) return { abstract, jobs }
+      const { processor, plugin } = flatProcessor
+      if (processor.syntax !== syntax
+        || !processor.read
+        || !processor.read.abstract
+      ) return recursiveProcess(rest, abstract, jobs)
+
+      const processed = processor.read.abstract(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.plugin = plugin.name)
+      return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
+    }
+
+    return recursiveProcess(processors, unprocessedAbstract)
+  })
+
+  const processedAbstracts = []
+  const abstractsJobs = []
+
+  processed.forEach(({ abstract, job}) => {
+    processedAbstracts.push(abstract)
+    abstractsJobs.push(job)
+  })
+
+  return { processedAbstracts, abstractsJobs }
+}
+
+export default readAbstracts

package/lib/readFolders.js

@@ -0,0 +1,28 @@
+/** @import {VotiveConfig, FlatProcessors, Jobs} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+/** @import {Dirent} from "node:fs" */
+
+import path from "node:path"
+
+/**
+ * @param {Dirent[]} folders
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ * @param {FlatProcessors} processors
+ * @returns {Jobs}
+ */
+function readFolders(folders, config, database, processors) {
+  if(!folders) return
+  return folders.flatMap(folder => {
+    return config.plugins.flatMap(plugin => {
+      return plugin.processors.flatMap(processor => {
+        const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
+        return processor.read
+          && processor.read.folder
+          && processor.read.folder(folderPath, database, config)
+      })
+    })
+  })
+}
+
+export default readFolders

package/lib/readSources.js

@@ -0,0 +1,142 @@
+import { decodeBuffer } from "encoding-sniffer"
+import fs from "node:fs/promises"
+import path from "node:path"
+import { visit } from "unist-util-visit"
+
+/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax} from "./bundle.js" */
+/** @import {Dirent} from "node:fs" */
+/** @import {Database} from "./createDatabase.js" */
+
+/**
+ * @typedef {object} ReadSourcesResult
+ * @property {Dirent[]} folders
+ * @property {ReadSourceFilePlugin[]} sources
+ */
+
+/**
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ * @param {FlatProcessors} processors
+ * @returns {Promise<ReadSourcesResult>}
+ */
+async function readSources(config, database, processors) {
+  const dirents = await fs.readdir(config.sourceFolder, {
+    withFileTypes: true,
+    recursive: true
+  })
+
+  // TODO: Check file modified time
+
+  const filteredDirents = (dirents || []).filter(fileFilter(config, database))
+  const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
+  if (!files) return { folders, sources: [] }
+  const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config))
+  const sources = readingSourceFiles && await Promise.all(readingSourceFiles)
+  return {
+    folders,
+    sources
+  }
+}
+
+/**
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ */
+function fileFilter(config, database) {
+  /** @param {Dirent} dirent */
+  return (dirent) => {
+    if (dirent.parentPath === config.destinationFolder) {
+      return false
+    } else if (dirent.parentPath.startsWith(config.destinationFolder + path.sep)) {
+      return false // Ignore destination folder
+    } else if (dirent.name.startsWith(".")) {
+      return false // Ignore hidden files
+    } else if (dirent.parentPath.includes(path.sep + ".")) {
+      return false // Ignore hidden folders
+    }
+    return true
+  }
+}
+
+/**
+ * @typedef {ReadSourceFilePlugin[]} ReadSourceFileResult
+ */
+
+/**
+ * @typedef {object} ReadSourceFilePlugin
+ * @property {Abstract} abstract
+ * @property {ProcessorSyntax} syntax
+ * @property {Jobs} jobs
+ * @property {string} destinationPath
+ */
+
+/**
+ * @param {{ plugin: VotivePlugin, processor: VotiveProcessor }[]} processors
+ * @param {Database} database
+ * @param {VotiveConfig} config
+ */
+function readSourceFile(processors, database, config) {
+  /**
+   * @param {import("node:fs").Dirent} dirent
+   * @returns {Promise<ReadSourceFilePlugin>[]}
+   */
+  return (dirent) => {
+    const { name, parentPath } = dirent
+    const filePath = path.join(parentPath, name)
+    const fileInfo = path.parse(filePath)
+
+    const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config))
+
+    /**
+     * @param {VotivePlugin} plugin
+     * @param {VotiveProcessor} processor
+     * @param {VotiveConfig} config
+     */
+    async function process(plugin, processor, config) {
+      const { read, filter, syntax } = processor
+      if (filter.extensions.includes(fileInfo.ext) && read && read.text) {
+
+        // Check modified time
+        const stat = await fs.stat(filePath)
+        const source = database.getSource(filePath)
+
+
+        if (source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
+
+        // Get destination route
+        const destinationPath = route(filePath, plugin)
+
+        // Set URL if exists
+        const buffer = await fs.readFile(path.format(fileInfo))
+        const data = decodeBuffer(buffer)
+        const { jobs, abstract, metadata } = read.text(data, database, config)
+        jobs && jobs.forEach(job => job.plugin = plugin.name)
+        // TODO: Cache: Check if file already exists
+
+        const timeStamp = stat.mtimeMs.toFixed()
+        database.createSource(filePath, destinationPath, Number(timeStamp))
+        database.createOrUpdateDestination({
+          metadata,
+          path: destinationPath,
+          abstract: abstract,
+          syntax: syntax
+        })
+
+        return { abstract, destinationPath, syntax, jobs }
+      }
+    }
+
+    return processing
+  }
+}
+
+/**
+ * @param {string} filePath
+ * @param {VotivePlugin} plugin
+ */
+function route(filePath, plugin) {
+  if (!plugin.router) return ""
+  return plugin.router(filePath) || ""
+}
+
+export default readSources

package/lib/runJobs.js

@@ -0,0 +1,23 @@
+import workerpool from "workerpool"
+
+/** @import {Job, Jobs, Database, VotiveConfig} from "./bundle.js" */
+
+const pool = workerpool.pool()
+
+/**
+ * @param {Jobs} jobs
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ */
+async function runJobs(jobs, config, database) {
+  const running = jobs.map(async job => {
+    if (!job) return
+    const plugin = config.plugins.find(plugin => plugin.name === job.plugin)
+    return pool.exec(plugin.runners[job.runner], [job.data])
+  })
+
+  const ran = await Promise.allSettled(running)
+  pool.terminate()
+}
+
+export default runJobs

package/lib/utils/index.js

@@ -0,0 +1,22 @@
+import { styleText } from "node:util"
+
+/** @param {string} label */
+export function stopwatch(label) {
+  const start = performance.now()
+
+  function stop() {
+    const end = performance.now()
+    const duration = end - start
+    console.info(`${label}: ${styleText("red", duration.toFixed(2) + "ms")}`)
+  }
+
+  return stop
+}
+
+/**
+ * @param {string} urlPath
+ */
+export function splitURL(urlPath) {
+  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
+  return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
+}

package/lib/writeDestinations.js

@@ -0,0 +1,42 @@
+import { mkdir, writeFile } from "node:fs/promises"
+import path from "node:path"
+
+/** @import {Abstract, Abstracts, VotiveConfig} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+
+
+/**
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ */
+async function writeDestinations(config, database) {
+  const writeProcessors = config && config.plugins && config.plugins.flatMap(plugin => (
+    plugin.processors && plugin.processors.map(({ write, syntax }) => write && { write, syntax }).filter(a => a)
+  )).filter(a => a)
+
+  if (!writeProcessors || !writeProcessors.length) throw "No write processor provided"
+
+  const destinations = database.getStaleDestinations()
+  if (!destinations) return
+
+  const writing = destinations.flatMap(destination => {
+    const destinationPath = path.join(config.destinationFolder, String(destination.path))
+    const { dir } = path.parse(destinationPath)
+    return writeProcessors.map(processor => {
+      if (processor.syntax === destination.syntax) {
+        const { data, encoding = 'utf-8' } = processor.write(destination, database, config)
+
+        async function write() {
+          await mkdir(dir, { recursive: true })
+          await writeFile(destinationPath, data, encoding)
+        }
+        
+        return write()
+      }
+    })
+  })
+
+  await Promise.all(writing)
+}
+
+export default writeDestinations

package/package.json

@@ -1,12 +1,30 @@
 {
   "name": "votive",
-  "version": "0.0.1",
-  "description": "tk",
+  "version": "0.0.3",
+  "description": "A file processor.",
+  "homepage": "https://github.com/samlfair/votive#readme",
+  "bugs": {
+    "url": "https://github.com/samlfair/votive/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/samlfair/votive.git"
+  },
+  "license": "MIT",
+  "author": "Sam Littlefair",
+  "type": "module",
   "main": "index.js",
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
+  "exports": {
+    ".": "./lib/bundle.js",
+    "./internals": "./lib/index.js"
+  },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node --test --watch tests/*.js"
+  },
+  "dependencies": {
+    "encoding-sniffer": "^0.2.1",
+    "unified": "^11.0.5",
+    "unist-util-visit": "^5.0.0",
+    "workerpool": "^10.0.1"
   }
 }

package/tests/index.js

@@ -0,0 +1,155 @@
+import test from "node:test"
+import assert from "node:assert/strict"
+import fs from "node:fs"
+import path from "node:path"
+import * as votive from "votive/internals"
+import { stopwatch } from "./../lib/utils/index.js"
+import { readFolders, runJobs } from "../lib/index.js"
+
+/** @import {VotiveConfig, VotivePlugin, FlatProcessors, VotiveProcessor, Runner, Job} from "./../lib/bundle.js" */
+
+process.chdir("./tests")
+
+test("empty directory", async () => {
+  const temp = fs.mkdtempDisposableSync("destination-")
+
+  try {
+    const stop = stopwatch("Bundle empty folder")
+    await votive.bundle({
+      sourceFolder: "./empty",
+      destinationFolder: temp.path
+    })
+
+    stop()
+
+    const dir = fs.readdirSync(temp.path)
+    assert(dir.length === 0, "Destination directory is not empty.")
+  } catch (error) {
+    temp.remove()
+    console.error(error)
+
+  }
+
+  temp.remove()
+})
+
+test("create empty database", () => {
+  const stop = stopwatch("Create empty database")
+  const database = votive.createDatabase()
+  stop()
+
+  database.createSource('exampleSource', 'exampleDestination', 123)
+  const [source] = database.getAllSources()
+  
+  assert(source.destination === 'exampleDestination')
+})
+
+test("read sources", async () => {
+  const temp = fs.mkdtempDisposableSync("destination-")
+
+  /** @type {Job} */
+  const testJob = {
+    data: "123",
+    runner: "testRunner"
+  }
+
+  /** @type {VotiveProcessor} */
+  const testProcessor = {
+    syntax: "txt",
+    filter: {
+      extensions: [".md"]
+    },
+    read: {
+      path: (filePath, database, config) => undefined,
+      text: (text, database, config) => ({ metadata: { foo: "bar", bang: "baz" }, abstract: { baz: "bang" } }),
+      abstract: (abstract, database, config) => ({ abstract: { ...abstract, newAddition: true }, jobs: [testJob] }),
+      folder: (folder, database, config) => [testJob]
+    },
+    write: (destination, database, config) => {
+      return {
+        data: "yo there",
+      }
+    }
+  }
+
+  /** @type {VotivePlugin} */
+  const testPlugin = {
+    name: "test plugin",
+    runners: {
+      testRunner
+    },
+    router: (path) => path,
+    processors: [testProcessor]
+  }
+
+  /** @type {Runner} */
+  async function testRunner(data, database) {
+    const waiting = await new Promise((resolve) => setTimeout(() => resolve(data), 1000))
+
+    return waiting
+  }
+
+  /** @type {VotiveConfig} */
+  const config = {
+    sourceFolder: "./markdown",
+    destinationFolder: temp.path,
+    plugins: [testPlugin]
+  }
+
+  /** @type {FlatProcessors} */
+  const processors = [
+    {
+      plugin: testPlugin,
+      processor: testProcessor
+    }
+  ]
+
+  try {
+    const database = votive.createDatabase()
+
+    const moresources = database.getAllSources()
+    console.log({ moresources })
+
+    const stop = stopwatch("Read sources")
+    const { folders, sources } = await votive.readSources(config, database, processors)
+    stop()
+
+    const sourcesJobs = sources.flatMap(source => source.jobs)
+    const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
+
+    const foldersJobs = readFolders(folders, config, database, processors)
+
+    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.txt", abstract: { c: 3 }, syntax: "txt" })
+    const destination = database.getDestinationDependently("abc.txt", ["a"], "def")
+    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "def.txt", abstract: { c: 5 }, syntax: "txt" })
+    database.createOrUpdateDestination({ metadata: { a: 3, b: 9 }, path: "abc.txt", abstract: { c: 4 }, syntax: "txt" })
+    database.createOrUpdateDestination({ metadata: { a: 3, b: 9 }, path: "abc/def.txt", abstract: { c: 4 }, syntax: "txt" })
+
+
+    const setting = database.setSetting("abc", "theme", "blue")
+    const newSetting = database.setSetting("def", "category", "Dog")
+    const sameSetting = database.setSetting("def", "category", "Dog")
+    const oldSetting = database.setSetting("abc", "theme", "green")
+
+    const settings = database.getSettings("abc", "def")
+
+    const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
+    runJobs(jobs, config, database)
+
+    const dbResults = database.getAllSources()
+
+    const written = await votive.writeDestinations(config, database)
+
+    // const everything = database.getEverything()
+    // console.log(everything)
+
+    const newsources = database.getAllSources()
+    console.log({ newsources })
+
+    assert(true, "Read test tk")
+    temp.remove()
+  } catch (error) {
+    temp.remove()
+    throw error
+  }
+})

package/index.js

@@ -1 +0,0 @@
-console.log("tk")