votive 0.0.5 → 0.0.7

package/README.md

@@ -4,7 +4,8 @@
 
 - [ ] ReadPaths
 - [ ] Flesh out job runners
-- [ ] Destination query filters
+- [x] Destination query filters
+- [x] Setters for metadata
 
 ## Helpers
 

package/lib/bundle.js

@@ -7,6 +7,7 @@ import writeDestinations from "./writeDestinations.js"
 import { default as createDatabase } from "./createDatabase.js"
 
 /** @import {Database} from "./createDatabase.js" */
+/** @import {ParsedPath} from "node:path" */
 
 /**
  * @typedef {object} VotivePlugin
@@ -38,6 +39,7 @@ import { default as createDatabase } from "./createDatabase.js"
 /**
  * @typedef {object} ProcessorRead
  * @property {ReadPath} [path]
+ * @property {ReadURL} [url]
  * @property {ReadText} [text]
  * @property {ReadAbstract} [abstract]
  * @property {ReadFolder} [folder]
@@ -49,23 +51,31 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {string} filePath
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {Job[] | undefined}
+ * @returns {Promise<ReadTextResult>}
+ */
+
+/**
+ * Reads a the response from a URL and returns arbitrary data.
+ * @callback ReadURL
+ * @param {Response} response
+ * @returns {object}
  */
 
 /**
  * @callback ReadText
  * @param {string} text
  * @param {string} filePath
+ * @param {string} destinationPath
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {ReadTextResult | undefined}
+ * @returns {ReadTextResult}
  */
 
 /**
  * @typedef {object} ReadTextResult
  * @property {Jobs} [jobs]
- * @property {object} [metadata]
- * @property {Abstract} [abstract]
+ * @property {object} metadata
+ * @property {Abstract} abstract
  */
 
 /**
@@ -100,9 +110,10 @@ import { default as createDatabase } from "./createDatabase.js"
 
 /**
  * @callback ReadFolder
- * @param {object} Folder
+ * @param {object} folder
  * @param {Database} database
  * @param {VotiveConfig} config
+ * @param {boolean} isRoot
  * @returns {{ jobs?: Jobs, destinations?: Destination[] }}
  */
 
@@ -125,7 +136,7 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {object} destination
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {{ data: string, encoding?: BufferEncoding = 'utf-8' }}
+ * @returns {{ data: string, buffer: Buffer, encoding?: BufferEncoding = 'utf-8' }}
  */
 
 /**
@@ -140,14 +151,22 @@ import { default as createDatabase } from "./createDatabase.js"
  * Run a job.
  * @typedef {object} Job
  * @property {object} data
- * @property {string} runner
- * @property {string} [plugin]
+ * @property {"text" | "blob" | "json"} format
+ * @property {string} [syntax]
+ */
+
+/**
+ * @typedef {object} PathInfo
+ * @property {string[]} dir
+ * @property {ParsedPath["name"]} name
+ * @property {ParsedPath["ext"]} ext
+ * @property {boolean} [inRootDir]
  */
 
 /**
  * @callback Router
- * @param {string} path
- * @returns {string | false | undefined}
+ * @param {PathInfo} path
+ * @returns {PathInfo | false | undefined}
  */
 
 /**
@@ -172,6 +191,7 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {Database | undefined} [cache]
  */
 async function bundle(config, cache) {
+  // TODO: Ensure all cached destinations exist as expected
 
   // Map out all processors
   const processors = config.plugins
@@ -213,9 +233,15 @@ async function bundle(config, cache) {
     ...foldersJobs
   ], config, database)
 
+
+  await writeDestinations(config, database)
+
   // Back up database (only if in-memory first run)
   await database.saveDB()
 
+  const stale = database.getStaleDestinations()
+  if(stale.length > 0) await bundle(config, cache)
+
   return database
 }
 

package/lib/createDatabase.js

@@ -2,6 +2,7 @@ import { DatabaseSync, backup } from "node:sqlite"
 import { splitURL, checkFile } from "./utils/index.js"
 import { statSync } from "node:fs"
 import path from "node:path"
+import createStatement from "./sqlite.js"
 
 /**
  * @typedef {ReturnType<createDatabase>} Database
@@ -35,7 +36,7 @@ function createDatabase(dbPath = ".votive.db") {
    * @param {string} urlPath
    */
   database.createFolder = (folderPath, urlPath) => {
-    return store.get`INSERT INTO folders (folderPath, urlPath) VALUES :${folderPath} :${urlPath} RETURNING *`
+    return store.get`INSERT OR IGNORE INTO folders (folderPath, urlPath) VALUES :${folderPath} :${urlPath} RETURNING *`
   }
 
   const createSource = databaseSync.prepare(`INSERT INTO sources (path, destination, lastModified) VALUES (?, ?, ?)`)
@@ -66,12 +67,12 @@ function createDatabase(dbPath = ".votive.db") {
   // These statements are cached, no need to refactor
   const createDest = databaseSync.prepare(`INSERT INTO destinations (path, dir, syntax, stale, abstract) VALUES (?, ?, ?, 1, ?)`)
   const updateDest = databaseSync.prepare(`UPDATE destinations SET abstract = ? WHERE path = ?`)
-  const createMeta = databaseSync.prepare(`INSERT INTO metadata (destination, label, value, type) VALUES (?, ?, ?, ?)`)
+  const createMeta = databaseSync.prepare(`INSERT OR REPLACE INTO metadata (destination, label, value, type) VALUES (?, ?, ?, ?)`)
   const updateMeta = databaseSync.prepare(`UPDATE metadata SET value = ? WHERE destination = ? AND label = ?`)
   const getDepends = databaseSync.prepare(`SELECT * FROM dependencies WHERE destination = ? AND property = ?`)
   const getDependenciesByDestination = databaseSync.prepare(`SELECT * FROM dependencies WHERE destination = ?`)
   const getAllDeps = databaseSync.prepare(`SELECT * FROM dependencies`)
-  const staleDepen = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path = ?`)
+  const staleDepen = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path = ? RETURNING *`)
   const freshDepen = databaseSync.prepare(`UPDATE destinations SET stale = 0 WHERE path = ? RETURNING *`)
   const staleDescendents = databaseSync.prepare(`UPDATE destinations SET stale = 1 WHERE path LIKE ? RETURNING *`)
   const getAllSettings = databaseSync.prepare(`SELECT * FROM settings`)
@@ -97,14 +98,21 @@ function createDatabase(dbPath = ".votive.db") {
    * @param {string} params.syntax - Destination abstract syntax
    */
   database.createOrUpdateDestination = ({ metadata, ...dest }) => {
-    const [dir] = dest.path ? splitURL(dest.path) : []
+    const dir = dest.path && splitURL(dest.path)
     const params = Object.keys(metadata)
     if (dest.abstract) params.push("abstract")
     const extant = database.getDestinationIndependently(dest.path, params)
 
     if (!extant) {
       createDest.get(dest.path, dir, dest.syntax, JSON.stringify(dest.abstract))
-      Object.entries(metadata).map(([k, v]) => createMeta.get(dest.path, k, v, typeof v))
+      Object.entries(metadata).map(([k, v]) => {
+        const type = typeof v
+        const value = type === "object"
+          ? JSON.stringify(v)
+          : v
+
+        createMeta.get(dest.path, k, value, typeof v)
+      })
       return
     }
 
@@ -121,7 +129,7 @@ function createDatabase(dbPath = ".votive.db") {
       cachedMetadata.splice(index, 1)
       if (JSON.stringify(metadata[key]) !== JSON.stringify(extant.metadata[key])) {
         changedMetadata = true
-        updateMeta.get(metadata[key], dest.path, key)
+        updateMeta.get(JSON.stringify(metadata[key]), dest.path, key)
         const dependencies = getDepends.all(dest.path, key)
         dependencies.forEach(({ dependent }) => {
           staleDepen.get(dependent)
@@ -158,11 +166,10 @@ function createDatabase(dbPath = ".votive.db") {
     return getMetadataByPath.get(path)
   }
 
-  /** @param {string[]} params */
-  database.getMetadataIndependently = (params) => {
+  database.getMetadataIndependently = () => {
     return databaseSync.prepare(`
       SELECT * FROM destinations d
-      LEFT JOIN metadata m ON d.path = m.destination AND m.label IN (${params.map(p => `'${p}'`).join(", ")})
+      LEFT JOIN metadata m ON d.path = m.destination
       WHERE d.path = ?
     `)
   }
@@ -174,16 +181,29 @@ function createDatabase(dbPath = ".votive.db") {
       `)
   }
 
+  function castMetadata({ label, value, type }) {
+    return [label,
+      type === "undefined"
+        ? undefined
+        : type === "boolean"
+          ? Boolean(value)
+          : value
+    ]
+  }
+
+  function parseMetadatas(metadatas) {
+    return Object.fromEntries(
+      metadatas.map(castMetadata)
+    )
+  }
+
   /**
    * @param {string} path
-   * @param {string[]} [params]
    */
-  database.getDestinationIndependently = (path, params) => {
+  database.getDestinationIndependently = (path) => {
     // TODO: Could potentially speed up the next two db queries by using the tag store
 
-    const metadatas = (params && params.length)
-      ? database.getMetadataIndependently(params).all(path)
-      : [database.getDestinationWithoutMetadata().get(path)]
+    const metadatas = database.getMetadataIndependently().all(path)
 
     if (!metadatas) return
 
@@ -191,52 +211,94 @@ function createDatabase(dbPath = ".votive.db") {
 
     if (!first) return
 
-    const metadata = Object.fromEntries(
-      metadatas.map(({ label, value, type }) => {
-        const originalValue = type === "undefined"
-          ? undefined
-          : type === "string"
-            ? String(value)
-            : type === "boolean"
-              ? Boolean(value)
-              : typeof value !== "string"
-                ? value
-                : JSON.parse(value)
-        return [label, originalValue]
-      })
-    )
-
+    const metadata = parseMetadatas(metadatas)
 
     const result = {
+      abstract: JSON.parse(first.abstract),
       path: first.path,
       dir: first.dir,
       syntax: first.syntax,
       metadata
     }
 
-    const includeAbstract = params && params.includes("abstract")
-    if (includeAbstract) result.abstract = JSON.parse(first.abstract)
-
     return result
   }
 
+  const createDependency = databaseSync.prepare(`INSERT OR IGNORE INTO dependencies (destination, property, dependent) VALUES (?, ?, ?) `)
+  const getDependency = databaseSync.prepare("SELECT * FROM dependencies WHERE destination = ? AND property = ?")
   /**
    * @param {string} path
-   * @param {string[]} properties
    * @param {string} dependent
    */
-  database.getDestinationDependently = (path, properties, dependent) => {
-    const params = []
-    const deps = []
+  database.getDestinationDependently = (path, dependent) => {
+    const result = database.getDestinationIndependently(path)
+
+    const { abstract, metadata, ...copy } = result
+
+    copy.metadata = {}
 
-    properties.forEach(p => {
-      params.push(p)
-      deps.push(`('${path}', '${p}', '${dependent}')`)
+    Object.defineProperty(copy, "abstract", {
+      enumerable: true,
+      get() {
+        createDependency.get(path, "abstract", dependent)
+        return abstract
+      }
     })
 
-    const result = database.getDestinationIndependently(path, params)
-    if (result.abstract) deps.push(`('${path}, 'abstract', '${dependent}')`)
-    databaseSync.prepare(`INSERT INTO dependencies (destination, property, dependent) VALUES ${deps.join(", ")} RETURNING *`).all()
+    const keys = Object.keys(metadata)
+
+    keys.forEach(key => {
+      Object.defineProperty(copy.metadata, key, {
+        enumerable: true,
+        get() {
+          createDependency.get(path, key, dependent)
+          return metadata[key]
+        }
+      })
+    })
+
+
+    return copy
+  }
+
+  /** @param {import("./sqlite.js").Query} query */
+  database.getDestinations = (query = {}, dependent) => {
+    const statement = createStatement(query)
+    const results = databaseSync.prepare(statement).all()
+    const destinations = Object.values(results.reduce((pv, cv) => {
+      if (!pv || !pv[cv.path]) {
+        pv[cv.path] = {
+          dir: cv.dir,
+          path: cv.path,
+          syntax: cv.syntax,
+          metadata: {}
+        }
+
+        Object.defineProperty(pv[cv.path], "abstract",
+          {
+            enumerable: true,
+            get() {
+              createDependency.get(cv.path, "abstract", dependent)
+              return cv.abstract
+            }
+          }
+        )
+      }
+
+      Object.defineProperty(pv[cv.path].metadata, cv.label,
+        {
+          enumerable: true,
+          get() {
+            createDependency.get(cv.path, cv.label, dependent)
+            return castMetadata(cv)[1]
+          }
+        }
+      )
+
+      return pv
+    }, {}))
+
+    return destinations
   }
 
   const selectSource = databaseSync.prepare(`SELECT * FROM sources WHERE path = ?`)
@@ -285,8 +347,6 @@ function createDatabase(dbPath = ".votive.db") {
       LEFT JOIN metadata m on d.path = m.destination
       WHERE stale = 1`
 
-    console.log({ metadatas })
-
     const grouped = metadatas.map((value, index, array) => {
       if (array.slice(0, index)
         .find(prior => prior.path === value.path)
@@ -298,6 +358,7 @@ function createDatabase(dbPath = ".votive.db") {
           dir: value.dir,
           syntax: value.syntax,
           stale: value.stale,
+          abstract: JSON.parse(value.abstract),
           metadata: Object.fromEntries(
             array.slice(index)
               .filter(following => following.path === value.path)
@@ -326,7 +387,7 @@ function createDatabase(dbPath = ".votive.db") {
     return grouped
   }
 
-  const createSetting = databaseSync.prepare(`INSERT INTO settings (destination, label, value, source) VALUES (?, ?, ?, ?) RETURNING *`)
+  const createSetting = databaseSync.prepare(`INSERT OR IGNORE INTO settings (destination, label, value, source) VALUES (?, ?, ?, ?) RETURNING *`)
   const selectSetting = databaseSync.prepare(`SELECT * FROM settings WHERE label = ? AND destination = ?`)
   const updateSetting = databaseSync.prepare(`UPDATE settings SET value = ? WHERE destination = ? AND label = ? RETURNING *`)
   const deleteSettings = databaseSync.prepare(`DELETE FROM settings WHERE source = ?`)
@@ -356,9 +417,14 @@ function createDatabase(dbPath = ".votive.db") {
   database.setSetting = (destinationFolder, key, value, source = "") => {
     const extant = selectSetting.get(key, destinationFolder)
 
-    if (!extant) return createSetting.get(destinationFolder, key, value, source)
-    if (extant.value === value) return
-    updateSetting.get(value, destinationFolder, key)
+    const type = typeof value
+    const safeValue = type === "object"
+      ? JSON.stringify(value)
+      : value
+
+    if (!extant) return createSetting.get(destinationFolder, key, safeValue, source)
+    if (extant.value === safeValue) return
+    updateSetting.get(safeValue, destinationFolder, key)
 
     destinationFolder === ""
       ? staleDescendents.all("%")
@@ -399,6 +465,29 @@ function createDatabase(dbPath = ".votive.db") {
     )
   }
 
+  const createURL = databaseSync.prepare(`INSERT OR IGNORE INTO urls (url, data) VALUES (?, ?) RETURNING *`)
+  const getURL = databaseSync.prepare(`SELECT data FROM urls WHERE url = ?`)
+
+  /**
+   * @param {string} url
+   * @param {string} data
+   * @param {string} destination
+   */
+  database.createURL = (url, data, destination) => {
+    const created = createURL.get(url, JSON.stringify(data))
+    const staled = staleDepen.get(destination)
+    // TODO: Better signal propagation here?
+  }
+
+  /**
+   * @param {string} url
+   */
+  database.getURL = (url) => {
+    const { data } = getURL.get(url) || {}
+    if(!data) return
+    return JSON.parse(data)
+  }
+
   return database
 }
 
@@ -414,7 +503,8 @@ CREATE TABLE IF NOT EXISTS dependencies (
   key INTEGER PRIMARY KEY,
   destination STRING NOT NULL,
   property STRING NOT NULL,
-  dependent STRING NOT NULL
+  dependent STRING NOT NULL,
+  UNIQUE(destination, property, dependent)
 );
 
 CREATE TABLE IF NOT EXISTS destinations (
@@ -440,7 +530,13 @@ CREATE TABLE IF NOT EXISTS settings (
   destination STRING,
   source STRING,
   label STRING,
-  value STRING
+  value STRING,
+  UNIQUE(destination, label, source)
+);
+
+CREATE TABLE IF NOT EXISTS urls (
+  url STRING PRIMARY KEY,
+  data STRING
 );
 
 `

package/lib/readAbstracts.js

@@ -38,7 +38,7 @@ function readAbstracts(abstracts, config, database, processors) {
       ) return recursiveProcess(rest, abstract, jobs)
 
       const processed = processor.read.abstract(abstract, database, config)
-      processed.jobs && processed.jobs.forEach(job => job.plugin = plugin.name)
+      processed.jobs && processed.jobs.forEach(job => job.syntax = processor.syntax)
       return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
     }
 

package/lib/readBuffers.js

@@ -0,0 +1,59 @@
+/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+/** @import {ReadSourceFileResult} from "./readSources.js"
+
+/**
+ * @typedef {object} ReadAbstractsResult
+ * @property {Jobs} jobs
+ * @property {ReadAbstractResult[]} abstracts
+ */
+
+/**
+ * @typedef {object} ReadAbstractResult
+ */
+
+/**
+ * @param {ReadSourceFileResult} abstracts
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ * @param {FlatProcessors} processors
+ * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
+ */
+function readBuffers(abstracts, config, database, processors) {
+
+  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
+
+    /**
+     * @param {FlatProcessors} processors
+     * @param {Abstract} abstract
+     * @param {Jobs} jobs
+     */
+    function recursiveProcess(processors, abstract, jobs = []) {
+      const [flatProcessor, ...rest] = processors
+      if (!flatProcessor) return { abstract, jobs }
+      const { processor, plugin } = flatProcessor
+      if (processor.syntax !== syntax
+        || !processor.read
+        || !processor.read.abstract
+      ) return recursiveProcess(rest, abstract, jobs)
+
+      const processed = processor.read.abstract(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.syntax = processor.syntax)
+      return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
+    }
+
+    return recursiveProcess(processors, unprocessedAbstract)
+  })
+
+  const processedAbstracts = []
+  const abstractsJobs = []
+
+  processed.forEach(({ abstract, jobs }) => {
+    processedAbstracts.push(abstract)
+    abstractsJobs.push(...jobs)
+  })
+
+  return { processedAbstracts, abstractsJobs }
+}
+
+export default readAbstracts

package/lib/readFilePaths.js

@@ -0,0 +1,59 @@
+/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
+/** @import {Database} from "./createDatabase.js" */
+/** @import {ReadSourceFileResult} from "./readSources.js"
+
+/**
+ * @typedef {object} ReadAbstractsResult
+ * @property {Jobs} jobs
+ * @property {ReadAbstractResult[]} abstracts
+ */
+
+/**
+ * @typedef {object} ReadAbstractResult
+ */
+
+/**
+ * @param {ReadSourceFileResult} abstracts
+ * @param {VotiveConfig} config
+ * @param {Database} database
+ * @param {FlatProcessors} processors
+ * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
+ */
+function readFilePaths(abstracts, config, database, processors) {
+
+  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
+
+    /**
+     * @param {FlatProcessors} processors
+     * @param {Abstract} abstract
+     * @param {Jobs} jobs
+     */
+    function recursiveProcess(processors, abstract, jobs = []) {
+      const [flatProcessor, ...rest] = processors
+      if (!flatProcessor) return { abstract, jobs }
+      const { processor, plugin } = flatProcessor
+      if (processor.syntax !== syntax
+        || !processor.read
+        || !processor.read.abstract
+      ) return recursiveProcess(rest, abstract, jobs)
+
+      const processed = processor.read.abstract(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.syntax = processor.syntax)
+      return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
+    }
+
+    return recursiveProcess(processors, unprocessedAbstract)
+  })
+
+  const processedAbstracts = []
+  const abstractsJobs = []
+
+  processed.forEach(({ abstract, jobs }) => {
+    processedAbstracts.push(abstract)
+    abstractsJobs.push(...jobs)
+  })
+
+  return { processedAbstracts, abstractsJobs }
+}
+
+export default readFilePaths

package/lib/readFolders.js

@@ -1,8 +1,9 @@
+import path from "node:path"
+
 /** @import {VotiveConfig, FlatProcessors} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
 /** @import {Dirent} from "node:fs" */
 
-import path from "node:path"
 
 /**
  * @param {Dirent[]} folders
@@ -17,11 +18,13 @@ function readFolders(folders = [], config, database, processors) {
   const processed = folderProcessors.flatMap(({ processor, plugin }) => {
 
     const jobs = folders.flatMap(folder => {
-      const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
+      let folderPath = path.join(folder.parentPath, folder.name)
+      if(folderPath) folderPath += path.sep
+
 
       /** @ts-ignore `.read` is throwing a warning, but it's guarded above */
       const { destinations, jobs } = processor.read.folder(folderPath, database, config)
-      jobs.forEach(job => job.plugin = plugin.name)
+      jobs.forEach(job => job.syntax = processor.syntax)
       if (destinations) {
         destinations.forEach(destination => {
           database.createOrUpdateDestination(destination)
@@ -30,7 +33,14 @@ function readFolders(folders = [], config, database, processors) {
       }
     })
 
-    const rootFolder = processor.read.folder("", database, config)
+    
+    const rootInfo = path.parse(config.sourceFolder)
+    const rootPath = rootInfo.name
+      ? path.format(rootInfo) + path.sep
+      : ""
+
+    // TODO: Add isRoot to type definition
+    const rootFolder = processor.read.folder(rootPath, database, config, true)
 
     if (rootFolder.destinations) {
       rootFolder.destinations.forEach(destination => {
@@ -38,7 +48,7 @@ function readFolders(folders = [], config, database, processors) {
       })
     }
 
-    rootFolder.jobs.forEach(job => job.plugin = plugin.name)
+    rootFolder.jobs.forEach(job => job.syntax = processor.syntax)
 
     if(rootFolder.jobs) jobs.push(... rootFolder.jobs)
     return jobs

package/lib/readSources.js

@@ -29,7 +29,7 @@ async function readSources(config, database, processors) {
   const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
   if (!files) return { folders, sources: [] }
   const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config))
-  const sources = readingSourceFiles && await Promise.all(readingSourceFiles)
+  const sources = readingSourceFiles && (await Promise.all(readingSourceFiles)).filter(a => a)
   return {
     folders,
     sources
@@ -43,6 +43,8 @@ async function readSources(config, database, processors) {
 function fileFilter(config, database) {
   /** @param {Dirent} dirent */
   return (dirent) => {
+    const isDestinationFolder = !path.relative(config.destinationFolder, path.join(dirent.parentPath, dirent.name))
+
     if (dirent.parentPath === config.destinationFolder) {
       return false
     } else if (dirent.parentPath.startsWith(config.destinationFolder + path.sep)) {
@@ -51,6 +53,10 @@ function fileFilter(config, database) {
       return false // Ignore hidden files
     } else if (dirent.parentPath.includes(path.sep + ".")) {
       return false // Ignore hidden folders
+    } else if(dirent.parentPath.match(/^\.\w/)) {
+      return false // Ignore hidden folders
+    } if(isDestinationFolder) {
+      return false
     }
     return true
   }
@@ -92,42 +98,85 @@ function readSourceFile(processors, database, config) {
      */
     async function process(plugin, processor, config) {
       const { read, filter, syntax } = processor
-      if (filter.extensions.includes(fileInfo.ext) && read && read.text) {
+      if (filter && filter.extensions.includes(fileInfo.ext) && read && (read.text || read.path)) {
 
         // Check modified time
         const stat = await fs.stat(filePath)
         const source = database.getSource(filePath)
 
 
-        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
+        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return null
 
         database.deleteSettings(filePath)
 
         // Get destination route
-        const destinationPath = route(filePath, plugin)
+        const destinationPath = route(filePath, plugin, config)
 
         // Set URL if exists
-        const buffer = await fs.readFile(path.format(fileInfo))
-        const data = decodeBuffer(buffer)
-        const { jobs, abstract, metadata } = read.text(data, filePath, database, config)
-        jobs && jobs.forEach(job => job.plugin = plugin.name)
+        const allJobs = []
+        const destination = {}
+
+        // TODO: Rename "path" to "asset"
+        if (read.path) {
+          const {metadata, abstract, jobs} = await read.path(filePath, database, config)
+
+          database.createOrUpdateDestination({
+            metadata,
+            abstract,
+            path: destinationPath,
+            syntax
+          })
+
+          if(Array.isArray(jobs)) {
+            allJobs.push(...jobs)
+          }
+        }
 
-        const timeStamp = stat.mtimeMs.toFixed()
+        if (read.text) {
+          const buffer = await fs.readFile(path.format(fileInfo))
+          const data = decodeBuffer(buffer)
 
-        if(source) {
-          database.updateSource(filePath, Number(timeStamp))
-        } else {
-          database.createSource(filePath, destinationPath, Number(timeStamp))
+          const { jobs, abstract, metadata } = read.text(data, filePath, destinationPath, database, config)
+
+          if(Array.isArray(jobs)) {
+            allJobs.push(...jobs)
+          }
+
+          database.createOrUpdateDestination({
+            abstract,
+            path: destinationPath,
+            metadata,
+            syntax
+          })
+
+          // TODO: DRY up
+          allJobs && allJobs.forEach(job => job.syntax = processor.syntax)
+          updateSource()
+          return { abstract, destinationPath, syntax, jobs: allJobs }
         }
 
-        database.createOrUpdateDestination({
-          metadata,
-          path: destinationPath,
-          abstract: abstract,
-          syntax: syntax
-        })
+        // TODO: DRY up
+        allJobs && allJobs.forEach(job => job.syntax = processor.syntax)
+        updateSource()
+
+        return {
+          jobs: allJobs,
+          abstract: null,
+          destinationPath: null,
+          metadata: null,
+          syntax
+        }
+
+        function updateSource() {
+          const timeStamp = stat.mtimeMs.toFixed()
+
+          if (source) {
+            database.updateSource(filePath, Number(timeStamp))
+          } else {
+            database.createSource(filePath, destinationPath, Number(timeStamp))
+          }
+        }
 
-        return { abstract, destinationPath, syntax, jobs }
       }
     }
 
@@ -138,10 +187,22 @@ function readSourceFile(processors, database, config) {
 /**
  * @param {string} filePath
  * @param {VotivePlugin} plugin
+ * @param {VotiveConfig} config
  */
-function route(filePath, plugin) {
-  if (!plugin.router) return ""
-  return plugin.router(filePath) || ""
+function route(filePath, plugin, config) {
+  const { dir, ...parsedPath } = path.parse(filePath)
+  const rooty = !path.relative(config.sourceFolder, dir)
+  const pathInfo = {
+    inRootDir: rooty,
+    dir: dir.split(path.sep),
+    ...parsedPath
+  }
+  if (!plugin.router) return "0"
+  const routedPath = plugin.router(pathInfo)
+  if (routedPath && typeof routedPath.dir !== "string") {
+    routedPath.dir = path.join(...routedPath.dir)
+  }
+  return routedPath ? path.format(routedPath) : "0"
 }
 
 export default readSources

package/lib/runJobs.js

@@ -1,8 +1,5 @@
-import workerpool from "workerpool"
-
 /** @import {Job, Jobs, Database, VotiveConfig} from "./bundle.js" */
 
-const pool = workerpool.pool()
 
 /**
  * @param {Jobs} jobs
@@ -10,14 +7,37 @@ const pool = workerpool.pool()
  * @param {Database} database
  */
 async function runJobs(jobs, config, database) {
-  const running = jobs.map(async job => {
+  const processors = config.plugins.flatMap(plugin => plugin.processors.flatMap(processor => processor.read?.url && ({ fetcher: processor.read.url, syntax: processor.syntax }))).filter(a => a)
+  const running = jobs.flatMap(async job => {
     if (!job) return
-    const plugin = config.plugins.find(plugin => plugin.name === job.plugin)
-    return pool.exec(plugin.runners[job.runner], [job.data])
+    const cachedURL = database.getURL(job.data)
+    if (cachedURL) return
+    const processing = processors.map(async processor => {
+      if (processor.syntax === job.syntax) {
+        try {
+          const response = await fetch(job.data)
+          // TODO: Change name of "job" to "url"
+          // TODO: Change name of "runner" to "format"
+          // TODO: Probably get rid of syntax filter
+          if (response.status >= 200 && response.status < 300) {
+            const data = await response[job.runner]()
+            const processed = processor.fetcher(data)
+            database.createURL(job.data, processed, job.destination)
+            return
+          } else {
+            console.warn(`Error fetching URL: ${job.data}`)
+          }
+        } catch (e) {
+          return
+        }
+      }
+    })
+
+    return Promise.allSettled(processing)
   })
 
-  const ran = await Promise.allSettled(running)
-  pool.terminate()
+  const settled = await Promise.allSettled(running)
+  return settled
 }
 
 export default runJobs

package/lib/sqlite.js

@@ -0,0 +1,76 @@
+// TODO: and, or, not, between, not in, order
+
+const operators = {
+  glob: "GLOB",
+  like: "LIKE",
+  in: "IN",
+  gt: ">",
+  lt: "<",
+  gte: ">=",
+  lte: "<=",
+  equal: "=",
+  notEqual: "!-"
+}
+
+const select = `SELECT * FROM destinations d`
+const join = `LEFT JOIN metadata m ON d.path = m.destination`
+
+/**
+ * @typedef {Condition[]} Filter
+ */
+
+/**
+ * @typedef {object} Condition
+ * @property {string} property
+ * @property {keyof operators} operator
+ * @property {string | number | array} value 
+ */
+
+/**
+ * @typedef {object} Query
+ * @property {number} [query.limit]
+ * @property {number} [query.offset]
+ * @property {string} [query.orderBy]
+ * @property {Filter} [query.filter]
+ */
+
+/**
+ * @param {Query} query
+ */
+export default function createStatement(query) {
+  const segments = [select, join]
+  if(query.limit) segments.push(`LIMIT ${query.limit}`)
+  if(query.offset) segments.push(`OFFSET ${query.offset}`)
+  if(query.orderBy) segments.push(`ORDER BY ${query.orderBy}`)
+  if(query.filter) {
+    const conditions = query.filter.map(condition => {
+      const { property, operator, value } = condition
+      
+      if(["dir", "abstract", "path"].includes(property)) {
+        
+      return `d.${property} ${operators[operator]} ${formatValue(value)}`
+      }
+      return `m.label = '${property}' AND m.value ${operators[operator]} ${formatValue(value)}`
+    })
+
+    segments.push(`WHERE ${conditions.join(" AND ")}`)
+  }
+
+  return segments.join(`\n`)
+}
+
+function formatProperty(property) {
+  if(property === "abstract") return `d.${property}`
+  else return `m.${property}`
+}
+
+/** @param {any} value */
+function formatValue(value) {
+  if(Array.isArray(value)) {
+    return `(${value.map(x => `'${x}'`).join(', ')})`
+  }
+
+  if(typeof value === "number") return value
+
+  return `'${value}'`
+}

package/lib/utils/index.js

@@ -1,5 +1,6 @@
 import { styleText } from "node:util"
 import { statSync } from "node:fs"
+import path from "path"
 
 /** @param {string} label */
 export function stopwatch(label) {
@@ -18,8 +19,11 @@ export function stopwatch(label) {
  * @param {string} urlPath
  */
 export function splitURL(urlPath) {
-  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
-  return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
+  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").filter(a => a).reverse()
+  const segments = urlDirSegmentsReversed.reverse()
+  segments.push('')
+  const folder = segments.join(path.sep)
+  return folder
 }
 
 /** @param {string} dbPath */

package/lib/writeDestinations.js

@@ -1,5 +1,6 @@
 import { mkdir, writeFile } from "node:fs/promises"
 import path from "node:path"
+import { checkFile } from "./utils/index.js"
 
 /** @import {Abstract, Abstracts, VotiveConfig} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
@@ -20,15 +21,29 @@ async function writeDestinations(config, database) {
   if (!destinations) return
 
   const writing = destinations.flatMap(destination => {
+    if(!destination.path) database.freshenDependency(destination.path)
     const destinationPath = path.join(config.destinationFolder, String(destination.path))
     const { dir } = path.parse(destinationPath)
-    return writeProcessors.map(processor => {
+    return writeProcessors.map(async processor => {
       if (processor.syntax === destination.syntax) {
-        const { data, encoding = 'utf-8' } = processor.write(destination, database, config)
+        const writeInfo = await processor.write(destination, database, config)
+        if(!writeInfo) return
+        const { data, buffer, encoding = 'utf-8' } = writeInfo
 
         async function write() {
-          await mkdir(dir, { recursive: true })
-          await writeFile(destinationPath, data, encoding)
+          const destinationExists = checkFile(dir)
+
+          // TODO: Avoid collisions
+          if(!destinationExists) {
+            await mkdir(dir, { recursive: true })
+          }
+
+          if(buffer) {
+            await writeFile(destinationPath, buffer)
+          } else if(data) {
+            await writeFile(destinationPath, data, encoding)
+          }
+
           database.freshenDependency(destination.path)
         }
         

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {

package/tests/index.js

@@ -56,7 +56,7 @@ test("internals", async (t) => {
 
 
   /** @type {ReadText} */
-  function exampleTextReader(text, filePath, database, config) {
+  function exampleTextReader(text, filePath, destinationPath, database, config) {
     const matches = text.match(/\b\w+\b/)
     const title = matches ? matches[0] : "Untitled"
 
@@ -86,7 +86,7 @@ test("internals", async (t) => {
       jobs: [createExampleJob()],
       destinations: [
         {
-          path: "/index.html",
+          path: "index.html",
           abstract: { content: "" },
           metadata: {
             title: "home"
@@ -116,9 +116,8 @@ test("internals", async (t) => {
   }
 
   /** @param {string} sourcePath */
-  function router(sourcePath) {
-    const { base, ...parsed }= path.parse(sourcePath)
-    return path.format({ ...parsed, ext: ".html" })
+  function router({ base, ...parsed }) {
+    return { ...parsed, ext: ".html" }
   }
 
   /** @type {VotivePlugin} */
@@ -207,7 +206,9 @@ test("internals", async (t) => {
     })
 
     database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
-    const destination = database.getDestinationDependently("abc.html", ["a"], "abc/def.html")
+    const destination = database.getDestinationDependently("abc.html", "abc/def.html")
+
+    console.info(`'abc/def.html' requests the property 'a' from 'abc.html', creating a dependency: ${destination.metadata.a}`)
 
     const dependencies = database.getDependencies()
 
@@ -293,7 +294,9 @@ test("internals", async (t) => {
     runJobs(jobs, config, database)
 
     const destinations = database.getAllDestinations()
-    database.getDestinationDependently("markdown/prunee.html", ["title"], "abc/def.html")
+    const prunee = database.getDestinationDependently("markdown/prunee.html", "abc/def.html")
+
+    console.info(`'abc/def.html' requests the property 'title' from 'markdown/prunee.html', creating a dependency: ${prunee.metadata.title}`)
     const newDependencies = database.getDependencies()
 
     fs.rmSync("markdown/prunee.md")
@@ -310,6 +313,20 @@ test("internals", async (t) => {
       assert(!prunedMetadata)
     })
 
+    const finalDestinations = database.getDestinations({
+      filter: [
+        {
+          property: "a",
+          operator: "gt",
+          value: 3
+        }
+      ]
+    }, "markdown/prunee.html")
+
+    const finalDependencies = database.getDependencies()
+
+    // TODO: Write a test for get destinations
+
     await database.saveDB()
 
     temp.remove()