votive 0.2.0 → 0.3.0

package/lib/readAbstracts.js

@@ -1,6 +1,6 @@
 /** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
-/** @import {ReadSourceFilesResult} from "./readSources.js"
+/** @import {ReadSourceFileResult} from "./readSources.js"
 
 /**
  * @typedef {object} ReadAbstractsResult
@@ -13,17 +13,15 @@
  */
 
 /**
- * @param {ReadSourceFilesResult} files
+ * @param {ReadSourceFileResult} abstracts
  * @param {VotiveConfig} config
  * @param {Database} database
  * @param {FlatProcessors} processors
  * @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
  */
-function readAbstracts(files, config, database, processors) {
+function readAbstracts(abstracts, config, database, processors) {
 
-  const processed = files.flatMap(file => {
-
-    const { abstract: unprocessedAbstract } = file
+  const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
 
     /**
      * @param {FlatProcessors} processors
@@ -34,14 +32,13 @@ function readAbstracts(files, config, database, processors) {
       const [flatProcessor, ...rest] = processors
       if (!flatProcessor) return { abstract, jobs }
       const { processor, plugin } = flatProcessor
-      if (!processor.extensions.includes(file.syntax)
-        || !processor.transformFile
+      if (processor.syntax !== syntax
+        || !processor.read
+        || !processor.read.abstract
       ) return recursiveProcess(rest, abstract, jobs)
 
-      const settings = database.setting.getByFolder(file.dir)
-      
-      const processed = processor.transformFile(abstract, database, config)
-      processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
+      const processed = processor.read.abstract(abstract, database, config)
+      processed.jobs && processed.jobs.forEach(job => job.plugin = plugin.name)
       return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
     }
 

package/lib/readFolders.js

@@ -1,9 +1,8 @@
-import path from "node:path"
-
 /** @import {VotiveConfig, FlatProcessors} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
 /** @import {Dirent} from "node:fs" */
 
+import path from "node:path"
 
 /**
  * @param {Dirent[]} folders
@@ -13,35 +12,33 @@ import path from "node:path"
  */
 function readFolders(folders = [], config, database, processors) {
 
-  const folderProcessors = processors.filter(({ processor }) => processor.readFolder)
+  const folderProcessors = processors.filter(({ processor }) => processor.read && processor.read.folder)
 
   const processed = folderProcessors.flatMap(({ processor, plugin }) => {
 
     const jobs = folders.flatMap(folder => {
-      let folderPath = path.join(folder.parentPath, folder.name)
-      if(folderPath) folderPath += path.sep
-
-      const { targets, jobs } = processor.readFolder(folderPath, database, config)
-      jobs.forEach(job => job.syntax = processor.syntax)
-      if (targets) {
-        targets.forEach(target => {
-          database.target.create(target)
+      const folderPath = path.relative(config.sourceFolder, path.join(folder.parentPath, folder.name))
+
+      /** @ts-ignore `.read` is throwing a warning, but it's guarded above */
+      const { destinations, jobs } = processor.read.folder(folderPath, database, config)
+      jobs.forEach(job => job.plugin = plugin.name)
+      if (destinations) {
+        destinations.forEach(destination => {
+          database.createOrUpdateDestination(destination)
         })
         return jobs
       }
     })
 
-    
-    const rootPath = path.relative(config.sourceFolder, "")
-    const rootFolder = processor.readFolder(rootPath, database, config, true)
+    const rootFolder = processor.read.folder("", database, config)
 
-    if (rootFolder.targets) {
-      rootFolder.targets.forEach(target => {
-        database.target.create(target)
+    if (rootFolder.destinations) {
+      rootFolder.destinations.forEach(destination => {
+        database.createOrUpdateDestination(destination)
       })
     }
 
-    rootFolder.jobs.forEach(job => job.syntax = processor.syntax)
+    rootFolder.jobs.forEach(job => job.plugin = plugin.name)
 
     if(rootFolder.jobs) jobs.push(... rootFolder.jobs)
     return jobs

package/lib/readSources.js

@@ -1,16 +1,16 @@
 import { decodeBuffer } from "encoding-sniffer"
 import fs from "node:fs/promises"
 import path from "node:path"
-import pLimit from "p-limit"
+import { visit } from "unist-util-visit"
 
-/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax, Router} from "./bundle.js" */
+/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax} from "./bundle.js" */
 /** @import {Dirent} from "node:fs" */
 /** @import {Database} from "./createDatabase.js" */
 
 /**
  * @typedef {object} ReadSourcesResult
  * @property {Dirent[]} folders
- * @property {ReadSourceFileResult[]} sources
+ * @property {ReadSourceFilePlugin[]} sources
  */
 
 /**
@@ -25,64 +25,24 @@ async function readSources(config, database, processors) {
     recursive: true
   })
 
-  const filteredDirents = (dirents || []).filter(fileFilter(config))
-
+  const filteredDirents = (dirents || []).filter(fileFilter(config, database))
   const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
-  const loadingFiles = files.map(a => path.normalize(path.format({ name: a.name, dir: a.parentPath })))
   if (!files) return { folders, sources: [] }
-  const limit = pLimit(5)
   const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config))
-
-  const reading = [
-    (await Promise.all(readingSourceFiles)).filter(a => a),
-    pruneDeletions(config, database, filteredDirents)
-  ]
-
-  const [sources, deletedSources] = await Promise.all(reading)
-
-  // const sources = readingSourceFiles && (await Promise.all(readingSourceFiles)).filter(a => a)
-
-  console.log({ sources })
-
-  // const deletedSources = await pruneDeletions(config, database, filteredDirents)
-
-  console.log({ deletedSources })
-
+  const sources = readingSourceFiles && await Promise.all(readingSourceFiles)
   return {
     folders,
-    sources: [...sources, ...deletedSources]
+    sources
   }
 }
 
 /**
  * @param {VotiveConfig} config
  * @param {Database} database
- * @param {Dirent[]} dirents
- */
-async function pruneDeletions(config, database, dirents) {
-  const sourceFilePaths = new Set(dirents.map(d => d.isFile() && path.join(d.parentPath, d.name)))
-  const sourceRecords = database.source.getAll()
-  const sourceRecordPaths = new Set(sourceRecords.map(r => r.path))
-
-  const deletions = sourceRecordPaths.difference(sourceFilePaths)
-
-  let deletedSources = []
-
-  deletions.forEach(deletion => {
-    deletedSources.push(database.source.delete(deletion))
-  })
-
-  return deletedSources.filter(a => a)
-}
-
-/**
- * @param {VotiveConfig} config
  */
-function fileFilter(config) {
+function fileFilter(config, database) {
   /** @param {Dirent} dirent */
   return (dirent) => {
-    const isDestinationFolder = !path.relative(config.destinationFolder, path.join(dirent.parentPath, dirent.name))
-
     if (dirent.parentPath === config.destinationFolder) {
       return false
     } else if (dirent.parentPath.startsWith(config.destinationFolder + path.sep)) {
@@ -91,28 +51,21 @@ function fileFilter(config) {
       return false // Ignore hidden files
     } else if (dirent.parentPath.includes(path.sep + ".")) {
       return false // Ignore hidden folders
-    } else if (dirent.parentPath.match(/^\.\w/)) {
-      return false // Ignore hidden folders
-    } if (isDestinationFolder) {
-      return false
     }
     return true
   }
 }
 
 /**
- * @typedef {ReadSourceFileResult[]} ReadSourceFilesResult
+ * @typedef {ReadSourceFilePlugin[]} ReadSourceFileResult
  */
 
 /**
- * @typedef {object} ReadSourceFileResult
+ * @typedef {object} ReadSourceFilePlugin
  * @property {Abstract} abstract
  * @property {ProcessorSyntax} syntax
- * @property {object} [metadata]
- * @property {string} [targetFilePath]
- * @property {string} [dir]
  * @property {Jobs} jobs
- * @property {string} sourceFilePath
+ * @property {string} destinationPath
  */
 
 /**
@@ -123,12 +76,12 @@ function fileFilter(config) {
 function readSourceFile(processors, database, config) {
   /**
    * @param {import("node:fs").Dirent} dirent
-   * @returns {Promise<ReadSourceFileResult>[]}
+   * @returns {Promise<ReadSourceFilePlugin>[]}
    */
   return (dirent) => {
     const { name, parentPath } = dirent
-    const sourceFilePath = path.join(parentPath, name)
-    const sourceFileInfo = path.parse(sourceFilePath)
+    const filePath = path.join(parentPath, name)
+    const fileInfo = path.parse(filePath)
 
     const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config))
 
@@ -138,98 +91,43 @@ function readSourceFile(processors, database, config) {
      * @param {VotiveConfig} config
      */
     async function process(plugin, processor, config) {
-      const { readFile: read, extensions: filter, format } = processor
-      if (filter.includes(sourceFileInfo.ext) && read) {
+      const { read, filter, syntax } = processor
+      if (filter.extensions.includes(fileInfo.ext) && read && read.text) {
 
         // Check modified time
-        const stat = await fs.stat(sourceFilePath)
-        const source = database.source.get(sourceFilePath)
-        const diff = source && source.lastModified - Number(Math.floor(stat.mtimeMs))
-
-        if (source && diff > -1) {
-          return null
-        }
-
-        database.setting.deleteBySource(sourceFilePath)
+        const stat = await fs.stat(filePath)
+        const source = database.getSource(filePath)
 
-        const targetFilePath = route(sourceFilePath, plugin, config)
-        const targetFileExtension = path.extname(targetFilePath)
 
-        // Set URL if exists
-        const allJobs = []
-
-        if (format === "buffer") {
-
-          // FIXME update this type
-          const data = read(sourceFilePath, database, config)
-          const { metadata, abstract, jobs } = data
-
-          const target = database.target.create({
-            metadata,
-            abstract,
-            path: targetFilePath,
-          })
-
-
-          if (Array.isArray(jobs)) {
-            allJobs.push(...jobs)
-          }
-        }
-
-        if (format === "text") {
-          const stats = await fs.stat(sourceFilePath)
-          const data = await fs.readFile(sourceFilePath, { encoding: "utf-8" })
-
-          const { jobs, abstract, metadata } = read(data, sourceFilePath, targetFilePath, database, config)
+        if (source && source.lastModified === Number(stat.mtimeMs.toFixed())) return { jobs: [] }
 
+        database.deleteSettings(filePath)
 
-          if (Array.isArray(jobs)) {
-            allJobs.push(...jobs)
-          }
+        // Get destination route
+        const destinationPath = route(filePath, plugin)
 
-          const target = database.target.create({
-            abstract,
-            path: targetFilePath,
-            metadata,
-          })
-
-          // FIXME this won't work with the refactor
-          allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
-          updateSource()
-          return {
-            abstract,
-            metadata,
-            syntax: targetFileExtension,
-            targetFilePath: target.path,
-            dir: target.dir,
-            sourceFilePath,
-            jobs: allJobs
-          }
-        }
+        // Set URL if exists
+        const buffer = await fs.readFile(path.format(fileInfo))
+        const data = decodeBuffer(buffer)
+        const { jobs, abstract, metadata } = read.text(data, filePath, database, config)
+        jobs && jobs.forEach(job => job.plugin = plugin.name)
 
-        // FIXME Job syntax
-        allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
-        updateSource()
+        const timeStamp = stat.mtimeMs.toFixed()
 
-        return {
-          jobs: allJobs,
-          abstract: null,
-          targetFilePath: null,
-          sourceFilePath: null,
-          metadata: null,
-          syntax: null
+        if(source) {
+          database.updateSource(filePath, Number(timeStamp))
+        } else {
+          database.createSource(filePath, destinationPath, Number(timeStamp))
         }
 
-        function updateSource() {
-          const timeStamp = stat.mtimeMs.toFixed()
-
-          if (source) {
-            database.source.updateTimestamp(sourceFilePath, Number(timeStamp))
-          } else {
-            database.source.create(sourceFilePath, targetFilePath, Number(timeStamp))
-          }
-        }
+        database.createOrUpdateDestination({
+          metadata,
+          path: destinationPath,
+          abstract: abstract,
+          syntax: syntax
+        })
 
+        return { abstract, destinationPath, syntax, jobs }
       }
     }
 
@@ -240,43 +138,10 @@ function readSourceFile(processors, database, config) {
 /**
  * @param {string} filePath
  * @param {VotivePlugin} plugin
- * @param {VotiveConfig} config
  */
-function route(filePath, plugin, config) {
-  const { dir, ...parsedPath } = path.parse(filePath)
-  const rooty = !path.relative(config.sourceFolder, dir)
-  const segments = ["", ...dir.split(path.sep).filter(a => a)]
-  const pathInfo = {
-    inRootDir: rooty,
-    dir: segments,
-    ...parsedPath
-  }
-
-  if (!plugin.router) return "0"
-
-  const routedPath = plugin.router(pathInfo)
-
-  if (!routedPath) return "0"
-
-  if (routedPath.hasOwnProperty("dir") && Array.isArray(routedPath.dir)) {
-    return path.normalize(path.format({
-      dir: path.join(...routedPath.dir),
-      root: routedPath.root || "",
-      base: routedPath.base,
-      name: routedPath.name,
-      ext: routedPath.ext
-    }))
-  }
-
-  const routedInfo = {
-    dir: routedPath.dir || "",
-    root: routedPath.root || "",
-    base: routedPath.base,
-    name: routedPath.name,
-    ext: routedPath.ext
-  }
-
-  return path.normalize(path.format(routedInfo))
+function route(filePath, plugin) {
+  if (!plugin.router) return ""
+  return plugin.router(filePath) || ""
 }
 
 export default readSources

package/lib/runJobs.js

@@ -1,5 +1,8 @@
+import workerpool from "workerpool"
+
 /** @import {Job, Jobs, Database, VotiveConfig} from "./bundle.js" */
 
+const pool = workerpool.pool()
 
 /**
  * @param {Jobs} jobs
@@ -7,35 +10,14 @@
  * @param {Database} database
  */
 async function runJobs(jobs, config, database) {
-  const processors = config.plugins.flatMap(plugin => plugin.processors.flatMap(processor => processor.read?.url && ({ fetcher: processor.read.url, syntax: processor.syntax }))).filter(a => a)
-  const running = jobs.flatMap(async job => {
+  const running = jobs.map(async job => {
     if (!job) return
-    const cachedURL = database.getURL(job.data)
-    if (cachedURL) return
-    const processing = processors.map(async processor => {
-      if (processor.syntax === job.syntax) {
-        try {
-          const response = await fetch(job.data)
-          if (response.status >= 200 && response.status < 300) {
-            const data = await response[job.runner]()
-            const processed = processor.fetcher(data)
-            database.createURL(job.data, processed, job.destination)
-            return
-          } else {
-            console.warn(`Error fetching URL: ${job.data}`)
-          }
-        } catch (e) {
-          console.error(e)
-          return
-        }
-      }
-    })
-
-    return Promise.allSettled(processing)
+    const plugin = config.plugins.find(plugin => plugin.name === job.plugin)
+    return pool.exec(plugin.runners[job.runner], [job.data])
   })
 
-  const settled = await Promise.allSettled(running)
-  return settled
+  const ran = await Promise.allSettled(running)
+  pool.terminate()
 }
 
 export default runJobs

package/lib/utils/index.js

@@ -1,19 +1,14 @@
 import { styleText } from "node:util"
 import { statSync } from "node:fs"
-import path from "path"
 
-/**
- * @param {string} label
- * @param {string} message
- * @param {boolean} verbose
- */
-export function stopwatch(label, message, verbose) {
+/** @param {string} label */
+export function stopwatch(label) {
   const start = performance.now()
 
   function stop() {
     const end = performance.now()
     const duration = end - start
-    if(verbose) console.info(`${styleText("dim", label + ":")} ${styleText("magenta", message)} ${styleText("magenta", duration.toFixed(2) + "ms")}`)
+    console.info(`${label}: ${styleText("red", duration.toFixed(2) + "ms")}`)
   }
 
   return stop
@@ -23,11 +18,8 @@ export function stopwatch(label, message, verbose) {
  * @param {string} urlPath
  */
 export function splitURL(urlPath) {
-  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").filter(a => a).reverse()
-  const segments = urlDirSegmentsReversed.reverse()
-  segments.push('')
-  const folder = path.relative("", segments.join(path.sep))
-  return folder
+  const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").reverse()
+  return [urlDirSegmentsReversed.reverse().join("/") || "/", urlFileName]
 }
 
 /** @param {string} dbPath */

package/lib/writeDestinations.js

@@ -1,6 +1,5 @@
 import { mkdir, writeFile } from "node:fs/promises"
 import path from "node:path"
-import { checkFile } from "./utils/index.js"
 
 /** @import {Abstract, Abstracts, VotiveConfig} from "./bundle.js" */
 /** @import {Database} from "./createDatabase.js" */
@@ -12,46 +11,27 @@ import { checkFile } from "./utils/index.js"
  */
 async function writeDestinations(config, database) {
   const writeProcessors = config && config.plugins && config.plugins.flatMap(plugin => (
-    plugin.processors && plugin.processors.map(processor => processor.writeFile && processor).filter(a => a)
+    plugin.processors && plugin.processors.map(({ write, syntax }) => write && { write, syntax }).filter(a => a)
   )).filter(a => a)
 
   if (!writeProcessors || !writeProcessors.length) throw "No write processor provided"
 
-  const targets = database.target.getStale()
+  const destinations = database.getStaleDestinations()
+  if (!destinations) return
 
-  if (!targets) return
-
-  const writing = targets.filter(({ path }) => path !== "0").flatMap(destination => {
-
-    /* FIXME the following line doesn't make sense */
-    if (!destination.path) database.target.markFresh(destination.path)
-    if (!Object.keys(destination.abstract).length) return
+  const writing = destinations.flatMap(destination => {
     const destinationPath = path.join(config.destinationFolder, String(destination.path))
     const { dir } = path.parse(destinationPath)
-    return writeProcessors.map(async processor => {
-      if (processor.extensions.includes(destination.syntax)) {
-        const writeInfo = await processor.writeFile(destination, database, config)
-        if (!writeInfo) return
-        const { data, encoding = 'utf-8' } = writeInfo
+    return writeProcessors.map(processor => {
+      if (processor.syntax === destination.syntax) {
+        const { data, encoding = 'utf-8' } = processor.write(destination, database, config)
 
         async function write() {
-          const destinationExists = checkFile(dir)
-
-          if (!destinationExists) {
-            await mkdir(dir, { recursive: true })
-          }
-
-          if (data) {
-            if (processor.format === "text") {
-              await writeFile(destinationPath, data, "utf-8")
-              database.target.markFresh(destination.path)
-            } else {
-              await writeFile(destinationPath, data)
-              database.target.markFresh(destination.path)
-            }
-          }
+          await mkdir(dir, { recursive: true })
+          await writeFile(destinationPath, data, encoding)
+          database.freshenDependency(destination.path)
         }
-
+        
         return write()
       }
     })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {
@@ -24,7 +24,6 @@
   },
   "dependencies": {
     "encoding-sniffer": "^0.2.1",
-    "p-limit": "^7.3.0",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "workerpool": "^10.0.1"