votive 0.0.8 → 0.0.9

package/README.md

@@ -7,12 +7,11 @@
 
 ## Roadmap
 
-- [ ] ReadPaths
 - [ ] Flesh out job runners
-- [x] Destination query filters
-- [x] Setters for metadata
 - [ ] Rename jobs and paths
+- [ ] Better dependency tracking
+- [ ] Better query filters
 
-## Helpers
+## Project: Jobs
 
-- Read
+Jobs was originally a generic concept, but after working it's clear that there are two main categories of jobs: async writes and data fetching. We can probably handle async writes with inbuilt logic. So, instead of a "job", we should have "read uri"? That way we can cache all the uris.

package/lib/bundle.js

@@ -5,9 +5,10 @@ import readSources from "./readSources.js"
 import runJobs from "./runJobs.js"
 import writeDestinations from "./writeDestinations.js"
 import { default as createDatabase } from "./createDatabase.js"
+import { stopwatch } from "./utils/index.js"
+import { styleText } from "node:util"
 
 /** @import {Database} from "./createDatabase.js" */
-/** @import {ParsedPath} from "node:path" */
 
 /**
  * @typedef {object} VotivePlugin
@@ -18,17 +19,40 @@ import { default as createDatabase } from "./createDatabase.js"
  */
 
 /**
- * @typedef {object} VotiveProcessor
- * @property {ProcessorFilter} [filter]
- * @property {ProcessorSyntax} syntax
- * @property {ProcessorRead} [read]
- * @property {ProcessorWrite} [write]
+ * @typedef {object} VotiveProcessorCommon
+ * @property {string[]} extensions
+ * @property {ReadResource} [readResource]
+ * @property {ProcessorWrite} [writeFile]
+ * @property {ReadFolder} [readFolder]
+ * @property {ReadAbstract} [transformFile]
+ */
+
+/**
+ * @typedef {VotiveProcessorCommon & {
+ *  format: "buffer",
+ *  readFile: ReadPath | undefined
+ * }} VotiveProcessorBuffer
+ */
+
+/**
+ * @typedef {VotiveProcessorCommon & {
+ *  format: "text",
+ *  readFile: ReadText | undefined
+ * }} VotiveProcessorText
+ */
+
+/**
+ * @typedef {VotiveProcessorBuffer | VotiveProcessorText} VotiveProcessor
+ */
+
+/**
+ * @typedef {object} ReadResource
  */
 
 /**
  * Filter for files that the processor will read from.
  * @typedef {object} ProcessorFilter
- * @property {string[]} extensions
+ * @property {string} extensions
  */
 
 /**
@@ -39,7 +63,6 @@ import { default as createDatabase } from "./createDatabase.js"
 /**
  * @typedef {object} ProcessorRead
  * @property {ReadPath} [path]
- * @property {ReadURL} [url]
  * @property {ReadText} [text]
  * @property {ReadAbstract} [abstract]
  * @property {ReadFolder} [folder]
@@ -51,14 +74,7 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {string} filePath
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {Promise<ReadTextResult>}
- */
-
-/**
- * Reads a the response from a URL and returns arbitrary data.
- * @callback ReadURL
- * @param {Response} response
- * @returns {object}
+ * @returns {Job[] | undefined}
  */
 
 /**
@@ -68,14 +84,14 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {string} destinationPath
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {ReadTextResult}
+ * @returns {ReadTextResult | undefined}
  */
 
 /**
  * @typedef {object} ReadTextResult
  * @property {Jobs} [jobs]
- * @property {object} metadata
- * @property {Abstract} abstract
+ * @property {object} [metadata]
+ * @property {Abstract} [abstract]
  */
 
 /**
@@ -110,15 +126,14 @@ import { default as createDatabase } from "./createDatabase.js"
 
 /**
  * @callback ReadFolder
- * @param {object} folder
+ * @param {object} Folder
  * @param {Database} database
  * @param {VotiveConfig} config
- * @param {boolean} isRoot
- * @returns {{ jobs?: Jobs, destinations?: Destination[] }}
+ * @returns {{ jobs?: Jobs, targets?: Target[] }}
  */
 
 /**
- * @typedef {object} Destination
+ * @typedef {object} Target
  * @property {string} path
  * @property {object} metadata
  * @property {Abstract} abstract
@@ -136,7 +151,7 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {object} destination
  * @param {Database} database
  * @param {VotiveConfig} config
- * @returns {{ data: string, buffer: Buffer, encoding?: BufferEncoding = 'utf-8' }}
+ * @returns {{ data: string, encoding?: BufferEncoding = 'utf-8' }}
  */
 
 /**
@@ -151,22 +166,18 @@ import { default as createDatabase } from "./createDatabase.js"
  * Run a job.
  * @typedef {object} Job
  * @property {object} data
- * @property {"text" | "blob" | "json"} format
- * @property {string} [syntax]
+ * @property {string} runner
+ * @property {string} [plugin]
  */
 
 /**
- * @typedef {object} PathInfo
- * @property {string[]} dir
- * @property {ParsedPath["name"]} name
- * @property {ParsedPath["ext"]} ext
- * @property {boolean} [inRootDir]
+ * @typedef {(Pick<path.ParsedPath, "root" | "base" | "ext" | "name") & { dir: string[] | string }} RouteInfo
  */
 
 /**
  * @callback Router
- * @param {PathInfo} path
- * @returns {PathInfo | false | undefined}
+ * @param {RouteInfo} path
+ * @returns {Partial<RouteInfo> | path.ParsedPath | false}
  */
 
 /**
@@ -174,6 +185,7 @@ import { default as createDatabase } from "./createDatabase.js"
  * @property {string} sourceFolder
  * @property {string} destinationFolder
  * @property {VotivePlugin[]} plugins
+ * @property {boolean} verbose
  */
 
 /**
@@ -191,13 +203,11 @@ import { default as createDatabase } from "./createDatabase.js"
  * @param {Database | undefined} [cache]
  */
 async function bundle(config, cache) {
-  // TODO: Ensure all cached destinations exist as expected
 
   // Map out all processors
   const processors = config.plugins
     && config.plugins.flatMap(plugin => plugin.processors && plugin.processors.map(processor => ({ plugin, processor })))
 
-  // Create database
   const database = cache || createDatabase(path.join(config.sourceFolder, ".votive.db"))
 
   /*
@@ -211,38 +221,94 @@ async function bundle(config, cache) {
     even out.
   */
 
+  // database.begin()
+
+  const sourceTime = stopwatch("build", "read sources in", config.verbose)
   // Read folders and source files
   const { folders, sources } = await readSources(config, database, processors)
 
-  // Map out jobs from source files
-  const sourcesJobs = sources.flatMap(source => source.jobs) || []
+  sourceTime()
 
-  // Process source file abstracts and map jobs
-  const { abstractsJobs } = readAbstracts(sources, config, database, processors)
+  if(config.verbose) console.info(`${styleText("dim", "build:")} ${styleText("magenta", `found ${sources.length} stale files`)}`)
 
-  // Scan folders and map out jobs
-  const foldersJobs = readFolders(folders, config, database, processors) || []
+  if (sources.length) {
+    // Map out jobs from source files
+    const sourcesJobs = sources.flatMap(source => source.jobs) || []
 
-  // Write destination files
-  await writeDestinations(config, database)
+    // Process source file abstracts and map jobs
+    const { abstractsJobs } = readAbstracts(sources, config, database, processors)
 
-  // Run all jobs
-  await runJobs([
-    ...sourcesJobs,
-    ...abstractsJobs,
-    ...foldersJobs
-  ], config, database)
+    // Scan folders and map out jobs
+    const foldersJobs = readFolders(folders, config, database, processors) || []
 
+    const writeTime = stopwatch("build", "wrote files in", config.verbose)
+    // Write destination files
+    await writeDestinations(config, database)
 
-  await writeDestinations(config, database)
+    writeTime()
 
-  // Back up database (only if in-memory first run)
-  await database.saveDB()
+    // database.commit()
 
-  const stale = database.getStaleDestinations()
-  if(stale.length > 0) await bundle(config, cache)
+    await database.saveDB(sources)
+  }
+  // Run all jobs
+  // await runJobs([
+  //   ...sourcesJobs,
+  //   ...abstractsJobs,
+  //   ...foldersJobs
+  // ], config, database)
+
+  // Back up database (only if in-memory first run)
 
   return database
 }
 
-export default bundle
+/**
+ * @param {VotiveConfig} config
+ */
+async function bundler(config) {
+  let queue = []
+  let cache
+
+  async function step() {
+    if (queue.length === 0) {
+      if(config.verbose) console.info(`${styleText("dim", "build:")} ${styleText("magenta", "starting build")}`)
+      /*
+        If queue is empty, bundle.
+      */
+      if (!cache) {
+        queue.push(bundle(config))
+        cache = await queue[0]
+      } else {
+        "cache"
+        queue.push(bundle(config, cache))
+        await queue[0]
+      }
+      queue.shift()
+      return cache
+    } else if (queue.length === 1) {
+      /*
+        If currently bundling, prepare another
+        bundle as cleanup.
+      */
+      if(config.verbose) console.info(`${styleText("dim", "build:")} ${styleText("magenta", "queueing build")}`)
+
+      let bundling
+      queue.push(bundling)
+      bundling = await bundle(config, cache)
+      queue.shift()
+      return cache
+    } else {
+      /*
+        If cleanup is already queued, do nothing.
+      */
+      if(config.verbose) console.info(`${styleText("dim", "build:")} ${styleText("magenta", "already queued")}`)
+      await Promise.all(queue)
+      return cache
+    }
+  }
+
+  return step
+}
+
+export default bundler