votive 0.0.7 → 0.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "votive",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "description": "A file processor.",
   "homepage": "https://github.com/samlfair/votive#readme",
   "bugs": {
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "encoding-sniffer": "^0.2.1",
+    "p-limit": "^7.3.0",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "workerpool": "^10.0.1"

package/tests/index.js

@@ -13,325 +13,365 @@ import { checkFile } from "./../lib/utils/index.js"
 
 process.chdir("./tests")
 
-test("empty directory", async () => {
-  const tempDest = fs.mkdtempDisposableSync("destination-")
-  const tempSource = fs.mkdtempDisposableSync("source-")
-
-  try {
-    const stop = stopwatch("Bundle empty folder")
-    await votive.bundle({
-      sourceFolder: tempSource.path,
-      destinationFolder: tempDest.path,
-      plugins: []
+test("database creation in memory", (t) => {
+  const database = votive.createDatabase(":memory:")
+
+  t.test("database functions", () => {
+    const databaseKeys = new Set(Object.keys(database))
+    const expectedKeys = new Set(["saveDB", "source", "setting", "dependency", "target", "url"])
+    const difference = databaseKeys.difference(expectedKeys)
+    assert(difference.size === 0)
+  })
+
+  t.test("create sources", () => {
+    database.source.create("abc.in", "abc.out", 1)
+    database.source.create("def.in", "def.out", 2)
+    const sources = database.source.getAll()
+    assert(sources.length === 2)
+  })
+
+  t.test("create targets", () => {
+    database.target.create({
+      path: "abc.out",
+      abstract: ["abc"],
+      metadata: {
+        color: "blue",
+      }
     })
 
-    stop()
-
-    const dir = fs.readdirSync(tempDest.path)
-    assert(dir.length === 0, "Destination directory is not empty.")
-  } catch (error) {
-    tempDest.remove()
-    tempSource.remove()
-    console.error(error)
-
-  }
-
-  tempDest.remove()
-  tempSource.remove()
-})
-
-test("internals", async (t) => {
-  const dbExists = checkFile(".votive.db")
-  if (dbExists) await rm(".votive.db")
-
-  const temp = fs.mkdtempDisposableSync("destination-")
-
-  /** @returns {Job} */
-  function createExampleJob() {
-    return {
-      data: Math.floor(Math.random() * 1000),
-      runner: "exampleRunner"
-    }
-  }
-
-
-  /** @type {ReadText} */
-  function exampleTextReader(text, filePath, destinationPath, database, config) {
-    const matches = text.match(/\b\w+\b/)
-    const title = matches ? matches[0] : "Untitled"
-
-    /** @type {ReadTextResult} */
-    return {
-      abstract: {
-        content: text,
-      },
+    database.target.create({
+      path: "def.out",
+      abstract: ["def"],
       metadata: {
-        title
-      },
-      jobs: [
-        createExampleJob()
-      ]
-    }
-  }
-
-  /** @type {ReadAbstract} */
-  function exampleAbstractReader(abstract, database, config) {
-    abstract.exampleAppend = true
-    return { abstract, jobs: [createExampleJob()] }
-  }
-
-  /** @type {ReadFolder} */
-  function exampleFolderReader(folder, database, config) {
-    return {
-      jobs: [createExampleJob()],
-      destinations: [
-        {
-          path: "index.html",
-          abstract: { content: "" },
-          metadata: {
-            title: "home"
-          },
-          syntax: "md"
-        }
-      ]
-    }
-  }
-
-  /** @type {VotiveProcessor} */
-  const exampleProcessor = {
-    syntax: "md",
-    filter: {
-      extensions: [".md"]
-    },
-    read: {
-      text: exampleTextReader,
-      abstract: exampleAbstractReader,
-      folder: exampleFolderReader
-    },
-    write: (destination, database, config) => {
-      return {
-        data: "lorem ipsum",
+        color: "red",
       }
-    }
-  }
-
-  /** @param {string} sourcePath */
-  function router({ base, ...parsed }) {
-    return { ...parsed, ext: ".html" }
-  }
+    })
 
-  /** @type {VotivePlugin} */
-  const examplePlugin = {
-    name: "example plugin",
-    runners: {
-      exampleRunner: exampleRunner
-    },
-    router,
-    processors: [exampleProcessor]
-  }
+    const targets = database.target.getAll()
 
-  /** @type {Runner} */
-  async function exampleRunner(data, database) {
-    return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
-  }
+    assert(targets.length === 2)
+  })
+})
 
-  /** @type {VotiveConfig} */
-  const config = {
-    sourceFolder: "./markdown",
-    destinationFolder: temp.path,
-    plugins: [examplePlugin]
-  }
+test("bundle", async () => {
+  const tempDest = fs.mkdtempDisposableSync("target-")
+  const tempSource = fs.mkdtempDisposableSync("source-")
 
-  /** @type {FlatProcessors} */
-  const processors = [
-    {
-      plugin: examplePlugin,
-      processor: exampleProcessor
-    }
-  ]
+  fs.writeFileSync(
+    path.join(tempSource.path, "abc.in"),
+    JSON.stringify({ abstract: ["abc"], metadata: { color: "blue" } }),
+    "utf-8"
+  )
 
   try {
-    fs.writeFileSync("markdown/prunee.md", "A little content")
-
-    const database = votive.createDatabase()
-    const sourcesOne = database.getAllSources()
+    const stop = stopwatch("Bundle folder")
+    const database = await votive.bundle({
+      sourceFolder: tempSource.path,
+      destinationFolder: tempDest.path,
+      plugins: [{
+        name: "test-plugin",
+        processors: [{
+          format: "text",
+          extensions: [".in"],
+          // readResource: null,
+          writeFile: (destination) => ({ data: destination.abstract[0] }),
+          readFile: (data) => ({ abstract: [data], metadata: { color: "green" } }),
+          // readFolder: null,
+          // transformFile: null
+        }],
+        router: ({ base, dir, root }) => ({ base, root, ext: ".out" }),
+      }]
 
-    t.test("no sources on startup", () => {
-      const isArray = Array.isArray(sourcesOne)
-      const isEmpty = !sourcesOne.length
-      assert(isArray && isEmpty)
     })
 
+    const sources = database.source.getAll()
 
-
-    const stop = stopwatch("Read sources")
-
-    const { folders, sources } = await votive.readSources(config, database, processors)
-
-    t.test('one folder exists', () => {
-      assert(folders.length === 1)
-    })
-
-    t.test('three sources exist', () => {
-      assert(sources.length === 4)
-    })
+    const targets = database.target.getAll()
 
     stop()
 
-    const sourcesJobs = sources.flatMap(source => source.jobs)
-
-    const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
-
-    t.test('transformation succeeded', () => {
-      assert(processedAbstracts.find(abstract => abstract.exampleAppend))
-    })
-
-    t.test('abstract jobs exist', () => {
-      assert(abstractsJobs.length > 0)
-      assert(abstractsJobs.every(job => job.data && job.runner))
-    })
-
-    const foldersJobs = readFolders(folders, config, database, processors)
+    const dir = fs.readdirSync(tempDest.path, { recursive: true })
+    assert(dir.length === 0, "Destination directory is not empty.")
+  } catch (error) {
+    console.error(error)
+    tempDest.remove()
+    tempSource.remove()
 
-    t.test("folders jobs exist", () => {
-      assert(foldersJobs.length > 0)
-      assert(foldersJobs.every(job => job.data && job.runner))
-    })
+  }
 
-    database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
-    const firstDestination = database.getDestinationIndependently("abc.html", [])
+  tempDest.remove()
+  tempSource.remove()
+})
 
-    t.test('first destination created', () => {
-      assert(firstDestination.path === 'abc.html')
-    })
+// test("internals", async (t) => {
+//   const dbExists = checkFile(".votive.db")
+//   if (dbExists) await rm(".votive.db")
+
+//   const temp = fs.mkdtempDisposableSync("destination-")
+
+//   /** @returns {Job} */
+//   function createExampleJob() {
+//     return {
+//       data: Math.floor(Math.random() * 1000),
+//       runner: "exampleRunner"
+//     }
+//   }
+
+
+//   /** @type {ReadText} */
+//   function exampleTextReader(text, filePath, destinationPath, database, config) {
+//     const matches = text.match(/\b\w+\b/)
+//     const title = matches ? matches[0] : "Untitled"
+
+//     /** @type {ReadTextResult} */
+//     return {
+//       abstract: {
+//         content: text,
+//       },
+//       metadata: {
+//         title
+//       },
+//       jobs: [
+//         createExampleJob()
+//       ]
+//     }
+//   }
+
+//   /** @type {ReadAbstract} */
+//   function exampleAbstractReader(abstract, database, config) {
+//     abstract.exampleAppend = true
+//     return { abstract, jobs: [createExampleJob()] }
+//   }
+
+//   /** @type {ReadFolder} */
+//   function exampleFolderReader(folder, database, config) {
+//     return {
+//       jobs: [createExampleJob()],
+//       destinations: [
+//         {
+//           path: "index.html",
+//           abstract: { content: "" },
+//           metadata: {
+//             title: "home"
+//           },
+//           syntax: "md"
+//         }
+//       ]
+//     }
+//   }
+
+//   /** @type {VotiveProcessor} */
+//   const exampleProcessor = {
+//     syntax: "md",
+//     filter: {
+//       extensions: [".md"]
+//     },
+//     read: {
+//       text: exampleTextReader,
+//       abstract: exampleAbstractReader,
+//       folder: exampleFolderReader
+//     },
+//     write: (destination, database, config) => {
+//       return {
+//         data: "lorem ipsum",
+//       }
+//     }
+//   }
+
+//   /** @param {string} sourcePath */
+//   function router({ base, ...parsed }) {
+//     return { ...parsed, ext: ".html" }
+//   }
+
+//   /** @type {VotivePlugin} */
+//   const examplePlugin = {
+//     name: "example plugin",
+//     runners: {
+//       exampleRunner: exampleRunner
+//     },
+//     router,
+//     processors: [exampleProcessor]
+//   }
+
+//   /** @type {Runner} */
+//   async function exampleRunner(data, database) {
+//     return await new Promise((resolve) => setTimeout(() => resolve(data), 1))
+//   }
+
+//   /** @type {VotiveConfig} */
+//   const config = {
+//     sourceFolder: "./markdown",
+//     destinationFolder: temp.path,
+//     plugins: [examplePlugin]
+//   }
+
+//   /** @type {FlatProcessors} */
+//   const processors = [
+//     {
+//       plugin: examplePlugin,
+//       processor: exampleProcessor
+//     }
+//   ]
+
+//   try {
+//     fs.writeFileSync("markdown/prunee.md", "A little content")
+
+//     const database = votive.createDatabase()
+//     const sourcesOne = database.getAllSources()
+
+//     t.test("no sources on startup", () => {
+//       const isArray = Array.isArray(sourcesOne)
+//       const isEmpty = !sourcesOne.length
+//       assert(isArray && isEmpty)
+//     })
+
+
+
+//     const stop = stopwatch("Read sources")
+
+//     const { folders, sources } = await votive.readSources(config, database, processors)
+
+//     t.test('one folder exists', () => {
+//       assert(folders.length === 1)
+//     })
+
+//     t.test('three sources exist', () => {
+//       assert(sources.length === 4)
+//     })
+
+//     stop()
+
+//     const sourcesJobs = sources.flatMap(source => source.jobs)
 
-    database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
-    const destination = database.getDestinationDependently("abc.html", "abc/def.html")
+//     const { processedAbstracts, abstractsJobs } = votive.readAbstracts(sources, config, database, processors)
+
+//     t.test('transformation succeeded', () => {
+//       assert(processedAbstracts.find(abstract => abstract.exampleAppend))
+//     })
 
-    console.info(`'abc/def.html' requests the property 'a' from 'abc.html', creating a dependency: ${destination.metadata.a}`)
+//     t.test('abstract jobs exist', () => {
+//       assert(abstractsJobs.length > 0)
+//       assert(abstractsJobs.every(job => job.data && job.runner))
+//     })
 
-    const dependencies = database.getDependencies()
+//     const foldersJobs = readFolders(folders, config, database, processors)
 
-    t.test('dependency created', () => {
-      assert(dependencies[0].dependent === 'abc/def.html')
-    })
+//     t.test("folders jobs exist", () => {
+//       assert(foldersJobs.length > 0)
+//       assert(foldersJobs.every(job => job.data && job.runner))
+//     })
 
+//     database.createOrUpdateDestination({ metadata: { a: 1, b: 2 }, path: "abc.html", abstract: { c: 3 }, syntax: "md" })
+//     const firstDestination = database.getDestinationIndependently("abc.html", [])
 
-    const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
-    const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-    const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
-    const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
+//     t.test('first destination created', () => {
+//       assert(firstDestination.path === 'abc.html')
+//     })
 
-    t.test('settings created', () => {
-      assert(setting.value === 'blue')
-      assert(newSetting.value === 'Dog')
-      assert(!sameSetting)
-    })
+//     database.createOrUpdateDestination({ metadata: { a: 3, b: 4 }, path: "abc/def.html", abstract: { c: 5 }, syntax: "md" })
 
-    const abcSettings = database.getSettings("abc.html")
+//     /* TODO: Test that dependencies are working */
 
-    t.test('settings retrieved', () => {
-      assert(abcSettings.theme[0] === "blue")
-    })
+//     const setting = database.setSetting("abc.html", "theme", "blue", "markdown/prunee.md")
+//     const newSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+//     const sameSetting = database.setSetting("abc", "category", "Dog", "markdown/prunee.md")
+//     const folderSetting = database.setSetting("abc", "theme", "red", "markdown/prunee.md")
 
-    const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
-    const defSettings = database.getSettings("abc/def.html")
+//     t.test('settings created', () => {
+//       assert(setting.value === 'blue')
+//       assert(newSetting.value === 'Dog')
+//       assert(!sameSetting)
+//     })
 
-    t.test('settings retrieved', () => {
-      assert(abcSettings.theme[0] === "blue")
-      assert(defSettings.theme[1] === "green")
-    })
+//     const abcSettings = database.getSettings("abc.html")
 
-    const staleDestinations = database.getStaleDestinations()
+//     t.test('settings retrieved', () => {
+//       assert(abcSettings.theme[0] === "blue")
+//     })
 
-    t.test('all destinations are stale', () => {
-      assert(staleDestinations.length === 7)
-    })
+//     const descendentSetting = database.setSetting("abc/def.html", "theme", "green", "abc.md")
+//     const defSettings = database.getSettings("abc/def.html")
 
-    const written = await votive.writeDestinations(config, database)
+//     t.test('settings retrieved', () => {
+//       assert(abcSettings.theme[0] === "blue")
+//       assert(defSettings.theme[1] === "green")
+//     })
 
-    const staleDestinationsAfterWriting = database.getStaleDestinations()
+//     const staleDestinations = database.getStaleDestinations()
 
-    t.test('all destinations are fresh', () => {
-      assert(staleDestinationsAfterWriting.length === 0)
-    })
+//     t.test('all destinations are stale', () => {
+//       assert(staleDestinations.length === 7)
+//     })
 
-    const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-    const staleAfterAbstractUpdate = database.getStaleDestinations()
+//     const written = await votive.writeDestinations(config, database)
 
-    t.test('updated document with no side effects is stale', () => {
-      assert(staleAfterAbstractUpdate.length === 1)
-    })
+//     const staleDestinationsAfterWriting = database.getStaleDestinations()
 
-    await votive.writeDestinations(config, database)
-    const staleAfterSecondWrite = database.getStaleDestinations()
+//     t.test('all destinations are fresh', () => {
+//       assert(staleDestinationsAfterWriting.length === 0)
+//     })
 
-    t.test('everything fresh again', () => {
-      assert(staleAfterSecondWrite.length === 0)
-    })
+//     const updated = database.createOrUpdateDestination({ metadata: { a: 1, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+//     const staleAfterAbstractUpdate = database.getStaleDestinations()
 
-    const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
-    const staleAfterSideEffects = database.getStaleDestinations()
+//     t.test('updated document with no side effects is stale', () => {
+//       assert(staleAfterAbstractUpdate.length === 1)
+//     })
 
-    t.test('side effects work', () => {
-      assert(staleAfterSideEffects.length === 2)
-    })
+//     await votive.writeDestinations(config, database)
+//     const staleAfterSecondWrite = database.getStaleDestinations()
 
-    const freshDestinations = database.getAllDestinations()
+//     t.test('everything fresh again', () => {
+//       assert(staleAfterSecondWrite.length === 0)
+//     })
 
-    await votive.writeDestinations(config, database)
+//     const updatedWithSideEffects = database.createOrUpdateDestination({ metadata: { a: 4, b: 9 }, path: "abc.html", abstract: { c: 4 }, syntax: "md" })
+//     const staleAfterSideEffects = database.getStaleDestinations()
 
-    const oldSetting = database.setSetting("abc", "theme", "green")
+//     t.test('side effects work', () => {
+//       assert(staleAfterSideEffects.length === 2)
+//     })
 
-    const staleAfterSettingsChange = database.getStaleDestinations()
 
-    t.test('setting affects descendents', () => {
-      assert(staleAfterSettingsChange.length === 1)
-    })
+//     await votive.writeDestinations(config, database)
 
-    // TODO: Write tests for jobs
-    const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
-    runJobs(jobs, config, database)
+//     const oldSetting = database.setSetting("abc", "theme", "green")
 
-    const destinations = database.getAllDestinations()
-    const prunee = database.getDestinationDependently("markdown/prunee.html", "abc/def.html")
+//     const staleAfterSettingsChange = database.getStaleDestinations()
 
-    console.info(`'abc/def.html' requests the property 'title' from 'markdown/prunee.html', creating a dependency: ${prunee.metadata.title}`)
-    const newDependencies = database.getDependencies()
+//     t.test('setting affects descendents', () => {
+//       assert(staleAfterSettingsChange.length === 1)
+//     })
 
-    fs.rmSync("markdown/prunee.md")
+//     // TODO: Write tests for jobs
+//     const jobs = [...sourcesJobs, ...abstractsJobs, ...foldersJobs]
+//     runJobs(jobs, config, database)
 
-    await votive.pruneSources(config, database)
+//     fs.rmSync("markdown/prunee.md")
 
-    const result = database.getDestinationIndependently("markdown/prunee.html")
-    const prunedDependencies = database.getDependencies()
-    const prunedMetadata = database.getAllMetadataByPath("markdown/prunee.html")
+//     await votive.pruneSources(config, database)
 
-    t.test('source pruned', () => {
-      assert(!result)
-      assert(newDependencies.length - prunedDependencies.length === 1)
-      assert(!prunedMetadata)
-    })
+//     const result = database.getDestinationIndependently("markdown/prunee.html")
 
-    const finalDestinations = database.getDestinations({
-      filter: [
-        {
-          property: "a",
-          operator: "gt",
-          value: 3
-        }
-      ]
-    }, "markdown/prunee.html")
+//     /* TODO: Check pruning metadata works (I removed it) */
 
-    const finalDependencies = database.getDependencies()
+//     const finalDestinations = database.getDestinations({
+//       filter: [
+//         {
+//           property: "a",
+//           operator: "gt",
+//           value: 3
+//         }
+//       ]
+//     }, "markdown/prunee.html")
 
-    // TODO: Write a test for get destinations
+//     // TODO: Write a test for get destinations
 
-    await database.saveDB()
+//     await database.saveDB()
 
-    temp.remove()
-  } catch (error) {
-    temp.remove()
-    throw error
-  }
-})
+//     temp.remove()
+//   } catch (error) {
+//     temp.remove()
+//     throw error
+//   }
+// })

package/tests/markdown/prunee.md

@@ -0,0 +1 @@
+A little content