wiki-server 0.25.10 → 0.26.0-rc.1

package/AUTHORS.txt

@@ -19,3 +19,4 @@ Eric Dobbs <eric@dobbse.net>
 Joshua Benuck <joshua_benuck@yahoo.com>
 Tom Lieber <tom@alltom.com>
 Andrew Shell <andrew@andrewshell.org>
+jon r <jon@allmende.io>

package/index.js

@@ -2,11 +2,7 @@
 // Simple file so that if you require this directory
 // in node it instead requires ./lib/server.coffee
 // with coffee-script already loaded.
-require('coffeescript');
-require('coffeescript/register');
+require('coffeescript')
+require('coffeescript/register')
 
-// set a default process exitCode, so we can diferentiate between exiting as
-// part of a reload, and an exit after an uncaught error
-process.exitCode = 1
-
-module.exports = require('./lib/server');
+module.exports = require('./lib/server')

package/lib/defaultargs.js

@@ -0,0 +1,83 @@
+/*
+ * Federated Wiki : Node Server
+ *
+ * Copyright Ward Cunningham and other contributors
+ * Licensed under the MIT license.
+ * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
+ */
+
+// **defaultargs.coffee** when called on the argv object this
+// module will create reasonable defaults for options not supplied,
+// based on what information is provided.
+const path = require('node:path')
+
+const getUserHome = () => {
+  return process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE
+}
+
+module.exports = argv => {
+  argv = argv || {}
+  argv.root ||= __dirname
+  // the directory that contains all the packages that makeup the wiki
+  argv.packageDir ||= path.join(argv.root, '..')
+  argv.port ||= 3000
+  argv.home ||= 'welcome-visitors'
+  argv.data ||= path.join(getUserHome(), '.wiki') // see also cli
+  argv.client ||= path.join(argv.packageDir, 'wiki-client', 'client')
+  argv.db ||= path.join(argv.data, 'pages')
+  argv.status ||= path.join(argv.data, 'status')
+  argv.assets ||= path.join(argv.data, 'assets')
+  argv.recycler ||= path.join(argv.data, 'recycle')
+  argv.commons ||= path.join(argv.data, 'commons')
+  argv.url ||= `http://localhost${argv.port === 80 ? '' : ':' + argv.port}`
+  argv.id ||= path.join(argv.status, 'owner.json')
+  argv.uploadLimit ||= '5mb'
+  argv.cookieSecret ||= require('crypto').randomBytes(64).toString('hex')
+  argv.secure_cookie ||= false
+  argv.session_duration ||= 7
+  argv.neighbors ||= ''
+  argv.debug ||= false
+  argv.test ||= false
+
+  if (typeof argv.database === 'string') {
+    argv.database = JSON.parse(argv.database)
+  }
+  argv.database ||= {}
+  argv.database.type ||= './page'
+  if (argv.database.type.charAt(0) === '.') {
+    if (argv.database.type != './page') {
+      console.log('\n\nWARNING: This storage option is depeciated.')
+      console.log('    See ReadMe for details of the changes required.\n\n')
+    }
+  } else {
+    argv.database.type = 'wiki-storage-' + argv.database.type
+  }
+
+  argv.security_type ||= './security'
+  if (argv.security_type === './security') {
+    console.log('\n\nINFORMATION: Using default security module.')
+  } else {
+    argv.security_type = 'wiki-security-' + argv.security_type
+  }
+  argv.security_legacy ||= false
+
+  // resolve all relative paths
+  argv.root = path.resolve(argv.root)
+  argv.packageDir = path.resolve(argv.packageDir)
+  argv.data = path.resolve(argv.data)
+  argv.client = path.resolve(argv.client)
+  argv.db = path.resolve(argv.db)
+  argv.status = path.resolve(argv.status)
+  argv.assets = path.resolve(argv.assets)
+  argv.recycler = path.resolve(argv.recycler)
+  argv.commons = path.resolve(argv.commons)
+  argv.id = path.resolve(argv.id)
+
+  if (/node_modules/.test(argv.data)) {
+    console.log('\n\nWARNING : The dafault data path is not a safe place.')
+    console.log('       : by using ', argv.data, ' your pages will be lost when packages are updated.')
+    console.log('       : You are strongly advised to use an alternative directory.')
+    console.log('       : See the wiki package ReadMe for how to do this.\n\n')
+  }
+  return argv
+}

package/lib/forward.js

@@ -1,12 +1,12 @@
 const init = (app, emitter) => {
   let sockets = []
-  app.io.on('connection', (socket) => {
+  app.io.on('connection', socket => {
     let listeners = []
     sockets.push(socket)
     console.log('client connected:', socket.handshake.address)
-    socket.on('disconnect', (reason) => {
+    socket.on('disconnect', reason => {
       console.log('client disconnected:', socket.handshake.address, reason)
-      for (let {sProducer, listener} of listeners) {
+      for (let { sProducer, listener } of listeners) {
         console.log('removing listener:', sProducer)
         emitter.removeListener(sProducer, listener)
       }
@@ -14,9 +14,9 @@ const init = (app, emitter) => {
       let i = sockets.indexOf(socket)
       sockets.splice(i, 1)
     })
-    socket.on('unsubscribe', (sProducer) => {
+    socket.on('unsubscribe', sProducer => {
       console.log('unsubscribing listener:', socket.handshake.address, sProducer)
-      for (let [i, {slugItem, listener}] of listeners.entries()) {
+      for (let [i, { slugItem, listener }] of listeners.entries()) {
         if (slugItem == sProducer) {
           console.log('removing listener:', sProducer)
           emitter.removeListener(sProducer, listener)
@@ -24,15 +24,15 @@ const init = (app, emitter) => {
         }
       }
     })
-    socket.on('subscribe', (sProducer) => {
-      let listener = (result) => {
+    socket.on('subscribe', sProducer => {
+      let listener = result => {
         console.log('forwarding:', socket.handshake.address, result)
-        socket.emit(sProducer, {slugItem: sProducer, result})
+        socket.emit(sProducer, { slugItem: sProducer, result })
       }
       console.log(`registering listener:`, socket.handshake.address, sProducer)
       emitter.on(sProducer, listener)
-      listeners.push({sProducer, listener})
+      listeners.push({ sProducer, listener })
     })
   })
 }
-module.exports = {init}
+module.exports = { init }

package/lib/page.js

@@ -0,0 +1,435 @@
+/*
+ * Federated Wiki : Node Server
+ *
+ * Copyright Ward Cunningham and other contributors
+ * Licensed under the MIT license.
+ * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
+ */
+// **page.coffee**
+// Module for interacting with pages persisted on the server.
+// Everything is stored using json flat files.
+
+// #### Requires ####
+const fs = require('fs')
+const path = require('path')
+const events = require('events')
+
+const async = require('async')
+
+const random_id = require('./random_id')
+const synopsis = require('wiki-client/lib/synopsis')
+
+const asSlug = name =>
+  name
+    .replace(/\s/g, '-')
+    .replace(/[^A-Za-z0-9-]/g, '')
+    .toLowerCase()
+
+// Export a function that generates a page handler
+// when called with options object.
+module.exports = exports = argv => {
+  const wikiName = new URL(argv.url).hostname
+
+  fs.mkdir(argv.db, { recursive: true }, e => {
+    if (e) throw e
+  })
+
+  // create a list of plugin pages.
+  const pluginPages = new Map()
+  Object.keys(require.main.require('./package').dependencies)
+    .filter(depend => depend.startsWith('wiki-plugin'))
+    .forEach(plugin => {
+      const pagesPath = path.join(
+        path.dirname(require.resolve(`${plugin}/package`, { paths: require.main.paths })),
+        'pages',
+      )
+      fs.readdir(pagesPath, { withFileTypes: true }, (err, entries) => {
+        if (err) return
+        entries.forEach(entry => {
+          if (entry.isFile() && !pluginPages.has(entry.name)) {
+            pluginPages.set(entry.name, { pluginName: plugin, pluginPath: entry.parentPath })
+          }
+        })
+      })
+    })
+
+  // #### Private utility methods. ####
+  const load_parse = (loc, cb, annotations = {}) => {
+    let page
+    fs.readFile(loc, (err, data) => {
+      if (err) return cb(err)
+      try {
+        page = JSON.parse(data)
+      } catch {
+        const errorPage = path.basename(loc)
+        const errorPagePath = path.dirname(loc)
+        const recyclePage = path.resolve(errorPagePath, '..', 'recycle', errorPage)
+        fs.access(path.dirname(recyclePage), fs.constants.F_OK, err => {
+          if (!err) {
+            fs.rename(loc, recyclePage, err => {
+              if (err) {
+                console.log(`ERROR: moving problem page ${loc} to recycler`, err)
+              } else {
+                console.log(`ERROR: problem page ${loc} moved to recycler`)
+              }
+            })
+          } else {
+            fs.mkdir(path.dirname(recyclePage), { recursive: true }, err => {
+              if (err) {
+                console.log('ERROR: creating recycler', err)
+              } else {
+                fs.rename(loc, recyclePage, err => {
+                  if (err) {
+                    console.log(`ERROR: moving problem page ${loc} to recycler`, err)
+                  } else {
+                    console.log(`ERROR: problem page ${loc} moved to recycler`)
+                  }
+                })
+              }
+            })
+          }
+        })
+
+        return cb(null, 'Error Parsing Page', 404)
+      }
+      for (const [key, val] of Object.entries(annotations)) {
+        page[key] = val
+      }
+      cb(null, page)
+    })
+  }
+
+  const load_parse_copy = (defloc, file, cb) => {
+    fs.readFile(defloc, (err, data) => {
+      if (err) cb(err)
+      let page
+      try {
+        page = JSON.parse(data)
+      } catch (e) {
+        return cb(e)
+      }
+      cb(null, page)
+      // TODO: what is happening here?! put will never be reached???
+      itself.put(file, page, err => {
+        if (err) cb(err)
+      })
+    })
+  }
+  // Reads and writes are async, but serially queued to avoid race conditions.
+  const queue = []
+
+  const tryDefaults = (file, cb) => {
+    const lastDefault = cb => {
+      const defloc = path.join(argv.root, 'default-data', 'pages', file)
+      fs.access(defloc, fs.constants.F_OK, err => {
+        if (!err) {
+          cb(defloc)
+        } else {
+          cb(null)
+        }
+      })
+    }
+    if (argv.defaults) {
+      const defloc = path.join(argv.data, '..', argv.defaults, 'pages', file)
+      fs.access(defloc, fs.constants.F_OK, err => {
+        if (!err) {
+          cb(defloc)
+        } else {
+          lastDefault(cb)
+        }
+      })
+    } else {
+      lastDefault(cb)
+    }
+  }
+
+  // Main file io function, when called without page it reads,
+  // when called with page it writes.
+  const fileio = (action, file, page, cb) => {
+    const loc = file.startsWith('recycler/') ? path.join(argv.recycler, file.split('/')[1]) : path.join(argv.db, file)
+
+    switch (action) {
+      case 'delete':
+        if (file.startsWith('recycler/')) {
+          // delete from recycler
+          fs.access(loc, fs.constants.F_OK, err => {
+            if (!err)
+              fs.unlink(loc, err => {
+                cb(err)
+              })
+          })
+        } else {
+          // move page to recycler
+          fs.access(loc, fs.constants.F_OK, err => {
+            if (!err) {
+              const recycleLoc = path.join(argv.recycler, file)
+              fs.access(path.dirname(recycleLoc), fs.constants.F_OK, err => {
+                if (!err) {
+                  fs.rename(loc, recycleLoc, err => {
+                    cb(err)
+                  })
+                } else {
+                  fs.mkdir(path.dirname(recycleLoc), { recursive: true }, err => {
+                    if (err) cb(err)
+                    fs.rename(loc, recycleLoc, err => {
+                      cb(err)
+                    })
+                  })
+                }
+              })
+            } else {
+              cb('page does not exist')
+            }
+          })
+        }
+        break
+      case 'recycle': {
+        const copyFile = (source, target, cb) => {
+          const done = err => {
+            if (!cbCalled) {
+              cb(err)
+              cbCalled = true
+            }
+            return
+          }
+
+          let cbCalled = false
+
+          const rd = fs.createReadStream(source)
+          rd.on('error', err => {
+            done(err)
+            return
+          })
+
+          const wr = fs.createWriteStream(target)
+          wr.on('error', err => {
+            done(err)
+            return
+          })
+          wr.on('close', () => {
+            done()
+            return
+          })
+          rd.pipe(wr)
+          return
+        }
+
+        fs.access(loc, fs.constants.F_OK, err => {
+          if (!err) {
+            const recycleLoc = path.join(argv.recycler, file)
+            fs.access(path.dirname(recycleLoc), fs.constants.F_OK, err => {
+              if (!err) {
+                copyFile(loc, recycleLoc, err => {
+                  cb(err)
+                })
+              } else {
+                fs.mkdir(path.dirname(recycleLoc), { recursive: true }, err => {
+                  if (err) cb(err)
+                  copyFile(loc, recycleLoc, err => {
+                    cb(err)
+                  })
+                })
+              }
+            })
+          } else {
+            cb('page does not exist')
+          }
+        })
+        break
+      }
+      case 'get':
+        fs.access(loc, fs.constants.F_OK, err => {
+          if (!err) {
+            load_parse(loc, cb, { plugin: undefined })
+          } else {
+            tryDefaults(file, defloc => {
+              if (defloc) {
+                load_parse(defloc, cb)
+              } else {
+                if (pluginPages.has(file)) {
+                  const { pluginName, pluginPath } = pluginPages.get(file)
+                  load_parse(path.join(pluginPath, file), cb, { plugin: pluginName.slice(12) })
+                } else {
+                  cb(null, 'Page not found', 404)
+                }
+              }
+            })
+          }
+        })
+        break
+      case 'put':
+        page = JSON.stringify(page, null, 2)
+        fs.access(path.dirname(loc), fs.constants.F_OK, err => {
+          if (!err) {
+            fs.writeFile(loc, page, err => {
+              if (err) {
+                console.log(`ERROR: write file ${loc} `, err)
+              }
+              cb(err)
+            })
+          } else {
+            fs.mkdir(path.dirname(loc), { recursive: true }, err => {
+              if (err) cb(err)
+              fs.writeFile(loc, page, err => {
+                if (err) {
+                  console.log(`ERROR: write file ${loc} `, err)
+                }
+                cb(err)
+              })
+            })
+          }
+        })
+        break
+      default:
+        console.log(`pagehandler: unrecognized action ${action}`)
+    }
+  }
+
+  // Control variable that tells if the serial queue is currently working.
+  // Set back to false when all jobs are complete.
+  let working = false
+
+  // Keep file io working on queued jobs, but don't block the main thread.
+  const serial = item => {
+    if (item) {
+      itself.start()
+      fileio(item.action, item.file, item.page, (err, data, status) => {
+        process.nextTick(() => {
+          serial(queue.shift())
+        })
+        item.cb(err, data, status)
+      })
+    } else {
+      itself.stop()
+    }
+  }
+
+  // #### Public stuff ####
+  // Make the exported object an instance of EventEmitter
+  // so other modules can tell if it is working or not.
+  const itself = new events.EventEmitter()
+
+  itself.start = () => {
+    working = true
+    itself.emit('working')
+  }
+
+  itself.stop = () => {
+    working = false
+    itself.emit('finished')
+  }
+
+  itself.isWorking = () => working
+
+  // get method takes a slug and a callback, adding them to the queue,
+  // starting serial if it isn't already working.
+  itself.get = (file, cb) => {
+    queue.push({ action: 'get', file, page: null, cb })
+    if (!working) serial(queue.shift())
+  }
+
+  // put takes a slugged name, the page as a json object, and a callback.
+  // adds them to the queue, and starts it unless it is working.
+  itself.put = (file, page, cb) => {
+    queue.push({ action: 'put', file, page, cb })
+    if (!working) serial(queue.shift())
+  }
+
+  itself.delete = (file, cb) => {
+    queue.push({ action: 'delete', file, page: null, cb })
+    if (!working) serial(queue.shift())
+  }
+
+  itself.saveToRecycler = (file, cb) => {
+    queue.push({ action: 'recycle', file, page: null, cb })
+    if (!working) serial(queue.shift())
+  }
+
+  const editDate = journal => {
+    if (!journal) return undefined
+    // find the last journal entry, that is not a fork, with a date.
+    const last = journal.findLast(action => {
+      return action.date && action.type != 'fork'
+    })
+    return last ? last.date : undefined
+  }
+
+  itself.pages = cb => {
+    const extractPageLinks = (collaborativeLinks, currentItem, currentIndex, array) => {
+      // extract collaborative links
+      // - this will need extending if we also extract the id of the item containing the link
+      try {
+        const linkRe = /\[\[([^\]]+)\]\]/g
+        let match = undefined
+        while ((match = linkRe.exec(currentItem.text)) != null) {
+          if (!collaborativeLinks.has(asSlug(match[1]))) {
+            collaborativeLinks.set(asSlug(match[1]), currentItem.id)
+          }
+        }
+        if ('reference' == currentItem.type) {
+          if (!collaborativeLinks.has(currentItem.slug)) {
+            collaborativeLinks.set(currentItem.slug, currentItem.id)
+          }
+        }
+      } catch (err) {
+        console.log(
+          `METADATA *** ${wikiName} Error extracting links from ${currentIndex} of ${JSON.stringify(array)}`,
+          err.message,
+        )
+      }
+      return collaborativeLinks
+    }
+    fs.readdir(argv.db, (e, files) => {
+      if (e) return cb(e)
+      // used to make sure all of the files are read
+      // and processesed in the site map before responding
+      const doSitemap = (file, cb) => {
+        itself.get(file, (e, page, status) => {
+          if (file.match(/^\./)) return cb()
+          if (e || status === 404) {
+            console.log('Problem building sitemap:', file, 'e: ', e, 'status:', status)
+            return cb() // Ignore errors in the pagehandler get.
+          }
+          let pageLinksMap
+          try {
+            pageLinksMap = page.story.reduce(extractPageLinks, new Map())
+          } catch (err) {
+            console.log(`METADATA *** ${wikiName} reduce to extract links on ${file} failed`, err.message)
+            pageLinksMap = []
+          }
+          //
+          const pageLinks = pageLinksMap.size > 0 ? Object.fromEntries(pageLinksMap) : undefined
+
+          cb(null, {
+            slug: file,
+            title: page.title,
+            date: editDate(page.journal),
+            synopsis: synopsis(page),
+            links: pageLinks,
+          })
+        })
+      }
+      async.map(files, doSitemap, (e, sitemap) => {
+        if (e) return cb(e)
+        cb(
+          null,
+          sitemap.filter(item => item != null),
+        )
+      })
+    })
+  }
+
+  itself.slugs = cb => {
+    fs.readdir(argv.db, { withFileTypes: true }, (e, files) => {
+      if (e) {
+        console.log('Problem reading pages directory', e)
+        return cb(e)
+      }
+
+      const onlyFiles = files.map(i => (i.isFile() ? i.name : null)).filter(i => i != null && !i?.startsWith('.'))
+      cb(null, onlyFiles)
+    })
+  }
+
+  return itself
+}

package/lib/plugins.js

@@ -0,0 +1,54 @@
+/*
+ * Federated Wiki : Node Server
+ *
+ * Copyright Ward Cunningham and other contributors
+ * Licensed under the MIT license.
+ * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
+ */
+
+// support server-side plugins
+
+const fs = require('node:fs')
+const { pathToFileURL } = require('node:url')
+// forward = require './forward'
+
+module.exports = exports = argv => {
+  // NOTE: plugins are now in their own package directories alongside this one...
+  // Plugins are in directories of the form wiki-package-*
+  // those with a server component will have a server directory
+
+  const plugins = {}
+
+  // http://stackoverflow.com/questions/10914751/loading-node-js-modules-dynamically-based-on-route
+
+  const startServer = (params, plugin) => {
+    const server = `${argv.packageDir}/${plugin}/server/server.js`
+    fs.access(server, fs.constants.F_OK, err => {
+      if (!err) {
+        console.log('starting plugin', plugin)
+        import(pathToFileURL(server))
+          .then(exported => {
+            plugins[plugin] = exported
+            plugins[plugin].startServer?.(params)
+          })
+          .catch(e => {
+            console.log('failed to start plugin', plugin, e?.stack || e)
+          })
+      }
+    })
+  }
+
+  const startServers = params => {
+    // emitter = new events.EventEmitter()
+    // forward.init params.app, emitter
+    // params.emitter = emitter
+
+    Object.keys(require.main.require('./package').dependencies)
+      .filter(depend => depend.startsWith('wiki-plugin'))
+      .forEach(plugin => {
+        startServer(params, plugin)
+      })
+  }
+
+  return { startServers }
+}

package/lib/random_id.js

@@ -0,0 +1,15 @@
+/*
+ * Federated Wiki : Node Server
+ *
+ * Copyright Ward Cunningham and other contributors
+ * Licensed under the MIT license.
+ * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
+ */
+
+// **random_id.coffee**
+// Simple random hex generator, takes an optional number of
+// chars that defaults to 16 and returns a random id.
+
+const random_id = (chars = 16) => [...Array(chars)].map(() => Math.floor(Math.random() * 16).toString(16)).join('')
+
+module.exports = random_id.random_id = random_id