wiki-server 0.23.0 → 0.24.1

package/.github/workflows/node.js.yml

@@ -16,11 +16,11 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x]
+        node-version: [18.x, 20.x, 21.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
       uses: actions/setup-node@v3
       with:

package/lib/page.coffee

@@ -15,7 +15,6 @@ path = require 'path'
 events = require 'events'
 glob = require 'glob'
 
-mkdirp = require 'mkdirp'
 async = require 'async'
 
 random_id = require './random_id'
@@ -31,7 +30,7 @@ module.exports = exports = (argv) ->
 
   wikiName = new URL(argv.url).hostname
 
-  mkdirp argv.db, (e) ->
+  fs.mkdir argv.db, { recursive: true }, (e) ->
     if e then throw e
 
   #### Private utility methods. ####
@@ -53,7 +52,7 @@ module.exports = exports = (argv) ->
                 console.log "ERROR: problem page #{loc} moved to recycler"
               )
           else
-            mkdirp(path.dirname(recyclePage), (err) ->
+            fs.mkdir(path.dirname(recyclePage), { recursive: true }, (err) ->
               if err
                 console.log "ERROR: creating recycler", err
               else
@@ -87,6 +86,25 @@ module.exports = exports = (argv) ->
   # Reads and writes are async, but serially queued to avoid race conditions.
   queue = []
 
+  tryDefaults = (file, cb) ->
+    lastDefault = (cb) ->
+      defloc = path.join(argv.root, 'default-data', 'pages', file)
+      fs.exists defloc, (exists) ->
+        if exists
+          cb(defloc)
+        else
+          cb(null)
+    if argv.defaults
+      defloc = path.join(argv.data, '..', argv.defaults, 'pages', file)
+      console.log 'firstDefault', defloc
+      fs.exists defloc, (exists) ->
+        if exists
+          cb(defloc)
+        else
+          lastDefault(cb)
+    else
+      lastDefault(cb)
+
   # Main file io function, when called without page it reads,
   # when called with page it writes.
   fileio = (action, file, page, cb) ->
@@ -115,7 +133,7 @@ module.exports = exports = (argv) ->
                     cb(err)
                   )
                 else
-                  mkdirp(path.dirname(recycleLoc), (err) ->
+                  fs.mkdir(path.dirname(recycleLoc), { recursive: true }, (err) ->
                     if err then cb(err)
                     fs.rename(loc, recycleLoc, (err) ->
                       cb(err)
@@ -160,7 +178,7 @@ module.exports = exports = (argv) ->
                   cb(err)
                 )
               else
-                mkdirp(path.dirname(recycleLoc), (err) ->
+                fs.mkdir(path.dirname(recycleLoc), { recursive: true }, (err) ->
                   if err then cb(err)
                   copyFile(loc, recycleLoc, (err) ->
                     cb(err)
@@ -175,12 +193,10 @@ module.exports = exports = (argv) ->
           if exists
             load_parse(loc, cb, {plugin: undefined})
           else
-            defloc = path.join(argv.root, 'default-data', 'pages', file)
-            fs.exists(defloc, (exists) ->
-              if exists
+            tryDefaults(file, (defloc) ->
+              if defloc
                 load_parse(defloc, cb)
               else
-
                 glob "wiki-plugin-*/pages", {cwd: argv.packageDir}, (e, plugins) ->
                   if e then return cb(e)
 
@@ -217,7 +233,7 @@ module.exports = exports = (argv) ->
               cb(err)
             )
           else
-            mkdirp(path.dirname(loc), (err) ->
+            fs.mkdir(path.dirname(loc), { recursive: true }, (err) ->
               if err then cb(err)
               fs.writeFile(loc, page, (err) ->
                 if err

package/lib/search.coffee

@@ -11,8 +11,8 @@
 fs = require 'fs'
 path = require 'path'
 events = require 'events'
+url = require 'node:url'
 writeFileAtomic = require 'write-file-atomic'
-mkdirp = require 'mkdirp'
 
 miniSearch = require 'minisearch'
 
@@ -46,9 +46,6 @@ module.exports = exports = (argv) ->
 
   searchPageUpdate = (slug, page, cb) ->
     # to update we have to remove the page first, and then readd it
-    timeLabel = "SITE INDEX update #{slug} - #{wikiName}"
-    console.time timeLabel
-    
     try
       pageText = page.story.reduce( extractPageText, '')
     catch err
@@ -66,37 +63,29 @@ module.exports = exports = (argv) ->
         'title': page.title
         'content': pageText
       }
-    console.timeEnd timeLabel
     cb()
 
   searchPageRemove = (slug, cb) ->
     # remove page from index
     timeLabel = "SITE INDEX page remove #{slug} - #{wikiName}"
-    console.time timeLabel
     try
       siteIndex.discard slug 
     catch err
       # swallow error, if the page was not in index
       console.log "removing #{slug} from index #{wikiName} failed", err unless err.message.includes('not in the index')
-    console.timeEnd timeLabel
     cb()
 
   searchSave = (siteIndex, cb) ->
     # save index to file
-    timeLabel = "SITE INDEX #{wikiName} : Saved"
-    console.time timeLabel
-
     fs.exists argv.status, (exists) ->
       if exists
         writeFileAtomic siteIndexLoc, JSON.stringify(siteIndex), (e) ->
-          console.timeEnd timeLabel
           return cb(e) if e
           touch indexUpdateFlag, (err) ->
             cb()
       else
-        mkdirp argv.status, ->
+        fs.mkdir argv.status, { recursive: true }, ->
           writeFileAtomic siteIndexLoc, JSON.stringify(siteIndex), (e) ->
-            console.timeEnd timeLabel
             return cb(e) if e
             touch indexUpdateFlag, (err) ->
               cb()
@@ -104,8 +93,6 @@ module.exports = exports = (argv) ->
 
   searchRestore = (cb) ->
     # restore index, or create if it doesn't already exist
-    timeLabel = "SITE INDEX #{wikiName} : Restored"
-    console.time timeLabel
     fs.exists siteIndexLoc, (exists) ->
       if exists
         fs.readFile(siteIndexLoc, (err, data) ->
@@ -115,7 +102,6 @@ module.exports = exports = (argv) ->
               fields: ['title', 'content']
           catch e
             return cb(e)
-          console.timeEnd timeLabel
           process.nextTick( ->
             serial(queue.shift())))
 
@@ -145,24 +131,39 @@ module.exports = exports = (argv) ->
         console.log "SITE INDEX *** save failed: " + e if e
         itself.stop()
 
+  extractItemText = (text) ->
+    return text.replace(/\[([^\]]*?)\][\[\(].*?[\]\)]/g, " $1 ")
+      .replace(/\[{2}|\[(?:[\S]+)|\]{1,2}/g, ' ')
+      .replace(/\n/g, ' ')
+      .replace(/<style.*?<\/style>/g, ' ')
+      .replace(/<(?:"[^"]*"['"]*|'[^']*'['"]*|[^'">])+>/g, ' ')
+      .replace(/<(?:[^>])+>/g, ' ')
+      .replace(/(https?.*?)(?=\p{White_Space}|\p{Quotation_Mark}|$)/gu, (match) ->
+        myUrl = url.parse(match)
+        return myUrl.hostname + myUrl.pathname)
+      .replace(/[\p{P}\p{Emoji}\p{Symbol}}]+/gu, ' ')
+      .replace /[\p{White_Space}\n\t]+/gu, ' '
+
+
   extractPageText = (pageText, currentItem, currentIndex, array) ->
     # console.log('extractPageText', pageText, currentItem, currentIndex, array)
     try
       if currentItem.text?
         switch currentItem.type
-          when 'paragraph'
-            pageText += ' ' + currentItem.text.replace /\[{2}|\[(?:[\S]+)|\]{1,2}/g, ''
-          when 'markdown'
-            # really need to extract text from the markdown, but for now just remove link brackets, urls...
-            pageText += ' ' + currentItem.text.replace /\[{2}|\[(?:[\S]+)|\]{1,2}|\\n/g, ' '
-          when 'html'
-            pageText += ' ' + currentItem.text.replace /<[^\>]*>?/g, ''
-          else
-            if currentItem.text?
-              for line in currentItem.text.split /\r\n?|\n/
-                pageText += ' ' + line.replace /\[{2}|\[(?:[\S]+)|\]{1,2}/g, '' unless line.match /^[A-Z]+[ ].*/
+          when 'paragraph', 'markdown', 'html', 'reference', 'image', 'pagefold', 'math', 'mathjax', 'code'
+            pageText += extractItemText currentItem.text
+          when 'audio', 'video', 'frame'
+            pageText += extractItemText(currentItem.text.split(/\r\n?|\n/)
+              .map((line) ->
+                firstWord = line.split(/\p{White_Space}/u)[0]
+                if firstWord.startsWith('http') or firstWord.toUpperCase() is firstWord or firstWord.startsWith('//')
+                  # line is markup
+                  return ''
+                else
+                  return line
+              ).join(' '))
     catch err
-      throw new Error("Error extracting text from #{currentIndex}, #{err}")
+      throw new Error("Error extracting text from #{currentIndex}, #{JSON.stringify(currentItem)} #{err}, #{err.stack}")
     pageText
 
 
@@ -192,8 +193,8 @@ module.exports = exports = (argv) ->
     # we save the pagehandler, so we can recreate the site index if it is removed
     searchPageHandler = pagehandler if !searchPageHandler?
 
-    timeLabel = "SITE INDEX #{wikiName} : Created"
-    console.time timeLabel
+    #timeLabel = "SITE INDEX #{wikiName} : Created"
+    #console.time timeLabel
 
     pagehandler.slugs (e, slugs) ->
       if e
@@ -227,7 +228,7 @@ module.exports = exports = (argv) ->
   
       Promise.all(indexPromises)
       .then () ->
-        console.timeEnd timeLabel
+        # console.timeEnd timeLabel
         process.nextTick ( ->
           serial(queue.shift()))
       

package/lib/server.coffee

@@ -26,14 +26,20 @@ url = require 'url'
 { pipeline } = require 'node:stream/promises'
 
 # From npm
-mkdirp = require 'mkdirp'
 express = require 'express'
 hbs = require 'express-hbs'
 glob = require 'glob'
 async = require 'async'
 f = require('flates')
-sanitize = require '@mapbox/sanitize-caja'
-fetch = require 'node-fetch'
+
+createDOMPurify = require('dompurify')
+{ JSDOM } = require('jsdom')
+
+window = new JSDOM('').window
+DOMPurify = createDOMPurify(window)
+
+# node-fetch is now ESM only
+fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
 
 # Express 4 middleware
 logger = require 'morgan'
@@ -71,7 +77,7 @@ render = (page) ->
             f.p(resolveClient.resolveLinks(story.text or story.caption or 'uploaded image'))
         else if story.type is 'html'
           f.div {class: "item html"},
-          f.p(resolveClient.resolveLinks(story.text or '', sanitize))
+          f.p(resolveClient.resolveLinks(story.text or '', DOMPurify.sanitize))
         else f.div {class: "item"}, f.p(resolveClient.resolveLinks(story.text or ''))
       ).join('\n')
 
@@ -145,7 +151,7 @@ module.exports = exports = (argv) ->
     # assume http, as we know no better at this point and we need to specify a protocol.
     remoteURL = new URL("http://#{remote}/#{slug}.json").toString()
     # set a two second timeout
-    fetch(remoteURL, {timeout: 2000})
+    fetch(remoteURL, {signal: AbortSignal.timeout(2000)})
     .then (res) ->
       if res.ok
         return res
@@ -182,7 +188,10 @@ module.exports = exports = (argv) ->
 
     # use logger, at least in development, probably needs a param to configure (or turn off).
     # use stream to direct to somewhere other than stdout.
-  app.use(logger('tiny'))
+  wikiName = new URL(argv.url).hostname
+  logger.token('vhost', () ->
+    return wikiName)
+  app.use(logger(':vhost :method :url :status :res[content-length] - :response-time ms'))
   app.use(cookieParser())
   app.use(bodyParser.json({ limit: argv.uploadLimit}))
   app.use(bodyParser.urlencoded({ extended: true, limit: argv.uploadLimit}))
@@ -425,7 +434,7 @@ module.exports = exports = (argv) ->
           res.send('Favicon Saved')
 
       else
-        mkdirp argv.status, ->
+        fs.mkdir argv.status, { recursive: true }, ->
           fs.writeFile favLoc, buf, (e) ->
             if e then return res.e e
             res.send('Favicon Saved')

package/lib/sitemap.coffee

@@ -12,11 +12,8 @@ fs = require 'fs'
 path = require 'path'
 events = require 'events'
 writeFileAtomic = require 'write-file-atomic'
-_ = require 'lodash'
 xml2js = require 'xml2js'
 
-mkdirp = require 'mkdirp'
-
 synopsis = require 'wiki-client/lib/synopsis'
 
 asSlug = (name) ->
@@ -100,7 +97,7 @@ module.exports = exports = (argv) ->
     idx = slugs.indexOf(file)
 
     if ~idx
-      _.pullAt(sitemap, idx)
+      sitemap.splice(idx,1)
 
     cb()
 
@@ -111,7 +108,7 @@ module.exports = exports = (argv) ->
           return cb(e) if e
           cb()
       else
-        mkdirp argv.status, ->
+        fs.mkdir argv.status, { recursive: true }, ->
           writeFileAtomic sitemapLoc, JSON.stringify(sitemap), (e) ->
             return cb(e) if e
             cb()
@@ -134,7 +131,7 @@ module.exports = exports = (argv) ->
 
   xmlSitemapSave = (sitemap, cb) ->
     xmlmap = []
-    _.each sitemap, (page) ->
+    sitemap.forEach (page) ->
       result = {}
       result["loc"] = argv.url + "/" + page.slug + ".html"
       if page.date?
@@ -151,7 +148,7 @@ module.exports = exports = (argv) ->
           return cb(e) if e
           cb()
       else
-        mkdirp argv.status, ->
+        fs.mkdir argv.status, { recursive: true }, ->
           writeFileAtomic xmlSitemapLoc, xml, (e) ->
             return cb(e) if e
             cb()

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "wiki-server",
   "description": "A Federated Wiki Server",
-  "version": "0.23.0",
+  "version": "0.24.1",
   "author": {
     "name": "Ward Cunningham",
     "email": "ward@c2.com",
@@ -20,43 +20,42 @@
     }
   ],
   "dependencies": {
-    "@mapbox/sanitize-caja": "^0.1.4",
     "async": "^3.2.4",
     "body-parser": "^1.20.2",
     "client-sessions": "^0.8.0",
     "coffeescript": "^2.5.0",
     "cookie-parser": "^1.4.4",
+    "dompurify": "^3.1.0",
     "errorhandler": "^1.5.1",
     "express": "^4.17.1",
-    "express-hbs": "^2.3.3",
+    "express-hbs": "^2.5.0",
     "flates": "0.0.5",
     "glob": "^7.2.3",
-    "lodash": "^4.17.19",
+    "jsdom": "^24.0.0",
     "method-override": "^3.0.0",
-    "minisearch": "^6.0.1",
-    "mkdirp": "^0.5.6",
+    "minisearch": "^6.3.0",
     "morgan": "^1.9.1",
-    "node-fetch": "^2.6.8",
+    "node-fetch": "^3.3.2",
     "write-file-atomic": "^5.0.0",
-    "xml2js": "^0.5.0"
+    "xml2js": "^0.6.2"
   },
   "scripts": {
     "test": "grunt mochaTest"
   },
   "devDependencies": {
     "grunt": "^1.6.1",
-    "grunt-contrib-watch": "^1.0.0",
+    "grunt-contrib-watch": "^1.1.0",
     "grunt-git-authors": "^3.2.0",
-    "grunt-mocha-test": "^0.13.2",
-    "mocha": "^10.2.0",
+    "grunt-mocha-test": "^0.13.3",
+    "mocha": "^10.4.0",
     "should": "^13.2.3",
-    "supertest": "^6.3.3",
-    "wiki-client": "^0.27.1",
+    "supertest": "^6.3.4",
+    "wiki-client": "^0.29.1",
     "wiki-plugin-activity": "^0.5.0",
-    "wiki-plugin-video": "^0.3.0"
+    "wiki-plugin-video": "^0.3.1"
   },
   "engines": {
-    "node": ">=12.x"
+    "node": ">=18.x"
   },
   "testling": {
     "harness": "mocha"

package/test/sitemap.coffee

@@ -19,6 +19,7 @@ describe 'sitemap', ->
 
 
   request = request('http://localhost:55556')
+  fs.mkdirSync path.join('/tmp', 'sfwtests', testid, 'pages'), {recursive: true}
 
   # location of the sitemap
   sitemapLoc = path.join('/tmp', 'sfwtests', testid, 'status', 'sitemap.json')