wiki-server 0.25.9 → 0.26.0-rc.0

package/lib/sitemap.js

@@ -0,0 +1,288 @@
+/*
+ * Federated Wiki : Node Server
+ *
+ * Copyright Ward Cunningham and other contributors
+ * Licensed under the MIT license.
+ * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
+ */
+
+// **sitemap.coffee**
+
+const fs = require('fs')
+const path = require('path')
+const events = require('events')
+const writeFileAtomic = require('write-file-atomic')
+const xml2js = require('xml2js')
+
+const synopsis = require('wiki-client/lib/synopsis')
+
+const asSlug = name =>
+  name
+    .replace(/\s/g, '-')
+    .replace(/[^A-Za-z0-9-]/g, '')
+    .toLowerCase()
+
+module.exports = exports = argv => {
+  const wikiName = new URL(argv.url).hostname
+
+  let sitemap = []
+
+  const queue = []
+
+  let sitemapPageHandler = null
+
+  // ms since last update we will remove sitemap from memory
+  const sitemapTimeoutMs = 120000
+  let sitemapTimeoutHandler = null
+
+  const sitemapLoc = path.join(argv.status, 'sitemap.json')
+  const xmlSitemapLoc = path.join(argv.status, 'sitemap.xml')
+
+  let working = false
+
+  const lastEdit = journal => {
+    if (!journal) return undefined
+    // find the last journal entry, that is not a fork, with a date.
+    const last = journal.findLast(action => {
+      return action.date && action.type != 'fork'
+    })
+    return last ? last.date : undefined
+  }
+
+  const sitemapUpdate = (file, page, cb) => {
+    let pageLinks, pageLinksMap
+    const extractPageLinks = (collaborativeLinks, currentItem, currentIndex, array) => {
+      // extract collaborative links
+      // - this will need extending if we also extract the id of the item containing the link
+      try {
+        const linkRe = /\[\[([^\]]+)\]\]/g
+        let match = undefined
+        while ((match = linkRe.exec(currentItem.text)) != null) {
+          if (!collaborativeLinks.has(asSlug(match[1]))) {
+            collaborativeLinks.set(asSlug(match[1]), currentItem.id)
+          }
+        }
+        if ('reference' == currentItem.type) {
+          if (!collaborativeLinks.has(currentItem.slug)) {
+            collaborativeLinks.set(currentItem.slug, currentItem.id)
+          }
+        }
+      } catch (err) {
+        console.log(
+          `METADATA *** ${wikiName} Error extracting links from ${currentIndex} of ${JSON.stringify(array)}`,
+          err.message,
+        )
+      }
+      return collaborativeLinks
+    }
+    try {
+      pageLinksMap = page.story.reduce(extractPageLinks, new Map())
+    } catch (err) {
+      console.log(`METADATA *** ${wikiName} reduce to extract links on ${file} failed`, err.message)
+      pageLinksMap = []
+    }
+    //
+    if (pageLinksMap.size > 0) {
+      pageLinks = Object.fromEntries(pageLinksMap)
+    } else {
+      pageLinks = undefined
+    }
+
+    const entry = {
+      slug: file,
+      title: page.title,
+      date: lastEdit(page.journal),
+      synopsis: synopsis(page),
+      links: pageLinks,
+    }
+
+    const slugs = sitemap.map(page => page.slug)
+
+    const idx = slugs.indexOf(file)
+
+    if (~idx) {
+      sitemap[idx] = entry
+    } else {
+      sitemap.push(entry)
+    }
+    cb()
+  }
+
+  const sitemapRemovePage = (file, cb) => {
+    const slugs = sitemap.map(page => page.slug)
+    const idx = slugs.indexOf(file)
+
+    if (~idx) {
+      sitemap.splice(idx, 1)
+    }
+    cb()
+  }
+
+  const sitemapSave = (sitemap, cb) => {
+    fs.exists(argv.status, exists => {
+      if (exists) {
+        writeFileAtomic(sitemapLoc, JSON.stringify(sitemap), e => {
+          if (e) return cb(e)
+          cb()
+        })
+      } else
+        fs.mkdir(argv.status, { recursive: true }, () => {
+          writeFileAtomic(sitemapLoc, JSON.stringify(sitemap), e => {
+            if (e) return cb(e)
+            cb()
+          })
+        })
+    })
+  }
+
+  const sitemapRestore = cb => {
+    fs.exists(sitemapLoc, exists => {
+      if (exists) {
+        fs.readFile(sitemapLoc, (err, data) => {
+          if (err) return cb(err)
+          try {
+            sitemap = JSON.parse(data)
+          } catch (e) {
+            return cb(e)
+          }
+          process.nextTick(() => {
+            serial(queue.shift())
+          })
+        })
+      } else {
+        // sitemap file does not exist, so needs creating
+        itself.createSitemap(sitemapPageHandler)
+      }
+    })
+  }
+
+  const xmlSitemapSave = (sitemap, cb) => {
+    const xmlmapPages = []
+    sitemap.forEach(page => {
+      const result = {}
+      result['loc'] = argv.url + '/' + page.slug + '.html'
+      if (page.date) {
+        const date = new Date(page.date)
+        if (!isNaN(date.valueOf())) {
+          result['lastmod'] = date.toISOString().substring(0, 10)
+        }
+      }
+      xmlmapPages.push(result)
+    })
+    const xmlmap = { urlset: { $: { xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9' }, url: xmlmapPages } }
+    const builder = new xml2js.Builder()
+    const xml = builder.buildObject(xmlmap)
+    fs.exists(argv.status, exists => {
+      if (exists) {
+        writeFileAtomic(xmlSitemapLoc, xml, e => {
+          if (e) return cb(e)
+          cb()
+        })
+      } else {
+        fs.mkdir(argv.status, { recursive: true }, () => {
+          writeFileAtomic(xmlSitemapLoc, xml, e => {
+            if (e) return cb(e)
+            cb()
+          })
+        })
+      }
+    })
+  }
+
+  const serial = item => {
+    if (item) {
+      switch (item.action) {
+        case 'update':
+          itself.start()
+          sitemapUpdate(item.file, item.page, e => process.nextTick(() => serial(queue.shift())))
+          break
+        case 'remove':
+          itself.start()
+          sitemapRemovePage(item.file, e => process.nextTick(() => serial(queue.shift())))
+          break
+        default:
+          console.log(`Sitemap unexpected action ${item.action} for ${item.page} in ${wikiName}`)
+          process.nextTick(() => serial(queue.shift))
+      }
+    } else
+      sitemapSave(sitemap, e => {
+        if (e) console.log(`Problems saving sitemap ${wikiName}: ` + e)
+        itself.stop()
+      })
+    xmlSitemapSave(sitemap, e => {
+      if (e) console.log(`Problems saving sitemap(xml) ${wikiName}`) + e
+    })
+  }
+
+  // #### Public stuff ####
+
+  const itself = new events.EventEmitter()
+  itself.start = () => {
+    clearTimeout(sitemapTimeoutHandler)
+    working = true
+    itself.emit('working')
+  }
+  itself.stop = () => {
+    const clearsitemap = () => {
+      console.log(`removing sitemap ${wikiName} from memory`)
+      sitemap.length = 0
+      clearTimeout(sitemapTimeoutHandler)
+    }
+    // don't clear sitemap when in test environment. It just delays the tests completing.
+    if (!argv.test) sitemapTimeoutHandler = setTimeout(clearsitemap, sitemapTimeoutMs)
+    working = false
+    itself.emit('finished')
+  }
+  itself.isWorking = () => {
+    working
+  }
+
+  itself.createSitemap = pagehandler => {
+    itself.start()
+    // we save the pagehandler, so we can recreate the sitemap if it is removed
+    if (!sitemapPageHandler) sitemapPageHandler = pagehandler
+
+    pagehandler.pages((e, newsitemap) => {
+      if (e) {
+        console.log(`createSitemap ${wikiName} : error ` + e)
+        itself.stop()
+        return e
+      }
+      sitemap = newsitemap
+
+      process.nextTick(() => {
+        serial(queue.shift())
+      })
+    })
+  }
+
+  itself.removePage = file => {
+    const action = 'remove'
+    queue.push({ action, file })
+    if (sitemap.length === 0 && !working) {
+      itself.start()
+      sitemapRestore(e => {
+        if (e) console.log(`Problems restoring sitemap ${wikiName} : ` + e)
+        itself.createSitemap(sitemapPageHandler)
+      })
+    } else {
+      if (!working) serial(queue.shift())
+    }
+  }
+
+  itself.update = (file, page) => {
+    const action = 'update'
+    queue.push({ action, file, page })
+    if (sitemap.length === 0 && !working) {
+      itself.start()
+      sitemapRestore(e => {
+        if (e) console.log(`Problems restoring sitemap ${wikiName} : ` + e)
+        itself.createSitemap(sitemapPageHandler)
+      })
+    } else {
+      if (!working) serial(queue.shift())
+    }
+  }
+
+  return itself
+}

package/package.json

@@ -1,23 +1,33 @@
 {
   "name": "wiki-server",
   "description": "A Federated Wiki Server",
-  "version": "0.25.9",
+  "version": "0.26.0-rc.0",
   "author": {
     "name": "Ward Cunningham",
     "email": "ward@c2.com",
     "url": "http://ward.fed.wiki.org"
   },
   "contributors": [
-    {
-      "name": "Nick Niemeir",
-      "email": "nick.niemeir@gmail.com",
-      "url": "http://nrn.io"
-    },
-    {
-      "name": "Paul Rodwell",
-      "email": "paul.rodwell@btinternet.com",
-      "url": "http://wiki-paul90.rhcloud.com"
-    }
+    "Austin King <shout@ozten.com>",
+    "Paul Rodwell <paul.rodwell@btinternet.com>",
+    "Nicholas Hallahan <nick@theoutpost.io>",
+    "Tom Lee <github@tomlee.co>",
+    "Erkan Yilmaz <erkan77@gmail.com>",
+    "Patrick Mueller <pmuellr@apache.org>",
+    "Nick Niemeir <nick.niemeir@gmail.com>",
+    "Ward Cunningham <ward@c2.com>",
+    "Christian Smith <smith@anvil.io>",
+    "Gui13 <guitreize@gmail.com>",
+    "Merlyn Albery-Speyer <merlyn@newrelic.com>",
+    "Marcin Cieslak <saper@saper.info>",
+    "enyst <engel.nyst@gmail.com>",
+    "Peter deHaan <pdehaan@users.noreply.github.com>",
+    "winckell benjamin <winckell.benjamin.isart@gmail.com>",
+    "Eric Dobbs <eric@dobbse.net>",
+    "Joshua Benuck <joshua_benuck@yahoo.com>",
+    "Tom Lieber <tom@alltom.com>",
+    "Andrew Shell <andrew@andrewshell.org>",
+    "jon r <jon@allmende.io>"
   ],
   "dependencies": {
     "async": "^3.2.4",
@@ -27,7 +37,7 @@
     "cookie-parser": "^1.4.4",
     "dompurify": "^3.1.0",
     "errorhandler": "^1.5.1",
-    "express": "^4.17.1",
+    "express": "^4.21.2",
     "express-hbs": "^2.5.0",
     "flates": "0.0.5",
     "glob": "^7.2.3",
@@ -40,25 +50,24 @@
     "xml2js": "^0.6.2"
   },
   "scripts": {
-    "test": "grunt mochaTest"
+    "prettier:format": "prettier --write './**/*.js'",
+    "prettier:check": "prettier --check ./**/*.js",
+    "test": "node --test",
+    "update-authors": "node scripts/update-authors.js"
   },
   "devDependencies": {
-    "grunt": "^1.6.1",
-    "grunt-contrib-watch": "^1.1.0",
+    "@eslint/js": "^9.29.0",
+    "eslint": "^9.29.0",
+    "globals": "^16.1.0",
     "grunt-git-authors": "^3.2.0",
-    "grunt-mocha-test": "^0.13.3",
-    "mocha": "^11.1.0",
-    "should": "^13.2.3",
+    "prettier": "^3.5.3",
     "supertest": "^7.0.0",
-    "wiki-client": "^0.31",
+    "wiki-client": "^0.31.2",
     "wiki-plugin-activity": "0.7",
     "wiki-plugin-video": "^0.4"
   },
   "engines": {
-    "node": ">=18.x"
-  },
-  "testling": {
-    "harness": "mocha"
+    "node": ">=20.x"
   },
   "license": "MIT",
   "repository": {

package/.github/workflows/node.js.yml

@@ -1,31 +0,0 @@
-# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
-
-name: Node.js CI
-
-on:
-  push:
-    branches: [ "master" ]
-  pull_request:
-    branches: [ "master" ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        node-version: [18.x, 20.x, 22.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-    - run: npm ci
-    - run: npm run build --if-present
-    - run: npm test

package/.mailmap

@@ -1 +0,0 @@
-Nick Niemeir <nick.niemeir@gmail.com> <nniemeir@newrelic.com>

package/Gruntfile.js

@@ -1,55 +0,0 @@
-module.exports = function( grunt ) {
-
-  "use strict";
-
-  grunt.loadNpmTasks('grunt-contrib-watch');
-  grunt.loadNpmTasks('grunt-mocha-test');
-  grunt.loadNpmTasks('grunt-git-authors');
-
-  grunt.initConfig({
-    pkg: grunt.file.readJSON('package.json'),
-
-    mochaTest: {
-      test: {
-        options: {
-          reporter: 'spec',
-          require: [
-            'coffeescript/register',
-            'should'
-          ]
-        },
-        src: [
-          'test/defaultargs.coffee',
-          'test/page.coffee',
-          'test/random.coffee',
-          'test/server.coffee',
-          'test/sitemap.coffee'
-        ]
-      }
-    },
-
-    authors: {
-      prior: [
-        "Ward Cunningham <ward@c2.com>",
-        "Nick Niemeir <nick.niemeir@gmail.com>",
-        "Patrick Mueller <pmuellr@apache.org>",
-        "Erkan Yilmaz <erkan77@gmail.com>",
-        "Tom Lee <github@tomlee.co>",
-        "Nicholas Hallahan <nick@theoutpost.io>",
-        "Paul Rodwell <paul.rodwell@btinternet.com>",
-        "Austin King <shout@ozten.com>"
-      ]
-    },
-
-    watch: {
-      all: {
-        files: ['lib/*.coffee', 'test/*.coffee'],
-        tasks: ['mochaTest']
-      }
-    }
-  });
-
-  grunt.registerTask('default', ['mochaTest']);
-  grunt.registerTask('check', ['retire']);
-
-}

package/lib/defaultargs.coffee

@@ -1,77 +0,0 @@
-###
- * Federated Wiki : Node Server
- *
- * Copyright Ward Cunningham and other contributors
- * Licensed under the MIT license.
- * https://github.com/fedwiki/wiki-server/blob/master/LICENSE.txt
-###
-
-
-# **defaultargs.coffee** when called on the argv object this
-# module will create reasonable defaults for options not supplied,
-# based on what information is provided.
-path = require 'path'
-
-getUserHome = ->
-  process.env.HOME or process.env.HOMEPATH or process.env.USERPROFILE
-
-module.exports = (argv) ->
-  argv or= {}
-  argv.root or= __dirname
-  # the directory that contains all the packages that makeup the wiki
-  argv.packageDir or= path.join(argv.root, "..")
-  argv.port or= 3000
-  argv.home or= 'welcome-visitors'
-  argv.data or= path.join(getUserHome(), '.wiki') # see also cli
-  argv.client or= path.join(argv.packageDir, 'wiki-client', 'client')
-  argv.db or= path.join(argv.data, 'pages')
-  argv.status or= path.join(argv.data, 'status')
-  argv.assets or= path.join(argv.data, 'assets')
-  argv.recycler or= path.join(argv.data, 'recycle')
-  argv.commons or= path.join(argv.data, 'commons')
-  argv.url or= 'http://localhost' + (if argv.port is 80 then '' else ':' + argv.port)
-  argv.id or= path.join(argv.status, 'owner.json')
-  argv.uploadLimit or= '5mb'
-  argv.cookieSecret or= require('crypto').randomBytes(64).toString('hex')
-  argv.secure_cookie or= false
-  argv.session_duration or= 7
-  argv.neighbors or= ''
-  argv.debug or= false
-
-  if typeof(argv.database) is 'string'
-    argv.database = JSON.parse(argv.database)
-  argv.database or= {}
-  argv.database.type or= './page'
-  if argv.database.type.charAt(0) is '.'
-    if argv.database.type != './page'
-      console.log "\n\nWARNING: This storage option is depeciated."
-      console.log "    See ReadMe for details of the changes required.\n\n"
-  else
-    argv.database.type = 'wiki-storage-' + argv.database.type
-
-  argv.security_type or= './security'
-  if argv.security_type is './security'
-    console.log "\n\nINFORMATION: Using default security module."
-  else
-    argv.security_type = 'wiki-security-' + argv.security_type
-  argv.security_legacy or= false
-
-  #resolve all relative paths
-  argv.root = path.resolve(argv.root)
-  argv.packageDir = path.resolve(argv.packageDir)
-  argv.data = path.resolve(argv.data)
-  argv.client = path.resolve(argv.client)
-  argv.db = path.resolve(argv.db)
-  argv.status = path.resolve(argv.status)
-  argv.assets = path.resolve(argv.assets)
-  argv.recycler = path.resolve(argv.recycler)
-  argv.commons = path.resolve(argv.commons)
-  argv.id = path.resolve(argv.id)
-
-  if /node_modules/.test(argv.data)
-    console.log "\n\nWARNING : The dafault data path is not a safe place."
-    console.log "       : by using ", argv.data, " your pages will be lost when packages are updated."
-    console.log "       : You are strongly advised to use an alternative directory."
-    console.log "       : See the wiki package ReadMe for how to do this.\n\n"
-
-  argv