solid-server 5.6.9-beta

package/lib/handlers/cors-proxy.js

@@ -0,0 +1,95 @@
+/* eslint-disable node/no-deprecated-api */
+
+module.exports = addCorsProxyHandler
+
+const { createProxyMiddleware } = require('http-proxy-middleware')
+const cors = require('cors')
+const debug = require('../debug')
+const url = require('url')
+const dns = require('dns')
+const isIp = require('is-ip')
+const ipRange = require('ip-range-check')
+const validUrl = require('valid-url')
+
+const CORS_SETTINGS = {
+  methods: 'GET',
+  exposedHeaders: 'Authorization, User, Location, Link, Vary, Last-Modified, Content-Length, Content-Location, MS-Author-Via, X-Powered-By',
+  maxAge: 1728000,
+  origin: true
+}
+const PROXY_SETTINGS = {
+  target: 'dynamic',
+  logLevel: 'silent',
+  changeOrigin: true,
+  followRedirects: true,
+  proxyTimeout: 10000,
+  router: req => req.destination.target,
+  pathRewrite: (path, req) => req.destination.path
+}
+// https://en.wikipedia.org/wiki/Reserved_IP_addresses
+const RESERVED_IP_RANGES = [
+  '127.0.0.0/8', // loopback
+  '::1/128', // loopback
+  '0.0.0.0/8', // current network (only valid as source address)
+  '169.254.0.0/16', // link-local
+  '10.0.0.0/8', // private network
+  '100.64.0.0/10', // Shared Address Space
+  '172.16.0.0/12', // private network
+  '192.0.0.0/24', // IETF Protocol Assignments
+  '192.0.2.0/24', // TEST-NET-1, documentation and examples
+  '192.88.99.0/24', // IPv6 to IPv4 relay (includes 2002::/16)
+  '192.168.0.0/16', // private network
+  '198.18.0.0/15', // network benchmark tests
+  '198.51.100.0/24', // TEST-NET-2, documentation and examples
+  '203.0.113.0/24', // TEST-NET-3, documentation and examples
+  '224.0.0.0/4', // IP multicast (former Class D network)
+  '240.0.0.0/4', // reserved (former Class E network)
+  '255.255.255.255', // broadcast
+  '64:ff9b::/96', // IPv4/IPv6 translation (RFC 6052)
+  '100::/64', // discard prefix (RFC 6666)
+  '2001::/32', // Teredo tunneling
+  '2001:10::/28', // deprecated (previously ORCHID
+  '2001:20::/28', // ORCHIDv2
+  '2001:db8::/32', // documentation and example source code
+  '2002::/16', // 6to4
+  'fc00::/7', // unique local address
+  'fe80::/10', // link-local address
+  'ff00::/8' // multicast
+]
+
+// Adds a CORS proxy handler to the application on the given path
+function addCorsProxyHandler (app, path) {
+  const corsHandler = cors(CORS_SETTINGS)
+  const proxyHandler = createProxyMiddleware(PROXY_SETTINGS)
+
+  debug.settings(`CORS proxy listening at ${path}?uri={uri}`)
+  app.get(path, extractProxyConfig, corsHandler, proxyHandler)
+}
+
+// Extracts proxy configuration parameters from the request
+function extractProxyConfig (req, res, next) {
+  // Retrieve and validate the destination URL
+  const uri = req.query.uri
+  debug.settings(`Proxy request for ${uri}`)
+  if (!validUrl.isUri(uri)) {
+    return res.status(400).send(`Invalid URL passed: ${uri || '(none)'}`)
+  }
+
+  // Parse the URL and retrieve its host's IP address
+  const { protocol, host, hostname, path } = url.parse(uri)
+  if (isIp(hostname)) {
+    addProxyConfig(null, hostname)
+  } else {
+    dns.lookup(hostname, addProxyConfig)
+  }
+
+  // Verifies and adds the proxy configuration to the request
+  function addProxyConfig (error, hostAddress) {
+    // Ensure the host is not a local IP
+    if (error || RESERVED_IP_RANGES.some(r => ipRange(hostAddress, r))) {
+      return res.status(400).send(`Cannot proxy ${uri}`)
+    }
+    req.destination = { path, target: `${protocol}//${host}` }
+    next()
+  }
+}

package/lib/handlers/delete.js

@@ -0,0 +1,23 @@
+module.exports = handler
+
+const debug = require('../debug').handlers
+
+async function handler (req, res, next) {
+  debug('DELETE -- Request on' + req.originalUrl)
+
+  const ldp = req.app.locals.ldp
+  try {
+    await ldp.delete(req)
+    debug('DELETE -- Ok.')
+    res.sendStatus(200)
+    next()
+  } catch (err) {
+    debug('DELETE -- Failed to delete: ' + err)
+
+    // method DELETE not allowed
+    if (err.status === 405) {
+      res.set('allow', 'OPTIONS, HEAD, GET, PATCH, POST, PUT')
+    }
+    next(err)
+  }
+}

package/lib/handlers/error-pages.js

@@ -0,0 +1,212 @@
+const debug = require('../debug').server
+const fs = require('fs')
+const util = require('../utils')
+const Auth = require('../api/authn')
+
+/**
+ * Serves as a last-stop error handler for all other middleware.
+ *
+ * @param err {Error}
+ * @param req {IncomingRequest}
+ * @param res {ServerResponse}
+ * @param next {Function}
+ */
+function handler (err, req, res, next) {
+  debug('Error page because of:', err)
+
+  const locals = req.app.locals
+  const authMethod = locals.authMethod
+  const ldp = locals.ldp
+
+  // If the user specifies this function,
+  // they can customize the error programmatically
+  if (ldp.errorHandler) {
+    debug('Using custom error handler')
+    return ldp.errorHandler(err, req, res, next)
+  }
+
+  const statusCode = statusCodeFor(err, req, authMethod)
+  switch (statusCode) {
+    case 401:
+      setAuthenticateHeader(req, res, err)
+      renderLoginRequired(req, res, err)
+      break
+    case 403:
+      renderNoPermission(req, res, err)
+      break
+    default:
+      if (ldp.noErrorPages) {
+        sendErrorResponse(statusCode, res, err)
+      } else {
+        sendErrorPage(statusCode, res, err, ldp)
+      }
+  }
+}
+
+/**
+ * Returns the HTTP status code for a given request error.
+ *
+ * @param err {Error}
+ * @param req {IncomingRequest}
+ * @param authMethod {string}
+ *
+ * @returns {number}
+ */
+function statusCodeFor (err, req, authMethod) {
+  let statusCode = err.status || err.statusCode || 500
+
+  if (authMethod === 'oidc') {
+    statusCode = Auth.oidc.statusCodeOverride(statusCode, req)
+  }
+
+  return statusCode
+}
+
+/**
+ * Dispatches the writing of the `WWW-Authenticate` response header (used for
+ * 401 Unauthorized responses).
+ *
+ * @param req {IncomingRequest}
+ * @param res {ServerResponse}
+ * @param err {Error}
+ */
+function setAuthenticateHeader (req, res, err) {
+  const locals = req.app.locals
+  const authMethod = locals.authMethod
+
+  switch (authMethod) {
+    case 'oidc':
+      Auth.oidc.setAuthenticateHeader(req, res, err)
+      break
+    case 'tls':
+      Auth.tls.setAuthenticateHeader(req, res)
+      break
+    default:
+      break
+  }
+}
+
+/**
+ * Sends the HTTP status code and error message in the response.
+ *
+ * @param statusCode {number}
+ * @param res {ServerResponse}
+ * @param err {Error}
+ */
+function sendErrorResponse (statusCode, res, err) {
+  res.status(statusCode)
+  res.header('Content-Type', 'text/plain;charset=utf-8')
+  res.send(err.message + '\n')
+}
+
+/**
+ * Sends the HTTP status code and error message as a custom error page.
+ *
+ * @param statusCode {number}
+ * @param res {ServerResponse}
+ * @param err {Error}
+ * @param ldp {LDP}
+ */
+function sendErrorPage (statusCode, res, err, ldp) {
+  const errorPage = ldp.errorPages + statusCode.toString() + '.html'
+
+  return new Promise((resolve) => {
+    fs.readFile(errorPage, 'utf8', (readErr, text) => {
+      if (readErr) {
+        // Fall back on plain error response
+        return resolve(sendErrorResponse(statusCode, res, err))
+      }
+
+      res.status(statusCode)
+      res.header('Content-Type', 'text/html')
+      res.send(text)
+      resolve()
+    })
+  })
+}
+
+/**
+ * Renders the databrowser
+ *
+ * @param req {IncomingRequest}
+ * @param res {ServerResponse}
+ */
+function renderDataBrowser (req, res) {
+  res.set('Content-Type', 'text/html')
+  const ldp = req.app.locals.ldp
+  const defaultDataBrowser = require.resolve('mashlib/dist/databrowser.html')
+  const dataBrowserPath = ldp.dataBrowserPath === 'default' ? defaultDataBrowser : ldp.dataBrowserPath
+  debug('   sending data browser file: ' + dataBrowserPath)
+  const dataBrowserHtml = fs.readFileSync(dataBrowserPath, 'utf8')
+  // Note: This must be done instead of sendFile because the test suite doesn't accept 412 responses
+  res.set('content-type', 'text/html')
+  res.send(dataBrowserHtml)
+}
+
+/**
+ * Renders a 401 response explaining that a login is required.
+ *
+ * @param req {IncomingRequest}
+ * @param res {ServerResponse}
+ */
+function renderLoginRequired (req, res, err) {
+  const currentUrl = util.fullUrlForReq(req)
+  debug(`Display login-required for ${currentUrl}`)
+  res.statusMessage = err.message
+  res.status(401)
+  if (req.accepts('html')) {
+    renderDataBrowser(req, res)
+  } else {
+    res.send('Not Authenticated')
+  }
+}
+
+/**
+ * Renders a 403 response explaining that the user has no permission.
+ *
+ * @param req {IncomingRequest}
+ * @param res {ServerResponse}
+ */
+function renderNoPermission (req, res, err) {
+  const currentUrl = util.fullUrlForReq(req)
+  debug(`Display no-permission for ${currentUrl}`)
+  res.statusMessage = err.message
+  res.status(403)
+  if (req.accepts('html')) {
+    renderDataBrowser(req, res)
+  } else {
+    res.send('Not Authorized')
+  }
+}
+
+/**
+ * Returns a response body for redirecting browsers to a Select Provider /
+ * login workflow page. Uses either a JS location.href redirect or an
+ * http-equiv type html redirect for no-script conditions.
+ *
+ * @param url {string}
+ *
+ * @returns {string} Response body
+ */
+function redirectBody (url) {
+  return `<!DOCTYPE HTML>
+<meta charset="UTF-8">
+<script>
+  window.location.href = "${url}" + encodeURIComponent(window.location.hash)
+</script>
+<noscript>
+  <meta http-equiv="refresh" content="0; url=${url}">
+</noscript>
+<title>Redirecting...</title>
+If you are not redirected automatically,
+follow the <a href='${url}'>link to login</a>
+`
+}
+
+module.exports = {
+  handler,
+  redirectBody,
+  sendErrorPage,
+  sendErrorResponse,
+  setAuthenticateHeader
+}

package/lib/handlers/get.js

@@ -0,0 +1,219 @@
+/* eslint-disable no-mixed-operators, no-async-promise-executor */
+
+module.exports = handler
+
+const fs = require('fs')
+const glob = require('glob')
+const _path = require('path')
+const $rdf = require('rdflib')
+const Negotiator = require('negotiator')
+const mime = require('mime-types')
+
+const debug = require('debug')('solid:get')
+const debugGlob = require('debug')('solid:glob')
+const allow = require('./allow')
+
+const translate = require('../utils.js').translate
+const error = require('../http-error')
+
+const RDFs = require('../ldp').mimeTypesAsArray()
+
+async function handler (req, res, next) {
+  const ldp = req.app.locals.ldp
+  const includeBody = req.method === 'GET'
+  const negotiator = new Negotiator(req)
+  const baseUri = ldp.resourceMapper.resolveUrl(req.hostname, req.path)
+  const path = res.locals.path || req.path
+  const requestedType = negotiator.mediaType()
+  const possibleRDFType = negotiator.mediaType(RDFs)
+
+  res.header('MS-Author-Via', 'SPARQL')
+
+  // Set live updates
+  if (ldp.live) {
+    res.header('Updates-Via', ldp.resourceMapper.resolveUrl(req.hostname).replace(/^http/, 'ws'))
+  }
+
+  debug(req.originalUrl + ' on ' + req.hostname)
+
+  const options = {
+    hostname: req.hostname,
+    path: path,
+    includeBody: includeBody,
+    possibleRDFType: possibleRDFType,
+    range: req.headers.range,
+    contentType: req.headers.accept
+  }
+
+  let ret
+  try {
+    ret = await ldp.get(options, req.accepts(['html', 'turtle', 'rdf+xml', 'n3', 'ld+json']) === 'html')
+  } catch (err) {
+    // use globHandler if magic is detected
+    if (err.status === 404 && glob.hasMagic(path)) {
+      debug('forwarding to glob request')
+      return globHandler(req, res, next)
+    } else {
+      debug(req.method + ' -- Error: ' + err.status + ' ' + err.message)
+      return next(err)
+    }
+  }
+
+  let stream
+  let contentType
+  let container
+  let contentRange
+  let chunksize
+
+  if (ret) {
+    stream = ret.stream
+    contentType = ret.contentType
+    container = ret.container
+    contentRange = ret.contentRange
+    chunksize = ret.chunksize
+  }
+
+  // Till here it must exist
+  if (!includeBody) {
+    debug('HEAD only')
+    res.setHeader('Content-Type', ret.contentType)
+    return res.status(200).send('OK')
+  }
+
+  // Handle dataBrowser
+  if (requestedType && requestedType.includes('text/html')) {
+    const { path: filename } = await ldp.resourceMapper.mapUrlToFile({ url: options })
+    const mimeTypeByExt = mime.lookup(_path.basename(filename))
+    const isHtmlResource = mimeTypeByExt && mimeTypeByExt.includes('html')
+    const useDataBrowser = ldp.dataBrowserPath && (
+      container ||
+      RDFs.includes(contentType) && !isHtmlResource && !ldp.suppressDataBrowser)
+
+    if (useDataBrowser) {
+      res.set('Content-Type', 'text/html')
+      const defaultDataBrowser = require.resolve('mashlib/dist/databrowser.html')
+      const dataBrowserPath = ldp.dataBrowserPath === 'default' ? defaultDataBrowser : ldp.dataBrowserPath
+      debug('   sending data browser file: ' + dataBrowserPath)
+      res.sendFile(dataBrowserPath)
+      return
+    } else if (stream) {
+      res.setHeader('Content-Type', contentType)
+      return stream.pipe(res)
+    }
+  }
+
+  // If request accepts the content-type we found
+  if (stream && negotiator.mediaType([contentType])) {
+    res.setHeader('Content-Type', contentType)
+    if (contentRange) {
+      const headers = { 'Content-Range': contentRange, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize }
+      res.writeHead(206, headers)
+      return stream.pipe(res)
+    } else {
+      return stream.pipe(res)
+    }
+  }
+
+  // If it is not in our RDFs we can't even translate,
+  // Sorry, we can't help
+  if (!possibleRDFType) {
+    return next(error(406, 'Cannot serve requested type: ' + contentType))
+  }
+
+  try {
+    // Translate from the contentType found to the possibleRDFType desired
+    const data = await translate(stream, baseUri, contentType, possibleRDFType)
+    debug(req.originalUrl + ' translating ' + contentType + ' -> ' + possibleRDFType)
+    res.setHeader('Content-Type', possibleRDFType)
+    res.send(data)
+    return next()
+  } catch (err) {
+    debug('error translating: ' + req.originalUrl + ' ' + contentType + ' -> ' + possibleRDFType + ' -- ' + 500 + ' ' + err.message)
+    return next(error(500, 'Error translating between RDF formats'))
+  }
+}
+
+async function globHandler (req, res, next) {
+  const { ldp } = req.app.locals
+
+  // Ensure this is a glob for all files in a single folder
+  // https://github.com/solid/solid-spec/pull/148
+  const requestUrl = await ldp.resourceMapper.getRequestUrl(req)
+  if (!/^[^*]+\/\*$/.test(requestUrl)) {
+    return next(error(404, 'Unsupported glob pattern'))
+  }
+
+  // Extract the folder on the file system from the URL glob
+  const folderUrl = requestUrl.substr(0, requestUrl.length - 1)
+  const folderPath = (await ldp.resourceMapper.mapUrlToFile({ url: folderUrl, searchIndex: false })).path
+
+  const globOptions = {
+    noext: true,
+    nobrace: true,
+    nodir: true
+  }
+
+  glob(`${folderPath}*`, globOptions, async (err, matches) => {
+    if (err || matches.length === 0) {
+      debugGlob('No files matching the pattern')
+      return next(error(404, 'No files matching glob pattern'))
+    }
+
+    // Matches found
+    const globGraph = $rdf.graph()
+
+    debugGlob('found matches ' + matches)
+    await Promise.all(matches.map(match => new Promise(async (resolve, reject) => {
+      const urlData = await ldp.resourceMapper.mapFileToUrl({ path: match, hostname: req.hostname })
+      fs.readFile(match, { encoding: 'utf8' }, function (err, fileData) {
+        if (err) {
+          debugGlob('error ' + err)
+          return resolve()
+        }
+        // Files should be Turtle
+        if (urlData.contentType !== 'text/turtle') {
+          return resolve()
+        }
+        // The agent should have Read access to the file
+        hasReadPermissions(match, req, res, function (allowed) {
+          if (allowed) {
+            try {
+              $rdf.parse(fileData, globGraph, urlData.url, 'text/turtle')
+            } catch (parseErr) {
+              debugGlob(`error parsing ${match}: ${parseErr}`)
+            }
+          }
+          return resolve()
+        })
+      })
+    })))
+
+    const data = $rdf.serialize(undefined, globGraph, requestUrl, 'text/turtle')
+    // TODO this should be added as a middleware in the routes
+    res.setHeader('Content-Type', 'text/turtle')
+    debugGlob('returning turtle')
+
+    res.send(data)
+    next()
+  })
+}
+
+// TODO: get rid of this ugly hack that uses the Allow handler to check read permissions
+function hasReadPermissions (file, req, res, callback) {
+  const ldp = req.app.locals.ldp
+
+  if (!ldp.webid) {
+    // FIXME: what is the rule that causes
+    // "Unexpected literal in error position of callback" in `npm run standard`?
+    // eslint-disable-next-line
+    return callback(true)
+  }
+
+  const root = ldp.resourceMapper.resolveFilePath(req.hostname)
+  const relativePath = '/' + _path.relative(root, file)
+  res.locals.path = relativePath
+  // FIXME: what is the rule that causes
+  // "Unexpected literal in error position of callback" in `npm run standard`?
+  // eslint-disable-next-line
+  allow('Read')(req, res, err => callback(!err))
+}

package/lib/handlers/index.js

@@ -0,0 +1,42 @@
+/* eslint-disable node/no-deprecated-api */
+
+module.exports = handler
+
+const path = require('path')
+const debug = require('debug')('solid:index')
+const Negotiator = require('negotiator')
+const url = require('url')
+const URI = require('urijs')
+
+async function handler (req, res, next) {
+  const indexFile = 'index.html'
+  const ldp = req.app.locals.ldp
+  const negotiator = new Negotiator(req)
+  const requestedType = negotiator.mediaType()
+
+  try {
+    const { path: filename } = await ldp.resourceMapper.mapUrlToFile({ url: req })
+
+    const stats = await ldp.stat(filename)
+    if (!stats.isDirectory()) {
+      return next()
+    }
+    // redirect to the right container if missing trailing /
+    if (req.path.lastIndexOf('/') !== req.path.length - 1) {
+      return res.redirect(301, URI.joinPaths(req.path, '/').toString())
+    }
+
+    if (requestedType && requestedType.indexOf('text/html') !== 0) {
+      return next()
+    }
+    debug('Looking for index in ' + req.path)
+
+    // Check if file exists in first place
+    await ldp.exists(req.hostname, path.join(req.path, indexFile))
+    res.locals.path = url.resolve(req.path, indexFile)
+    debug('Found an index for current path')
+  } catch (e) {
+    // Ignore errors
+  }
+  next()
+}

package/lib/handlers/options.js

@@ -0,0 +1,33 @@
+/* eslint-disable node/no-deprecated-api */
+
+const addLink = require('../header').addLink
+const url = require('url')
+
+module.exports = handler
+
+function handler (req, res, next) {
+  linkServiceEndpoint(req, res)
+  linkAuthProvider(req, res)
+  linkSparqlEndpoint(res)
+
+  res.status(204)
+
+  next()
+}
+
+function linkAuthProvider (req, res) {
+  const locals = req.app.locals
+  if (locals.authMethod === 'oidc') {
+    const oidcProviderUri = locals.host.serverUri
+    addLink(res, oidcProviderUri, 'http://openid.net/specs/connect/1.0/issuer')
+  }
+}
+
+function linkServiceEndpoint (req, res) {
+  const serviceEndpoint = url.resolve(req.app.locals.ldp.resourceMapper.resolveUrl(req.hostname, req.path), '.well-known/solid')
+  addLink(res, serviceEndpoint, 'service')
+}
+
+function linkSparqlEndpoint (res) {
+  res.header('Accept-Patch', 'application/sparql-update')
+}

package/lib/handlers/patch/n3-patch-parser.js

@@ -0,0 +1,49 @@
+// Parses a text/n3 patch
+
+module.exports = parsePatchDocument
+
+const $rdf = require('rdflib')
+const error = require('../../http-error')
+
+const PATCH_NS = 'http://www.w3.org/ns/solid/terms#'
+const PREFIXES = `PREFIX solid: <${PATCH_NS}>\n`
+
+// Parses the given N3 patch document
+async function parsePatchDocument (targetURI, patchURI, patchText) {
+  // Parse the N3 document into triples
+  const patchGraph = $rdf.graph()
+  try {
+    $rdf.parse(patchText, patchGraph, patchURI, 'text/n3')
+  } catch (err) {
+    throw error(400, `Patch document syntax error: ${err}`)
+  }
+
+  // Query the N3 document for insertions and deletions
+  let firstResult
+  try {
+    firstResult = await queryForFirstResult(patchGraph, `${PREFIXES}
+    SELECT ?insert ?delete ?where WHERE {
+      ?patch solid:patches <${targetURI}>.
+      OPTIONAL { ?patch solid:inserts ?insert. }
+      OPTIONAL { ?patch solid:deletes ?delete. }
+      OPTIONAL { ?patch solid:where   ?where.  }
+    }`)
+  } catch (err) {
+    throw error(400, `No patch for ${targetURI} found.`, err)
+  }
+
+  // Return the insertions and deletions as an rdflib patch document
+  const { '?insert': insert, '?delete': deleted, '?where': where } = firstResult
+  if (!insert && !deleted) {
+    throw error(400, 'Patch should at least contain inserts or deletes.')
+  }
+  return { insert, delete: deleted, where }
+}
+
+// Queries the store with the given SPARQL query and returns the first result
+function queryForFirstResult (store, sparql) {
+  return new Promise((resolve, reject) => {
+    const query = $rdf.SPARQLToQuery(sparql, false, store)
+    store.query(query, resolve, null, () => reject(new Error('No results.')))
+  })
+}

package/lib/handlers/patch/sparql-update-parser.js

@@ -0,0 +1,16 @@
+// Parses an application/sparql-update patch
+
+module.exports = parsePatchDocument
+
+const $rdf = require('rdflib')
+const error = require('../../http-error')
+
+// Parses the given SPARQL UPDATE document
+async function parsePatchDocument (targetURI, patchURI, patchText) {
+  const baseURI = patchURI.replace(/#.*/, '')
+  try {
+    return $rdf.sparqlUpdateParser(patchText, $rdf.graph(), baseURI)
+  } catch (err) {
+    throw error(400, `Patch document syntax error: ${err}`)
+  }
+}