reachable-url 1.8.4 → 2.1.0

package/README.md

@@ -33,7 +33,82 @@ The target URL to be resolved.
 
 #### options
 
-Same as [got#options](https://github.com/sindresorhus/got#goturl-options)
+Same as [got#options](https://github.com/sindresorhus/got#goturl-options), plus:
+
+##### maxBody
+
+Type: `number`<br>
+Default: `0`
+
+How many bytes of the body to keep when the download would otherwise be cancelled. A non-negative integer or `Infinity`; anything else throws.
+
+The default answers reachability from the status and headers alone, never reading a body. Ask for more when the bytes themselves decide something:
+
+```js
+// one byte is enough to tell an image from an HTML error page served as one
+const response = await reachableUrl('https://example.com/favicon.png', { maxBody: 1 })
+response.body[0] === 60 // => `<`, so the server answered with markup
+```
+
+Asking for more than one byte drops the `Range` header, since a server that honors it would answer with just that byte. `Infinity` is the extreme of that, and keeps the whole entity:
+
+```js
+const response = await reachableUrl('https://example.com/favicon.svg', { maxBody: Infinity })
+response.body // => the whole entity
+```
+
+Passing `cache` keeps the whole body too, since a cache entry is only written once the body has been read in full:
+
+```js
+const cache = new Map()
+const response = await reachableUrl('https://example.com/video.mp4', { cache })
+response.body // => the whole entity, so it can be cached
+```
+
+`cache` needs [@kikobeats/cacheable-request](https://github.com/Kikobeats/cacheable-request): the version [got](https://github.com/sindresorhus/got) pulls in never settles when the origin keeps the connection alive, and no timeout recovers from it. Declare the override, otherwise passing `cache` throws:
+
+```yaml
+# pnpm-workspace.yaml
+overrides:
+  got>cacheable-request: npm:@kikobeats/cacheable-request
+```
+
+#### returns
+
+The [got response](https://github.com/sindresorhus/got#response), plus `requestUrl`, `redirectUrls`, `redirectStatusCodes` and the `followRedirect` in effect.
+
+By default the request asks for a single byte (`Range: bytes=0-0`, which [`maxBody`](#maxbody) drops when it wants more). When a server ignores that and starts sending the whole entity, the download is cancelled: the status and headers already say whether the URL is reachable, so `body` is `undefined` on those responses unless [`maxBody`](#maxbody) asked for some of it.
+
+A `206` that did answer the range is reported as the `200` it stands for, with `content-length` taken from `content-range`.
+
+### reachableUrl.isReachable(response)
+
+#### response
+
+*Required*<br>
+Type: `object`
+
+The response returned by `reachableUrl`, which echoes back `followRedirect` so it can be handed straight over.
+
+A URL is reachable when the response is a final 2xx.
+
+A redirect status is the final answer only when redirects were not being followed:
+
+```js
+const response = await reachableUrl('https://example.com', { followRedirect: false })
+reachableUrl.isReachable(response) // => true, the 3xx is the destination
+```
+
+With redirect following on (the default), a 3xx is the hop the follow stopped at (a `beforeRedirect` hook threw, `maxRedirects` ran out), meaning the URL was never reached:
+
+```js
+const response = await reachableUrl('https://example.com', {
+  hooks: { beforeRedirect: [() => { throw new Error('refused') }] }
+})
+reachableUrl.isReachable(response) // => false
+```
+
+A partial object missing `followRedirect` is judged as if redirects were being followed, so a bare `{ statusCode: 302 }` is unreachable.
 
 ## License
 

package/index.js

@@ -1,14 +1,15 @@
 'use strict'
 
-const pReflect = require('p-reflect')
 const { URL } = require('url')
 
+const RANGE_LENGTH = 1
+
 const got = require('got').extend({
   decompress: false,
   responseType: 'buffer',
   retry: 1,
   headers: {
-    Range: 'bytes=0-0'
+    Range: `bytes=0-${RANGE_LENGTH - 1}`
   },
   hooks: {
     beforeRetry: [
@@ -19,16 +20,95 @@ const got = require('got').extend({
   }
 })
 
-const mergeResponse = (responseOrigin = {}, responseDestination = {}) => ({
+// `headers` is a prototype getter on IncomingMessage, so spreading never carries it.
+const toResponse = response => ({
   statusMessage: 'Not Found',
   statusCode: 404,
-  headers: { ...responseOrigin.headers, ...responseDestination.headers },
-  ...responseOrigin,
-  ...responseDestination
+  ...response,
+  headers: { ...response?.headers }
 })
 
-const reachableUrl = async (url, opts) => {
-  const req = got(url, opts)
+const CACHE_ERROR = `The \`cache\` option needs @kikobeats/cacheable-request.
+
+got resolved the unpatched cacheable-request@7, which never settles when the
+origin keeps the connection alive, and no timeout recovers from it. Add the
+override to your pnpm-workspace.yaml:
+
+  overrides:
+    got>cacheable-request: npm:@kikobeats/cacheable-request`
+
+const loadCacheableRequestManifest = () =>
+  require(require.resolve('cacheable-request/package.json', { paths: [require.resolve('got')] }))
+
+// The override is an alias, which keeps the real package name, so the manifest says
+// which one got resolved. Only a positive match rejects: the lookup can legitimately
+// fail under a bundler, and refusing to run then is worse than missing the warning.
+const assertCacheSupport = (load = loadCacheableRequestManifest) => {
+  let name
+  try {
+    name = load().name
+  } catch {}
+  if (name === 'cacheable-request') throw new TypeError(CACHE_ERROR)
+}
+
+const honoredRange = res =>
+  res.statusCode === 206 && Number(res.headers['content-length']) <= RANGE_LENGTH
+
+const willRetry = res => {
+  const { retry, method } = res.request.options
+  return (
+    res.retryCount < retry.limit &&
+    retry.methods.includes(method) &&
+    retry.statusCodes.includes(res.statusCode) &&
+    // got declines a 413 outright unless it carries a `retry-after`.
+    (res.statusCode !== 413 || res.headers['retry-after'] !== undefined)
+  )
+}
+
+// `responseType: 'buffer'` would read the whole entity a server sent despite
+// `Range`; the status and headers already answer reachability. A cache entry is
+// written on `end`, a retry needs a retryable error, and `maxBody: Infinity`
+// asked for the entity, so all three keep the body.
+const shouldAbortDownload = (res, maxBody) =>
+  maxBody !== Infinity && !res.request.options.cache && !honoredRange(res) && !willRetry(res)
+
+// Asking for a range while wanting more of the body than it holds is
+// contradictory: a server honoring it would answer with that one byte.
+const withoutRange = opts => ({ ...opts, headers: { ...opts.headers, Range: undefined } })
+
+const cancelDownload = (req, res) => {
+  // `phases.total` is filled on `end`, which a cancelled request never emits,
+  // and the timer does not record the cancel either.
+  res.timings.phases.total = Date.now() - res.timings.start
+  req.cancel()
+}
+
+const keepFirstBytes = (req, res, maxBody) => {
+  const chunks = []
+  let read = 0
+  // The cancel is not immediate, so chunks already in flight keep arriving.
+  const onData = chunk => {
+    chunks.push(chunk)
+    if ((read += chunk.length) < maxBody) return
+    res.off('data', onData)
+    res.body = Buffer.concat(chunks, maxBody)
+    cancelDownload(req, res)
+  }
+  res.on('data', onData)
+}
+
+// The count reaches `Buffer.concat` inside a stream callback, where a throw is
+// uncatchable, so a value it cannot take is refused before the socket opens.
+const assertMaxBody = maxBody => {
+  if (maxBody === Infinity || (Number.isInteger(maxBody) && maxBody >= 0)) return
+  throw new TypeError('`maxBody` needs to be a non-negative integer or `Infinity`.')
+}
+
+const reachableUrl = async (url, { maxBody = 0, ...opts } = {}) => {
+  assertMaxBody(maxBody)
+  if (opts.cache) assertCacheSupport()
+  const followRedirect = opts.followRedirect ?? got.defaults.options.followRedirect
+  const req = got(url, maxBody > RANGE_LENGTH ? withoutRange(opts) : opts)
 
   const redirectStatusCodes = []
   const redirectUrls = []
@@ -36,6 +116,9 @@ const reachableUrl = async (url, opts) => {
 
   req.on('response', res => {
     response = res
+    if (!shouldAbortDownload(res, maxBody)) return
+    if (maxBody === 0) return cancelDownload(req, res)
+    keepFirstBytes(req, res, maxBody)
   })
 
   req.on('redirect', res => {
@@ -43,34 +126,45 @@ const reachableUrl = async (url, opts) => {
     redirectStatusCodes.push(res.statusCode)
   })
 
-  const { isFulfilled, value, reason: error } = await pReflect(req)
+  let errorResponse
+  try {
+    await req
+  } catch (error) {
+    errorResponse = error.response
+  }
 
-  const mergedResponse = mergeResponse(isFulfilled ? value : error.response, response)
+  // A retried or cancelled request settles with the attempt before it, whose body
+  // is not this response's, so a response we saw outranks the error's. The error
+  // still carries one when nothing else did, as MaxRedirects does.
+  const resolvedResponse = toResponse(response ?? errorResponse)
 
-  if (mergedResponse.statusCode === 206) {
-    const contentRange = mergedResponse.headers['content-range']
+  if (resolvedResponse.statusCode === 206) {
+    const contentRange = resolvedResponse.headers['content-range']
     if (typeof contentRange === 'string') {
       let contentLength = contentRange.split('/')
       if (contentLength.length > 1) {
         contentLength = contentLength[contentLength.length - 1]
-        mergedResponse.statusCode = 200
-        mergedResponse.statusMessage = 'OK'
-        mergedResponse.headers['content-length'] = contentLength
-        mergedResponse.headers['content-range'] = undefined
+        resolvedResponse.statusCode = 200
+        resolvedResponse.statusMessage = 'OK'
+        resolvedResponse.headers['content-length'] = contentLength
+        resolvedResponse.headers['content-range'] = undefined
       }
     }
   }
 
   return {
     url,
-    ...mergedResponse,
+    ...resolvedResponse,
+    followRedirect,
     redirectUrls,
     redirectStatusCodes,
     requestUrl: url
   }
 }
 
-const isReachable = ({ statusCode }) => statusCode >= 200 && statusCode < 400
+// While following, a 3xx is the hop the follow stopped at, never the destination.
+const isReachable = ({ statusCode, followRedirect = true }) =>
+  statusCode >= 200 && statusCode < (followRedirect ? 300 : 400)
 
 module.exports = async (url, opts) => {
   if (/^\/\//.test(url)) url = `https:${url}`
@@ -78,3 +172,4 @@ module.exports = async (url, opts) => {
 }
 
 module.exports.isReachable = isReachable
+module.exports.assertCacheSupport = assertCacheSupport

package/package.json

@@ -2,7 +2,7 @@
   "name": "reachable-url",
   "description": "Resolve a URL as fast as possible and report whether the destination is reachable over HTTP.",
   "homepage": "https://github.com/Kikobeats/reachable-url",
-  "version": "1.8.4",
+  "version": "2.1.0",
   "main": "index.js",
   "author": {
     "email": "josefrancisco.verdu@gmail.com",
@@ -30,8 +30,7 @@
     "url"
   ],
   "dependencies": {
-    "got": "~11.8.0",
-    "p-reflect": "~2.1.0"
+    "got": "~11.8.0"
   },
   "devDependencies": {
     "@commitlint/cli": "latest",
@@ -51,7 +50,7 @@
     "standard-version": "latest"
   },
   "engines": {
-    "node": ">=8"
+    "node": ">=14"
   },
   "files": [
     "index.js"