reachable-url 2.0.0 → 2.1.0

package/README.md

@@ -33,9 +33,31 @@ The target URL to be resolved.
 
 #### options
 
-Same as [got#options](https://github.com/sindresorhus/got#goturl-options)
+Same as [got#options](https://github.com/sindresorhus/got#goturl-options), plus:
 
-Passing `cache` keeps the whole body, since a cache entry is only written once the body has been read in full:
+##### maxBody
+
+Type: `number`<br>
+Default: `0`
+
+How many bytes of the body to keep when the download would otherwise be cancelled. A non-negative integer or `Infinity`; anything else throws.
+
+The default answers reachability from the status and headers alone, never reading a body. Ask for more when the bytes themselves decide something:
+
+```js
+// one byte is enough to tell an image from an HTML error page served as one
+const response = await reachableUrl('https://example.com/favicon.png', { maxBody: 1 })
+response.body[0] === 60 // => `<`, so the server answered with markup
+```
+
+Asking for more than one byte drops the `Range` header, since a server that honors it would answer with just that byte. `Infinity` is the extreme of that, and keeps the whole entity:
+
+```js
+const response = await reachableUrl('https://example.com/favicon.svg', { maxBody: Infinity })
+response.body // => the whole entity
+```
+
+Passing `cache` keeps the whole body too, since a cache entry is only written once the body has been read in full:
 
 ```js
 const cache = new Map()
@@ -55,7 +77,7 @@ overrides:
 
 The [got response](https://github.com/sindresorhus/got#response), plus `requestUrl`, `redirectUrls`, `redirectStatusCodes` and the `followRedirect` in effect.
 
-The request asks for a single byte (`Range: bytes=0-0`). When a server ignores that and starts sending the whole entity, the download is cancelled: the status and headers already say whether the URL is reachable, so `body` is `undefined` on those responses.
+By default the request asks for a single byte (`Range: bytes=0-0`, which [`maxBody`](#maxbody) drops when it wants more). When a server ignores that and starts sending the whole entity, the download is cancelled: the status and headers already say whether the URL is reachable, so `body` is `undefined` on those responses unless [`maxBody`](#maxbody) asked for some of it.
 
 A `206` that did answer the range is reported as the `200` it stands for, with `content-length` taken from `content-range`.
 

package/index.js

@@ -67,14 +67,48 @@ const willRetry = res => {
 
 // `responseType: 'buffer'` would read the whole entity a server sent despite
 // `Range`; the status and headers already answer reachability. A cache entry is
-// written on `end` and a retry needs a retryable error, so both keep the body.
-const shouldAbortDownload = res =>
-  !res.request.options.cache && !honoredRange(res) && !willRetry(res)
+// written on `end`, a retry needs a retryable error, and `maxBody: Infinity`
+// asked for the entity, so all three keep the body.
+const shouldAbortDownload = (res, maxBody) =>
+  maxBody !== Infinity && !res.request.options.cache && !honoredRange(res) && !willRetry(res)
+
+// Asking for a range while wanting more of the body than it holds is
+// contradictory: a server honoring it would answer with that one byte.
+const withoutRange = opts => ({ ...opts, headers: { ...opts.headers, Range: undefined } })
+
+const cancelDownload = (req, res) => {
+  // `phases.total` is filled on `end`, which a cancelled request never emits,
+  // and the timer does not record the cancel either.
+  res.timings.phases.total = Date.now() - res.timings.start
+  req.cancel()
+}
+
+const keepFirstBytes = (req, res, maxBody) => {
+  const chunks = []
+  let read = 0
+  // The cancel is not immediate, so chunks already in flight keep arriving.
+  const onData = chunk => {
+    chunks.push(chunk)
+    if ((read += chunk.length) < maxBody) return
+    res.off('data', onData)
+    res.body = Buffer.concat(chunks, maxBody)
+    cancelDownload(req, res)
+  }
+  res.on('data', onData)
+}
+
+// The count reaches `Buffer.concat` inside a stream callback, where a throw is
+// uncatchable, so a value it cannot take is refused before the socket opens.
+const assertMaxBody = maxBody => {
+  if (maxBody === Infinity || (Number.isInteger(maxBody) && maxBody >= 0)) return
+  throw new TypeError('`maxBody` needs to be a non-negative integer or `Infinity`.')
+}
 
-const reachableUrl = async (url, opts) => {
-  if (opts?.cache) assertCacheSupport()
-  const followRedirect = opts?.followRedirect ?? got.defaults.options.followRedirect
-  const req = got(url, opts)
+const reachableUrl = async (url, { maxBody = 0, ...opts } = {}) => {
+  assertMaxBody(maxBody)
+  if (opts.cache) assertCacheSupport()
+  const followRedirect = opts.followRedirect ?? got.defaults.options.followRedirect
+  const req = got(url, maxBody > RANGE_LENGTH ? withoutRange(opts) : opts)
 
   const redirectStatusCodes = []
   const redirectUrls = []
@@ -82,11 +116,9 @@ const reachableUrl = async (url, opts) => {
 
   req.on('response', res => {
     response = res
-    if (!shouldAbortDownload(res)) return
-    // `phases.total` is filled on `end`, which a cancelled request never emits,
-    // and the timer does not record the cancel either.
-    res.timings.phases.total = Date.now() - res.timings.start
-    req.cancel()
+    if (!shouldAbortDownload(res, maxBody)) return
+    if (maxBody === 0) return cancelDownload(req, res)
+    keepFirstBytes(req, res, maxBody)
   })
 
   req.on('redirect', res => {

package/package.json

@@ -2,7 +2,7 @@
   "name": "reachable-url",
   "description": "Resolve a URL as fast as possible and report whether the destination is reachable over HTTP.",
   "homepage": "https://github.com/Kikobeats/reachable-url",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "main": "index.js",
   "author": {
     "email": "josefrancisco.verdu@gmail.com",
@@ -50,7 +50,7 @@
     "standard-version": "latest"
   },
   "engines": {
-    "node": ">=8"
+    "node": ">=14"
   },
   "files": [
     "index.js"