shared-http-cache 1.0.3 → 1.0.5

package/index.js

@@ -9,11 +9,12 @@ class SharedHttpCache {
     }
     /**
      * Fetch multiple resources with HTTP cache support.
-     * @param {Array<{url:string,integrity?:string,options?:RequestInit,callback:(result:{buffer:Buffer,headers:Headers,fromCache:boolean})=>void}>} requests
-     * @returns {Promise<this|{url: string, headers?: Headers, error: Error }[]>}
+     * @param {Array<{url:string,integrity?:string,options?:RequestInit,callback:(response:{buffer:Buffer,headers:Headers,fromCache:boolean,index:number})=>void}>} requests
+     * @returns {Promise<this|{url:string,headers?:Headers,error:Error,index:number}[]>}
      * @see [RequestInit](https://developer.mozilla.org/en-US/docs/Web/API/RequestInit), [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) MDN references
      */
     async fetch(requests) {
+        if (!Array.isArray(requests)) return Promise.reject([{ error: new TypeError('requests must be an array.') }]);
         const errors = [];
         const parseHeader = (string) => {
             if (!string || typeof string !== 'string') return {};
@@ -47,8 +48,9 @@ class SharedHttpCache {
             return false;
         };
         await Promise.all(
-            requests.map(async (request) => {
+            requests.map(async (request, index) => {
                 const { url, options = {}, integrity, callback } = request;
+                if (typeof url !== 'string') return errors.push({ error: new Error('Malformed request, url undefined.'), index });
                 if (!options.method) options.method = 'GET';
                 if (!options.headers) options.headers = {};
                 Object.keys(options.headers).forEach((key) => /\p{Lu}/u.test(key) && ((options.headers[key.toLowerCase()] = options.headers[key]), delete options.headers[key]));
@@ -64,7 +66,7 @@ class SharedHttpCache {
                         if (!isFreshFile || requestCacheControl['no-cache'] || responseCacheControl['no-cache']) fromCache = false;
                         if (requestCacheControl['max-stale'] && (responseCacheControl['must-revalidate'] || responseCacheControl['proxy-revalidate'])) fromCache = false;
                         if (fromCache) {
-                            buffer = (await this.store.get(this.cacheDir, url)).data;
+                            buffer = integrity && !requestCacheControl['max-stale'] ? await this.store.get.byDigest(this.cacheDir, integrity) : (await this.store.get(this.cacheDir, url)).data;
                             headers = file.metadata.headers;
                         }
                     } else {
@@ -72,6 +74,7 @@ class SharedHttpCache {
                         if (requestCacheControl['only-if-cached']) throw new Error('HTTP error! status: 504 Only-If-Cached');
                     }
                     if (!fromCache) {
+                        if (!file && integrity) options.integrity = integrity;
                         if (file && file.metadata.headers['etag']) options.headers['if-none-match'] = file.metadata.headers['etag'];
                         if (file && file.metadata.headers['last-modified']) options.headers['if-modified-since'] = file.metadata.headers['last-modified'];
                         response = await fetch(url, options);
@@ -83,7 +86,7 @@ class SharedHttpCache {
                             buffer = Buffer.from(await response.arrayBuffer());
                             headers = Object.fromEntries(response.headers.entries());
                         } else {
-                            if (response.status === 410) {
+                            if (file && response.status === 410) {
                                 this.store.rm.entry(this.cacheDir, url, { removeFully: true });
                                 this.store.rm.content(this.cacheDir, file.integrity);
                             }
@@ -91,7 +94,7 @@ class SharedHttpCache {
                         }
                     }
                     // chance to preform content validation before saving it to disk
-                    if (typeof callback === 'function') callback({ buffer, headers, fromCache });
+                    if (typeof callback === 'function') callback({ buffer, headers, fromCache, index });
                     if (!fromCache || response?.status === 304) {
                         const responseCacheControl = parseHeader(headers['cache-control']);
                         if (options.method !== 'GET') return;
@@ -100,12 +103,13 @@ class SharedHttpCache {
                         if (requestCacheControl['no-store'] || options.headers['authorization']) return;
                         const store = async () => {
                             await this.store.rm.entry(this.cacheDir, url, { removeFully: true });
-                            await this.store.put(this.cacheDir, url, buffer, integrity ? { metadata: { headers }, integrity } : { metadata: { headers } });
+                            if (file && integrity && file.integrity !== integrity) this.store.rm.content(this.cacheDir, file.integrity);
+                            await this.store.put(this.cacheDir, url, buffer, integrity ? { metadata: { headers }, integrity, algorithms: [integrity.split('-')[0]] } : { metadata: { headers } });
                         };
                         this.awaitStorage ? await store() : store();
                     }
                 } catch (error) {
-                    errors.push({ url, headers, error });
+                    errors.push({ url, headers, error, index });
                 }
             }),
         );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shared-http-cache",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "Node.Js Utility for fetching multiple HTTP resources with browser-like cache management.",
   "keywords": [
     "http-cache",

package/readme.md

@@ -1,7 +1,9 @@
 ---
+
 title: Shared HTTP Cache
 
 description: Node.Js Utility for fetching multiple HTTP resources with browser-like cache management.
+
 ---
 
 ## Overview
@@ -15,7 +17,7 @@ This implementation models a shared HTTP cache that follows the semantics define
 The cache is shared (not private) and applies shared-cache rules.
 
 - Request `methods` other than `GET` are not stored.
-- Private responses (request `Authorization` header, response `Cache-Control="private"` and `Set-Cookie` headers) are not stored.
+- Private responses (request `Authorization`, response `Cache-Control="private"` and `Set-Cookie` headers) are not stored.
 - Variant responses (response header `Vary` present) are not stored.
 - Partial content (response `Content-Range` header) is not stored.
 - Time calculations rely exclusively on locally recorded timestamps, not on server-provided `Date`.
@@ -93,6 +95,16 @@ Even that request `Cache-Control` header's `max-stale` directive allows use of s
 
 On revalidation, if the cached content includes `ETag` or `Last-Modified` automatically `If-None-Match` and `If-Modified-Since` headers are added to the request. Revalidated entries are explicitly replaced during each successful fetch to avoid unbounded growth in the index.
 
+### Subresource integrity
+
+The [subresosurce integrity specifications](https://developer.mozilla.org/en-US/docs/Web/Security/Defenses/Subresource_Integrity) are implemented on both, fetch and storage:
+1. if a request does not provide an `integrity hash`, then `cacache` will compute it using its default algorithm: `sha512`, and subsequent requests may use that `hash` to retrieve the resource directly from cache using `store.get.byDigest` function along with the regular search by url `store.get`.
+1. if a request does provide an `integrity hash`:
+    - if the related resource is not stored, then `node:fetch` will use it to verify the incomming response,
+    - if the related resource is stored and fresh, or is stale but revalidated with origin server, then `cacache` will use the `hash` to:
+        - get the resource directly from cache if the stored `integrity hash` matches the one provided, or,
+        - recompute the path and rebase the resource on the new path if the provided `integrity hash` is different. In other words, multiple `integrity hashes` may validate a resource, but only the last provided `hash` is responsible for its storage path, as `cacache` can work with a single `algorithm` at a time. This situation may only be encountered when a resource was initially stored without an `integrity hash` provided in the request, or when a different `integrity hash` is provided in the request for the same resource. An exception for resource rebase is the case when a different `integrity hash` is provided in the request along with `Cache-Control` header's `max-stale` directive.
+
 ### Origin request outcomes
 
 When a request is sent to the origin:
@@ -191,7 +203,7 @@ Errors encountered during fetches are collected, and the returned `promise` eith
 The response is converted into a [Buffer](https://nodejs.org/api/buffer.html) served to `callback`, then stored in the `cache` along with the `response headers`.
 
 ```ts
-callback({ buffer: Buffer, headers: Headers, fromCache: boolean }) -> void
+callback({ buffer: Buffer, headers: Headers, fromCache: boolean, index: number }) -> void
 ```
 
 The `callback` provided for each request is executed before storing new content, allowing implementers to inspect, transform or validate the data before it's cached. The errors thrown by the `callback` are also catched and stored in the `errors` delivered by the `Promise.reject()`.
@@ -201,30 +213,29 @@ await sharedHttpCache
     .fetch([
         {
             url: 'https://example.com/data.txt',
-            callback: ({ buffer, headers, fromCache }) => {
+            callback: ({ buffer, headers, fromCache, index }) => {
                 console.log(buffer.toString());
                 console.log(headers);
-                console.log(fromCache);
+                console.log(index, fromCache);
             },
         },
     ])
-    .catch((errors) => errors.forEach((entry) => console.error(entry.url, entry.error)));
+    .catch((errors) => errors.forEach((entry) => console.error(entry.index, entry.url, entry.error.message)));
 ```
 
 ### Fetch multiple files
 
 ```js
 const urls = ['https://example.com/file1', 'https://example.com/file2'];
-const parser = ({ url, buffer, headers, fromCache }) => {
-    console.log(url);
+const parser = ({ url, buffer, headers, fromCache, index }) => {
+    console.log(index, fromCache, url);
     console.log(headers);
-    console.log(fromCache);
     console.log(buffer.toString());
 };
 
 const requests = urls.map((url) => ({ url, callback: (response) => parser({ ...response, url }) }));
 
-sharedHttpCache.fetch(requests).catch((errors) => errors.forEach((entry) => console.error(entry.url, entry.error)));
+sharedHttpCache.fetch(requests).catch((errors) => errors.forEach((entry) => console.error(entry.index, entry.url, entry.error.message)));
 ```
 
 ### Fetch with integrity
@@ -239,8 +250,6 @@ await sharedHttpCache.fetch([
 ]);
 ```
 
-**Note:** `integrity `affects storage, not `callback` execution.
-
 ### Fetch options
 
 ```ts
@@ -306,6 +315,16 @@ The underlying cache store (`cacache`) is exposed directly.
 sharedHttpCache.store -> cacache
 ```
 
+**Listing (example with promise)**
+
+```js
+sharedHttpCache
+    .fetch(requests)
+    .then((sharedHttpCache) => sharedHttpCache.store.ls(sharedHttpCache.cacheDir))
+    .then(console.log)
+    .catch((errors) => console.error('Errors:', errors));
+```
+
 **Compacting (example with await)**
 
 ```ts
@@ -313,20 +332,38 @@ sharedHttpCache.store.verify(cacheDir) -> Promise<Object>
 ```
 
 ```js
-await sharedHttpCache.store.verify(sharedHttpCache.cacheDir);
+// deadbeef collected, because of invalid checksum.
+sharedHttpCache.store.verify(sharedHttpCache.cacheDir).then((stats) => {
+    console.log('cache is much nicer now! stats:', stats);
+});
 ```
 
-**Listing (example with promise)**
+**Basic cleanup strategy**
 
 ```js
-sharedHttpCache
-    .fetch(requests)
-    .then((sharedHttpCache) => sharedHttpCache.store.ls(sharedHttpCache.cacheDir))
-    .then(console.log)
-    .catch((errors) => console.error('Errors:', errors));
+const SharedHttpCache = require('shared-http-cache');
+// only-if-cached also means ... and is not stale!
+(async () => {
+    const cache = new SharedHttpCache({ cacheDir: '.cache', awaitStorage: true });
+    const entries = await cache.store.ls(cache.cacheDir);
+    const requests = Object.keys(entries).map((url) => ({ url, options: { headers: { 'cache-control': 'only-if-cached' } } }));
+    await cache.fetch(requests).catch(async (errors) => {
+        for (const { url } of errors) {
+            const file = url && await cache.store.get.info(cache.cacheDir, url);
+            if (file) {
+                await cache.store.rm.entry(cache.cacheDir, url, { removeFully: true });
+                await cache.store.rm.content(cache.cacheDir, file.integrity);
+            }
+        }
+    });
+})();
 ```
 
-Other available operations
+**Note:**
+
+- This is a fully `RFC 9111` compliant strategy that cleans up all the resources that can be determined as expired based on the stored response headers. For a more flexible approach, `max-stale=$acceptedStaleness` directive can be used in conjunction with `only-if-cached`. Cleanup strategies that rely on empirical calculations, such as `least recently used`, are `NOT RECOMMENDED`.
+
+**Other available operations**
 
 - sharedHttpCache.store.put(...)
 - sharedHttpCache.store.get(...)
@@ -335,3 +372,11 @@ Other available operations
 - sharedHttpCache.store.rm.content(...)
 
 See full list of [cacache options](https://github.com/npm/cacache?tab=readme-ov-file#api).
+
+## Bottom line
+
+- `max-stale` is intended to be used: many servers enforce `max-age=0`, but clients usually know how much staleness they can tolerate. Using `max-stale` (recommended up to 24 h) can significantly reduce network requests.
+- providing `integrity` on requests enables fast loads by allowing cached content to be read directly from store.
+- `SharedHttpCache` instantiation with `awaitStorage: true` is important when `fetch` is continued with `store` actions.
+- private or sensitive content is served, but not stored.
+- cache cleanup and eviction are deliberately left to the consumer; a well-chosen cleanup strategy is essential for maintaining good performance.

package/z.js

@@ -1 +0,0 @@
-sharedHttpCache.fetch(requests).catch((errors) => errors.forEach((entry) => console.error(entry.url, entry.error)));