polystore 0.15.11 → 0.15.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "polystore",
-  "version": "0.15.11",
+  "version": "0.15.13",
   "description": "A small compatibility layer for many popular KV stores like localStorage, Redis, FileSystem, etc.",
   "homepage": "https://polystore.dev/",
   "repository": "https://github.com/franciscop/polystore.git",
@@ -56,6 +56,7 @@
     }
   },
   "jest": {
+    "testTimeout": 15000,
     "testEnvironment": "jsdom",
     "setupFiles": [
       "./test/setup.js"

package/readme.md

@@ -36,8 +36,8 @@ Available clients for the KV store:
 - [**Cookies** `"cookie"`](#cookies) (fe): persist the data using cookies
 - [**LocalForage** `localForage`](#local-forage) (fe): persist the data on IndexedDB
 - [**Fetch API** `"https://..."`](#fetch-api) (fe+be): call an API to save/retrieve the data
-- [**File** `new URL('file:///...')`](#file) (be): store the data in a single JSON file in your FS
-- [**Folder** `new URL('file:///...')`](#folder) (be): store each key in a folder as json files
+- [**File** `"file:///[...].json"`](#file) (be): store the data in a single JSON file in your FS
+- [**Folder** `"file:///[...]/"`](#folder) (be): store each key in a folder as json files
 - [**Redis Client** `redisClient`](#redis-client) (be): use the Redis instance that you connect to
 - [**Cloudflare KV** `env.KV_NAMESPACE`](#cloudflare-kv) (be): use Cloudflare's KV store
 - [**Level** `new Level('example', { valueEncoding: 'json' })`](#level) (fe+be): support the whole Level ecosystem
@@ -435,8 +435,8 @@ const store = kv(new Map());
 const session = kv(new Map());
 
 // Two file-stores
-const users = kv(new URL(`file://${import.meta.dirname}/users.json`));
-const books = kv(new URL(`file://${import.meta.dirname}/books.json`));
+const users = kv(`file://${import.meta.dirname}/users.json`);
+const books = kv(`file://${import.meta.dirname}/books.json`);
 ```
 
 The main reason this is not stable is because [_some_ store engines don't allow for atomic deletion of keys given a prefix](https://stackoverflow.com/q/4006324/938236). While we do still clear them internally in those cases, that is a non-atomic operation and it could have some trouble if some other thread is reading/writing the data _at the same time_.
@@ -646,32 +646,39 @@ console.log(await store.get("key1"));
 
 > Note: the API client expire resolution is in the seconds, so times shorter than 1 second like `expires: 0.02` (20 ms) don't make sense for this storage method and won't properly save them.
 
-> Note: see the reference implementation in src/server.js
+> Note: see the [reference implementation in src/server.js](https://github.com/franciscop/polystore/blob/master/src/server.js)
 
 
 ### File
 
-Treat a JSON file in your filesystem as the source for the KV store:
+Treat a JSON file in your filesystem as the source for the KV store. Pass it an absolute `file://` url or a `new URL('file://...')` instance:
 
 ```js
 import kv from "polystore";
 
-const store = kv(new URL("file:///Users/me/project/cache.json"));
+// Path is "/Users/me/project/cache.json"
+const store = kv("file:///Users/me/project/cache.json");
 
 await store.set("key1", "Hello world", { expires: "1h" });
 console.log(await store.get("key1"));
 // "Hello world"
 ```
 
-> Note: an extension is needed, to disambiguate with "folder"
+> Note: an extension is needed, to disambiguate with ["folder"](#folder)
 
 You can also create multiple stores:
 
 ```js
 // Paths need to be absolute, but you can use process.cwd() to make
 // it relative to the current process:
+const store1 = kv(`file://${process.cwd()}/cache.json`);
+const store2 = kv(`file://${import.meta.dirname}/data.json`);
+```
+
+You can also pass a `URL` instance:
+
+```js
 const store1 = kv(new URL(`file://${process.cwd()}/cache.json`));
-const store2 = kv(new URL(`file://${import.meta.dirname}/data.json`));
 ```
 
 <details>
@@ -705,7 +712,7 @@ Treat a single folder in your filesystem as the store, where each key is a file:
 ```js
 import kv from "polystore";
 
-const store = kv(new URL("file:///Users/me/project/data/"));
+const store = kv("file:///Users/me/project/data/");
 
 await store.set("key1", "Hello world", { expires: "1h" });
 // Writes "./data/key1.json"
@@ -713,19 +720,26 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
-> Note: the ending slash `/` is needed, to disambiguate with "file"
+> Note: the ending slash `/` is needed, to disambiguate with ["file"](#file)
 
 You can also create multiple stores:
 
 ```js
 // Paths need to be absolute, but you can use `process.cwd()` to make
 // it relative to the current process, or `import.meta.dirname`:
-const store1 = kv(new URL(`file://${process.cwd()}/cache/`));
-const store2 = kv(new URL(`file://${import.meta.dirname}/data/`));
+const store1 = kv(`file://${process.cwd()}/cache/`);
+const store2 = kv(`file://${import.meta.dirname}/data/`);
 ```
 
 The folder is created if it doesn't exist. When a key is deleted, the corresponding file is also deleted. The data is serialized as JSON, with a meta wrapper to store the expiration date.
 
+
+You can also pass a `URL` instance:
+
+```js
+const store1 = kv(new URL(`file://${process.cwd()}/cache/`));
+```
+
 <details>
   <summary>Why use polystore with a folder?</summary>
   <p>These benefits are for wrapping a folder with polystore:</p>
@@ -767,6 +781,8 @@ export default {
 };
 ```
 
+It expects that you pass the namespace from Cloudflare straight as a `kv()` argument. This is unfortunately not available outside of the `fetch()` method.
+
 <details>
   <summary>Why use polystore with Cloudflare's KV?</summary>
   <p>These benefits are for wrapping Cloudflare's KV with polystore:</p>
@@ -804,6 +820,8 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+You will need to set the `valueEncoding` to `"json"` for the store to work as expected.
+
 <details>
   <summary>Why use polystore with Level?</summary>
   <p>These benefits are for wrapping Level with polystore:</p>
@@ -835,6 +853,8 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+You'll need to be running the etcd store for this to work as expected.
+
 <details>
   <summary>Why use polystore with Etcd?</summary>
   <p>These benefits are for wrapping Etcd with polystore:</p>
@@ -938,7 +958,7 @@ class MyClient {
 
 Note that this is NOT the public API, it's the internal **client** API. It's simpler than the public API since we do some of the heavy lifting as an intermediate layer (e.g. for the client, the `expires` will always be a `null` or `number`, never `undefined` or a `string`), but also it differs from polystore's public API, like `.add()` has a different signature, and the group methods all take a explicit prefix.
 
-**Expires**: if you set the `EXPIRES = true`, then you are indicating that the client WILL manage the lifecycle of the data. This includes all methods, for example if an item is expired, then its key should not be returned in `.keys()`, it's value should not be returned in `.values()`, and the method `.has()` will return `false`. The good news is that you will always receive the option `expires`, which is either `null` (no expiration) or a `number` indicating the time when it will expire.
+**Expires**: if you set the `EXPIRES = true`, then you are indicating that the client WILL manage the lifecycle of the data. This includes all methods, for example if an item is expired, then its key should not be returned in `.keys()`, it's value should not be returned in `.values()`, and the method `.has()` will return `false`. The good news is that you will always receive the option `expires`, which is either `null` (no expiration) or a `number` indicating the **seconds** for the key/value to will expire.
 
 **Prefix**: we manage the `prefix` as an invisible layer on top, you only need to be aware of it in the `.add()` method, as well as in the group methods:
 
@@ -966,7 +986,7 @@ client.keys = (prefix) => {
 
 While the signatures are different, you can check each entries on the output of Polystore API to see what is expected for the methods of the client to do, e.g. `.clear()` will remove all of the items that match the prefix (or everything if there's no prefix).
 
-**Example: Plain Object client**
+### Example: Plain Object client
 
 This is a good example of how simple a store can be, however do not use it literally since it behaves the same as the already-supported `new Map()`, only use it as the base for your own clients:
 
@@ -996,7 +1016,7 @@ class MyClient {
 
 We don't set `EXPIRES` to true since plain objects do NOT support expiration natively. So by not adding the `EXPIRES` property, it's the same as setting it to `false`, and polystore will manage all the expirations as a layer on top of the data. We could be more explicit and set it to `EXPIRES = false`, but it's not needed in this case.
 
-**Example: custom ID generation**
+### Example: custom ID generation
 
 You might want to provide your custom key generation algorithm, which I'm going to call `customId()` for example purposes. The only place where `polystore` generates IDs is in `add`, so you can provide your client with a custom generator:
 
@@ -1028,3 +1048,100 @@ const id = await store.add({ hello: "world" });
 const id2 = await store.prefix("hello:").add({ hello: "world" });
 // this is `hello:{your own custom id}`
 ```
+
+### Example: serializing the data
+
+If you need to serialize the data before storing it, you can do it within your custom client. Here's an example of how you can handle data serialization when setting values:
+
+```js
+class MyClient {
+  get(key) {
+    const data = dataSource[key];
+    return data ? JSON.parse(data) : null;
+  }
+
+  set(key, value) {
+    dataSource[key] = JSON.stringify(value);
+  }
+
+  *iterate(prefix) {
+    for (const [key, value] of Object.entries(dataSource)) {
+      if (key.startsWith(prefix) && value) {
+        yield [key, JSON.parse(value)];
+      }
+    }
+  }
+}
+```
+
+### Example: Cloudflare API calls
+
+In this example on one of my projects, I needed to use Cloudflare's REST API since I didn't have access to any KV store I was happy with on Netlify's Edge Functions. So I created it like this:
+
+> Warning: this code snippet is an experimental example and hasn't gone through rigurous testing as the rest of the library, so please treat with caution.
+
+```js
+const {
+  CLOUDFLARE_ACCOUNT,
+  CLOUDFLARE_NAMESPACE,
+  CLOUDFLARE_EMAIL,
+  CLOUDFLARE_API_KEY,
+} = process.env;
+
+const baseUrl = `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT}/storage/kv/namespaces/${CLOUDFLARE_NAMESPACE}`;
+const headers = {
+  "X-Auth-Email": CLOUDFLARE_EMAIL,
+  "X-Auth-Key": CLOUDFLARE_API_KEY,
+};
+
+class CloudflareCustom {
+  EXPIRES = true;
+
+  async get(key) {
+    const res = await fetch(`${baseUrl}/values/${key}`, { headers });
+    if (res.status === 404) return null; // It does not exist
+    const data = await (res.headers.get("content-type").includes("json")
+      ? res.json()
+      : res.text());
+    if (!data) return null;
+    return JSON.parse(data);
+  }
+
+  async set(key, body, { expires }) {
+    const expiration = expires ? `expiration_ttl=${expires}&` : "";
+    await fetch(`${baseUrl}/values/${key}?${expiration}`, {
+      method: "PUT",
+      headers,
+      body: JSON.stringify(body),
+    });
+    return key;
+  }
+
+  async keys(prefix) {
+    const res = await fetch(`${baseUrl}/keys`, { headers });
+    const data = await res.json();
+    return data.result
+      .map((it) => it.name)
+      .filter((key) => key.startsWith(prefix));
+  }
+
+  async *iterate(prefix) {
+    const keys = await this.keys(prefix);
+
+    // A list of promises. Requests them all in parallel, but will start
+    // yielding them as soon as they are available (in order)
+    const pairs = keys.map(async (key) => [key, await this.get(key)]);
+    for (let prom of pairs) {
+      const pair = await prom;
+      // Some values could have been nullified from reading of the keys to
+      // reading of the value
+      if (!pair[1]) continue;
+      yield await pair;
+    }
+  }
+}
+
+const store = kv(CloudflareCustom);
+````
+
+It's lacking few things, so make sure to adapt to your needs, but it worked for my very simple cache needs.

package/src/clients/api.js

@@ -13,7 +13,11 @@ export default class Api extends Client {
     const headers = { accept: "application/json" };
     if (body) headers["content-type"] = "application/json";
     const res = await fetch(url, { method, headers, body });
-    return res.ok ? res.json() : null;
+    if (!res.ok) return null;
+    if (res.headers.get("content-type")?.includes("application/json")) {
+      return res.json();
+    }
+    return res.text();
   };
 
   get = (key) => this.#api(key);

package/src/server.js

@@ -2,6 +2,10 @@
 import http from "node:http";
 import kv from "./index.js";
 
+// Add/remove the key whether you want the API to be behind a key
+const key = null;
+// const key = 'MY-SECRET-KEY';
+
 // Modify this to use any sub-store as desired. It's nice
 // to use polystore itself for the polystore server library!'
 const store = kv(new Map());
@@ -51,6 +55,9 @@ async function fetch({ method, url, body }) {
 
 // http or express server-like handler:
 async function server(req, res) {
+  // Secure it behind a key (optional)
+  if (key && req.headers.get("x-api-key") !== key) return res.send(401);
+
   const url = new URL(req.url, "http://localhost:3000/").href;
   const reply = await fetch({ ...req, url });
   res.writeHead(reply.status, null, reply.headers || {});