polystore 0.14.0 → 0.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "polystore",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "A small compatibility layer for many popular KV stores like localStorage, Redis, FileSystem, etc.",
   "homepage": "https://polystore.dev/",
   "repository": "https://github.com/franciscop/polystore.git",
@@ -14,11 +14,11 @@
     "src/"
   ],
   "scripts": {
-    "size": "echo $(gzip -c src/index.js | wc -c) bytes",
     "lint": "check-dts test/index.types.ts",
     "start": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch --coverage --detectOpenHandles",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --ci --watchAll=false --detectOpenHandles",
-    "db": "etcd"
+    "db": "etcd",
+    "server": "bun ./src/server.js"
   },
   "keywords": [
     "kv",
@@ -29,10 +29,10 @@
     "value"
   ],
   "license": "MIT",
-  "dependencies": {},
   "devDependencies": {
     "@deno/kv": "^0.8.1",
     "check-dts": "^0.8.0",
+    "cross-fetch": "^4.0.0",
     "dotenv": "^16.3.1",
     "edge-mock": "^0.0.15",
     "etcd3": "^1.1.2",

package/readme.md

@@ -35,6 +35,7 @@ Available clients for the KV store:
 - [**Session Storage** `sessionStorage`](#session-storage) (fe): persist the data in the browser's sessionStorage.
 - [**Cookies** `"cookie"`](#cookies) (fe): persist the data using cookies
 - [**LocalForage** `localForage`](#local-forage) (fe): persist the data on IndexedDB
+- [**Fetch API** `"https://..."`](#fetch-api) (fe+be): call an API to save/retrieve the data
 - [**File** `new URL('file:///...')`](#file) (be): store the data in a single JSON file in your FS
 - [**Folder** `new URL('file:///...')`](#folder) (be): store each key in a folder as json files
 - [**Redis Client** `redisClient`](#redis-client) (be): use the Redis instance that you connect to
@@ -462,13 +463,23 @@ console.log(await store.get("key1"));
 
 <details>
   <summary>Why use polystore with <code>new Map()</code>?</summary>
-  <p>Besides the other benefits already documented elsewhere, these are the ones specifically for wrapping Map() with polystore:</p>
+  <p>These benefits are for wrapping Map() with polystore:</p>
   <ul>
     <li><strong>Expiration</strong>: you can now set lifetime to your values so that they are automatically evicted when the time passes. <a href="#expiration-explained">Expiration explained</a>.</li>
     <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
   </ul>
 </details>
 
+```js
+// GOOD - with polystore
+await store.set("key1", { name: "Francisco" }, { expires: "2days" });
+
+// COMPLEX - With sessionStorage
+const data = new Map();
+data.set("key1", { name: "Francisco" });
+// Expiration not supported
+```
+
 ### Local Storage
 
 The traditional localStorage that we all know and love, this time with a unified API, and promises:
@@ -485,6 +496,26 @@ console.log(await store.get("key1"));
 
 Same limitations as always apply to localStorage, if you think you are going to use too much storage try instead our integration with [Local Forage](#local-forage)!
 
+<details>
+  <summary>Why use polystore with <code>localStorage</code>?</summary>
+  <p>These benefits are for wrapping localStorage with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Expiration</strong>: you can now set lifetime to your values so that they are automatically evicted when the time passes. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
+```js
+// GOOD - with polystore
+await store.set("key1", { name: "Francisco" }, { expires: "2days" });
+
+// COMPLEX - With localStorage
+const serialValue = JSON.stringify({ name: "Francisco" });
+localStorage.set("key1", serialValue);
+// Expiration not supported
+```
+
 ### Session Storage
 
 Same as localStorage, but now for the session only:
@@ -499,6 +530,26 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+<details>
+  <summary>Why use polystore with <code>sessionStorage</code>?</summary>
+  <p>These benefits are for wrapping sessionStorage with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Expiration</strong>: you can now set lifetime to your values so that they are automatically evicted when the time passes. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
+```js
+// GOOD - with polystore
+await store.set("key1", { name: "Francisco" }, { expires: "2days" });
+
+// COMPLEX - With sessionStorage
+const serialValue = JSON.stringify({ name: "Francisco" });
+sessionStorage.set("key1", serialValue);
+// Expiration not supported
+```
+
 ### Cookies
 
 Supports native browser cookies, including setting the expire time:
@@ -517,6 +568,16 @@ It is fairly limited for how powerful cookies are, but in exchange it has the sa
 
 > Note: the cookie expire resolution is in the seconds, so times shorter than 1 second like `expires: 0.02` (20 ms) don't make sense for this storage method and won't properly save them.
 
+<details>
+  <summary>Why use polystore with <code>cookies</code>?</summary>
+  <p>These benefits are for wrapping cookies with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
 ### Local Forage
 
 Supports localForage (with any driver it uses) so that you have a unified API. It also _adds_ the `expires` option to the setters!
@@ -532,6 +593,15 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+<details>
+  <summary>Why use polystore with <code>localStorage</code>?</summary>
+  <p>These benefits are for wrapping localStorage with polystore:</p>
+  <ul>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
 ### Redis Client
 
 Supports the official Node Redis Client. You can pass either the client or the promise:
@@ -551,6 +621,34 @@ You don't need to `await` for the connect or similar, this will process it prope
 
 > Note: the Redis client expire resolution is in the seconds, so times shorter than 1 second like `expires: 0.02` (20 ms) don't make sense for this storage method and won't properly save them.
 
+<details>
+  <summary>Why use polystore with <code>Redis</code>?</summary>
+  <p>These benefits are for wrapping Redis with polystore:</p>
+  <ul>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
+### Fetch API
+
+Calls an API to get/put the data:
+
+```js
+import kv from "polystore";
+
+const store = kv("https://kv.example.com/");
+
+await store.set("key1", "Hello world", { expires: "1h" });
+console.log(await store.get("key1"));
+// "Hello world"
+```
+
+> Note: the API client expire resolution is in the seconds, so times shorter than 1 second like `expires: 0.02` (20 ms) don't make sense for this storage method and won't properly save them.
+
+> Note: see the reference implementation in src/server.js
+
+
 ### File
 
 Treat a JSON file in your filesystem as the source for the KV store:
@@ -565,7 +663,7 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
-> Note: an extension is needed, to desambiguate with "folder"
+> Note: an extension is needed, to disambiguate with "folder"
 
 You can also create multiple stores:
 
@@ -576,6 +674,30 @@ const store1 = kv(new URL(`file://${process.cwd()}/cache.json`));
 const store2 = kv(new URL(`file://${import.meta.dirname}/data.json`));
 ```
 
+<details>
+  <summary>Why use polystore with a file?</summary>
+  <p>These benefits are for wrapping a file with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Expiration</strong>: you can now set lifetime to your values so that they are automatically evicted when the time passes. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
+```js
+// GOOD - with polystore
+await store.set("key1", { name: "Francisco" }, { expires: "2days" });
+
+// COMPLEX - With native file managing
+const file = './data/users.json';
+const str = await fsp.readFile(file, "utf-8");
+const data = JSON.parse(str);
+data["key1"] = { name: "Francisco" };
+const serialValue = JSON.stringify(data);
+await fsp.writeFile(file, serialValue);
+// Expiration not supported (and error handling not shown)
+```
+
 ### Folder
 
 Treat a single folder in your filesystem as the source for the KV store, with each key being within a file:
@@ -590,7 +712,7 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
-> Note: the ending slash `/` is needed, to desambiguate with "file"
+> Note: the ending slash `/` is needed, to disambiguate with "file"
 
 You can also create multiple stores:
 
@@ -601,6 +723,27 @@ const store1 = kv(new URL(`file://${process.cwd()}/cache/`));
 const store2 = kv(new URL(`file://${import.meta.dirname}/data/`));
 ```
 
+<details>
+  <summary>Why use polystore with a folder?</summary>
+  <p>These benefits are for wrapping a folder with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Expiration</strong>: you can now set lifetime to your values so that they are automatically evicted when the time passes. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
+```js
+// GOOD - with polystore
+await store.set("key1", { name: "Francisco" }, { expires: "2days" });
+
+// COMPLEX - With native folder
+const file = './data/user/key1.json';
+const serialValue = JSON.stringify({ name: "Francisco" });
+await fsp.writeFile(file, serialValue);
+// Expiration not supported (and error handling not shown)
+```
+
 ### Cloudflare KV
 
 Supports the official Cloudflare's KV stores. Follow [the official guide](https://developers.cloudflare.com/kv/get-started/), then load it like this:
@@ -621,7 +764,15 @@ export default {
 };
 ```
 
-Why use polystore? The Cloudflare native KV store only accepts strings and has you manually calculating timeouts, but as usual with `polystore` you can set/get any serializable value and set the timeout in a familiar format:
+<details>
+  <summary>Why use polystore with Cloudflare's KV?</summary>
+  <p>These benefits are for wrapping Cloudflare's KV with polystore:</p>
+  <ul>
+    <li><strong>Data structures</strong>: with Polystore you can pass more complex data structures and we'll handle the serialization/deserialization.</li>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
 
 ```js
 // GOOD - with polystore
@@ -650,7 +801,13 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
-Why use polystore? The main reason is that we add expiration on top of Level, which is not supported by Level:
+<details>
+  <summary>Why use polystore with Level?</summary>
+  <p>These benefits are for wrapping Level with polystore:</p>
+  <ul>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+  </ul>
+</details>
 
 ```js
 // GOOD - with polystore
@@ -675,6 +832,15 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+<details>
+  <summary>Why use polystore with Etcd?</summary>
+  <p>These benefits are for wrapping Etcd with polystore:</p>
+  <ul>
+    <li><strong>Intuitive expirations</strong>: use plain English to specify the expiration time like <code>10min</code>. <a href="#expiration-explained">Expiration explained</a>.</li>
+    <li><strong>Substores</strong>: you can also create substores and manage partial data with ease. <a href="#prefix">Details about substores</a>.</li>
+  </ul>
+</details>
+
 ### Custom store
 
 Please see the [creating a store](#creating-a-store) section for all the details!
@@ -782,8 +948,7 @@ const value = await store.get("a");
 // client.get("hello:world:a");
 
 // User calls this, then the client is called with that:
-for await (const entry of store.iterate()) {
-}
+for await (const [key, value] of store) {}
 // client.iterate("hello:world:");
 ```
 

package/src/clients/api.js

@@ -0,0 +1,71 @@
+// Use fetch()
+export default class Api {
+  // Indicate that the file handler does NOT handle expirations
+  EXPIRES = true;
+
+  // Check whether the given store is a FILE-type
+  static test(client) {
+    return (
+      typeof client === "string" &&
+      (client.startsWith("https://") || client.startsWith("http://"))
+    );
+  }
+
+  constructor(client) {
+    client = client.replace(/\/$/, "") + "/";
+    this.client = async (path, opts = {}) => {
+      const query = Object.entries(opts.query || {})
+        .map(([k, v]) => `${k}=${encodeURIComponent(v)}`)
+        .join("&");
+      let url = client + path.replace(/^\//, "") + "?" + query;
+      opts.headers = opts.headers || {};
+      opts.headers.accept = "application/json";
+      if (opts.body) opts.headers["content-type"] = "application/json";
+      const res = await fetch(url, opts);
+      if (!res.ok) return null;
+      if (!res.headers["content-type"] !== "application/json") {
+        console.warn("Not a JSON API");
+      }
+      return res.json();
+    };
+  }
+
+  async get(key) {
+    return await this.client(`/${key}`);
+  }
+
+  async set(key, value, { expires } = {}) {
+    return await this.client(`/${encodeURIComponent(key)}`, {
+      query: { expires },
+      method: "put",
+      body: JSON.stringify(value),
+    });
+  }
+
+  async del(key) {
+    return await this.client(`/${encodeURIComponent(key)}`, {
+      method: "delete",
+    });
+  }
+
+  // Since we have pagination, we don't want to get all of the
+  // keys at once if we can avoid it
+  async *iterate(prefix = "") {
+    const data = await this.client("/", { query: { prefix } });
+    if (!data) return [];
+    for (let [key, value] of Object.entries(data)) {
+      yield [prefix + key, value];
+    }
+  }
+
+  async keys(prefix = "") {
+    const data = await this.client(`/`, { query: { prefix } });
+    if (!data) return [];
+    return Object.keys(data).map((k) => prefix + k);
+  }
+
+  async clear(prefix = "") {
+    const list = await this.keys(prefix);
+    return Promise.all(list.map((k) => this.del(k)));
+  }
+}

package/src/clients/folder.js

@@ -23,7 +23,7 @@ export default class Folder {
   constructor(folder) {
     this.folder =
       typeof folder === "string"
-        ? folder.slice("folder://".length).replace(/\/$/, "") + "/"
+        ? folder.slice("file://".length).replace(/\/$/, "") + "/"
         : folder.pathname.replace(/\/$/, "") + "/";
 
     // Run this once on launch; import the FS module and reset the file
@@ -48,7 +48,7 @@ export default class Folder {
   async set(key, value) {
     const fsp = await this.promise;
     const file = this.folder + key + ".json";
-    await fsp.writeFile(file, JSON.stringify(value), "utf8");
+    await fsp.writeFile(file, JSON.stringify(value, null, 2), "utf8");
     return file;
   }
 
@@ -61,15 +61,10 @@ export default class Folder {
 
   async *iterate(prefix = "") {
     const fsp = await this.promise;
-    const all = await fsp.readdir(this.folder, { withFileTypes: true });
-    const files = all.filter((f) => !f.isDirectory());
-    const keys = files
-      .map((file) =>
-        (file.path.replace(/\/$/, "") + "/" + file.name)
-          .replace(this.folder, "")
-          .replace(".json", ""),
-      )
-      .filter((k) => k.startsWith(prefix));
+    const all = await fsp.readdir(this.folder);
+    const keys = all
+      .filter((f) => f.startsWith(prefix) && f.endsWith(".json"))
+      .map((name) => name.slice(0, -".json".length));
     for (const key of keys) {
       const data = await this.get(key);
       yield [key, data];

package/src/clients/index.js

@@ -1,3 +1,4 @@
+import api from "./api.js";
 import cloudflare from "./cloudflare.js";
 import cookie from "./cookie.js";
 import etcd from "./etcd.js";
@@ -10,6 +11,7 @@ import redis from "./redis.js";
 import storage from "./storage.js";
 
 export default {
+  api,
   cloudflare,
   cookie,
   etcd,

package/src/server.js

@@ -0,0 +1,75 @@
+// This is an example server implementation of the HTTP library!
+import http from "node:http";
+import kv from "./index.js";
+import { parse } from "./utils.js";
+
+// Modify this to use any sub-store as desired. It's nice
+// to use polystore itself for the polystore server library!'
+const store = kv(new Map());
+
+// Some reply helpers
+const notFound = () => new Response(null, { status: 404 });
+const sendJson = (data, status = 200) => {
+  const body = JSON.stringify(data);
+  const headers = { "content-type": "application/json" };
+  return new Response(body, { status, headers });
+};
+
+async function fetch({ method, url, body }) {
+  method = method.toLowerCase();
+  url = new URL(url);
+  let [, id] = url.pathname.split("/");
+  id = decodeURIComponent(id);
+  const expires = Number(url.searchParams.get("expires")) || null;
+  const prefix = url.searchParams.get("prefix") || null;
+
+  let local = store;
+  if (prefix) local = store.prefix(prefix);
+
+  if (method === "get") {
+    if (id === "ping") return new Response(null, { status: 200 });
+    if (!id) return sendJson(await local.all());
+    const data = await local.get(id);
+    if (data === null) return notFound();
+    return sendJson(data);
+  }
+
+  if (method === "put") {
+    if (!id) return notFound();
+    const data = await new Response(body).json();
+    if (!data) return notFound();
+    await local.set(id, data, { expires });
+    return sendJson(id);
+  }
+
+  if (method === "delete" && id) {
+    await local.del(id);
+    return sendJson(id);
+  }
+
+  return notFound();
+}
+
+// http or express server-like handler:
+async function server(req, res) {
+  const url = new URL(req.url, "http://localhost:3000/").href;
+  const reply = await fetch({ ...req, url });
+  res.writeHead(reply.status, null, reply.headers || {});
+  if (reply.body) res.write(reply.body);
+  res.end();
+}
+
+function start(port = 3000) {
+  return new Promise((resolve, reject) => {
+    const server = http.createServer(server);
+    server.on("clientError", (error, socket) => {
+      reject(error);
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+    server.listen(port, resolve);
+    return () => server.close();
+  });
+}
+
+export { fetch, server, start };
+export default { fetch, server, start };