polystore 0.12.1 → 0.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "polystore",
-  "version": "0.12.1",
+  "version": "0.14.0",
   "description": "A small compatibility layer for many popular KV stores like localStorage, Redis, FileSystem, etc.",
   "homepage": "https://polystore.dev/",
   "repository": "https://github.com/franciscop/polystore.git",
@@ -17,7 +17,8 @@
     "size": "echo $(gzip -c src/index.js | wc -c) bytes",
     "lint": "check-dts test/index.types.ts",
     "start": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch --coverage --detectOpenHandles",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --ci --watchAll=false --detectOpenHandles"
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --ci --watchAll=false --detectOpenHandles",
+    "db": "etcd"
   },
   "keywords": [
     "kv",

package/readme.md

@@ -1,4 +1,4 @@
-# Polystore [![npm install polystore](https://img.shields.io/badge/npm%20install-polystore-blue.svg)](https://www.npmjs.com/package/polystore) [![test badge](https://github.com/franciscop/polystore/workflows/tests/badge.svg "test badge")](https://github.com/franciscop/polystore/blob/master/.github/workflows/tests.yml) [![gzip size](https://badgen.net/bundlephobia/minzip/polystore?label=gzip&color=green)](https://github.com/franciscop/polystore/blob/master/src/index.js)
+# Polystore [![npm install polystore](https://img.shields.io/badge/npm%20install-polystore-blue.svg)](https://www.npmjs.com/package/polystore) [![test badge](https://github.com/franciscop/polystore/workflows/tests/badge.svg "test badge")](https://github.com/franciscop/polystore/blob/master/.github/workflows/tests.yml) [![gzip size](https://badgen.net/bundlephobia/minzip/polystore?label=gzip&color=green)](https://bundlephobia.com/package/polystore)
 
 A key-value library to unify the API of [many clients](#clients), like localStorage, Redis, FileSystem, etc:
 
@@ -24,7 +24,9 @@ These are all the methods of the [API](#api) (they are all `async`):
 - [`.all()`](#all): get an object with the key:values mapped.
 - [`.clear()`](#clear): delete ALL of the data in the store, effectively resetting it.
 - [`.close()`](#close): (only _some_ stores) ends the connection to the store.
-- [`.prefix(prefix)`](#prefix): create a sub-store that only manages the keys with the given prefix.
+- [`.prefix(prefix)`](#prefix): create a sub-store that manages the keys with that prefix.
+
+> This library has very high performance with the item methods (GET/SET/ADD/HAS/DEL). For other methods or to learn more, see [the performance considerations](#performance) and read the docs on your specific client.
 
 Available clients for the KV store:
 
@@ -34,15 +36,14 @@ Available clients for the KV store:
 - [**Cookies** `"cookie"`](#cookies) (fe): persist the data using cookies
 - [**LocalForage** `localForage`](#local-forage) (fe): persist the data on IndexedDB
 - [**File** `new URL('file:///...')`](#file) (be): store the data in a single JSON file in your FS
+- [**Folder** `new URL('file:///...')`](#folder) (be): store each key in a folder as json files
 - [**Redis Client** `redisClient`](#redis-client) (be): use the Redis instance that you connect to
 - [**Cloudflare KV** `env.KV_NAMESPACE`](#cloudflare-kv) (be): use Cloudflare's KV store
 - [**Level** `new Level('example', { valueEncoding: 'json' })`](#level) (fe+be): support the whole Level ecosystem
 - [**Etcd** `new Etcd3()`](#etcd) (be): the Microsoft's high performance KV store.
-- [**_Custom_** `{}`](#creating-a-store) (?): create your own store with just 3 methods!
-
-> This library should be as performant as the client you use with the item methods (GET/SET/ADD/HAS/DEL). For other and advanced cases, see [the performance considerations](#performance) and read the docs on your client.
+- [**_Custom_** `{}`](#creating-a-store) (fe+be): create your own store with just 3 methods!
 
-I made this library to be used as a "building block" of other libraries, so that _your library_ can accept many cache stores effortlessly! It's isomorphic (Node.js, Bun and the Browser) and tiny (~2KB). For example, let's say you create an API library, then you can accept the stores from your client:
+I made this library to be used as a "building block" of other libraries, so that _your library_ can accept many cache stores effortlessly! It's universal (Node.js, Bun and the Browser) and tiny (~3KB). For example, let's say you create an API library, then you can accept the stores from your client:
 
 ```js
 import MyApi from "my-api";
@@ -127,9 +128,9 @@ console.log(await store.get("key2"));  // ["my", "grocery", "list"]
 console.log(await store.get("key3"));  // { name: "Francisco" }
 ```
 
-If the value is returned, it can be a simple type like `boolean`, `string` or `number`, or it can be a plain Object or Array, or a combination of those.
+If the value is returned, it can be a simple type like `boolean`, `string` or `number`, or it can be a plain `Object` or `Array`, or any combination of those.
 
-> The value cannot be more complex or non-serializable values like a `Date()`, `Infinity`, `undefined`, a `Symbol`, etc.
+When there's no value (either never set, or expired), `null` will be returned from the operation.
 
 ### .set()
 
@@ -143,7 +144,7 @@ await store.set("key2", ["my", "grocery", "list"], { expires: "1h" });
 await store.set("key3", { name: "Francisco" }, { expires: 60 * 60 });
 ```
 
-The value can be a simple type like `boolean`, `string` or `number`, or it can be a plain Object or Array, or a combination of those. It **cannot** be a more complex or non-serializable values like a `Date()`, `Infinity`, `undefined` (casted to `null`), a `Symbol`, etc.
+The value can be a simple type like `boolean`, `string` or `number`, or it can be a plain `Object` or `Array`, or a combination of those. It **cannot** be a more complex or non-serializable values like a `Date()`, `Infinity`, `undefined` (casted to `null`), a `Symbol`, etc.
 
 - By default the keys _don't expire_.
 - Setting the `value` to `null`, or the `expires` to `0` is the equivalent of deleting the key+value.
@@ -182,11 +183,16 @@ const key2 = await store.add(["my", "grocery", "list"], { expires: "1h" });
 const key3 = await store.add({ name: "Francisco" }, { expires: 60 * 60  });
 ```
 
-The generated key is 24 AlphaNumeric characters (including upper and lower case) generated with random cryptography to make sure it's unguessable, high entropy and safe to use in most contexts like URLs, queries, etc. We use [`nanoid`](https://github.com/ai/nanoid/) with a custom dictionary, so you can check the entropy [in this dictionary](https://zelark.github.io/nano-id-cc/) by removing the "\_" and "-", and setting it to 24 characters.
+The options and details are similar to [`.set()`](#set), except for the lack of the first argument, since `.add()` will generate the key automatically.
 
-Here is the safety: "If you generate 1 million keys/second, it will take ~14 million years in order to have a 1% probability of at least one collision."
+The default key will be 24 AlphaNumeric characters (upper+lower case), however this can change if you are using a `.prefix()` or some clients might generate it differently (only custom clients can do that right now).
 
-> Note: please make sure to read the [`.set()`](#set) section for all the details, since `.set()` and `.add()` behave the same way except for the first argument.
+<details>
+  <summary>Key Generation details</summary>
+  The default key will be 24 AlphaNumeric characters (including upper and lower case) generated with random cryptography to make sure it's unguessable, high entropy and safe to use in most contexts like URLs, queries, etc. We use [`nanoid`](https://github.com/ai/nanoid/) with a custom dictionary, so you can check the entropy [in this dictionary](https://zelark.github.io/nano-id-cc/) by removing the "\_" and "-", and setting it to 24 characters.
+
+  Here is the safety: "If you generate 1 million keys/second, it will take ~14 million years in order to have a 1% probability of at least one collision."
+</details>
 
 The main reason why `.add()` exists is to allow it to work with the prefix seamlessly:
 
@@ -307,29 +313,38 @@ There are also methods to retrieve all of the keys, values, or entries at once b
 
 ### .keys()
 
-Get all of the keys in the store, optionally filtered by a prefix:
+Get all of the keys in the store as a simple array of strings:
 
 ```js
-await store.keys(filter?: string);
+await store.keys();
+// ["keyA", "keyB", "keyC", ...]
+```
+
+If you want to filter for a particular prefix, use `.prefix()`, which will return the values with the keys with that prefix (the keys have the prefix stripped!):
+
+```js
+const sessions = await store.prefix("session:").keys();
+// ["keyA", "keyB"]
 ```
 
 > We ensure that all of the keys returned by this method are _not_ expired, while discarding any potentially expired key. See [**expiration explained**](#expiration-explained) for more details.
 
 ### .values()
 
-Get all of the values in the store, optionally filtered by a **key** prefix:
+Get all of the values in the store as a simple array with all the values:
 
 ```js
-await store.values(filter?: string);
+await store.values();
+// ["valueA", "valueB", { hello: "world" }, ...]
 ```
 
-This is useful specially when you already have the id/key within the value as an object, then you can just get a list of all of them:
+If you want to filter for a particular prefix, use `.prefix()`, which will return the values with the keys with that prefix:
 
 ```js
-const sessions = await store.values("session:");
+const sessions = await store.prefix("session:").values();
 // A list of all the sessions
 
-const companies = await store.values("company:");
+const companies = await store.prefix("company:").values();
 // A list of all the companies
 ```
 
@@ -337,24 +352,48 @@ const companies = await store.values("company:");
 
 ### .entries()
 
-Get all of the entries (key:value tuples) in the store, optionally filtered by a **key** prefix:
+Get all of the entries (key:value tuples) in the store:
+
+```js
+const entries = await store.entries();
+// [["keyA", "valueA"], ["keyB", "valueB"], ["keyC", { hello: "world" }], ...]
+```
+
+It's in the same format as `Object.entries(obj)`, so it's an array of [key, value] tuples.
+
+If you want to filter for a particular prefix, use `.prefix()`, which will return the entries that have that given prefix (the keys have the prefix stripped!):
 
 ```js
-await store.entries(filter?: string);
+const sessionEntries = await store.prefix('session:').entries();
+// [["keyA", "valueA"], ["keyB", "valueB"]]
 ```
 
-It is in a format that you can easily build an object out of it:
+> We ensure that all of the entries returned by this method are _not_ expired, while discarding any potentially expired key. See [**expiration explained**](#expiration-explained) for more details.
+
+### .all()
+
+Get all of the entries (key:value) in the store as an object:
 
 ```js
-const sessionEntries = await store.entries("session:");
-const sessions = Object.fromEntries(sessionEntries);
+const obj = await store.all(filter?: string);
+// { keyA: "valueA", keyB: "valueB", keyC: { hello: "world" }, ... }
+```
+
+It's in the format of a normal key:value object, where the object key is the store's key and the object value is the store's value.
+
+If you want to filter for a particular prefix, use `.prefix()`, which will return the object with only the keys that have that given prefix (stripping the keys of the prefix!):
+
+```js
+const sessionObj = await store.prefix('session:').entries();
+// { keyA: "valueA", keyB: "valueB" }
 ```
 
 > We ensure that all of the entries returned by this method are _not_ expired, while discarding any potentially expired key. See [**expiration explained**](#expiration-explained) for more details.
 
+
 ### .clear()
 
-Remove all of the data from the store:
+Remove all of the data from the store and resets it to the original state:
 
 ```js
 await store.clear();
@@ -382,6 +421,9 @@ await session.del("key4"); // store.del('session:key4');
 await session.keys(); // store.keys(); + filter
 // ['key1', 'key2', ...]   Note no prefix here
 await session.clear(); // delete only keys with the prefix
+for await (const [key, value] of session) {
+  console.log(key, value);
+}
 ```
 
 Different clients have better/worse support for substores, and in some cases some operations might be slower. This should be documented on each client's documentation (see below). As an alternative, you can always create two different stores instead of a substore:
@@ -523,6 +565,8 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+> Note: an extension is needed, to desambiguate with "folder"
+
 You can also create multiple stores:
 
 ```js
@@ -532,6 +576,31 @@ const store1 = kv(new URL(`file://${process.cwd()}/cache.json`));
 const store2 = kv(new URL(`file://${import.meta.dirname}/data.json`));
 ```
 
+### Folder
+
+Treat a single folder in your filesystem as the source for the KV store, with each key being within a file:
+
+```js
+import kv from "polystore";
+
+const store = kv(new URL("file:///Users/me/project/data/"));
+
+await store.set("key1", "Hello world", { expires: "1h" });
+console.log(await store.get("key1"));
+// "Hello world"
+```
+
+> Note: the ending slash `/` is needed, to desambiguate with "file"
+
+You can also create multiple stores:
+
+```js
+// Paths need to be absolute, but you can use process.cwd() to make
+// it relative to the current process:
+const store1 = kv(new URL(`file://${process.cwd()}/cache/`));
+const store2 = kv(new URL(`file://${import.meta.dirname}/data/`));
+```
+
 ### Cloudflare KV
 
 Supports the official Cloudflare's KV stores. Follow [the official guide](https://developers.cloudflare.com/kv/get-started/), then load it like this:
@@ -581,6 +650,16 @@ console.log(await store.get("key1"));
 // "Hello world"
 ```
 
+Why use polystore? The main reason is that we add expiration on top of Level, which is not supported by Level:
+
+```js
+// GOOD - with polystore
+await store.set("user", { hello: 'world' }, { expires: "2days" });
+
+// With Level:
+?? // Just not possible
+```
+
 ### Etcd
 
 Connect to Microsoft's Etcd Key-Value store:
@@ -598,15 +677,15 @@ console.log(await store.get("key1"));
 
 ### Custom store
 
-Please see the [creating a store](#creating-a-store) section for more details!
+Please see the [creating a store](#creating-a-store) section for all the details!
 
 ## Performance
 
-> TL;DR: if you only use the item operations (add,set,get,has,del) and your client supports expiration natively, you have nothing to worry about!
+> TL;DR: if you only use the item operations (add,set,get,has,del) and your client supports expiration natively, you have nothing to worry about! Otherwise, please read on.
 
 While all of our stores support `expires`, `.prefix()` and group operations, the nature of those makes them to have different performance characteristics.
 
-**Expires** we polyfill expiration when the underlying client library does not support it. The impact on read/write operations and on data size of each key should be minimal. However, it can have a big impact in storage size, since the expired keys are not evicted automatically. Note that when attempting to read an expired key, polystore **will delete that key**. However, if an expired key is never read, it would remain in the datastore and could create some old-data issues. This is **especially important where sensitive data is involved**! To fix this, the easiest way is calling `await store.entries();` on a cron job and that should evict all of the old keys (this operation is O(n) though, so not suitable for calling it on EVERY API call, see the next point).
+**Expires** we polyfill expiration when the underlying client library does not support it. The impact on read/write operations and on data size of each key should be minimal. However, it can have a big impact in storage size, since the expired keys are not evicted automatically. Note that when attempting to read *an expired key*, polystore **will delete that key**. However, if an expired key is never read, it would remain in the datastore and could create some old-data issues. This is **especially important where sensitive data is involved**! To fix this, the easiest way is calling `await store.entries();` on a cron job and that should evict all of the old keys (this operation is O(n) though, so not suitable for calling it on EVERY API call, see the next point).
 
 **Group operations** these are there mostly for small datasets only, for one-off scripts or for dev purposes, since by their own nature they can _never_ be high performance in the general case. But this is normal if you think about traditional DBs, reading a single record by its ID is O(1), while reading all of the IDs in the DB into an array is going to be O(n). Same applies with polystore.
 

package/src/clients/file.js

@@ -2,8 +2,17 @@
 export default class File {
   // Check if this is the right class for the given client
   static test(client) {
-    if (typeof client === "string" && client.startsWith("file:")) return true;
-    return client instanceof URL && client.protocol === "file:";
+    if (
+      typeof client === "string" &&
+      client.startsWith("file:") &&
+      client.includes(".")
+    )
+      return true;
+    return (
+      client instanceof URL &&
+      client.protocol === "file:" &&
+      client.pathname.includes(".")
+    );
   }
 
   constructor(file) {

package/src/clients/folder.js

@@ -0,0 +1,88 @@
+const noFileOk = (error) => {
+  if (error.code === "ENOENT") return null;
+  throw error;
+};
+
+// A client that uses a single file (JSON) as a store
+export default class Folder {
+  // Check if this is the right class for the given client
+  static test(client) {
+    if (
+      typeof client === "string" &&
+      client.startsWith("file:") &&
+      client.endsWith("/")
+    )
+      return true;
+    return (
+      client instanceof URL &&
+      client.protocol === "file:" &&
+      client.pathname.endsWith("/")
+    );
+  }
+
+  constructor(folder) {
+    this.folder =
+      typeof folder === "string"
+        ? folder.slice("folder://".length).replace(/\/$/, "") + "/"
+        : folder.pathname.replace(/\/$/, "") + "/";
+
+    // Run this once on launch; import the FS module and reset the file
+    this.promise = (async () => {
+      const fsp = await import("node:fs/promises");
+
+      // Make sure the folder already exists, so attempt to create it
+      // It fails if it already exists, hence the catch case
+      await fsp.mkdir(this.folder, { recursive: true }).catch((err) => {});
+      return fsp;
+    })();
+  }
+
+  async get(key) {
+    const fsp = await this.promise;
+    const file = this.folder + key + ".json";
+    const text = await fsp.readFile(file, "utf8").catch(noFileOk);
+    if (!text) return null;
+    return JSON.parse(text);
+  }
+
+  async set(key, value) {
+    const fsp = await this.promise;
+    const file = this.folder + key + ".json";
+    await fsp.writeFile(file, JSON.stringify(value), "utf8");
+    return file;
+  }
+
+  async del(key) {
+    const file = this.folder + key + ".json";
+    const fsp = await this.promise;
+    await fsp.unlink(file).catch(noFileOk);
+    return file;
+  }
+
+  async *iterate(prefix = "") {
+    const fsp = await this.promise;
+    const all = await fsp.readdir(this.folder, { withFileTypes: true });
+    const files = all.filter((f) => !f.isDirectory());
+    const keys = files
+      .map((file) =>
+        (file.path.replace(/\/$/, "") + "/" + file.name)
+          .replace(this.folder, "")
+          .replace(".json", ""),
+      )
+      .filter((k) => k.startsWith(prefix));
+    for (const key of keys) {
+      const data = await this.get(key);
+      yield [key, data];
+    }
+  }
+
+  // async clear(prefix = "") {
+  //   const data = await this.#read();
+  //   for (let key in data) {
+  //     if (key.startsWith(prefix)) {
+  //       delete data[key];
+  //     }
+  //   }
+  //   await this.#write(data);
+  // }
+}

package/src/clients/index.js

@@ -2,6 +2,7 @@ import cloudflare from "./cloudflare.js";
 import cookie from "./cookie.js";
 import etcd from "./etcd.js";
 import file from "./file.js";
+import folder from "./folder.js";
 import forage from "./forage.js";
 import level from "./level.js";
 import memory from "./memory.js";
@@ -13,6 +14,7 @@ export default {
   cookie,
   etcd,
   file,
+  folder,
   forage,
   level,
   memory,

package/src/index.js

@@ -1,5 +1,5 @@
 import clients from "./clients/index.js";
-import { createId, isClass, parse } from "./utils.js";
+import { createId, isClass, parse, unix } from "./utils.js";
 
 class Store {
   PREFIX = "";
@@ -42,18 +42,13 @@ class Store {
       for (let method of ["has", "keys", "values"]) {
         if (client[method]) {
           throw new Error(
-            `You can only define client.${method}() when the client manages the expiration; otherwise please do NOT define .${method}() and let us manage it`
+            `You can only define client.${method}() when the client manages the expiration; otherwise please do NOT define .${method}() and let us manage it`,
           );
         }
       }
     }
   }
 
-  #unix(expires) {
-    const now = new Date().getTime();
-    return expires === null ? null : now + expires * 1000;
-  }
-
   // Check if the given data is fresh or not; if
   #isFresh(data, key) {
     // Should never happen, but COULD happen; schedule it for
@@ -76,30 +71,29 @@ class Store {
 
   async add(value, options = {}) {
     await this.promise;
-    const expires = parse(options.expire ?? options.expires);
+    let expires = parse(options.expire ?? options.expires);
 
     // Use the underlying one from the client if found
     if (this.client.add) {
       if (this.client.EXPIRES) {
-        return this.client.add(this.PREFIX, value, { expires });
+        return await this.client.add(this.PREFIX, value, { expires });
       }
 
       // In the data we need the timestamp since we need it "absolute":
-      return this.client.add(this.PREFIX, {
-        expires: this.#unix(expires),
-        value,
-      });
+      expires = unix(expires);
+      const key = await this.client.add(this.PREFIX, { expires, value });
+      return key;
     }
 
-    const id = createId();
-    await this.set(id, value, { expires });
-    return id; // The plain one without the prefix
+    const key = createId();
+    await this.set(key, value, { expires });
+    return key; // The plain one without the prefix
   }
 
   async set(key, value, options = {}) {
     await this.promise;
     const id = this.PREFIX + key;
-    const expires = parse(options.expire ?? options.expires);
+    let expires = parse(options.expire ?? options.expires);
 
     // Quick delete
     if (value === null || (typeof expires === "number" && expires <= 0)) {
@@ -114,7 +108,8 @@ class Store {
     }
 
     // In the data we need the timestamp since we need it "absolute":
-    await this.client.set(id, { expires: this.#unix(expires), value });
+    expires = unix(expires);
+    await this.client.set(id, { expires, value });
     return key;
   }
 
@@ -258,7 +253,7 @@ class Store {
 
   prefix(prefix = "") {
     const store = new Store(
-      Promise.resolve(this.promise).then((client) => client || this.client)
+      Promise.resolve(this.promise).then((client) => client || this.client),
     );
     store.PREFIX = this.PREFIX + prefix;
     return store;

package/src/utils.js

@@ -47,3 +47,8 @@ export function isClass(func) {
     /^class\s/.test(Function.prototype.toString.call(func))
   );
 }
+
+export function unix(expires) {
+  const now = new Date().getTime();
+  return expires === null ? null : now + expires * 1000;
+}