polystore 0.10.0 → 0.11.0

package/assets/home.html

@@ -14,7 +14,7 @@
       <a
         class="pseudo button"
         href="https://superpeer.com/francisco/-/javascript-and-react-help"
-        >Professional JS help</a
+        >Get JS help</a
       >
     </div>
   </div>
@@ -180,8 +180,10 @@ console.log(await store.get(key));
         </svg>
         <h3>Intuitive expirations</h3>
       </header>
-      <p>Write the expiration as <code>100s</code>, <code>1week</code>, etc.</p>
-      and forget time-related bugs.
+      <p>
+        Write the expiration as <code>100s</code>, <code>1week</code>, etc. and
+        forget time-related bugs.
+      </p>
     </div>
   </div>
 </section>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "polystore",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "A small compatibility layer for many popular KV stores like localStorage, Redis, FileSystem, etc.",
   "homepage": "https://polystore.dev/",
   "repository": "https://github.com/franciscop/polystore.git",
@@ -24,6 +24,7 @@
     "value"
   ],
   "license": "MIT",
+  "dependencies": {},
   "devDependencies": {
     "@deno/kv": "^0.8.1",
     "check-dts": "^0.8.0",

package/readme.md

@@ -241,6 +241,26 @@ Remove a single key from the store and return the key itself:
 await store.del(key: string);
 ```
 
+It will ignore the operation if the key or value don't exist already (but won't thorw).
+
+### _Iterator_
+
+You can iterate over the whole store with an async iterator:
+
+```js
+for await (const [key, value] of store) {
+  // ...
+}
+```
+
+This is very useful for performance resons. You can also iterate on a subset of the entries with .prefix:
+
+```js
+for await (const [key, value] of store.prefix("session:")) {
+  // ...
+}
+```
+
 ### .keys()
 
 Get all of the keys in the store, optionally filtered by a prefix:
@@ -615,7 +635,7 @@ class MyClient {
   // Mandatory methods
   get (key): Promise<any>;
   set (key, value, { expires: null|number }): Promise<null>;
-  entries (prefix): Promise<[string, any][]>;
+  iterate(prefix): AyncIterator<[string, any]>
 
   // Optional item methods (for optimization or customization)
   add (prefix, data, { expires: null|number }): Promise<string>;
@@ -623,6 +643,7 @@ class MyClient {
   del (key): Promise<null>;
 
   // Optional group methods
+  entries (prefix): Promise<[string, any][]>;
   keys (prefix): Promise<string[]>;
   values (prefix): Promise<any[]>;
   clear (prefix): Promise<null>;
@@ -647,8 +668,9 @@ const value = await store.get("a");
 // client.get("hello:world:a");
 
 // User calls this, then the client is called with that:
-const value = await store.entries();
-// client.entries("hello:world:");
+for await (const entry of store.iterate()) {
+}
+// client.iterate("hello:world:");
 ```
 
 > Note: all of the _group methods_ that return keys, should return them **with the prefix**:
@@ -680,10 +702,12 @@ class MyClient {
   }
 
   // Filter them by the prefix, note that `prefix` will always be a string
-  entries(prefix) {
-    const entries = Object.entries(dataSource);
-    if (!prefix) return entries;
-    return entries.filter(([key, value]) => key.startsWith(prefix));
+  *iterate(prefix) {
+    for (const [key, value] of Object.entries(dataSource)) {
+      if (key.startsWith(prefix)) {
+        yield [key, value];
+      }
+    }
   }
 }
 ```

package/src/clients/cloudflare.js

@@ -31,9 +31,33 @@ export default class Cloudflare {
     return this.client.delete(key);
   }
 
+  // Since we have pagination, we don't want to get all of the
+  // keys at once if we can avoid it
+  async *iterate(prefix = "") {
+    let cursor;
+    do {
+      const raw = await this.client.list({ prefix, cursor });
+      const keys = raw.keys.map((k) => k.name);
+      for (let key of keys) {
+        const value = await this.get(key);
+        // By the time this specific value is read, it could
+        // already be gone!
+        if (!value) continue;
+        yield [key, value];
+      }
+      cursor = raw.list_complete ? null : raw.cursor;
+    } while (cursor);
+  }
+
   async keys(prefix = "") {
-    const raw = await this.client.list({ prefix });
-    return raw.keys.map((k) => k.name);
+    const keys = [];
+    let cursor;
+    do {
+      const raw = await this.client.list({ prefix, cursor });
+      keys.push(...raw.keys.map((k) => k.name));
+      cursor = raw.list_complete ? null : raw.cursor;
+    } while (cursor);
+    return keys;
   }
 
   async entries(prefix = "") {

package/src/clients/cookie.js

@@ -8,9 +8,25 @@ export default class Cookie {
     return client === "cookie" || client === "cookies";
   }
 
+  // Group methods
+  #read() {
+    const all = {};
+    for (let entry of document.cookie.split(";")) {
+      try {
+        const [rawKey, rawValue] = entry.split("=");
+        const key = decodeURIComponent(rawKey.trim());
+        const value = JSON.parse(decodeURIComponent(rawValue.trim()));
+        all[key] = value;
+      } catch (error) {
+        // no-op (some 3rd party can set cookies independently)
+      }
+    }
+    return all;
+  }
+
   // For cookies, an empty value is the same as null, even `""`
   get(key) {
-    return this.all()[key] || null;
+    return this.#read()[key] || null;
   }
 
   set(key, data = null, { expires } = {}) {
@@ -33,23 +49,10 @@ export default class Cookie {
     return key;
   }
 
-  // Group methods
-  all(prefix = "") {
-    const all = {};
-    for (let entry of document.cookie.split(";")) {
-      const [key, data] = entry.split("=");
-      const name = decodeURIComponent(key.trim());
-      if (!name.startsWith(prefix)) continue;
-      try {
-        all[name] = JSON.parse(decodeURIComponent(data.trim()));
-      } catch (error) {
-        // no-op (some 3rd party can set cookies independently)
-      }
+  async *iterate(prefix = "") {
+    for (let [key, value] of Object.entries(this.#read())) {
+      if (!key.startsWith(prefix)) continue;
+      yield [key, value];
     }
-    return all;
-  }
-
-  entries(prefix = "") {
-    return Object.entries(this.all(prefix));
   }
 }

package/src/clients/etcd.js

@@ -21,6 +21,13 @@ export default class Etcd {
     await this.client.put(key).value(JSON.stringify(value));
   }
 
+  async *iterate(prefix = "") {
+    const keys = await this.client.getAll().prefix(prefix).keys();
+    for (const key of keys) {
+      yield [key, await this.get(key)];
+    }
+  }
+
   async entries(prefix = "") {
     const keys = await this.client.getAll().prefix(prefix).keys();
     const values = await Promise.all(keys.map((k) => this.get(k)));

package/src/clients/file.js

@@ -53,10 +53,12 @@ export default class File {
     return key;
   }
 
-  // Group methods
-  async entries(prefix = "") {
+  async *iterate(prefix = "") {
     const data = await this.#read();
-    return Object.entries(data).filter((p) => p[0].startsWith(prefix));
+    const entries = Object.entries(data).filter((p) => p[0].startsWith(prefix));
+    for (const entry of entries) {
+      yield entry;
+    }
   }
 
   async clear(prefix = "") {

package/src/clients/forage.js

@@ -22,12 +22,18 @@ export default class Forage {
     return key;
   }
 
+  async *iterate(prefix = "") {
+    const keys = await this.client.keys();
+    const list = keys.filter((k) => k.startsWith(prefix));
+    for (const key of list) {
+      yield [key, await this.get(key)];
+    }
+  }
+
   async entries(prefix = "") {
     const all = await this.client.keys();
     const keys = all.filter((k) => k.startsWith(prefix));
-    const values = await Promise.all(
-      keys.map((key) => this.client.getItem(key))
-    );
+    const values = await Promise.all(keys.map((key) => this.get(key)));
     return keys.map((key, i) => [key, values[i]]);
   }
 

package/src/clients/level.js

@@ -26,6 +26,14 @@ export default class Level {
     return this.client.del(key);
   }
 
+  async *iterate(prefix = "") {
+    const keys = await this.client.keys().all();
+    const list = keys.filter((k) => k.startsWith(prefix));
+    for (const key of list) {
+      yield [key, await this.get(key)];
+    }
+  }
+
   async entries(prefix = "") {
     const keys = await this.client.keys().all();
     const list = keys.filter((k) => k.startsWith(prefix));

package/src/clients/memory.js

@@ -21,6 +21,14 @@ export default class Memory {
     this.client.delete(key);
   }
 
+  *iterate(prefix = "") {
+    const entries = this.entries();
+    for (const entry of entries) {
+      if (!entry[0].startsWith(prefix)) continue;
+      yield entry;
+    }
+  }
+
   // Group methods
   entries(prefix = "") {
     const entries = [...this.client.entries()];

package/src/clients/redis.js

@@ -37,12 +37,33 @@ export default class Redis {
     return this.client.keys(prefix + "*");
   }
 
+  // Go through each of the [key, value] in the set
+  async *iterate(prefix = "") {
+    const MATCH = prefix + "*";
+    for await (const key of this.client.scanIterator({ MATCH })) {
+      const value = await this.get(key);
+      yield [key, value];
+    }
+  }
+
+  // Optimizing the retrieval of them all in bulk by loading the values
+  // in parallel
   async entries(prefix = "") {
-    const keys = await this.client.keys(prefix + "*");
+    const keys = await this.keys(prefix);
     const values = await Promise.all(keys.map((k) => this.get(k)));
     return keys.map((k, i) => [k, values[i]]);
   }
 
+  // Optimizing the retrieval of them by not getting their values
+  async keys(prefix = "") {
+    const MATCH = prefix + "*";
+    const keys = [];
+    for await (const key of this.client.scanIterator({ MATCH })) {
+      keys.push(key);
+    }
+    return keys;
+  }
+
   async clear(prefix = "") {
     if (!prefix) return this.client.flushAll();
 

package/src/clients/storage.js

@@ -25,6 +25,13 @@ export default class WebStorage {
     return key;
   }
 
+  *iterate(prefix = "") {
+    const entries = this.entries(prefix);
+    for (const entry of entries) {
+      yield entry;
+    }
+  }
+
   // Group methods
   entries(prefix = "") {
     const entries = Object.entries(this.client);

package/src/index.js

@@ -43,9 +43,9 @@ class Store {
 
   // #region #validate()
   #validate(client) {
-    if (!client.set || !client.get || !client.entries) {
+    if (!client.set || !client.get || !client.iterate) {
       throw new Error(
-        "A client should have at least a .get(), .set() and .entries()"
+        "A client should have at least a .get(), .set() and .iterate()"
       );
     }
 
@@ -68,6 +68,31 @@ class Store {
     }
   }
 
+  #unix(expires) {
+    const now = new Date().getTime();
+    return expires === null ? null : now + expires * 1000;
+  }
+
+  // Check if the given data is fresh or not; if
+  #isFresh(data, key) {
+    // Should never happen, but COULD happen; schedule it for
+    // removal and mark it as stale
+    if (!data || !data.value || typeof data !== "object") {
+      if (key) this.del(key);
+      return false;
+    }
+
+    // It never expires, so keep it
+    if (data.expires === null) return true;
+
+    // It's fresh, keep it
+    if (data.expires > Date.now()) return true;
+
+    // It's expired, remove it
+    if (key) this.del(key);
+    return false;
+  }
+
   // #region .add()
   /**
    * Save the data on an autogenerated key, can add expiration as well:
@@ -94,9 +119,10 @@ class Store {
       }
 
       // In the data we need the timestamp since we need it "absolute":
-      const now = new Date().getTime();
-      const expDiff = expires === null ? null : now + expires * 1000;
-      return this.client.add(this.PREFIX, { expires: expDiff, value });
+      return this.client.add(this.PREFIX, {
+        expires: this.#unix(expires),
+        value,
+      });
     }
 
     const id = createId();
@@ -126,7 +152,7 @@ class Store {
     const expires = parse(options.expire ?? options.expires);
 
     // Quick delete
-    if (value === null) {
+    if (value === null || (typeof expires === "number" && expires <= 0)) {
       await this.del(id);
       return key;
     }
@@ -137,16 +163,8 @@ class Store {
       return key;
     }
 
-    // Already expired, then delete it
-    if (expires === 0) {
-      await this.del(id);
-      return key;
-    }
-
     // In the data we need the timestamp since we need it "absolute":
-    const now = new Date().getTime();
-    const expDiff = expires === null ? null : now + expires * 1000;
-    await this.client.set(id, { expires: expDiff, value });
+    await this.client.set(id, { expires: this.#unix(expires), value });
     return key;
   }
 
@@ -180,23 +198,8 @@ class Store {
     // so we can assume it's the raw user data
     if (this.client.EXPIRES) return data;
 
-    // Make sure that if there's no data by now, empty is returned
-    if (!data) return null;
-
-    // We manage expiration manually, so we know it should have this structure
-    // TODO: ADD A CHECK HERE
-    const { expires, value } = data;
-
-    // It never expires
-    if (expires === null) return value ?? null;
-
-    // Already expired! Return nothing, and remove the whole key
-    if (expires <= new Date().getTime()) {
-      await this.del(key);
-      return null;
-    }
-
-    return value;
+    if (!this.#isFresh(data, key)) return null;
+    return data.value;
   }
 
   // #region .has()
@@ -256,6 +259,19 @@ class Store {
     return key;
   }
 
+  async *[Symbol.asyncIterator]() {
+    await this.promise;
+
+    for await (const [name, data] of this.client.iterate(this.PREFIX)) {
+      const key = name.slice(this.PREFIX.length);
+      if (this.client.EXPIRES) {
+        yield [key, data];
+      } else if (this.#isFresh(data, key)) {
+        yield [key, data.value];
+      }
+    }
+  }
+
   // #region .entries()
   /**
    * Return an array of the entries, in the [key, value] format:
@@ -274,36 +290,25 @@ class Store {
   async entries() {
     await this.promise;
 
-    const entries = await this.client.entries(this.PREFIX);
-    const list = entries.map(([key, data]) => [
-      key.slice(this.PREFIX.length),
-      data,
-    ]);
+    let list = [];
+    if (this.client.entries) {
+      list = (await this.client.entries(this.PREFIX)).map(([key, value]) => [
+        key.slice(this.PREFIX.length),
+        value,
+      ]);
+    } else {
+      for await (const [key, value] of this.client.iterate(this.PREFIX)) {
+        list.push([key.slice(this.PREFIX.length), value]);
+      }
+    }
 
     // The client already manages the expiration, so we can assume
     // that at this point, all entries are not-expired
     if (this.client.EXPIRES) return list;
 
     // We need to do manual expiration checking
-    const now = new Date().getTime();
     return list
-      .filter(([key, data]) => {
-        // Should never happen
-        if (!data || data.value === null) return false;
-
-        // It never expires, so keep it
-        const { expires } = data;
-        if (expires === null) return true;
-
-        // It's expired, so remove it
-        if (expires <= now) {
-          this.del(key);
-          return false;
-        }
-
-        // It's not expired, keep it
-        return true;
-      })
+      .filter(([key, data]) => this.#isFresh(data, key))
       .map(([key, data]) => [key, data.value]);
   }
 
@@ -356,23 +361,8 @@ class Store {
     if (this.client.values) {
       const list = this.client.values(this.PREFIX);
       if (this.client.EXPIRES) return list;
-      const now = new Date().getTime();
       return list
-        .filter((data) => {
-          // There's no data, so remove this
-          if (!data || data.value === null) return false;
-
-          // It never expires, so keep it
-          const { expires } = data;
-          if (expires === null) return true;
-
-          // It's expired, so remove it
-          // We cannot unfortunately evict it since we don't know the key!
-          if (expires <= now) return false;
-
-          // It's not expired, keep it
-          return true;
-        })
+        .filter((data) => this.#isFresh(data))
         .map((data) => data.value);
     }
 

package/src/index.test.js

@@ -11,8 +11,6 @@ import kv from "./index.js";
 import customFull from "./test/customFull.js";
 import customSimple from "./test/customSimple.js";
 
-global.setImmediate = global.setImmediate || ((cb) => setTimeout(cb, 0));
-
 const stores = [];
 stores.push(["kv()", kv()]);
 stores.push(["kv(new Map())", kv(new Map())]);
@@ -30,6 +28,7 @@ if (process.env.REDIS) {
   stores.push(["kv(redis)", kv(createClient())]);
 }
 if (process.env.ETCD) {
+  // Note: need to add to .env "ETCD=true" and run `etcd` in the terminal
   stores.push(["kv(new Etcd3())", kv(new Etcd3())]);
 }
 
@@ -41,7 +40,7 @@ const delay = (t) => new Promise((done) => setTimeout(done, t));
 class Base {
   get() {}
   set() {}
-  entries() {}
+  *iterate() {}
 }
 
 global.console = {
@@ -56,7 +55,7 @@ describe("potato", () => {
 
   it("an empty object is not a valid store", async () => {
     await expect(() => kv({}).get("any")).rejects.toThrow({
-      message: "A client should have at least a .get(), .set() and .entries()",
+      message: "A client should have at least a .get(), .set() and .iterate()",
     });
   });
 
@@ -343,6 +342,42 @@ for (let [name, store] of stores) {
       expect(await store.get("a")).toBe("b");
     });
 
+    describe("iteration", () => {
+      beforeEach(async () => {
+        await store.clear();
+      });
+
+      it("supports raw iteration", async () => {
+        await store.set("a", "b");
+        await store.set("c", "d");
+
+        const entries = [];
+        for await (const entry of store) {
+          entries.push(entry);
+        }
+        expect(entries).toEqual([
+          ["a", "b"],
+          ["c", "d"],
+        ]);
+      });
+
+      it("supports raw prefix iteration", async () => {
+        await store.set("a:a", "b");
+        await store.set("b:a", "d");
+        await store.set("a:c", "d");
+        await store.set("b:c", "d");
+
+        const entries = [];
+        for await (const entry of store.prefix("a:")) {
+          entries.push(entry);
+        }
+        expect(entries.sort()).toEqual([
+          ["a", "b"],
+          ["c", "d"],
+        ]);
+      });
+    });
+
     describe("expires", () => {
       // The mock implementation does NOT support expiration 😪
       if (name === "kv(new KVNamespace())") return;

package/src/test/customFull.js

@@ -19,6 +19,13 @@ export default class MyClient {
     delete dataSource[key];
   }
 
+  *iterate(prefix) {
+    const entries = this.entries(prefix);
+    for (const entry of entries) {
+      yield entry;
+    }
+  }
+
   // Filter them by the prefix, note that `prefix` will always be a string
   entries(prefix) {
     const entries = Object.entries(dataSource);

package/src/test/customSimple.js

@@ -15,9 +15,13 @@ export default class MyClient {
   }
 
   // Filter them by the prefix, note that `prefix` will always be a string
-  entries(prefix) {
-    const entries = Object.entries(dataSource);
-    if (!prefix) return entries;
-    return entries.filter(([key, value]) => key.startsWith(prefix));
+  *iterate(prefix) {
+    const raw = Object.entries(dataSource);
+    const entries = prefix
+      ? raw.filter(([key, value]) => key.startsWith(prefix))
+      : raw;
+    for (const entry of entries) {
+      yield entry;
+    }
   }
 }

package/src/test/setup.js

@@ -14,3 +14,10 @@ if (typeof TextDecoder === "undefined") {
     value: util.TextDecoder,
   });
 }
+
+if (typeof setImmediate === "undefined") {
+  Object.defineProperty(window, "setImmediate", {
+    writable: true,
+    value: (cb) => setTimeout(cb, 0),
+  });
+}