tapeworm_persistence_store_hybrid 1.0.1 → 1.0.2

package/dist/HybridPartition.d.ts

@@ -0,0 +1,22 @@
+import BluebirdPromise from 'bluebird';
+import { Commit } from 'tapeworm';
+import { LoggingOptions, Partition, queryStreamCallback, truncateStreamFromCallback } from './HybridPersistence';
+import { Snapshot } from './utils';
+declare class HybridPartition implements Partition {
+    private localPartition;
+    private remotePartition;
+    private snapshotLifeTime;
+    private loggingOptions;
+    constructor(localPartition: Partition, remotePartition: Partition, autoCleanLocalPartition: boolean, snapshotLifeTime: number, loggingOptions: LoggingOptions);
+    private cleanPartition;
+    open(): BluebirdPromise<this>;
+    append(commit: Commit<Record<string, any>>): BluebirdPromise<import("./utils").Commit<Record<string, any>>[]>;
+    storeSnapshot(streamId: string, snapshot: Snapshot, version: number): BluebirdPromise<void>;
+    private checkIsSnapshotToOld;
+    loadSnapshot(streamId: string, callback?: (err: Error, snapshot: Snapshot) => void): BluebirdPromise<Snapshot>;
+    queryStream(streamId: string, fromEventSequence?: number | queryStreamCallback, callback?: queryStreamCallback): BluebirdPromise<any[]>;
+    removeSnapshot(streamId: string): BluebirdPromise<void>;
+    truncateStreamFrom(streamId: string, commit: Commit<Record<string, any>>, remove: boolean | truncateStreamFromCallback, callback?: truncateStreamFromCallback): BluebirdPromise<void>;
+    applyCommitHeader(streamId: string, commit: Commit<Record<string, any>>, remove: any, callback: () => void): BluebirdPromise<void>;
+}
+export default HybridPartition;

package/dist/HybridPersistence.d.ts

@@ -0,0 +1,38 @@
+import BluebirdPromise from 'bluebird';
+import { Commit, Snapshot } from './utils';
+export declare type truncateStreamFromCallback = (error: Error, result: Record<string, any>) => void;
+declare type TruncateStreamFrom = (streamId: string, commitSequence: number, remove: boolean | truncateStreamFromCallback, callback?: truncateStreamFromCallback) => BluebirdPromise<void>;
+export declare type queryStreamCallback = (err: Error, steam: Commit<Record<string, any>>[]) => void;
+declare type QueryStream = (streamId: string, fromEventSequence?: number | queryStreamCallback, callback?: queryStreamCallback) => BluebirdPromise<Commit<Record<string, any>>[]>;
+export declare type Partition = {
+    open: () => BluebirdPromise<Partition>;
+    storeSnapshot: (streamId: string, snapshot: Snapshot['snapshot'], version: number) => BluebirdPromise<void>;
+    loadSnapshot: (streamId: string, callback?: (err: Error, snapshot: Snapshot | undefined) => void) => BluebirdPromise<Snapshot | undefined>;
+    queryStream: QueryStream;
+    append: (commit: Commit<Record<string, any>>) => BluebirdPromise<Commit<Record<string, any>>[]>;
+    removeSnapshot: (streamId: string) => BluebirdPromise<void>;
+    removeSnapshots?: (streamIds: string[]) => BluebirdPromise<void>;
+    truncateStreamFrom: TruncateStreamFrom;
+    applyCommitHeader: (streamId: string, commit: Commit<Record<string, any>>, remove: any, callback: () => void) => BluebirdPromise<void>;
+    querySnapshotsOlderThanMaxDate?: (dateTime: string) => BluebirdPromise<Snapshot[]>;
+};
+export interface LoggingOptions {
+    loggingEnabled: boolean;
+    loadSnapshotMaxTime?: number;
+}
+export declare type Persistence = {
+    openPartition: (partitionId?: string) => BluebirdPromise<Partition>;
+};
+declare class HybridPersistence {
+    private localPersistence;
+    private remotePersistence;
+    private autoCleanLocalPartition;
+    private snapshotLifeTime;
+    private loggingOptions;
+    private partitions;
+    constructor(localPersistence: Persistence, remotePersistence: Persistence, autoCleanLocalPartition: boolean, snapshotLifeTime: number, loggingOptions: LoggingOptions);
+    openPartition(partitionId?: string): BluebirdPromise<unknown>;
+    private getPartition;
+    private setPartition;
+}
+export default HybridPersistence;

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import HybridPartition from './HybridPartition';
+import HybridPersistence from './HybridPersistence';
+export { HybridPartition, HybridPersistence };

package/dist/utils.d.ts

@@ -0,0 +1,24 @@
+interface Event<Payload = Record<string, any>> {
+    type: string;
+    aggregateId: string;
+    correlationId: string;
+    payload: Payload;
+    id?: string;
+    dateTime?: string;
+    headers?: Record<string, any>;
+    version?: number;
+}
+export interface Commit<Payload> {
+    id: string;
+    partitionId: string;
+    streamId: string;
+    commitSequence: number;
+    events: Event<Payload>[];
+}
+export interface Snapshot {
+    id: string;
+    version: number;
+    storedDateTime?: string;
+    snapshot?: Record<string, any>;
+}
+export {};

package/package.json

@@ -1,39 +1,39 @@
-{
-  "name": "tapeworm_persistence_store_hybrid",
-  "version": "1.0.1",
-  "description": "",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "prettier": "prettier-config-surikaterna",
-  "scripts": {
-    "test": "jest",
-    "prepare": "npm test && npm run build",
-    "build": "tsc"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/surikaterna/tapeworm_persistence_store_hybrid.git"
-  },
-  "author": "Gustav Albertsson",
-  "license": "ISC",
-  "bugs": {
-    "url": "https://github.com/surikaterna/tapeworm_persistence_store_hybrid/issues"
-  },
-  "homepage": "https://github.com/surikaterna/tapeworm_persistence_store_hybrid#readme",
-  "devDependencies": {
-    "@types/bluebird": "^3.5.36",
-    "@types/jest": "^27.0.1",
-    "fake-indexeddb": "^3.1.3",
-    "jest": "^27.3.1",
-    "prettier": "^2.3.2",
-    "prettier-config-surikaterna": "^1.0.1",
-    "tapeworm": "^0.3.5",
-    "tapeworm_persistence_store_indexeddb": "^0.2.6",
-    "ts-jest": "^27.0.5",
-    "typescript": "^4.4.2"
-  },
-  "dependencies": {
-    "bluebird": "^3.7.2",
-    "slf": "^1.0.3"
-  }
-}
+{
+  "name": "tapeworm_persistence_store_hybrid",
+  "version": "1.0.2",
+  "description": "",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "prettier": "prettier-config-surikaterna",
+  "scripts": {
+    "test": "jest",
+    "prepare": "npm test && npm run build",
+    "build": "tsc"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/surikaterna/tapeworm_persistence_store_hybrid.git"
+  },
+  "author": "Gustav Albertsson",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/surikaterna/tapeworm_persistence_store_hybrid/issues"
+  },
+  "homepage": "https://github.com/surikaterna/tapeworm_persistence_store_hybrid#readme",
+  "devDependencies": {
+    "@types/bluebird": "^3.5.36",
+    "@types/jest": "^27.0.1",
+    "fake-indexeddb": "^3.1.3",
+    "jest": "^27.3.1",
+    "prettier": "^2.3.2",
+    "prettier-config-surikaterna": "^1.0.1",
+    "tapeworm": "^0.3.5",
+    "tapeworm_persistence_store_indexeddb": "^0.2.6",
+    "ts-jest": "^27.0.5",
+    "typescript": "^4.4.2"
+  },
+  "dependencies": {
+    "bluebird": "^3.7.2",
+    "slf": "^1.0.3"
+  }
+}

package/src/HybridPartition.ts

@@ -0,0 +1,162 @@
+import BluebirdPromise from 'bluebird';
+// @ts-ignore
+import { Commit } from 'tapeworm';
+import { LoggingOptions, Partition, queryStreamCallback, truncateStreamFromCallback } from './HybridPersistence';
+import { LoggerFactory } from 'slf';
+import { Snapshot } from './utils';
+
+const log = LoggerFactory.getLogger('tapeworm-persistence-hybrid:hybrid-partition');
+
+const DEFAULT_LOAD_SNAPSHOT_MAX_TIME = 10;
+
+class HybridPartition implements Partition {
+  private loggingOptions: LoggingOptions;
+
+  /**
+   * @param localPartition
+   * @param remotePartition
+   * @param autoCleanLocalPartition If set to true the partition will automatically clean out data older than dataLifeTime minutes dataLifeTime.
+   * @param snapshotLifeTime Number of minutes before data will be considered old.
+   * @param loggingOptions Options for logging
+   */
+  constructor(
+    private localPartition: Partition,
+    private remotePartition: Partition,
+    autoCleanLocalPartition: boolean,
+    private snapshotLifeTime: number,
+    loggingOptions: LoggingOptions
+  ) {
+    this.loggingOptions = loggingOptions;
+    if (!this.loggingOptions.loadSnapshotMaxTime) {
+      this.loggingOptions.loadSnapshotMaxTime = DEFAULT_LOAD_SNAPSHOT_MAX_TIME;
+    }
+
+    if (autoCleanLocalPartition) {
+      if (!localPartition.querySnapshotsOlderThanMaxDate) {
+        throw new Error('Unable to start auto clean of local partition since it does not support it.');
+      }
+
+      log.info('Starting auto clean of local partition');
+      setInterval(() => {
+        this.cleanPartition();
+      }, 1000 * 60 * 30); //Clean every 30 minutes
+    }
+  }
+
+  private cleanPartition() {
+    log.info('Auto cleaning partition');
+    const snapshotTTLDate = new Date();
+    snapshotTTLDate.setMinutes(snapshotTTLDate.getMinutes() - this.snapshotLifeTime);
+    this.localPartition.querySnapshotsOlderThanMaxDate(snapshotTTLDate.toISOString()).then((snapshots) => {
+      log.info('cleaning %s snapshots', snapshots.length);
+      const snapshotIdsToRemove = snapshots.map((snapshot) => snapshot.id);
+      this.localPartition.removeSnapshots(snapshotIdsToRemove);
+      snapshotIdsToRemove.forEach((snapshotId) => this.localPartition.truncateStreamFrom(snapshotId, 0, true));
+    });
+  }
+
+  open() {
+    return BluebirdPromise.resolve(this);
+  }
+
+  append(commit: Commit<Record<string, any>>) {
+    return this.localPartition.append(commit);
+  }
+
+  storeSnapshot(streamId: string, snapshot: Snapshot, version: number) {
+    return this.localPartition.storeSnapshot(streamId, snapshot, version);
+  }
+
+  private checkIsSnapshotToOld(snapshot?: Snapshot) {
+    if (!snapshot) {
+      return true;
+    }
+
+    if (snapshot.storedDateTime) {
+      const storedDateEpoch = new Date(snapshot.storedDateTime).getTime();
+      const nowEpoch = new Date().getTime();
+
+      const elapsedTimeInMilliSeconds = nowEpoch - storedDateEpoch;
+      const snapshotLifetimeMilliSeconds = this.snapshotLifeTime * 60 * 1000;
+
+      return elapsedTimeInMilliSeconds > snapshotLifetimeMilliSeconds;
+    }
+
+    return false;
+  }
+
+  loadSnapshot(streamId: string, callback?: (err: Error, snapshot: Snapshot) => void) {
+    const startTime = Date.now();
+    return new BluebirdPromise<Snapshot>((resolve) => {
+      this.localPartition.loadSnapshot(streamId).then((localSnapshot) => {
+        const isSnapshotToOld = this.checkIsSnapshotToOld(localSnapshot);
+        if (localSnapshot && !isSnapshotToOld) {
+          log.info('loadSnapshot: Using local snapshot');
+          callback?.(null, localSnapshot);
+          resolve(localSnapshot);
+          return;
+        }
+
+        this.remotePartition
+          .loadSnapshot(streamId)
+          .then((remoteSnapshot) => {
+            log.info('loadSnapshot: Using remote snapshot');
+            if (remoteSnapshot) {
+              this.localPartition.storeSnapshot(remoteSnapshot.id, remoteSnapshot.snapshot, remoteSnapshot.version);
+              this.localPartition.truncateStreamFrom(remoteSnapshot.id, 0, true);
+            }
+            callback?.(null, remoteSnapshot);
+            resolve(remoteSnapshot);
+          })
+          .catch((error) => {
+            log.warn('Failed to load snapshot with stream id %s from remote partition', streamId);
+          });
+      });
+    }).finally(() => {
+      const endTime = Date.now();
+      const queryTime = (endTime - startTime) / 1000;
+      if (this.loggingOptions.loggingEnabled && queryTime > this.loggingOptions.loadSnapshotMaxTime) {
+        log.warn('Loading snapshot with id %s, took %s seconds. Allowed max time is %s seconds.', streamId, queryTime, this.loggingOptions.loadSnapshotMaxTime);
+      }
+    });
+  }
+
+  queryStream(streamId: string, fromEventSequence?: number | queryStreamCallback, callback?: queryStreamCallback) {
+    log.debug('queryStream, streamId %s ,fromEventSequence %s', streamId, fromEventSequence);
+    return new BluebirdPromise<Commit<Record<string, any>>[]>((resolve) => {
+      this.localPartition.loadSnapshot(streamId).then((localSnapshot) => {
+        let currentPartition = this.localPartition;
+        const isSnapshotToOld = this.checkIsSnapshotToOld(localSnapshot);
+
+        if (!localSnapshot || isSnapshotToOld) {
+          currentPartition = this.remotePartition;
+        }
+
+        currentPartition.queryStream(streamId, fromEventSequence).then((queryStream) => {
+          callback?.(null, queryStream);
+          resolve(queryStream);
+          return;
+        });
+      });
+    });
+  }
+
+  removeSnapshot(streamId: string) {
+    return this.localPartition.removeSnapshot(streamId);
+  }
+
+  truncateStreamFrom(
+    streamId: string,
+    commit: Commit<Record<string, any>>,
+    remove: boolean | truncateStreamFromCallback,
+    callback?: truncateStreamFromCallback
+  ) {
+    return this.localPartition.truncateStreamFrom(streamId, commit, remove, callback);
+  }
+
+  applyCommitHeader(streamId: string, commit: Commit<Record<string, any>>, remove: any, callback: () => void) {
+    return this.localPartition.applyCommitHeader(streamId, commit, remove, callback);
+  }
+}
+
+export default HybridPartition;

package/src/HybridPersistence.ts

@@ -0,0 +1,82 @@
+import BluebirdPromise from 'bluebird';
+import HybridPartition from './HybridPartition';
+import { Commit, Snapshot } from './utils';
+
+export type truncateStreamFromCallback = (error: Error, result: Record<string, any>) => void;
+type TruncateStreamFrom = (
+  streamId: string,
+  commitSequence: number,
+  remove: boolean | truncateStreamFromCallback,
+  callback?: truncateStreamFromCallback
+) => BluebirdPromise<void>;
+
+export type queryStreamCallback = (err: Error, steam: Commit<Record<string, any>>[]) => void;
+type QueryStream = (
+  streamId: string,
+  fromEventSequence?: number | queryStreamCallback,
+  callback?: queryStreamCallback
+) => BluebirdPromise<Commit<Record<string, any>>[]>;
+
+export type Partition = {
+  open: () => BluebirdPromise<Partition>;
+  storeSnapshot: (streamId: string, snapshot: Snapshot['snapshot'], version: number) => BluebirdPromise<void>;
+  loadSnapshot: (streamId: string, callback?: (err: Error, snapshot: Snapshot | undefined) => void) => BluebirdPromise<Snapshot | undefined>;
+  queryStream: QueryStream;
+  append: (commit: Commit<Record<string, any>>) => BluebirdPromise<Commit<Record<string, any>>[]>;
+  removeSnapshot: (streamId: string) => BluebirdPromise<void>;
+  removeSnapshots?: (streamIds: string[]) => BluebirdPromise<void>;
+  truncateStreamFrom: TruncateStreamFrom;
+  applyCommitHeader: (streamId: string, commit: Commit<Record<string, any>>, remove: any, callback: () => void) => BluebirdPromise<void>;
+  querySnapshotsOlderThanMaxDate?: (dateTime: string) => BluebirdPromise<Snapshot[]>;
+};
+
+export interface LoggingOptions {
+  loggingEnabled: boolean;
+  loadSnapshotMaxTime?: number;
+}
+
+export type Persistence = {
+  openPartition: (partitionId?: string) => BluebirdPromise<Partition>;
+};
+
+const DEFAULT_PARTITION = 'master';
+
+class HybridPersistence {
+  private partitions: Record<string, Partition> = {};
+
+  constructor(
+    private localPersistence: Persistence,
+    private remotePersistence: Persistence,
+    private autoCleanLocalPartition: boolean,
+    private snapshotLifeTime: number,
+    private loggingOptions: LoggingOptions
+  ) {}
+
+  openPartition(partitionId: string = DEFAULT_PARTITION) {
+    return new BluebirdPromise(async (resolve) => {
+      const partition = this.getPartition(partitionId);
+      if (partition) {
+        resolve(partition);
+      } else {
+        const localPartition = await this.localPersistence.openPartition(partitionId);
+        const remotePartition = await this.remotePersistence.openPartition(partitionId);
+        const newPartition = new HybridPartition(localPartition, remotePartition, this.autoCleanLocalPartition, this.snapshotLifeTime, this.loggingOptions);
+        this.setPartition(partitionId, newPartition);
+        newPartition.open().then(() => {
+          resolve(newPartition);
+        });
+      }
+    });
+  }
+
+  private getPartition(partitionId: string = DEFAULT_PARTITION) {
+    return this.partitions[partitionId];
+  }
+
+  private setPartition(partitionId?: string, partition?: Partition) {
+    this.partitions[partitionId || DEFAULT_PARTITION] = partition;
+    return partition;
+  }
+}
+
+export default HybridPersistence;

package/src/index.ts

@@ -0,0 +1,4 @@
+import HybridPartition from './HybridPartition';
+import HybridPersistence from './HybridPersistence';
+
+export { HybridPartition, HybridPersistence };

package/src/utils.ts

@@ -0,0 +1,25 @@
+interface Event<Payload = Record<string, any>> {
+  type: string;
+  aggregateId: string;
+  correlationId: string;
+  payload: Payload;
+  id?: string;
+  dateTime?: string;
+  headers?: Record<string, any>;
+  version?: number;
+}
+
+export interface Commit<Payload> {
+  id: string;
+  partitionId: string;
+  streamId: string;
+  commitSequence: number;
+  events: Event<Payload>[];
+}
+
+export interface Snapshot {
+  id: string;
+  version: number;
+  storedDateTime?: string;
+  snapshot?: Record<string, any>;
+}