woonplan-packages-redishelper 1.0.93 → 2.0.1

package/.gitattributes

@@ -1,2 +1,2 @@
-# Auto detect text files and perform LF normalization
-* text=auto
+# Auto detect text files and perform LF normalization
+* text=auto

package/jest.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+roots: ['<rootDir>/src'
+],
+testMatch: [
+    "**/__tests__/**/*.+(ts|tsx|js)",
+    "**/?(*.)+(spec|test).+(ts|tsx|js)"
+],
+transform: {
+    "^.+\\.(ts|tsx)$": "ts-jest"
+},
+testTimeout: 500,
+globals: {
+    'ts-jest': {
+    isolatedModules: true
+    }
+},
+preset: 'ts-jest/presets/js-with-ts'
+}

package/package.json

@@ -1,25 +1,39 @@
 {
   "name": "woonplan-packages-redishelper",
-  "version": "1.0.93",
+  "version": "2.0.1",
   "description": "",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "scripts": {
-    "prepublish": "npm run build",
-    "build": "tsc",
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
+  "author": "Jasper Denk <jasper@e-trias.nl>",
+  "repository": "https://github.com/e-line-websolutions/woonplan-micro-energyio",
+  "license": "MIT",
   "keywords": [],
-  "author": "Jasper Denk",
-  "license": "ISC",
-  "devDependencies": {
-    "@types/node": "^16.11.13",
-    "@types/uuid": "^8.3.4",
-    "typescript": "^4.2.3"
+  "main": "./dist/tsc/main.js",
+  "types": "./dist/tsc/main.d.ts",
+  "scripts": {
+    "dev": "ts-node --esm src/main.ts",
+    "lint": "eslint src/ --ext .js,.jsx,.ts,.tsx",
+    "test": "jest",
+    "clean": "rm -rf dist build package node_modules package-lock.json && npm i && jest",
+    "ts-node": "ts-node",
+    "docs": "typedoc --entryPoints src/main.ts",
+    "build": "tsc -p tsconfig.json",
+    "start": "npm run build && node dist/tsc/main.js"
   },
   "dependencies": {
-    "@types/ioredis": "5.0.4",
-    "ioredis": "^5.0.4",
-    "uuid": "^8.3.2"
+    "@e-trias/woonplan": "1.0.82",
+    "@types/ioredis": "^4.28.8",
+    "@types/jest": "^27.4.0",
+    "@types/node": "^17.0.18",
+    "@types/pg": "^8.6.5",
+    "@typescript-eslint/eslint-plugin": "^5.12.0",
+    "@typescript-eslint/parser": "^5.12.0",
+    "class-validator": "^0.13.2",
+    "dotenv": "^16.0.0",
+    "ioredis": "^5.2.3",
+    "ioredis-mock": "^8.2.2",
+    "jest": "^27.5.1",
+    "rollbar": "^2.25.1",
+    "ts-jest": "^27.1.3",
+    "ts-node": "^10.5.0",
+    "typescript": "^4.5.5"
   }
 }

package/src/__tests__/Broker.test.ts

@@ -0,0 +1,223 @@
+
+
+import Broker from '../classes/Broker'
+import Listener from '../classes/Listener'
+import { DecypheredMessage, RedisConfig, RollbarConfig } from '../types'
+import { stringifyMap } from "../services/utils"
+import Redis from "ioredis-mock"
+
+jest.mock( 'ioredis', () => require("ioredis-mock") )
+// jest.mock( '../classes/Listener' )
+
+const cb = () => {}
+
+
+const rollbarConfig:RollbarConfig = {
+  environment : '',
+  accessToken : ''
+}
+const redisConfig:RedisConfig = {
+  REDISURL : ''
+}
+
+const service = "testservice",
+      consumer = "testconsumer"
+
+xdescribe("addListener", () => {
+  afterEach(()=>jest.clearAllMocks())
+  it("should create a Listener class", () => {
+    new Broker( 
+      redisConfig,
+      rollbarConfig,
+      service,
+      consumer
+    ).addListener( '', cb )
+
+    expect( Listener ).toHaveBeenCalled( )
+  })
+  it("should be chainable", () => {
+    new Broker( 
+      redisConfig,
+      rollbarConfig,
+      service,
+      consumer
+    ).addListener( '', cb ).addListener( '', cb ).addListener( '', cb )
+
+    expect( Listener ).toHaveBeenCalledTimes( 3 )
+  })
+})
+
+describe("sanitize",()=>{
+  const broker = new Broker( redisConfig, rollbarConfig,
+    service,
+    consumer )
+  it("should return a json map if a map is given" ,() => {
+    expect( broker.sanitizeValue( new Map() ) ).toStrictEqual( stringifyMap(new Map()))
+  })
+  it("should return a number if a number is given" ,() => {
+    expect( broker.sanitizeValue( 1 )).toStrictEqual( 1 ) 
+  })
+  it("should return a string if a string is given" ,() => {
+    expect( broker.sanitizeValue( 'foo' ) ).toStrictEqual( 'foo' )
+  })
+  it("should return a JSON array if an array is given" ,() => {
+    expect( broker.sanitizeValue( ['foo'] ) ).toStrictEqual( JSON.stringify( ['foo']))
+  })
+  it("should return a JSON object if an object is given" ,() => {
+    expect( broker.sanitizeValue( {'foo':'bar'} ) ).toStrictEqual( JSON.stringify( {'foo':'bar'} ))
+  })
+})
+
+describe("createRedisMessage", () => {
+  const broker = new Broker( redisConfig, rollbarConfig,
+    service,
+    consumer )
+  it("should create a redis message array from an object ", () => {
+    expect( broker.createRedisMessage({
+      foo : 'bar',
+      bar : ['baz'],
+      hello : new Map( [['world','foo']])
+    })).toStrictEqual(
+      ['foo','bar','bar',JSON.stringify( ['baz']),'hello', stringifyMap(new Map( [['world','foo']]))]
+    )
+  })
+})
+
+jest.setTimeout( 2500 )
+
+describe("getRequest", () => {
+  it("should add a subscription" , async () => {
+    const broker = new Broker( redisConfig, rollbarConfig,
+      service,
+      consumer )
+    
+    broker.getRequest( 'foo', 'bar', {})
+    const mockresult = new Redis()
+    mockresult.publish( broker.subscriptions[0], 'baz')
+    expect( broker.subscriptions.length > 0 ).toBe( true )
+    await broker 
+  })
+  it("should return a response" , async () => {
+    const broker = new Broker( redisConfig, rollbarConfig,
+      service,
+      consumer )
+    
+    const result = broker.getRequest( 'foo', 'bar', {})
+
+    const mockresult = new Redis()
+    mockresult.publish( broker.subscriptions[0], 'baz')
+    expect( await result ).toStrictEqual( 'baz' )
+  })
+  it("should return a timeout and null if no response is given within timeout" , async () => {
+    const broker = new Broker( redisConfig, rollbarConfig,
+      service,
+      consumer )
+    
+    const result = broker.getRequest( 'foo', 'bar', {})
+    expect( await result ).toStrictEqual( null )
+    
+  })
+})
+
+// Unable to test this untill ioredis-mock has a xreadgroup
+xdescribe("requester",()=>{
+  it("should send a request and be able to receive", async () => {
+    const receiver = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+    
+    const requestEndpoint = jest.fn(() => {
+      console.log( 'requestendpoint hit')
+      return 'baz'
+    })
+    receiver.setRequestEndpoint( requestEndpoint )
+
+    const requester = new Broker( redisConfig, rollbarConfig, 'bar', consumer )
+
+    const request = await requester.getRequest( 'foo', 'key', {} )
+
+    expect( request ).toStrictEqual( 'baz' )
+
+
+  })
+})
+
+xdescribe( "getRequest", () => {
+  afterEach(() => jest.clearAllMocks())
+  it("should subscribe to a channel", async ()=>{
+    const spy = jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
+    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+    
+
+    jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
+      resolve()
+      clearTimeout( timeout )
+    })
+    broker.getRequest( 'foo', 'bar', {} )
+
+    expect( spy ).toBeCalled( )
+  })
+  it("should setup a message response", async ()=>{
+    jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
+    
+
+    const spy = jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
+      console.log( 1 )
+      resolve()
+      clearTimeout( timeout )
+    })
+
+    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+    broker.getRequest( 'foo', 'bar', {} )
+    expect( spy ).toBeCalled( )
+  })
+  it("should setup a timeout", async ()=>{
+    jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
+
+    jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
+      resolve()
+      clearTimeout( timeout )
+    })
+
+    const spy = jest.spyOn( Broker.prototype, 'setupTimeout' ).mockImplementation((_resolve, channel) => {
+      return setTimeout(() => {
+        
+      }, 200);
+    })
+
+    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+    broker.getRequest( 'foo', 'bar', {} )
+    expect( spy ).toBeCalled( )
+  })
+})
+
+
+
+const responses = [['foo',[['bar',['id','baz']],['bar',['id','baz1']],['bar',['id','baz2']]]]]
+
+describe("filterStream", () => {
+  it("should filter the stream and only give back the messages with the request parameters", async () => {
+    jest.spyOn( Broker.prototype, 'getStreamInfo' ).mockResolvedValue( [0,1,2,3,4,5,6,7,8,responses] )
+    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+
+    expect( await broker.filterStream( 'foo', 'id', 'baz') ).toStrictEqual([{
+      id : 'bar',
+      parameters : {
+        id : 'baz'
+      }
+
+    }])
+
+  })
+})
+
+describe("addListListener", () => {
+  
+  const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
+
+  const listcb = ( id:string ) => {
+    console.log( id )
+  }
+  const messagecb = ( message:DecypheredMessage ) => listcb( message.id ) 
+
+
+  broker.addListListener('measuredataschanged', messagecb )
+})

package/src/__tests__/Listener.test.ts

@@ -0,0 +1,89 @@
+import Broker from "../classes/Broker"
+import Listener from "../classes/Listener"
+import { RedisConfig, RollbarConfig } from "../types"
+
+const cb = jest.fn()
+
+
+const rollbarConfig:RollbarConfig = {
+  environment : '',
+  accessToken : ''
+}
+
+const redisConfig:RedisConfig = {
+  REDISURL : ''
+}
+
+const responses = [['foo',[['bar',['id','baz']]]]]
+
+jest.mock( 'ioredis', () => jest.fn().mockImplementation(() => {
+  return {
+    xread: jest.fn(( _a,_b,_c,_d,e) => {
+      if( e == 'bar' ) return null
+      return responses
+    }),
+    
+    xreadgroup: jest.fn(() => {
+      return null
+    })
+
+  }
+}) )
+
+describe("listenToStream", () => {
+  afterEach(() => jest.clearAllMocks())
+  it("should call xreadgroup if a groupname is given", async () => {
+    const broker = new Broker( 
+      redisConfig,
+      rollbarConfig
+    ).addListener( 'stream', cb, 'groupname' )
+    
+    await new Promise((r) => setTimeout(r, 100))
+    expect( broker.listeners.get( 'stream')?.client.xreadgroup ).toHaveBeenCalled()
+    expect( broker.listeners.get( 'stream')?.client.xread ).not.toHaveBeenCalled()
+  })
+  it("should call xread if a groupname is not given", async () => {
+    const broker = new Broker( 
+      redisConfig,
+      rollbarConfig
+    ).addListener( 'stream', cb )
+    
+    await new Promise((r) => setTimeout(r, 100))
+
+    expect( broker.listeners.get( 'stream')?.client.xreadgroup ).not.toHaveBeenCalled()
+    expect( broker.listeners.get( 'stream')?.client.xread ).toHaveBeenCalled()
+  })
+
+  it("should call the callback if a message comes in from the stream ", async ()=>{
+    
+    new Broker( 
+      redisConfig,
+      rollbarConfig
+    ).addListener( '', cb )
+    
+    await new Promise((r) => setTimeout(r, 100))
+
+    expect( cb ).toHaveBeenCalledWith( 'bar', {
+      id : 'baz'
+    })
+
+  })
+
+
+  it("should report an error", async () => {
+
+    const errormock = jest.fn( () => {
+      throw new Error('foo')
+    })
+
+    const spy = jest.spyOn( Broker.prototype, 'throwError')
+    new Broker( 
+      redisConfig,
+      rollbarConfig
+    ).addListener( 'stream', errormock )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    expect( spy ).toHaveBeenCalled( )
+  })
+})

package/src/classes/Broker.ts

@@ -0,0 +1,224 @@
+import { isArray, isObject } from "class-validator"
+import IORedis, { Redis } from "ioredis"
+import Rollbar from "rollbar"
+import { isMap } from "util/types"
+import { capitalizeFirstLetter, stringifyMap } from "../services/utils"
+import Listener from "./Listener"
+import ListListener from "./ListListener"
+import { v4 as uuidv4 } from 'uuid'
+
+export default class Broker{ 
+  redisConfig : RedisConfig
+  rollbar ?: Rollbar
+  consumername : string  = ''
+  listeners: Map<string,Listener> = new Map()
+  writer : Redis
+  reader : Redis
+  listprefix : string = 'listUpdated'
+  service: string 
+  subscriptions : string[] = []
+
+  requestendpoint ?: Function
+
+  constructor( redisConfig:RedisConfig, rollbarConfig:RollbarConfig, service:string, consumer : string  ){
+    this.redisConfig = redisConfig
+    this.rollbar = new Rollbar({
+      accessToken: rollbarConfig.accessToken,
+      environment: rollbarConfig.environment,
+    })
+    this.writer = new IORedis( redisConfig.REDISURL as string )
+    this.reader = new IORedis( redisConfig.REDISURL as string )
+    this.consumername = consumer
+    this.service = service
+  }
+
+  get requeststream(){
+    return this.getRequestStream( this.service )
+  }
+
+  getRequestStream( service:string ){
+    return `keyRequestedFrom${capitalizeFirstLetter(service)}Service`
+  }
+
+  setRequestEndpoint( callback:Function ){
+    return this.addListener( this.requeststream, this.getRequestCallback( callback ), this.service )
+  }
+
+  getRequestCallback( callback:Function ){
+    return async ( id:string, parameters:DecypheredParameters ) => {
+      const result = await callback( id, parameters )
+      if( !parameters.messageid ) return null
+      const channel = this.getRequestSubscriptionName(parameters.messageid)
+      return this.publish( channel, result )
+    }
+  }
+
+  publish( channel:string, result:string ){
+    return this.writer.publish( channel, result )
+  }
+
+  addListener( stream:string, callback:Function, group ?: string){
+    const client = new IORedis( this.redisConfig.REDISURL as string )
+    this.listeners.set( stream, new Listener( this, client, stream, callback, group ))
+    return this
+  }
+
+  addListListener( event:string, callback:Function ){
+    const client = new IORedis( this.redisConfig.REDISURL as string )
+    const channel = this.getListChannel( event )
+    this.listeners.set( channel, new ListListener( this, client, channel, callback ))
+
+    return this
+  }
+
+  throwError( error:Error ){
+    if(!this.rollbar) throw new Error('Rollbar not initialized')
+    this.rollbar.error( error )
+  }
+
+  sendMessage( stream:string, data:Struct){
+    return this.writer.xadd( stream, '*',  ...this.createRedisMessage( data ))
+  }
+
+  async getRequest( targetservice:string, key:string, data:Struct ){
+    // create a message id to subscribe to
+    const messageid  = uuidv4()
+
+    //subscribe to message response     
+    const channel = this.getRequestSubscriptionName(messageid)
+    await this.subscribe( channel )
+
+    // send the message to the correct service
+    this.sendMessage( this.getRequestStream(targetservice), {
+      request : key, 
+      messageid : messageid,
+      data : data 
+    })
+
+    let resolver 
+    // create a promise to be able to pass on to resolve later
+    const promise = new Promise((r) => {
+      resolver = r
+    })
+
+    if( !resolver ) return null 
+
+    //setup a timeout
+    const timeout = this.setupTimeout( resolver, channel )
+
+    //setup a response
+    this.requestMessageResponse( resolver, timeout )
+    
+    // return a promise that will resolve when the message returns or times out
+    return promise
+
+  }
+
+  requestMessageResponse( resolve:(value?: any) => void, timeout:NodeJS.Timeout  ){
+    this.reader.once( 'message', (channel,message) => {      
+      
+      if( message.length ) resolve( message );
+      else resolve( null )
+
+      this.unsubscribe( channel )
+      clearTimeout( timeout )
+
+    })
+  }
+
+  unsubscribe( channel:string ){
+    this.reader.unsubscribe( channel )
+    this.subscriptions = this.subscriptions.filter( s => s != channel )
+  }
+
+  setupTimeout( resolve:(value?: any) => void, channel:string, n:number = 2000 ){
+    return setTimeout( () => {
+      if( !this.subscriptions.includes( channel )) return
+
+      console.log( `sub timedout: ${channel}`)
+      resolve( null )
+      this.unsubscribe( channel )
+    }, n )
+  }
+
+  subscribe( channel:string ){
+    this.subscriptions.push( channel )
+    return this.reader.subscribe( channel )
+  }
+
+  getRequestSubscriptionName( messageid:string ){
+    return `messageresponse${messageid}`
+  }
+
+  async sendListEvent( event:string, listitems:any[], data:Struct = {} ){
+    const list = await this.addToList.call( this, listitems )
+    return this.sendMessage( 
+      this.getListChannel( event ) , {
+      ...data,
+      listname : list
+    } )
+  }
+
+  async addToList( listitems:any[], listname ?: string ){
+    const list = listname??uuidv4()
+    await this.writer.lpush( list , ...listitems.map( this.sanitizeValue) )
+    return list
+  }
+
+  getListChannel( event:string ){
+    return `${this.listprefix}${event}`
+  }
+
+  sanitizeValue( value:any ){
+    if( isMap( value )) return stringifyMap( value  )
+    if( isObject( value )) return JSON.stringify( value )
+    if( isArray( value )) return JSON.stringify( value )
+    return value
+  }
+
+  createRedisMessage( struct:Struct ){
+    return Object.keys( struct ).reduce( (arr:string[],key:keyof Struct) => [...arr,key,this.sanitizeValue( struct[key] )] , [] )
+  }
+
+  async getStreamMessages( stream:string ){
+    const streaminfo = await this.getStreamInfo.call( this, stream )
+    if( !streaminfo || !isArray( streaminfo ) || streaminfo.length < 10 ) return []
+    const streamresponses = streaminfo[9] as StreamResponse[]
+    return this.decypherResponse( ...streamresponses ).flatMap( r => r.messages )
+  }
+  
+  getStreamInfo( stream:string, count:number = 0 ){
+    return this.reader.xinfo('STREAM', stream ,'FULL', 'COUNT', count ) as Promise< null | any[]>
+  }
+
+  async filterStream( stream:string, key : string, value:any ){
+    const messages = await this.getStreamMessages.call( this, stream )
+    return messages.filter( message => message.parameters?.[key] != null && message.parameters[key] == value )
+  }
+
+  decypherResponse( ...responses:StreamResponse[] ):DecypheredResponse[]{    
+    return responses.reduce( (resp:DecypheredResponse[], response:StreamResponse) => 
+      [
+        ...resp,
+        {
+          stream : response[0],
+          messages : response[1].map( (message) => ({
+            id : message[0],
+            parameters : this.decypherParameters( message[1] )
+          }) as DecypheredMessage )
+        } as DecypheredResponse
+      ]
+    , [] as DecypheredResponse[] )  
+  }
+
+  
+  decypherParameters( parameters:string[] ):DecypheredParameters{
+    return parameters.reduce( ( params, v:string, n:number ) => (  n == 0 || ( n % 2 == 0)) && parameters.length >= n+1  ? ({
+      ...params,
+      [v] : parameters[n+1]
+    }) : params , {} as DecypheredParameters)
+  }
+
+}
+
+

package/src/classes/BrokerClient.ts

@@ -0,0 +1,63 @@
+import { Redis } from "ioredis"
+import Broker from "./Broker"
+
+export default abstract class BrokerClient{ 
+  broker : Broker
+  client : Redis  
+  group : string | null = null
+  stream : string | null = null
+  list : string | null = null
+  callback:Function
+
+  constructor( broker:Broker, client:Redis, callback:Function, list:string | null, stream:string | null = null, group: string | null = null   ){
+    this.broker = broker
+    this.client = client
+    this.list = list
+    this.stream = stream
+    this.group = group
+    this.callback = callback
+
+    this.listenToStream.call( this )
+  }
+
+  throwError( error:any ){
+    return this.broker.throwError( error )
+  }
+
+  decypherResponse( ...responses:StreamResponse[] ):DecypheredResponse[]{
+    return this.broker.decypherResponse( ...responses )
+  }
+  
+  async getGroupResponse():Promise< StreamResponse[] | null >{
+    if( !this.group || !this.stream ) return null
+    return await this.client.xreadgroup( 'GROUP', this.group, this.broker.consumername, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', this.stream, '>'  ) as StreamResponse[]
+  }
+
+  async getResponse( lastid:string ):Promise< StreamResponse[] | null >{
+    if( !this.stream ) return null
+    return await this.client.xread("BLOCK", 0, "STREAMS", this.stream, lastid )
+  }
+
+  
+  async listenToStream( lastid:string = '$'):Promise<Function | null>{
+    const responses = await this.getResponse.call( this, lastid ) 
+    if( !responses ) return null
+
+    const streamResponses = this.decypherResponse( ...responses )
+    const messages = streamResponses.flatMap( r => r.messages )
+    await Promise.all( messages.map( async (message) => {
+      try{
+        await this.streamCallback( message )
+
+      }catch( e:any ){
+        this.broker.throwError( e )
+      }finally{
+        
+      }
+    }) )
+    
+    return this.listenToStream.call( this, messages[messages.length-1]?.id??'$' )
+  }
+
+  abstract streamCallback( message:DecypheredMessage ):Promise<any | null>
+}