woonplan-packages-redishelper 2.0.0 → 2.0.3

package/src/__tests__/Broker.test.ts

@@ -1,223 +0,0 @@
-
-
-import Broker from '../classes/Broker'
-import Listener from '../classes/Listener'
-import { DecypheredMessage, RedisConfig, RollbarConfig } from '../types'
-import { stringifyMap } from "../services/utils"
-import Redis from "ioredis-mock"
-
-jest.mock( 'ioredis', () => require("ioredis-mock") )
-// jest.mock( '../classes/Listener' )
-
-const cb = () => {}
-
-
-const rollbarConfig:RollbarConfig = {
-  environment : '',
-  accessToken : ''
-}
-const redisConfig:RedisConfig = {
-  REDISURL : ''
-}
-
-const service = "testservice",
-      consumer = "testconsumer"
-
-xdescribe("addListener", () => {
-  afterEach(()=>jest.clearAllMocks())
-  it("should create a Listener class", () => {
-    new Broker( 
-      redisConfig,
-      rollbarConfig,
-      service,
-      consumer
-    ).addListener( '', cb )
-
-    expect( Listener ).toHaveBeenCalled( )
-  })
-  it("should be chainable", () => {
-    new Broker( 
-      redisConfig,
-      rollbarConfig,
-      service,
-      consumer
-    ).addListener( '', cb ).addListener( '', cb ).addListener( '', cb )
-
-    expect( Listener ).toHaveBeenCalledTimes( 3 )
-  })
-})
-
-describe("sanitize",()=>{
-  const broker = new Broker( redisConfig, rollbarConfig,
-    service,
-    consumer )
-  it("should return a json map if a map is given" ,() => {
-    expect( broker.sanitizeValue( new Map() ) ).toStrictEqual( stringifyMap(new Map()))
-  })
-  it("should return a number if a number is given" ,() => {
-    expect( broker.sanitizeValue( 1 )).toStrictEqual( 1 ) 
-  })
-  it("should return a string if a string is given" ,() => {
-    expect( broker.sanitizeValue( 'foo' ) ).toStrictEqual( 'foo' )
-  })
-  it("should return a JSON array if an array is given" ,() => {
-    expect( broker.sanitizeValue( ['foo'] ) ).toStrictEqual( JSON.stringify( ['foo']))
-  })
-  it("should return a JSON object if an object is given" ,() => {
-    expect( broker.sanitizeValue( {'foo':'bar'} ) ).toStrictEqual( JSON.stringify( {'foo':'bar'} ))
-  })
-})
-
-describe("createRedisMessage", () => {
-  const broker = new Broker( redisConfig, rollbarConfig,
-    service,
-    consumer )
-  it("should create a redis message array from an object ", () => {
-    expect( broker.createRedisMessage({
-      foo : 'bar',
-      bar : ['baz'],
-      hello : new Map( [['world','foo']])
-    })).toStrictEqual(
-      ['foo','bar','bar',JSON.stringify( ['baz']),'hello', stringifyMap(new Map( [['world','foo']]))]
-    )
-  })
-})
-
-jest.setTimeout( 2500 )
-
-describe("getRequest", () => {
-  it("should add a subscription" , async () => {
-    const broker = new Broker( redisConfig, rollbarConfig,
-      service,
-      consumer )
-    
-    broker.getRequest( 'foo', 'bar', {})
-    const mockresult = new Redis()
-    mockresult.publish( broker.subscriptions[0], 'baz')
-    expect( broker.subscriptions.length > 0 ).toBe( true )
-    await broker 
-  })
-  it("should return a response" , async () => {
-    const broker = new Broker( redisConfig, rollbarConfig,
-      service,
-      consumer )
-    
-    const result = broker.getRequest( 'foo', 'bar', {})
-
-    const mockresult = new Redis()
-    mockresult.publish( broker.subscriptions[0], 'baz')
-    expect( await result ).toStrictEqual( 'baz' )
-  })
-  it("should return a timeout and null if no response is given within timeout" , async () => {
-    const broker = new Broker( redisConfig, rollbarConfig,
-      service,
-      consumer )
-    
-    const result = broker.getRequest( 'foo', 'bar', {})
-    expect( await result ).toStrictEqual( null )
-    
-  })
-})
-
-// Unable to test this untill ioredis-mock has a xreadgroup
-xdescribe("requester",()=>{
-  it("should send a request and be able to receive", async () => {
-    const receiver = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-    
-    const requestEndpoint = jest.fn(() => {
-      console.log( 'requestendpoint hit')
-      return 'baz'
-    })
-    receiver.setRequestEndpoint( requestEndpoint )
-
-    const requester = new Broker( redisConfig, rollbarConfig, 'bar', consumer )
-
-    const request = await requester.getRequest( 'foo', 'key', {} )
-
-    expect( request ).toStrictEqual( 'baz' )
-
-
-  })
-})
-
-xdescribe( "getRequest", () => {
-  afterEach(() => jest.clearAllMocks())
-  it("should subscribe to a channel", async ()=>{
-    const spy = jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
-    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-    
-
-    jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
-      resolve()
-      clearTimeout( timeout )
-    })
-    broker.getRequest( 'foo', 'bar', {} )
-
-    expect( spy ).toBeCalled( )
-  })
-  it("should setup a message response", async ()=>{
-    jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
-    
-
-    const spy = jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
-      console.log( 1 )
-      resolve()
-      clearTimeout( timeout )
-    })
-
-    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-    broker.getRequest( 'foo', 'bar', {} )
-    expect( spy ).toBeCalled( )
-  })
-  it("should setup a timeout", async ()=>{
-    jest.spyOn( Broker.prototype, 'subscribe' ).mockResolvedValueOnce( null )
-
-    jest.spyOn( Broker.prototype, 'requestMessageResponse' ).mockImplementation((resolve, timeout) => {
-      resolve()
-      clearTimeout( timeout )
-    })
-
-    const spy = jest.spyOn( Broker.prototype, 'setupTimeout' ).mockImplementation((_resolve, channel) => {
-      return setTimeout(() => {
-        
-      }, 200);
-    })
-
-    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-    broker.getRequest( 'foo', 'bar', {} )
-    expect( spy ).toBeCalled( )
-  })
-})
-
-
-
-const responses = [['foo',[['bar',['id','baz']],['bar',['id','baz1']],['bar',['id','baz2']]]]]
-
-describe("filterStream", () => {
-  it("should filter the stream and only give back the messages with the request parameters", async () => {
-    jest.spyOn( Broker.prototype, 'getStreamInfo' ).mockResolvedValue( [0,1,2,3,4,5,6,7,8,responses] )
-    const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-
-    expect( await broker.filterStream( 'foo', 'id', 'baz') ).toStrictEqual([{
-      id : 'bar',
-      parameters : {
-        id : 'baz'
-      }
-
-    }])
-
-  })
-})
-
-describe("addListListener", () => {
-  
-  const broker = new Broker( redisConfig, rollbarConfig, 'foo', consumer )
-
-  const listcb = ( id:string ) => {
-    console.log( id )
-  }
-  const messagecb = ( message:DecypheredMessage ) => listcb( message.id ) 
-
-
-  broker.addListListener('measuredataschanged', messagecb )
-})

package/src/__tests__/Listener.test.ts

@@ -1,89 +0,0 @@
-import Broker from "../classes/Broker"
-import Listener from "../classes/Listener"
-import { RedisConfig, RollbarConfig } from "../types"
-
-const cb = jest.fn()
-
-
-const rollbarConfig:RollbarConfig = {
-  environment : '',
-  accessToken : ''
-}
-
-const redisConfig:RedisConfig = {
-  REDISURL : ''
-}
-
-const responses = [['foo',[['bar',['id','baz']]]]]
-
-jest.mock( 'ioredis', () => jest.fn().mockImplementation(() => {
-  return {
-    xread: jest.fn(( _a,_b,_c,_d,e) => {
-      if( e == 'bar' ) return null
-      return responses
-    }),
-    
-    xreadgroup: jest.fn(() => {
-      return null
-    })
-
-  }
-}) )
-
-describe("listenToStream", () => {
-  afterEach(() => jest.clearAllMocks())
-  it("should call xreadgroup if a groupname is given", async () => {
-    const broker = new Broker( 
-      redisConfig,
-      rollbarConfig
-    ).addListener( 'stream', cb, 'groupname' )
-    
-    await new Promise((r) => setTimeout(r, 100))
-    expect( broker.listeners.get( 'stream')?.client.xreadgroup ).toHaveBeenCalled()
-    expect( broker.listeners.get( 'stream')?.client.xread ).not.toHaveBeenCalled()
-  })
-  it("should call xread if a groupname is not given", async () => {
-    const broker = new Broker( 
-      redisConfig,
-      rollbarConfig
-    ).addListener( 'stream', cb )
-    
-    await new Promise((r) => setTimeout(r, 100))
-
-    expect( broker.listeners.get( 'stream')?.client.xreadgroup ).not.toHaveBeenCalled()
-    expect( broker.listeners.get( 'stream')?.client.xread ).toHaveBeenCalled()
-  })
-
-  it("should call the callback if a message comes in from the stream ", async ()=>{
-    
-    new Broker( 
-      redisConfig,
-      rollbarConfig
-    ).addListener( '', cb )
-    
-    await new Promise((r) => setTimeout(r, 100))
-
-    expect( cb ).toHaveBeenCalledWith( 'bar', {
-      id : 'baz'
-    })
-
-  })
-
-
-  it("should report an error", async () => {
-
-    const errormock = jest.fn( () => {
-      throw new Error('foo')
-    })
-
-    const spy = jest.spyOn( Broker.prototype, 'throwError')
-    new Broker( 
-      redisConfig,
-      rollbarConfig
-    ).addListener( 'stream', errormock )
-
-    await new Promise((r) => setTimeout(r, 100))
-
-    expect( spy ).toHaveBeenCalled( )
-  })
-})

package/src/classes/Broker.ts

@@ -1,225 +0,0 @@
-import { isArray, isObject } from "class-validator"
-import IORedis, { Redis } from "ioredis"
-import Rollbar from "rollbar"
-import { isMap } from "util/types"
-import { capitalizeFirstLetter, stringifyMap } from "../services/utils"
-import { RollbarConfig, RedisConfig, Struct, DecypheredParameters, StreamResponse, DecypheredResponse, DecypheredMessage } from "../types"
-import Listener from "./Listener"
-import ListListener from "./ListListener"
-import { v4 as uuidv4 } from 'uuid'
-
-export default class Broker{ 
-  redisConfig : RedisConfig
-  rollbar ?: Rollbar
-  consumername : string  = ''
-  listeners: Map<string,Listener> = new Map()
-  writer : Redis
-  reader : Redis
-  listprefix : string = 'listUpdated'
-  service: string 
-  subscriptions : string[] = []
-
-  requestendpoint ?: Function
-
-  constructor( redisConfig:RedisConfig, rollbarConfig:RollbarConfig, service:string, consumer : string  ){
-    this.redisConfig = redisConfig
-    this.rollbar = new Rollbar({
-      accessToken: rollbarConfig.accessToken,
-      environment: rollbarConfig.environment,
-    })
-    this.writer = new IORedis( redisConfig.REDISURL as string )
-    this.reader = new IORedis( redisConfig.REDISURL as string )
-    this.consumername = consumer
-    this.service = service
-  }
-
-  get requeststream(){
-    return this.getRequestStream( this.service )
-  }
-
-  getRequestStream( service:string ){
-    return `keyRequestedFrom${capitalizeFirstLetter(service)}Service`
-  }
-
-  setRequestEndpoint( callback:Function ){
-    return this.addListener( this.requeststream, this.getRequestCallback( callback ), this.service )
-  }
-
-  getRequestCallback( callback:Function ){
-    return async ( id:string, parameters:DecypheredParameters ) => {
-      const result = await callback( id, parameters )
-      if( !parameters.messageid ) return null
-      const channel = this.getRequestSubscriptionName(parameters.messageid)
-      return this.publish( channel, result )
-    }
-  }
-
-  publish( channel:string, result:string ){
-    return this.writer.publish( channel, result )
-  }
-
-  addListener( stream:string, callback:Function, group ?: string){
-    const client = new IORedis( this.redisConfig.REDISURL as string )
-    this.listeners.set( stream, new Listener( this, client, stream, callback, group ))
-    return this
-  }
-
-  addListListener( event:string, callback:Function ){
-    const client = new IORedis( this.redisConfig.REDISURL as string )
-    const channel = this.getListChannel( event )
-    this.listeners.set( channel, new ListListener( this, client, channel, callback ))
-
-    return this
-  }
-
-  throwError( error:Error ){
-    if(!this.rollbar) throw new Error('Rollbar not initialized')
-    this.rollbar.error( error )
-  }
-
-  sendMessage( stream:string, data:Struct){
-    return this.writer.xadd( stream, '*',  ...this.createRedisMessage( data ))
-  }
-
-  async getRequest( targetservice:string, key:string, data:Struct ){
-    // create a message id to subscribe to
-    const messageid  = uuidv4()
-
-    //subscribe to message response     
-    const channel = this.getRequestSubscriptionName(messageid)
-    await this.subscribe( channel )
-
-    // send the message to the correct service
-    this.sendMessage( this.getRequestStream(targetservice), {
-      request : key, 
-      messageid : messageid,
-      data : data 
-    })
-
-    let resolver 
-    // create a promise to be able to pass on to resolve later
-    const promise = new Promise((r) => {
-      resolver = r
-    })
-
-    if( !resolver ) return null 
-
-    //setup a timeout
-    const timeout = this.setupTimeout( resolver, channel )
-
-    //setup a response
-    this.requestMessageResponse( resolver, timeout )
-    
-    // return a promise that will resolve when the message returns or times out
-    return promise
-
-  }
-
-  requestMessageResponse( resolve:(value?: any) => void, timeout:NodeJS.Timeout  ){
-    this.reader.once( 'message', (channel,message) => {      
-      
-      if( message.length ) resolve( message );
-      else resolve( null )
-
-      this.unsubscribe( channel )
-      clearTimeout( timeout )
-
-    })
-  }
-
-  unsubscribe( channel:string ){
-    this.reader.unsubscribe( channel )
-    this.subscriptions = this.subscriptions.filter( s => s != channel )
-  }
-
-  setupTimeout( resolve:(value?: any) => void, channel:string, n:number = 2000 ){
-    return setTimeout( () => {
-      if( !this.subscriptions.includes( channel )) return
-
-      console.log( `sub timedout: ${channel}`)
-      resolve( null )
-      this.unsubscribe( channel )
-    }, n )
-  }
-
-  subscribe( channel:string ){
-    this.subscriptions.push( channel )
-    return this.reader.subscribe( channel )
-  }
-
-  getRequestSubscriptionName( messageid:string ){
-    return `messageresponse${messageid}`
-  }
-
-  async sendListEvent( event:string, listitems:any[], data:Struct = {} ){
-    const list = await this.addToList.call( this, listitems )
-    return this.sendMessage( 
-      this.getListChannel( event ) , {
-      ...data,
-      listname : list
-    } )
-  }
-
-  async addToList( listitems:any[], listname ?: string ){
-    const list = listname??uuidv4()
-    await this.writer.lpush( list , ...listitems.map( this.sanitizeValue) )
-    return list
-  }
-
-  getListChannel( event:string ){
-    return `${this.listprefix}${event}`
-  }
-
-  sanitizeValue( value:any ){
-    if( isMap( value )) return stringifyMap( value  )
-    if( isObject( value )) return JSON.stringify( value )
-    if( isArray( value )) return JSON.stringify( value )
-    return value
-  }
-
-  createRedisMessage( struct:Struct ){
-    return Object.keys( struct ).reduce( (arr:string[],key:keyof Struct) => [...arr,key,this.sanitizeValue( struct[key] )] , [] )
-  }
-
-  async getStreamMessages( stream:string ){
-    const streaminfo = await this.getStreamInfo.call( this, stream )
-    if( !streaminfo || !isArray( streaminfo ) || streaminfo.length < 10 ) return []
-    const streamresponses = streaminfo[9] as StreamResponse[]
-    return this.decypherResponse( ...streamresponses ).flatMap( r => r.messages )
-  }
-  
-  getStreamInfo( stream:string, count:number = 0 ){
-    return this.reader.xinfo('STREAM', stream ,'FULL', 'COUNT', count ) as Promise< null | any[]>
-  }
-
-  async filterStream( stream:string, key : string, value:any ){
-    const messages = await this.getStreamMessages.call( this, stream )
-    return messages.filter( message => message.parameters?.[key] != null && message.parameters[key] == value )
-  }
-
-  decypherResponse( ...responses:StreamResponse[] ):DecypheredResponse[]{    
-    return responses.reduce( (resp:DecypheredResponse[], response:StreamResponse) => 
-      [
-        ...resp,
-        {
-          stream : response[0],
-          messages : response[1].map( (message) => ({
-            id : message[0],
-            parameters : this.decypherParameters( message[1] )
-          }) as DecypheredMessage )
-        } as DecypheredResponse
-      ]
-    , [] as DecypheredResponse[] )  
-  }
-
-  
-  decypherParameters( parameters:string[] ):DecypheredParameters{
-    return parameters.reduce( ( params, v:string, n:number ) => (  n == 0 || ( n % 2 == 0)) && parameters.length >= n+1  ? ({
-      ...params,
-      [v] : parameters[n+1]
-    }) : params , {} as DecypheredParameters)
-  }
-
-}
-
-

package/src/classes/BrokerClient.ts

@@ -1,64 +0,0 @@
-import { Redis } from "ioredis"
-import { DecypheredMessage, DecypheredResponse, StreamResponse } from "../types"
-import Broker from "./Broker"
-
-export default abstract class BrokerClient{ 
-  broker : Broker
-  client : Redis  
-  group : string | null = null
-  stream : string | null = null
-  list : string | null = null
-  callback:Function
-
-  constructor( broker:Broker, client:Redis, callback:Function, list:string | null, stream:string | null = null, group: string | null = null   ){
-    this.broker = broker
-    this.client = client
-    this.list = list
-    this.stream = stream
-    this.group = group
-    this.callback = callback
-
-    this.listenToStream.call( this )
-  }
-
-  throwError( error:any ){
-    return this.broker.throwError( error )
-  }
-
-  decypherResponse( ...responses:StreamResponse[] ):DecypheredResponse[]{
-    return this.broker.decypherResponse( ...responses )
-  }
-  
-  async getGroupResponse():Promise< StreamResponse[] | null >{
-    if( !this.group || !this.stream ) return null
-    return await this.client.xreadgroup( 'GROUP', this.group, this.broker.consumername, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', this.stream, '>'  ) as StreamResponse[]
-  }
-
-  async getResponse( lastid:string ):Promise< StreamResponse[] | null >{
-    if( !this.stream ) return null
-    return await this.client.xread("BLOCK", 0, "STREAMS", this.stream, lastid )
-  }
-
-  
-  async listenToStream( lastid:string = '$'):Promise<Function | null>{
-    const responses = await this.getResponse.call( this, lastid ) 
-    if( !responses ) return null
-
-    const streamResponses = this.decypherResponse( ...responses )
-    const messages = streamResponses.flatMap( r => r.messages )
-    await Promise.all( messages.map( async (message) => {
-      try{
-        await this.streamCallback( message )
-
-      }catch( e:any ){
-        this.broker.throwError( e )
-      }finally{
-        
-      }
-    }) )
-    
-    return this.listenToStream.call( this, messages[messages.length-1]?.id??'$' )
-  }
-
-  abstract streamCallback( message:DecypheredMessage ):Promise<any | null>
-}

package/src/classes/ListListener.ts

@@ -1,20 +0,0 @@
-import { Redis } from "ioredis"
-import { DecypheredMessage } from "../types"
-import Broker from "./Broker"
-import BrokerClient from "./BrokerClient"
-import ListRunner from "./ListRunner"
-
-export default class Listener extends BrokerClient{
-  constructor( broker:Broker, client:Redis, list:string, callback:Function ){
-    super( broker, client, callback, list )
-  }  
-
-  async streamCallback( message:DecypheredMessage ){
-    const listrunnercallback = await this.callback( message.id, message.parameters )
-        
-    if( message.parameters.listname )
-      return new ListRunner( this.client, message.parameters.listname, listrunnercallback)
-
-    return null
-  }
-}

package/src/classes/ListRunner.ts

@@ -1,43 +0,0 @@
-import { Redis } from "ioredis"
-
-export default class ListRunner{
-  listname:string 
-  itemsDone:number = 0
-  itemsPerCall:number
-  callback:Function
-  client : Redis
-  
-  
-  constructor( client:Redis, listname:string, callback:Function, itemsPerCall : number = 1 ){
-    this.client = client
-    this.callback = callback
-    this.listname = listname
-    this.itemsPerCall = itemsPerCall
-    
-    this.run.call( this )
-  }  
-
-
-  async run():Promise<Function|void>{
-    const items = await this.nextitems.call( this, this.listname )
-    if( !items ){
-      console.log( `finished with ${this.itemsDone} jobs`)
-      return
-    }  
-
-    try{
-      await this.callback( items )
-    }catch( e:any ){
-      console.log( e )
-
-    }finally{
-      this.itemsDone += this.itemsPerCall
-      return this.run.call( this )
-    }
-  }
-
-  async nextitems( listname:string ):Promise<string[] | null>{
-    return this.client.lpop( listname, this.itemsPerCall )
-  }
-
-} 

package/src/classes/Listener.ts

@@ -1,14 +0,0 @@
-import { Redis } from "ioredis"
-import { DecypheredMessage } from "../types"
-import Broker from "./Broker"
-import BrokerClient from "./BrokerClient"
-
-export default class Listener extends BrokerClient{
-  constructor( broker:Broker, client:Redis, stream:string, callback:Function, group ?: string ){
-    super( broker, client, callback, null, stream, group )
-  }  
-
-  streamCallback(message: DecypheredMessage): Promise<any> {
-    return this.callback( message.id, message.parameters )
-  }
-}

package/src/classes/Requester.ts

@@ -1,5 +0,0 @@
-import Broker from "./Broker";
-
-export default class Requester extends Broker{
-  
-}